<commit_before>package namedwebsockets

import (
	"fmt"
	"log"
	"math/rand"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/gorilla/websocket"
	"github.com/richtr/mdns"
)

// Regular expression matchers

var NetworkServiceMatcher = regexp.MustCompile("^([A-Za-z0-9\\._-]{1,255})\\[[0-9]+\\]( \\([0-9]+\\))?$")

/** DISCOVERYCLIENT interface **/

type DiscoveryClient struct {
	serviceType string
	Port        int
	server      *mdns.Server
}

func NewDiscoveryClient(service *NamedWebSocket_Service, serviceType string, port int) *DiscoveryClient {
	discoveryClient := &DiscoveryClient{
		serviceType: serviceType,
		Port:        port,
	}

	discoveryClient.Register(service, "local")

	return discoveryClient
}

func (dc *DiscoveryClient) Register(service *NamedWebSocket_Service, domain string) {

	rand.Seed(time.Now().UTC().UnixNano())

	dnssdServiceName := fmt.Sprintf("%s[%d]", dc.serviceType, rand.Int())

	s := &mdns.MDNSService{
		Instance: dnssdServiceName,
		Service:  "_ws._tcp",
		Domain:   domain,
		Port:     dc.Port,
		Info:     fmt.Sprintf("path=/network/%s", dc.serviceType),
	}
	if err := s.Init(); err != nil {
		log.Fatalf("err: %v", err)
	}

	serv, err := mdns.NewServer(&mdns.Config{Zone: s})
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	dc.server = serv

	service.advertisedServiceNames[dnssdServiceName] = true

	log.Printf("Network websocket advertised as '%s' in %s network", fmt.Sprintf("%s._ws._tcp", dnssdServiceName), domain)
}

func (dc *DiscoveryClient) Shutdown() {
	if dc.server != nil {
		dc.server.Shutdown()
	}
}

/** DISCOVERYSERVER interface **/

type DiscoveryServer struct {
	Host   string
	Port   int
	closed bool
}

func (ds *DiscoveryServer) Browse(service *NamedWebSocket_Service) {

	entries := make(chan *mdns.ServiceEntry, 255)

	timeout := 20 * time.Second

	params := &mdns.QueryParam{
		Service: "_ws._tcp",
		Domain:  "local",
		Timeout: timeout,
		Entries: entries,
	}

	go func() {
		complete := false
		finish := time.After(timeout)

		// Wait for responses until timeout
		for !complete {
			select {
			case e, ok := <-entries:

				if !ok {
					continue
				}

				nameComponents := strings.Split(e.Name, ".")
				shortName := ""

				for i := len(nameComponents) - 1; i >= 0; i-- {
					if nameComponents[i] == "_ws" {
						shortName = strings.Join(nameComponents[:i], ".")
						break
					}
				}

				// DEBUG
				//log.Printf("Found proxy web socket [%s] @ [%s:%d] TXT[%s]", shortName, e.Host, e.Port, e.Info)

				// Is this a NetworkWebSocket service?
				if isValid := NetworkServiceMatcher.MatchString(shortName); !isValid {
					continue
				}

				// Ignore our own NetworkWebSocket services
				if isOwned := service.advertisedServiceNames[shortName]; isOwned {
					continue
				}

				// Ignore previously discovered NetworkWebSocket services
				if isRegistered := service.registeredServiceNames[shortName]; isRegistered {
					continue
				}

				// Build websocket data from returned information
				servicePath := "/"
				serviceParts := strings.FieldsFunc(e.Info, func(r rune) bool {
					return r == '=' || r == ',' || r == ';' || r == ' '
				})
				if len(serviceParts) > 1 {
					for i := 0; i < len(serviceParts); i += 2 {
						if strings.ToLower(serviceParts[i]) == "path" {
							servicePath = serviceParts[i+1]
							break
						}
					}
				}

				// Generate unique id for connection
				rand.Seed(time.Now().UTC().UnixNano())
				newPeerId := rand.Int()

				serviceName := path.Base(servicePath)

				// Resolve websocket connection
				sock := service.namedWebSockets[servicePath]
				if sock == nil {
					sock = NewNamedWebSocket(service, serviceName, false, ds.Port)
					service.namedWebSockets[servicePath] = sock
				}

				hosts := [...]string{e.Host, e.AddrV4.String(), e.AddrV6.String()}

				for i := 0; i < len(hosts); i++ {

					// Build URL
					remoteWSUrl := &url.URL{
						Scheme: "ws",
						Host:   fmt.Sprintf("%s:%d", hosts[i], e.Port),
						Path:   fmt.Sprintf("%s/%d", servicePath, newPeerId),
					}

					log.Printf("Establishing proxy network websocket connection to ws://%s%s", remoteWSUrl.Host, remoteWSUrl.Path)

					ws, _, nErr := websocket.DefaultDialer.Dial(remoteWSUrl.String(), map[string][]string{
						"Origin":                   []string{ds.Host},
						"X-NetworkWebSocket-Proxy": []string{"true"},
					})
					if nErr != nil {
						log.Printf("Proxy network websocket connection failed: %s", nErr)
						continue
					}

					proxyConn := NewProxyConnection(newPeerId, ws, false)

					proxyConn.addConnection(sock)

					service.registeredServiceNames[shortName] = true

					break

				}

			case <-finish:
				complete = true
			}
		}
	}()

	// Run the mDNS query
	err := mdns.Query(params)
	if err != nil {
		log.Fatalf("err: %v", err)
	}
}

func (ds *DiscoveryServer) Shutdown() {
	ds.closed = true
}
<commit_msg>Revert "Do not advertise incoming network services"<commit_after>package namedwebsockets

import (
	"fmt"
	"log"
	"math/rand"
	"net/url"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/gorilla/websocket"
	"github.com/richtr/mdns"
)

// Regular expression matchers

var NetworkServiceMatcher = regexp.MustCompile("^([A-Za-z0-9\\._-]{1,255})\\[[0-9]+\\]( \\([0-9]+\\))?$")

/** DISCOVERYCLIENT interface **/

type DiscoveryClient struct {
	serviceType string
	Port        int
	server      *mdns.Server
}

func NewDiscoveryClient(service *NamedWebSocket_Service, serviceType string, port int) *DiscoveryClient {
	discoveryClient := &DiscoveryClient{
		serviceType: serviceType,
		Port:        port,
	}

	discoveryClient.Register(service, "local")

	return discoveryClient
}

func (dc *DiscoveryClient) Register(service *NamedWebSocket_Service, domain string) {

	rand.Seed(time.Now().UTC().UnixNano())

	dnssdServiceName := fmt.Sprintf("%s[%d]", dc.serviceType, rand.Int())

	s := &mdns.MDNSService{
		Instance: dnssdServiceName,
		Service:  "_ws._tcp",
		Domain:   domain,
		Port:     dc.Port,
		Info:     fmt.Sprintf("path=/network/%s", dc.serviceType),
	}
	if err := s.Init(); err != nil {
		log.Fatalf("err: %v", err)
	}

	serv, err := mdns.NewServer(&mdns.Config{Zone: s})
	if err != nil {
		log.Fatalf("err: %v", err)
	}

	dc.server = serv

	service.advertisedServiceNames[dnssdServiceName] = true

	log.Printf("Network websocket advertised as '%s' in %s network", fmt.Sprintf("%s._ws._tcp", dnssdServiceName), domain)
}

func (dc *DiscoveryClient) Shutdown() {
	if dc.server != nil {
		dc.server.Shutdown()
	}
}

/** DISCOVERYSERVER interface **/

type DiscoveryServer struct {
	Host   string
	Port   int
	closed bool
}

func (ds *DiscoveryServer) Browse(service *NamedWebSocket_Service) {

	entries := make(chan *mdns.ServiceEntry, 255)

	timeout := 20 * time.Second

	params := &mdns.QueryParam{
		Service: "_ws._tcp",
		Domain:  "local",
		Timeout: timeout,
		Entries: entries,
	}

	go func() {
		complete := false
		finish := time.After(timeout)

		// Wait for responses until timeout
		for !complete {
			select {
			case e, ok := <-entries:

				if !ok {
					continue
				}

				nameComponents := strings.Split(e.Name, ".")
				shortName := ""

				for i := len(nameComponents) - 1; i >= 0; i-- {
					if nameComponents[i] == "_ws" {
						shortName = strings.Join(nameComponents[:i], ".")
						break
					}
				}

				// DEBUG
				//log.Printf("Found proxy web socket [%s] @ [%s:%d] TXT[%s]", shortName, e.Host, e.Port, e.Info)

				// Is this a NetworkWebSocket service?
				if isValid := NetworkServiceMatcher.MatchString(shortName); !isValid {
					continue
				}

				// Ignore our own NetworkWebSocket services
				if isOwned := service.advertisedServiceNames[shortName]; isOwned {
					continue
				}

				// Ignore previously discovered NetworkWebSocket services
				if isRegistered := service.registeredServiceNames[shortName]; isRegistered {
					continue
				}

				// Build websocket data from returned information
				servicePath := "/"
				serviceParts := strings.FieldsFunc(e.Info, func(r rune) bool {
					return r == '=' || r == ',' || r == ';' || r == ' '
				})
				if len(serviceParts) > 1 {
					for i := 0; i < len(serviceParts); i += 2 {
						if strings.ToLower(serviceParts[i]) == "path" {
							servicePath = serviceParts[i+1]
							break
						}
					}
				}

				// Generate unique id for connection
				rand.Seed(time.Now().UTC().UnixNano())
				newPeerId := rand.Int()

				serviceName := path.Base(servicePath)

				// Resolve websocket connection
				sock := service.namedWebSockets[servicePath]
				if sock == nil {
					sock = NewNamedWebSocket(service, serviceName, true, ds.Port)
					service.namedWebSockets[servicePath] = sock
				}

				hosts := [...]string{e.Host, e.AddrV4.String(), e.AddrV6.String()}

				for i := 0; i < len(hosts); i++ {

					// Build URL
					remoteWSUrl := &url.URL{
						Scheme: "ws",
						Host:   fmt.Sprintf("%s:%d", hosts[i], e.Port),
						Path:   fmt.Sprintf("%s/%d", servicePath, newPeerId),
					}

					log.Printf("Establishing proxy network websocket connection to ws://%s%s", remoteWSUrl.Host, remoteWSUrl.Path)

					ws, _, nErr := websocket.DefaultDialer.Dial(remoteWSUrl.String(), map[string][]string{
						"Origin":                   []string{ds.Host},
						"X-NetworkWebSocket-Proxy": []string{"true"},
					})
					if nErr != nil {
						log.Printf("Proxy network websocket connection failed: %s", nErr)
						continue
					}

					proxyConn := NewProxyConnection(newPeerId, ws, false)

					proxyConn.addConnection(sock)

					service.registeredServiceNames[shortName] = true

					break

				}

			case <-finish:
				complete = true
			}
		}
	}()

	// Run the mDNS query
	err := mdns.Query(params)
	if err != nil {
		log.Fatalf("err: %v", err)
	}
}

func (ds *DiscoveryServer) Shutdown() {
	ds.closed = true
}
<|endoftext|>


<commit_before>package http

import (
	"io/ioutil"

	. "gopkg.in/check.v1"
	"gopkg.in/src-d/go-git.v4/clients/common"
	"gopkg.in/src-d/go-git.v4/core"
)

type RemoteSuite struct {
	Endpoint common.Endpoint
}

var _ = Suite(&RemoteSuite{})

func (s *RemoteSuite) SetUpSuite(c *C) {
	var err error
	s.Endpoint, err = common.NewEndpoint("https://github.com/tyba/git-fixture")
	c.Assert(err, IsNil)
}

func (s *RemoteSuite) TestConnect(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)
}

func (s *RemoteSuite) TestConnectWithAuth(c *C) {
	auth := &BasicAuth{}
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.ConnectWithAuth(auth), IsNil)
}

type mockAuth struct{}

func (*mockAuth) Name() string   { return "" }
func (*mockAuth) String() string { return "" }

func (s *RemoteSuite) TestConnectWithAuthWrongType(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.ConnectWithAuth(&mockAuth{}), Equals, common.ErrInvalidAuthMethod)
}

func (s *RemoteSuite) TestDefaultBranch(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	info, err := r.Info()
	c.Assert(err, IsNil)
	c.Assert(info.Capabilities.SymbolicReference("HEAD"), Equals, "refs/heads/master")
}

func (s *RemoteSuite) TestCapabilities(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	info, err := r.Info()
	c.Assert(err, IsNil)
	c.Assert(info.Capabilities.Get("agent").Values, HasLen, 1)
}

func (s *RemoteSuite) TestFetch(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	req := &common.GitUploadPackRequest{}
	req.Want(core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))

	reader, err := r.Fetch(req)
	c.Assert(err, IsNil)

	b, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Assert(b, HasLen, 85374)
}

func (s *RemoteSuite) TestFetchMulti(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	req := &common.GitUploadPackRequest{}
	req.Want(core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))
	req.Want(core.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"))

	reader, err := r.Fetch(req)
	c.Assert(err, IsNil)

	b, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Assert(len(b), HasLen, 85585)
}
<commit_msg>clients/http: fix tests<commit_after>package http

import (
	"io/ioutil"

	. "gopkg.in/check.v1"
	"gopkg.in/src-d/go-git.v4/clients/common"
	"gopkg.in/src-d/go-git.v4/core"
)

type RemoteSuite struct {
	Endpoint common.Endpoint
}

var _ = Suite(&RemoteSuite{})

func (s *RemoteSuite) SetUpSuite(c *C) {
	var err error
	s.Endpoint, err = common.NewEndpoint("https://github.com/tyba/git-fixture")
	c.Assert(err, IsNil)
}

func (s *RemoteSuite) TestConnect(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)
}

func (s *RemoteSuite) TestConnectWithAuth(c *C) {
	auth := &BasicAuth{}
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.ConnectWithAuth(auth), IsNil)
}

type mockAuth struct{}

func (*mockAuth) Name() string   { return "" }
func (*mockAuth) String() string { return "" }

func (s *RemoteSuite) TestConnectWithAuthWrongType(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.ConnectWithAuth(&mockAuth{}), Equals, common.ErrInvalidAuthMethod)
}

func (s *RemoteSuite) TestDefaultBranch(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	info, err := r.Info()
	c.Assert(err, IsNil)
	c.Assert(info.Capabilities.SymbolicReference("HEAD"), Equals, "refs/heads/master")
}

func (s *RemoteSuite) TestCapabilities(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	info, err := r.Info()
	c.Assert(err, IsNil)
	c.Assert(info.Capabilities.Get("agent").Values, HasLen, 1)
}

func (s *RemoteSuite) TestFetch(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	req := &common.GitUploadPackRequest{}
	req.Want(core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))

	reader, err := r.Fetch(req)
	c.Assert(err, IsNil)

	b, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Assert(b, HasLen, 85374)
}

func (s *RemoteSuite) TestFetchMulti(c *C) {
	r := NewGitUploadPackService(s.Endpoint)
	c.Assert(r.Connect(), IsNil)

	req := &common.GitUploadPackRequest{}
	req.Want(core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"))
	req.Want(core.NewHash("e8d3ffab552895c19b9fcf7aa264d277cde33881"))

	reader, err := r.Fetch(req)
	c.Assert(err, IsNil)

	b, err := ioutil.ReadAll(reader)
	c.Assert(err, IsNil)
	c.Assert(b, HasLen, 85585)
}
<|endoftext|>


<commit_before>package queryme

import (
	"fmt"
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

/*
predicates    = predicate *("," predicate)
predicate     = (not / and / or / eq / lt / le / gt / ge)
not           = "not" "(" predicate ")"
and           = "and" "(" predicates ")"
or            = "or" "(" predicates ")"
eq            = "eq" "(" field "," values ")"
lt            = "lt" "(" field "," value ")"
le            = "le" "(" field "," value ")"
gt            = "gt" "(" field "," value ")"
ge            = "ge" "(" field "," value ")"
fts           = "fts" "(" field "," string ")"

values        = value *("," value)
value         = (null / boolean / number / string / date)
null          = "null"
boolean       = "true" / "false"
number        = 1*(DIGIT / "." / "e" / "E" / "+" / "-")
string        = "'" *(unreserved / pct-encoded) "'"
date          = 4DIGIT "-" 2DIGIT "-" 2DIGIT *1("T" 2DIGIT ":" 2DIGIT ":" 2DIGIT *1("." 3DIGIT) "Z")

fieldorders   = *1(fieldorder *("," fieldorder))
fieldorder    = *1"!" field
field         = *(unreserved / pct-encoded)

unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
pct-encoded   = "%" HEXDIG HEXDIG
sub-delims    = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
query         = *( pchar / "/" / "?" )
*/

var (
	SortOrderSeparatorExpected error = errors.New("Expected seperator ',' after sorted order.")
	IdentifierExpected error = errors.New("Expected identifier.")
	ValueExpected error = errors.New("Expected value.")
	EndOfStringExpected error = errors.New("Expected end of string.")
	StringExpected error = errors.New("Expected string.")
	OperatorExpected error = errors.New("Expected operator.")
	UnexpectedEndOfPredicate error = errors.New("Unexpected end of predicate.")
	UnexpectedEndOfSortOrders error = errors.New("Unexpected end of sort orders.")

	characters []byte
)

func init() {
	characters = make([]byte, 128)

	characters[int('=')] = 1
	characters[int('&')] = 1

	characters[int('!')] = 2
	characters[int('$')] = 2
	characters[int('(')] = 2
	characters[int(')')] = 2
	characters[int('*')] = 2
	characters[int(',')] = 2
	characters[int(';')] = 2
	characters[int('/')] = 2
	characters[int('?')] = 2
	characters[int('@')] = 2

	characters[int('\'')] = 3
	characters[int('+')] = 3
	characters[int(':')] = 3

	// 'pct-encoded' characters
	characters[int('%')] = 4

	// 'unreserved' characters
	characters[int('-')] = 5
	characters[int('.')] = 5
	characters[int('_')] = 5
	characters[int('~')] = 5

	for i := int('0'); i <= int('9'); i++ {
		characters[i] = 5
	}

	for i := int('a'); i <= int('z'); i++ {
		characters[i] = 5
	}

	for i := int('A'); i <= int('Z'); i++ {
		characters[i] = 5
	}
}

func firstCharClass(s string) byte {
	r, _ := utf8.DecodeRuneInString(s)
	if r > 127 {
		return 0
	} else {
		return characters[r]
	}
}

func charClassDetector(min byte, max byte) func(r rune) bool {
	return func(r rune) bool {
		i := int(r)
		if i > 127 {
			return false
		}
		c := characters[i]
		return c >= min && c <= max
	}
}

// QueryString is a parsed query part of a URL.
type QueryString struct {
	fields map[string]string
}

// NewFromRawQuery creates a new QueryString from a raw query string.
func NewFromRawQuery(rawQuery string) *QueryString {
	qs := new(QueryString)
	qs.fields = make(map[string]string)

	for {
		i := strings.IndexRune(rawQuery, '=')
		if i == -1 {
			break
		}
		name := rawQuery[:i]
		rawQuery = rawQuery[i+1:]

		i = strings.IndexFunc(rawQuery, charClassDetector(1, 1))
		var value string
		if i == -1 {
			value = rawQuery
		} else {
			value = rawQuery[:i]
			rawQuery = rawQuery[i+1:]
		}

		qs.fields[name] = value

		if i == -1 {
			break
		}
	}

	return qs
}

// NewFromRawQuery creates a new QueryString from an existing URL object.
func NewFromURL(url *url.URL) *QueryString {
	return NewFromRawQuery(url.RawQuery)
}

// Tests if specified name has been found in query string.
func (q *QueryString) Contains(name string) bool {
	_, ok := q.fields[name]
	return ok
}

// Predicate parses the given component of the query as a predicate, then returns it.
func (q *QueryString) Predicate(name string) (p Predicate, err error) {
	defer func() {
		if rec := recover(); rec != nil {
			err = rec.(error)
		}
	}()

	raw, ok := q.fields[name]
	if !ok {
		return nil, fmt.Errorf("field not found: %q", name)
	}

	p, raw = parsePredicate(raw)
	if len(raw) != 0 {
		p = nil
		err = UnexpectedEndOfPredicate
	}

	return
}

// Predicate parses the given component of the query as a sort order, then returns it.
func (q *QueryString) SortOrder(name string) (os []*SortOrder, err error) {
	defer func() {
		if rec := recover(); rec != nil {
			err = rec.(error)
		}
	}()

	raw, ok := q.fields[name]
	if !ok {
		return nil, fmt.Errorf("field not found: %q", name)
	}

	os, raw = parseSortOrders(raw)
	if len(raw) != 0 {
		os = nil
		err = UnexpectedEndOfSortOrders
	}

	return
}

func parsePredicate(s string) (p Predicate, n string) {
	if len(s) == 0 {
		panic(OperatorExpected)
	}

	var op string
	op, n = parseIdentifier(s)

	n = parseLiteral(n, "(")

	var f string
	var ps []Predicate
	var vs []Value
	var v Value

	switch op {
		case "not":
			var operand Predicate
			operand, n = parsePredicate(n)
			p = Not{operand}
		case "and":
			ps, n = parsePredicates(n)
			p = And(ps)
		case "or":
			ps, n = parsePredicates(n)
			p = Or(ps)
		case "eq":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			vs, n = parseValues(n)
			p = Eq{Field(f), vs}
		case "gt":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Gt{Field(f), v}
		case "ge":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Ge{Field(f), v}
		case "lt":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Lt{Field(f), v}
		case "le":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Le{Field(f), v}
		case "fts":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			s, n = parseString(n)
			p = Fts{Field(f), s}
		default:
			panic(fmt.Errorf("Invalid operator: %q", op))
	}

	n = parseLiteral(n, ")")

	return
}

func parsePredicates(s string) (ps []Predicate, n string) {
	ps = make([]Predicate, 0, 4)

	if len(s) > 0 && firstCharClass(s) > 2 {
		n = s
		for {
			var operand Predicate
			operand, n = parsePredicate(n)
			ps = append(ps, operand)

			if len(n) > 0 && n[0] == ',' {
				n = n[1:]
			} else {
				break
			}
		}
	}

	return
}

func ParseValues(s string) ([]Value, error) {
	vs, n := parseValues(s)
	if n != "" {
		return vs, EndOfStringExpected
	}
	return vs, nil
}

func parseValues(s string) (vs []Value, n string) {
	vs = make([]Value, 0, 4)

	if len(s) > 0 && firstCharClass(s) > 2 {
		n = s
		for {
			var operand interface{}
			operand, n = parseValue(n)
			vs = append(vs, operand)

			if len(n) > 0 && n[0] == ',' {
				n = n[1:]
			} else {
				break
			}
		}
	}

	return
}

func parseString(s string) (v string, n string) {
	if len(s) == 0 || s[0] != '\'' {
		panic(StringExpected)
	}

	s = s[1:]

	l := strings.IndexFunc(s, charClassDetector(1, 2))

	if l == -1 {
		l = len(s)
	}

	if s[l - 1] != '\'' {
		panic(EndOfStringExpected)
	}

	var err error
	if v, err = url.QueryUnescape(s[:l-1]); err != nil {
		panic(err)
	}

	n = s[l:]
	return
}

func ParseValue(s string) (Value, error) {
	v, n := parseValue(s)
	if n != "" {
		return v, EndOfStringExpected
	}
	return v, nil
}

func parseValue(s string) (v Value, n string) {
	if len(s) == 0 {
		panic(ValueExpected)
	}

	r, l := utf8.DecodeRuneInString(s)

	switch(r) {
		case 'n':
			n = parseLiteral(s, "null")
			v = nil
		case 't':
			n = parseLiteral(s, "true")
			v = true
		case 'f':
			n = parseLiteral(s, "false")
			v = false
		case '\'':
			v, n = parseString(s)
		default:
			if l = strings.IndexFunc(s, charClassDetector(1, 2)); l == -1 {
				l = len(s)
			}

			if (l == 10 || ((l == 20 || (l == 24 && s[19] == '.')) && s[10] == 'T' && s[13] == ':' && s[16] == ':' && s[l-1] == 'Z')) && s[4] == '-' && s[7] == '-' {
				var err error
				var yr, mo, dy, hr, mn, sc, ms int64 = 0, 0, 0, 0, 0, 0, 0

				if yr, err = strconv.ParseInt(s[0:4], 10, 32); err != nil {
					panic(err)
				}
				if mo, err = strconv.ParseInt(s[5:7], 10, 32); err != nil {
					panic(err)
				}
				if dy, err = strconv.ParseInt(s[8:10], 10, 32); err != nil {
					panic(err)
				}

				if l >= 20 {
					if hr, err = strconv.ParseInt(s[11:13], 10, 32); err != nil {
						panic(err)
					}
					if mn, err = strconv.ParseInt(s[14:16], 10, 32); err != nil {
						panic(err)
					}
					if sc, err = strconv.ParseInt(s[17:19], 10, 32); err != nil {
						panic(err)
					}

					if l == 24 {
						if ms, err = strconv.ParseInt(s[20:23], 10, 32); err != nil {
							panic(err)
						}
					}
				}

				v = time.Date(int(yr), time.Month(mo), int(dy), int(hr), int(mn), int(sc), int(ms) * 1000000, time.UTC)
			} else {
				if f, err := strconv.ParseFloat(s[:l], 64); err != nil {
					panic(err)
				} else {
					v = f
				}
			}

			n = s[l:]
	}

	return
}

func parseLiteral(s string, expected string) (n string) {
	if s[:len(expected)] != expected {
		panic(fmt.Errorf("expected: %q", expected))
	}

	return s[len(expected):]
}

func parseSortOrders(s string) (os []*SortOrder, n string) {
	os = make([]*SortOrder, 0, 4)

	if len(s) > 0 {
		for {
			var o *SortOrder
			o, s = parseSortOrder(s)
			os = append(os, o)

			if len(s) == 0 {
				break
			}

			if r, l := utf8.DecodeRuneInString(s); r != ',' {
				panic(SortOrderSeparatorExpected)
			} else {
				s = s[l:]
			}
		}
	}

	n = s
	return
}

func parseSortOrder(s string) (o *SortOrder, n string) {
	o = new(SortOrder)

	if r, _ := utf8.DecodeRuneInString(s); r == '!' {
		s = s[1:]
	} else {
		o.Ascending = true
	}

	f, n := parseIdentifier(s)
	o.Field = Field(f)
	return
}

func ParseIdentifier(s string) (Value, error) {
	v, n := parseIdentifier(s)
	if n != "" {
		return v, EndOfStringExpected
	}
	return v, nil
}

func parseIdentifier(s string) (id string, n string) {
	if len(s) == 0 {
		panic(IdentifierExpected)
	}

	i := strings.IndexFunc(s, charClassDetector(1, 3))

	if i == 0 {
		panic(IdentifierExpected)
	}

	if i == -1 {
		n = ""
	} else {
		n = s[i:]
		s = s[:i]
	}

	var err error
	if id, err = url.QueryUnescape(s); err != nil {
		panic(err)
	}

	return
}
<commit_msg>Implement Raw function in QueryString.<commit_after>package queryme

import (
	"fmt"
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

/*
predicates    = predicate *("," predicate)
predicate     = (not / and / or / eq / lt / le / gt / ge)
not           = "not" "(" predicate ")"
and           = "and" "(" predicates ")"
or            = "or" "(" predicates ")"
eq            = "eq" "(" field "," values ")"
lt            = "lt" "(" field "," value ")"
le            = "le" "(" field "," value ")"
gt            = "gt" "(" field "," value ")"
ge            = "ge" "(" field "," value ")"
fts           = "fts" "(" field "," string ")"

values        = value *("," value)
value         = (null / boolean / number / string / date)
null          = "null"
boolean       = "true" / "false"
number        = 1*(DIGIT / "." / "e" / "E" / "+" / "-")
string        = "'" *(unreserved / pct-encoded) "'"
date          = 4DIGIT "-" 2DIGIT "-" 2DIGIT *1("T" 2DIGIT ":" 2DIGIT ":" 2DIGIT *1("." 3DIGIT) "Z")

fieldorders   = *1(fieldorder *("," fieldorder))
fieldorder    = *1"!" field
field         = *(unreserved / pct-encoded)

unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
pct-encoded   = "%" HEXDIG HEXDIG
sub-delims    = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
query         = *( pchar / "/" / "?" )
*/

var (
	SortOrderSeparatorExpected error = errors.New("Expected seperator ',' after sorted order.")
	IdentifierExpected error = errors.New("Expected identifier.")
	ValueExpected error = errors.New("Expected value.")
	EndOfStringExpected error = errors.New("Expected end of string.")
	StringExpected error = errors.New("Expected string.")
	OperatorExpected error = errors.New("Expected operator.")
	UnexpectedEndOfPredicate error = errors.New("Unexpected end of predicate.")
	UnexpectedEndOfSortOrders error = errors.New("Unexpected end of sort orders.")

	characters []byte
)

func init() {
	characters = make([]byte, 128)

	characters[int('=')] = 1
	characters[int('&')] = 1

	characters[int('!')] = 2
	characters[int('$')] = 2
	characters[int('(')] = 2
	characters[int(')')] = 2
	characters[int('*')] = 2
	characters[int(',')] = 2
	characters[int(';')] = 2
	characters[int('/')] = 2
	characters[int('?')] = 2
	characters[int('@')] = 2

	characters[int('\'')] = 3
	characters[int('+')] = 3
	characters[int(':')] = 3

	// 'pct-encoded' characters
	characters[int('%')] = 4

	// 'unreserved' characters
	characters[int('-')] = 5
	characters[int('.')] = 5
	characters[int('_')] = 5
	characters[int('~')] = 5

	for i := int('0'); i <= int('9'); i++ {
		characters[i] = 5
	}

	for i := int('a'); i <= int('z'); i++ {
		characters[i] = 5
	}

	for i := int('A'); i <= int('Z'); i++ {
		characters[i] = 5
	}
}

func firstCharClass(s string) byte {
	r, _ := utf8.DecodeRuneInString(s)
	if r > 127 {
		return 0
	} else {
		return characters[r]
	}
}

func charClassDetector(min byte, max byte) func(r rune) bool {
	return func(r rune) bool {
		i := int(r)
		if i > 127 {
			return false
		}
		c := characters[i]
		return c >= min && c <= max
	}
}

// QueryString is a parsed query part of a URL.
type QueryString struct {
	fields map[string]string
}

// NewFromRawQuery creates a new QueryString from a raw query string.
func NewFromRawQuery(rawQuery string) *QueryString {
	qs := new(QueryString)
	qs.fields = make(map[string]string)

	for {
		i := strings.IndexRune(rawQuery, '=')
		if i == -1 {
			break
		}
		name := rawQuery[:i]
		rawQuery = rawQuery[i+1:]

		i = strings.IndexFunc(rawQuery, charClassDetector(1, 1))
		var value string
		if i == -1 {
			value = rawQuery
		} else {
			value = rawQuery[:i]
			rawQuery = rawQuery[i+1:]
		}

		qs.fields[name] = value

		if i == -1 {
			break
		}
	}

	return qs
}

// NewFromRawQuery creates a new QueryString from an existing URL object.
func NewFromURL(url *url.URL) *QueryString {
	return NewFromRawQuery(url.RawQuery)
}

// Tests if specified name has been found in query string.
func (q *QueryString) Contains(name string) bool {
	_, ok := q.fields[name]
	return ok
}

// Returns raw query string value.
func (q *QueryString) Raw(name string) (string, bool) {
	return q.fields[name]
}

// Predicate parses the given component of the query as a predicate, then returns it.
func (q *QueryString) Predicate(name string) (p Predicate, err error) {
	defer func() {
		if rec := recover(); rec != nil {
			err = rec.(error)
		}
	}()

	raw, ok := q.fields[name]
	if !ok {
		return nil, fmt.Errorf("field not found: %q", name)
	}

	p, raw = parsePredicate(raw)
	if len(raw) != 0 {
		p = nil
		err = UnexpectedEndOfPredicate
	}

	return
}

// Predicate parses the given component of the query as a sort order, then returns it.
func (q *QueryString) SortOrder(name string) (os []*SortOrder, err error) {
	defer func() {
		if rec := recover(); rec != nil {
			err = rec.(error)
		}
	}()

	raw, ok := q.fields[name]
	if !ok {
		return nil, fmt.Errorf("field not found: %q", name)
	}

	os, raw = parseSortOrders(raw)
	if len(raw) != 0 {
		os = nil
		err = UnexpectedEndOfSortOrders
	}

	return
}

func parsePredicate(s string) (p Predicate, n string) {
	if len(s) == 0 {
		panic(OperatorExpected)
	}

	var op string
	op, n = parseIdentifier(s)

	n = parseLiteral(n, "(")

	var f string
	var ps []Predicate
	var vs []Value
	var v Value

	switch op {
		case "not":
			var operand Predicate
			operand, n = parsePredicate(n)
			p = Not{operand}
		case "and":
			ps, n = parsePredicates(n)
			p = And(ps)
		case "or":
			ps, n = parsePredicates(n)
			p = Or(ps)
		case "eq":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			vs, n = parseValues(n)
			p = Eq{Field(f), vs}
		case "gt":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Gt{Field(f), v}
		case "ge":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Ge{Field(f), v}
		case "lt":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Lt{Field(f), v}
		case "le":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			v, n = parseValue(n)
			p = Le{Field(f), v}
		case "fts":
			f, n = parseIdentifier(n)
			n = parseLiteral(n, ",")
			s, n = parseString(n)
			p = Fts{Field(f), s}
		default:
			panic(fmt.Errorf("Invalid operator: %q", op))
	}

	n = parseLiteral(n, ")")

	return
}

func parsePredicates(s string) (ps []Predicate, n string) {
	ps = make([]Predicate, 0, 4)

	if len(s) > 0 && firstCharClass(s) > 2 {
		n = s
		for {
			var operand Predicate
			operand, n = parsePredicate(n)
			ps = append(ps, operand)

			if len(n) > 0 && n[0] == ',' {
				n = n[1:]
			} else {
				break
			}
		}
	}

	return
}

func ParseValues(s string) ([]Value, error) {
	vs, n := parseValues(s)
	if n != "" {
		return vs, EndOfStringExpected
	}
	return vs, nil
}

func parseValues(s string) (vs []Value, n string) {
	vs = make([]Value, 0, 4)

	if len(s) > 0 && firstCharClass(s) > 2 {
		n = s
		for {
			var operand interface{}
			operand, n = parseValue(n)
			vs = append(vs, operand)

			if len(n) > 0 && n[0] == ',' {
				n = n[1:]
			} else {
				break
			}
		}
	}

	return
}

func parseString(s string) (v string, n string) {
	if len(s) == 0 || s[0] != '\'' {
		panic(StringExpected)
	}

	s = s[1:]

	l := strings.IndexFunc(s, charClassDetector(1, 2))

	if l == -1 {
		l = len(s)
	}

	if s[l - 1] != '\'' {
		panic(EndOfStringExpected)
	}

	var err error
	if v, err = url.QueryUnescape(s[:l-1]); err != nil {
		panic(err)
	}

	n = s[l:]
	return
}

func ParseValue(s string) (Value, error) {
	v, n := parseValue(s)
	if n != "" {
		return v, EndOfStringExpected
	}
	return v, nil
}

func parseValue(s string) (v Value, n string) {
	if len(s) == 0 {
		panic(ValueExpected)
	}

	r, l := utf8.DecodeRuneInString(s)

	switch(r) {
		case 'n':
			n = parseLiteral(s, "null")
			v = nil
		case 't':
			n = parseLiteral(s, "true")
			v = true
		case 'f':
			n = parseLiteral(s, "false")
			v = false
		case '\'':
			v, n = parseString(s)
		default:
			if l = strings.IndexFunc(s, charClassDetector(1, 2)); l == -1 {
				l = len(s)
			}

			if (l == 10 || ((l == 20 || (l == 24 && s[19] == '.')) && s[10] == 'T' && s[13] == ':' && s[16] == ':' && s[l-1] == 'Z')) && s[4] == '-' && s[7] == '-' {
				var err error
				var yr, mo, dy, hr, mn, sc, ms int64 = 0, 0, 0, 0, 0, 0, 0

				if yr, err = strconv.ParseInt(s[0:4], 10, 32); err != nil {
					panic(err)
				}
				if mo, err = strconv.ParseInt(s[5:7], 10, 32); err != nil {
					panic(err)
				}
				if dy, err = strconv.ParseInt(s[8:10], 10, 32); err != nil {
					panic(err)
				}

				if l >= 20 {
					if hr, err = strconv.ParseInt(s[11:13], 10, 32); err != nil {
						panic(err)
					}
					if mn, err = strconv.ParseInt(s[14:16], 10, 32); err != nil {
						panic(err)
					}
					if sc, err = strconv.ParseInt(s[17:19], 10, 32); err != nil {
						panic(err)
					}

					if l == 24 {
						if ms, err = strconv.ParseInt(s[20:23], 10, 32); err != nil {
							panic(err)
						}
					}
				}

				v = time.Date(int(yr), time.Month(mo), int(dy), int(hr), int(mn), int(sc), int(ms) * 1000000, time.UTC)
			} else {
				if f, err := strconv.ParseFloat(s[:l], 64); err != nil {
					panic(err)
				} else {
					v = f
				}
			}

			n = s[l:]
	}

	return
}

func parseLiteral(s string, expected string) (n string) {
	if s[:len(expected)] != expected {
		panic(fmt.Errorf("expected: %q", expected))
	}

	return s[len(expected):]
}

func parseSortOrders(s string) (os []*SortOrder, n string) {
	os = make([]*SortOrder, 0, 4)

	if len(s) > 0 {
		for {
			var o *SortOrder
			o, s = parseSortOrder(s)
			os = append(os, o)

			if len(s) == 0 {
				break
			}

			if r, l := utf8.DecodeRuneInString(s); r != ',' {
				panic(SortOrderSeparatorExpected)
			} else {
				s = s[l:]
			}
		}
	}

	n = s
	return
}

func parseSortOrder(s string) (o *SortOrder, n string) {
	o = new(SortOrder)

	if r, _ := utf8.DecodeRuneInString(s); r == '!' {
		s = s[1:]
	} else {
		o.Ascending = true
	}

	f, n := parseIdentifier(s)
	o.Field = Field(f)
	return
}

func ParseIdentifier(s string) (Value, error) {
	v, n := parseIdentifier(s)
	if n != "" {
		return v, EndOfStringExpected
	}
	return v, nil
}

func parseIdentifier(s string) (id string, n string) {
	if len(s) == 0 {
		panic(IdentifierExpected)
	}

	i := strings.IndexFunc(s, charClassDetector(1, 3))

	if i == 0 {
		panic(IdentifierExpected)
	}

	if i == -1 {
		n = ""
	} else {
		n = s[i:]
		s = s[:i]
	}

	var err error
	if id, err = url.QueryUnescape(s); err != nil {
		panic(err)
	}

	return
}
<|endoftext|>


<commit_before>package msgpack

import (
    "io"
    "os"
    "unsafe"
    "reflect"
)

func readByte(reader io.Reader) (v uint8, err os.Error) {
    data := [1]byte{}
    _, e := reader.Read(data[0:])
    if e != nil { return 0, e }
    return data[0], nil
}

func readUint16(reader io.Reader) (v uint16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint16(data[0]) >> 8) | uint16(data[1]), n, nil
}

func readUint32(reader io.Reader) (v uint32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint32(data[0]) << 24) | (uint32(data[1]) << 16) | (uint32(data[2]) << 8) | uint32(data[1]), n, nil
}

func readUint64(reader io.Reader) (v uint64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint64(data[0]) << 56) | (uint64(data[1]) << 48) | (uint64(data[2]) << 40) | (uint64(data[3]) << 32) | (uint64(data[4]) << 24) | (uint64(data[5]) << 16) | (uint64(data[6]) << 8) | uint64(data[7]), n, nil
}

func readInt16(reader io.Reader) (v int16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int16(data[0]) << 8) | int16(data[1]), n, nil
}

func readInt32(reader io.Reader) (v int32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int32(data[0]) << 24) | (int32(data[1]) << 16) | (int32(data[2]) << 8) | int32(data[1]), n, nil
}

func readInt64(reader io.Reader) (v int64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int64(data[0]) << 56) | (int64(data[1]) << 48) | (int64(data[2]) << 40) | (int64(data[3]) << 32) | (int64(data[4]) << 24) | (int64(data[5]) << 16) | (int64(data[6]) << 8) | int64(data[7]), n, nil
}

func unpackArray(reader io.Reader, nelems uint) (v reflect.Value, n int, err os.Error) {
    retval := make([]interface{}, nelems)
    nbytesread := 0
    var i uint
    for i = 0; i < nelems; i++ {
        v, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval[i] = v.Interface()
    }
    return reflect.NewValue(retval), nbytesread, nil
}

func unpackMap(reader io.Reader, nelems uint) (v reflect.Value, n int, err os.Error) {
    retval := make(map [interface{}] interface{})
    nbytesread := 0
    var i uint
    for i = 0; i < nelems; i++ {
        k, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        v, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval[k.Interface()] = v.Interface()
    }
    return reflect.NewValue(retval), nbytesread, nil
}

func Unpack(reader io.Reader) (v reflect.Value, n int, err os.Error) {
    var retval reflect.Value
    var nbytesread int = 0

    c, e := readByte(reader)
    if e != nil { return nil, 0, e }
    nbytesread += 1
    if c < 0x80 || c >= 0xe0 {
        retval = reflect.NewValue(int8(c))
    } else if c >= 0x80 && c <= 0x8f {
        retval, n, e = unpackMap(reader, uint(c & 0xf))
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        nbytesread += n
    } else if c >= 0x90 && c <= 0x9f {
        retval, n, e = unpackArray(reader, uint(c & 0xf))
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        nbytesread += n
    } else if c >= 0xa0 && c <= 0xbf {
        data := make([]byte, c & 0xf);
        n, e := reader.Read(data)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval = reflect.NewValue(data)
    } else {
        switch c {
        case 0xc0: retval = reflect.NewValue(nil)
        case 0xc2: retval = reflect.NewValue(false)
        case 0xc3: retval = reflect.NewValue(true)
        case 0xca:
            data, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(*(*float32)(unsafe.Pointer(&data)))
        case 0xcb:
            data, n, e := readUint64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(*(*float64)(unsafe.Pointer(&data)))
        case 0xcc:
            data, e := readByte(reader)
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(uint8(data))
            nbytesread += 1
        case 0xcd:
            data, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xce:
            data, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xcf:
            data, n, e := readUint64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd0:
            data, e := readByte(reader)
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(int8(data))
            nbytesread += 1
        case 0xd1:
            data, n, e := readInt16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd2:
            data, n, e := readInt32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd3:
            data, n, e := readInt64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xda:
            nbytestoread, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            data := make([]byte, nbytestoread)
            n, e = reader.Read(data)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xdb:
            nbytestoread, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            data := make([]byte, nbytestoread)
            n, e = reader.Read(data)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xdc:
            nelemstoread, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval, n, e = unpackArray(reader, uint(nelemstoread))
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
        case 0xdd:
            nelemstoread, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval, n, e = unpackArray(reader, uint(nelemstoread))
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
        }
    }
    return retval, nbytesread, nil
}
<commit_msg>Fix bugs in unpacker.<commit_after>package msgpack

import (
    "io"
    "os"
    "unsafe"
    "reflect"
)

func readByte(reader io.Reader) (v uint8, err os.Error) {
    data := [1]byte{}
    _, e := reader.Read(data[0:])
    if e != nil { return 0, e }
    return data[0], nil
}

func readUint16(reader io.Reader) (v uint16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint16(data[0]) << 8) | uint16(data[1]), n, nil
}

func readUint32(reader io.Reader) (v uint32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint32(data[0]) << 24) | (uint32(data[1]) << 16) | (uint32(data[2]) << 8) | uint32(data[3]), n, nil
}

func readUint64(reader io.Reader) (v uint64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (uint64(data[0]) << 56) | (uint64(data[1]) << 48) | (uint64(data[2]) << 40) | (uint64(data[3]) << 32) | (uint64(data[4]) << 24) | (uint64(data[5]) << 16) | (uint64(data[6]) << 8) | uint64(data[7]), n, nil
}

func readInt16(reader io.Reader) (v int16, n int, err os.Error) {
    data := [2]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int16(data[0]) << 8) | int16(data[1]), n, nil
}

func readInt32(reader io.Reader) (v int32, n int, err os.Error) {
    data := [4]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int32(data[0]) << 24) | (int32(data[1]) << 16) | (int32(data[2]) << 8) | int32(data[3]), n, nil
}

func readInt64(reader io.Reader) (v int64, n int, err os.Error) {
    data := [8]byte{}
    n, e := reader.Read(data[0:])
    if e != nil { return 0, n, e }
    return (int64(data[0]) << 56) | (int64(data[1]) << 48) | (int64(data[2]) << 40) | (int64(data[3]) << 32) | (int64(data[4]) << 24) | (int64(data[5]) << 16) | (int64(data[6]) << 8) | int64(data[7]), n, nil
}

func unpackArray(reader io.Reader, nelems uint) (v reflect.Value, n int, err os.Error) {
    retval := make([]interface{}, nelems)
    nbytesread := 0
    var i uint
    for i = 0; i < nelems; i++ {
        v, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval[i] = v.Interface()
    }
    return reflect.NewValue(retval), nbytesread, nil
}

func unpackMap(reader io.Reader, nelems uint) (v reflect.Value, n int, err os.Error) {
    retval := make(map [interface{}] interface{})
    nbytesread := 0
    var i uint
    for i = 0; i < nelems; i++ {
        k, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        v, n, e := Unpack(reader)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval[k.Interface()] = v.Interface()
    }
    return reflect.NewValue(retval), nbytesread, nil
}

func Unpack(reader io.Reader) (v reflect.Value, n int, err os.Error) {
    var retval reflect.Value
    var nbytesread int = 0

    c, e := readByte(reader)
    if e != nil { return nil, 0, e }
    nbytesread += 1
    if c < 0x80 || c >= 0xe0 {
        retval = reflect.NewValue(int8(c))
    } else if c >= 0x80 && c <= 0x8f {
        retval, n, e = unpackMap(reader, uint(c & 0xf))
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        nbytesread += n
    } else if c >= 0x90 && c <= 0x9f {
        retval, n, e = unpackArray(reader, uint(c & 0xf))
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        nbytesread += n
    } else if c >= 0xa0 && c <= 0xbf {
        data := make([]byte, c & 0xf);
        n, e := reader.Read(data)
        nbytesread += n
        if e != nil { return nil, nbytesread, e }
        retval = reflect.NewValue(data)
    } else {
        switch c {
        case 0xc0: retval = reflect.NewValue(nil)
        case 0xc2: retval = reflect.NewValue(false)
        case 0xc3: retval = reflect.NewValue(true)
        case 0xca:
            data, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(*(*float32)(unsafe.Pointer(&data)))
        case 0xcb:
            data, n, e := readUint64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(*(*float64)(unsafe.Pointer(&data)))
        case 0xcc:
            data, e := readByte(reader)
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(uint8(data))
            nbytesread += 1
        case 0xcd:
            data, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xce:
            data, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xcf:
            data, n, e := readUint64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd0:
            data, e := readByte(reader)
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(int8(data))
            nbytesread += 1
        case 0xd1:
            data, n, e := readInt16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd2:
            data, n, e := readInt32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xd3:
            data, n, e := readInt64(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xda:
            nbytestoread, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            data := make([]byte, nbytestoread)
            n, e = reader.Read(data)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xdb:
            nbytestoread, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            data := make([]byte, nbytestoread)
            n, e = reader.Read(data)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval = reflect.NewValue(data)
        case 0xdc:
            nelemstoread, n, e := readUint16(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval, n, e = unpackArray(reader, uint(nelemstoread))
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
        case 0xdd:
            nelemstoread, n, e := readUint32(reader)
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
            retval, n, e = unpackArray(reader, uint(nelemstoread))
            nbytesread += n
            if e != nil { return nil, nbytesread, e }
        }
    }
    return retval, nbytesread, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides access to DWARF debugging information
// loaded from executable files, as defined in the DWARF 2.0 Standard
// at http://dwarfstd.org/dwarf-2.0.0.pdf.
package dwarf

import (
	"encoding/binary"
	"os"
)

// Data represents the DWARF debugging information
// loaded from an executable file (for example, an ELF or Mach-O executable).
type Data struct {
	// raw data
	abbrev   []byte
	aranges  []byte
	frame    []byte
	info     []byte
	line     []byte
	pubnames []byte
	ranges   []byte
	str      []byte

	// parsed data
	abbrevCache map[uint32]abbrevTable
	addrsize    int
	order       binary.ByteOrder
	typeCache   map[Offset]Type
	unit        []unit
}

// New returns a new Data object initialized from the given parameters.
// Clients should typically use [TODO(rsc): method to be named later] instead of calling
// New directly.
//
// The []byte arguments are the data from the corresponding debug section
// in the object file; for example, for an ELF object, abbrev is the contents of
// the ".debug_abbrev" section.
func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, os.Error) {
	d := &Data{
		abbrev:      abbrev,
		aranges:     aranges,
		frame:       frame,
		info:        info,
		line:        line,
		pubnames:    pubnames,
		ranges:      ranges,
		str:         str,
		abbrevCache: make(map[uint32]abbrevTable),
		typeCache:   make(map[Offset]Type),
	}

	// Sniff .debug_info to figure out byte order.
	// bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3).
	if len(d.info) < 6 {
		return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
	}
	x, y := d.info[4], d.info[5]
	switch {
	case x == 0 && y == 0:
		return nil, DecodeError{"info", 4, "unsupported version 0"}
	case x == 0:
		d.order = binary.BigEndian
	case y == 0:
		d.order = binary.LittleEndian
	default:
		return nil, DecodeError{"info", 4, "cannot determine byte order"}
	}

	u, err := d.parseUnits()
	if err != nil {
		return nil, err
	}
	d.unit = u
	return d, nil
}
<commit_msg>debug/dwarf: update PDF link. Fixes issue 881.<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides access to DWARF debugging information
// loaded from executable files, as defined in the DWARF 2.0 Standard
// at http://dwarfstd.org/doc/dwarf-2.0.0.pdf
package dwarf

import (
	"encoding/binary"
	"os"
)

// Data represents the DWARF debugging information
// loaded from an executable file (for example, an ELF or Mach-O executable).
type Data struct {
	// raw data
	abbrev   []byte
	aranges  []byte
	frame    []byte
	info     []byte
	line     []byte
	pubnames []byte
	ranges   []byte
	str      []byte

	// parsed data
	abbrevCache map[uint32]abbrevTable
	addrsize    int
	order       binary.ByteOrder
	typeCache   map[Offset]Type
	unit        []unit
}

// New returns a new Data object initialized from the given parameters.
// Clients should typically use [TODO(rsc): method to be named later] instead of calling
// New directly.
//
// The []byte arguments are the data from the corresponding debug section
// in the object file; for example, for an ELF object, abbrev is the contents of
// the ".debug_abbrev" section.
func New(abbrev, aranges, frame, info, line, pubnames, ranges, str []byte) (*Data, os.Error) {
	d := &Data{
		abbrev:      abbrev,
		aranges:     aranges,
		frame:       frame,
		info:        info,
		line:        line,
		pubnames:    pubnames,
		ranges:      ranges,
		str:         str,
		abbrevCache: make(map[uint32]abbrevTable),
		typeCache:   make(map[Offset]Type),
	}

	// Sniff .debug_info to figure out byte order.
	// bytes 4:6 are the version, a tiny 16-bit number (1, 2, 3).
	if len(d.info) < 6 {
		return nil, DecodeError{"info", Offset(len(d.info)), "too short"}
	}
	x, y := d.info[4], d.info[5]
	switch {
	case x == 0 && y == 0:
		return nil, DecodeError{"info", 4, "unsupported version 0"}
	case x == 0:
		d.order = binary.BigEndian
	case y == 0:
		d.order = binary.LittleEndian
	default:
		return nil, DecodeError{"info", 4, "cannot determine byte order"}
	}

	u, err := d.parseUnits()
	if err != nil {
		return nil, err
	}
	d.unit = u
	return d, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"encoding/csv"
	"math/rand"
	"strconv"

)

const trainPath = "/Users/plorenzo/dev/uni/rna/final/train.csv"
const validatePath = "/Users/plorenzo/dev/uni/rna/final/validate.csv"
const testPath = "/Users/plorenzo/dev/uni/rna/final/test.csv"

// TODO: refacto to improve performance (S.O. QUESTION)
func readCSV(filepath string) ([][]float64, []float64) {

	csvfile, err := os.Open(filepath)
	if err != nil {
		return nil, nil
	}

	reader := csv.NewReader(csvfile)
	reader.Comma = ';'
	stringMatrix, err := reader.ReadAll()

	csvfile.Close()

	matrix := make([][]float64, len(stringMatrix))
	expectedY := make([]float64, len(stringMatrix))

	//Parse string matrix into float64
	for i := range stringMatrix {
		matrix[i] = make([]float64, len(stringMatrix[0]))
		for j := range stringMatrix[i] {
			if j < 8 {
				matrix[i][j], err = strconv.ParseFloat(stringMatrix[i][j], 64)
			} else {
				//Extract expected output date from file (last column)
				expectedY[i], err = strconv.ParseFloat(stringMatrix[i][j], 64)
				matrix[i][j] = 1
			}

		}
	}
	return matrix, expectedY
}

//This also inits the threshold
func initWeights(length int) []float64 {

	weights := make([]float64, length)
	//Inits the slice with random numbers betwen [-1, 1]
	for index := range weights {
		w := rand.Float64()
		s := rand.Float64()

		if s < 0.5 {
			weights[index] = w
		} else {
			weights[index] = w * -1
		}
	}
	return weights
}

func main() {

	//Read data from csv file
	data, expectedY := readCSV(trainPath)
	validateData, valExpectedY := readCSV(validatePath)
	testData, testExpectedY := readCSV(testPath)

	//PARAMETERS
	var cylces int = 100
	var learningRate float64 = 0.2

	weights := initWeights(len(data[0]))

	var estimate float64
	var errorData float64
	var errorsTrain []float64
	var errorsValidate []float64
	var errorsTest float64

	// Learning
	for i := 0; i < cylces; i++ {
		for j := range data {
			//Calculate estimate
			estimate = 0
			for x := range data[j]{
				estimate += data[j][x] * weights[x]
			}

			// Update weights (range passes values as a copy)
			for x := 0; x < len(weights); x++ {
				weights[x] += learningRate * (expectedY[j] - estimate) * data[j][x]
			}
		}

		// Compute cylce train error
		errorData = 0
		for j := range data {
			estimate = 0
			for x := range data[j] {
				estimate += data[j][x] * weights[x]
			}
			// Cuadratic error E = (Yd - Ye)^2
			errorData += (expectedY[j] - estimate) * (expectedY[j] - estimate)
		}
		errorsTrain = append(errorsTrain, errorData / float64(len(data)))

		// Compute cylce validate error
		errorData = 0
		for j := range validateData {
			estimate = 0
			for x := range validateData[j] {
				estimate += validateData[j][x] * weights[x]
			}
			errorData += (valExpectedY[j] - estimate) * (valExpectedY[j] - estimate)
		}
		errorsValidate = append(errorsValidate, errorData / float64(len(validateData)))
	}

	// Compute test error
	errorData = 0
	for j := range testData {
		estimate = 0
		for x := range testData[j] {
			estimate += testData[j][x] * weights[x]
		}
		errorData += (testExpectedY[j] - estimate) * (testExpectedY[j] - estimate)
	}
	errorsTest = errorData / float64(len(testData))
	fmt.Println(errorsTest)
}

<commit_msg>Moved error calculation to a function<commit_after>package main

import (
	"fmt"
	"os"
	"encoding/csv"
	"math/rand"
	"strconv"
)

const trainPath = "/Users/plorenzo/dev/uni/rna/final/train.csv"
const validatePath = "/Users/plorenzo/dev/uni/rna/final/validate.csv"
const testPath = "/Users/plorenzo/dev/uni/rna/final/test.csv"

// TODO: refacto to improve performance (S.O. QUESTION)
func readCSV(filepath string) ([][]float64, []float64) {

	csvfile, err := os.Open(filepath)
	if err != nil {
		return nil, nil
	}

	reader := csv.NewReader(csvfile)
	reader.Comma = ';'
	stringMatrix, err := reader.ReadAll()

	csvfile.Close()

	matrix := make([][]float64, len(stringMatrix))
	expectedY := make([]float64, len(stringMatrix))

	//Parse string matrix into float64
	for i := range stringMatrix {
		matrix[i] = make([]float64, len(stringMatrix[0]))
		for j := range stringMatrix[i] {
			if j < 8 {
				matrix[i][j], err = strconv.ParseFloat(stringMatrix[i][j], 64)
			} else {
				//Extract expected output date from file (last column)
				expectedY[i], err = strconv.ParseFloat(stringMatrix[i][j], 64)
				matrix[i][j] = 1
			}

		}
	}
	return matrix, expectedY
}

//This also inits the threshold
func initWeights(length int) []float64 {

	weights := make([]float64, length)
	//Inits the slice with random numbers betwen [-1, 1]
	for index := range weights {
		w := rand.Float64()
		s := rand.Float64()

		if s < 0.5 {
			weights[index] = w
		} else {
			weights[index] = w * -1
		}
	}
	return weights
}

func createCSV(train []float64, validate []float64) {

	//TODO Add to existing file instead of creating one each time

	file, _ := os.Create("/Users/plorenzo/dev/uni/rna/errores/total.csv")
    defer file.Close()

    writer := csv.NewWriter(file)
    defer writer.Flush()

    var strings []string
    var strings1 []string

    for i := range train {
    	strings = append(strings, strconv.FormatFloat(train[i], 'f', 6, 64))
    }
    for i := range validate {
    	strings1 = append(strings1, strconv.FormatFloat(validate[i], 'f', 6, 64))
    }
    writer.Write(strings)
    writer.Write(strings1)
}

func computeError(data [][]float64, expected []float64, weights []float64) float64 {

	var errors float64
	var errorSum, estimate float64 = 0, 0

	for i := range data {
		estimate = 0
		for j := range data[i] {
			estimate += data[i][j] * weights[j]
		}
		// Cuadratic error E = (Yd - Ye)^2
		errorSum += (expected[i] - estimate) * (expected[i] - estimate)
	}
	errors = errorSum / float64(len(data))

	return errors
}

func main() {

	//Read data from csv file
	data, expectedY := readCSV(trainPath)
	validateData, valExpectedY := readCSV(validatePath)
	testData, testExpectedY := readCSV(testPath)

	//PARAMETERS
	var cylces int = 57
	var learningRate float64 = 0.01

	weights := initWeights(len(data[0]))

	var estimate float64
	var errorsTrain []float64
	var errorsValidate []float64
	var errorsTest float64

	// Learning
	for i := 0; i < cylces; i++ {
		for j := range data {
			//Calculate estimate
			estimate = 0
			for x := range data[j]{
				estimate += data[j][x] * weights[x]
			}

			// Update weights (range passes values as a copy)
			for x := 0; x < len(weights); x++ {
				weights[x] += learningRate * (expectedY[j] - estimate) * data[j][x]
			}
		}

		// Compute cylce train error
		errorsTrain = append(errorsTrain, computeError(data, expectedY, weights))
		errorsValidate = append(errorsValidate, computeError(validateData, valExpectedY, weights))

	}

	errorsTest = computeError(testData, testExpectedY, weights)

	fmt.Println("Test error: ")
	fmt.Println(errorsTest)

	createCSV(errorsTrain, errorsValidate)
}

<|endoftext|>


<commit_before>package governor

import (
	"context"
	vaultapi "github.com/hashicorp/vault/api"
	"github.com/spf13/viper"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type (
	// ConfigOpts is the server base configuration
	ConfigOpts struct {
		DefaultFile string
		Appname     string
		Description string
		Version     string
		VersionHash string
		EnvPrefix   string
	}

	// Config is the complete server configuration including the dynamic
	// (runtime) options
	Config struct {
		config        *viper.Viper
		vault         *vaultapi.Client
		vaultK8sAuth  bool
		vaultExpire   int64
		mu            sync.RWMutex
		Appname       string
		Version       string
		VersionHash   string
		LogLevel      int
		LogOutput     io.Writer
		Port          string
		BaseURL       string
		PublicDir     string
		MaxReqSize    string
		FrontendProxy []string
		Origins       []string
		RouteRewrite  map[string]string
	}
)

func newConfig(conf ConfigOpts) *Config {
	v := viper.New()
	v.SetDefault("mode", "INFO")
	v.SetDefault("logoutput", "STDOUT")
	v.SetDefault("port", "8080")
	v.SetDefault("baseurl", "/")
	v.SetDefault("publicdir", "public")
	v.SetDefault("templatedir", "templates")
	v.SetDefault("maxreqsize", "2M")
	v.SetDefault("frontendproxy", []string{})
	v.SetDefault("alloworigins", []string{})
	v.SetDefault("vault.addr", "")
	v.SetDefault("vault.k8s.auth", false)
	v.SetDefault("vault.k8s.role", "")
	v.SetDefault("vault.k8s.loginpath", "/auth/kubernetes/login")
	v.SetDefault("vault.k8s.jwtpath", "/var/run/secrets/kubernetes.io/serviceaccount/token")

	v.SetConfigName(conf.DefaultFile)
	v.AddConfigPath(".")
	v.AddConfigPath(path.Join(".", "config"))
	if cfgdir, err := os.UserConfigDir(); err == nil {
		v.AddConfigPath(path.Join(cfgdir, conf.Appname))
	}
	v.SetConfigType("yaml")

	v.SetEnvPrefix(conf.EnvPrefix)
	v.AutomaticEnv()

	return &Config{
		config:      v,
		Appname:     conf.Appname,
		Version:     conf.Version,
		VersionHash: conf.VersionHash,
	}
}

func (c *Config) setConfigFile(file string) {
	c.config.SetConfigFile(file)
}

func (c *Config) init() error {
	if err := c.config.ReadInConfig(); err != nil {
		return NewError("Failed to read in config", http.StatusInternalServerError, err)
	}
	c.LogLevel = envToLevel(c.config.GetString("mode"))
	c.LogOutput = envToLogOutput(c.config.GetString("logoutput"))
	c.Port = c.config.GetString("port")
	c.BaseURL = c.config.GetString("baseurl")
	c.PublicDir = c.config.GetString("publicdir")
	c.MaxReqSize = c.config.GetString("maxreqsize")
	c.FrontendProxy = c.config.GetStringSlice("frontendproxy")
	c.Origins = c.config.GetStringSlice("alloworigins")
	c.RouteRewrite = c.config.GetStringMapString("routerewrite")
	return nil
}

func (c *Config) initvault(ctx context.Context, l Logger) error {
	vconfig := c.config.GetStringMapString("vault")
	vaultconfig := vaultapi.DefaultConfig()
	if err := vaultconfig.Error; err != nil {
		l.Warn("error creating vault config", map[string]string{
			"phase":      "init",
			"error":      err.Error(),
			"actiontype": "vaultdefaultconfig",
		})
	}
	if vaddr := vconfig["addr"]; vaddr != "" {
		vaultconfig.Address = vaddr
	}
	vault, err := vaultapi.NewClient(vaultconfig)
	if err != nil {
		return NewError("Failed to create vault client", http.StatusInternalServerError, err)
	}
	c.vault = vault
	if c.config.GetBool("vault.k8s.auth") {
		c.vaultK8sAuth = true
		if err := c.authk8s(); err != nil {
			return err
		}
	}
	return nil
}

func (c *Config) ensureValidAuth() error {
	if !c.vaultK8sAuth {
		return nil
	}
	if c.authk8sValid() {
		return nil
	}
	return c.authk8s()
}

func (c *Config) authk8sValidLocked() bool {
	return c.vaultExpire-time.Now().Round(0).Unix() > 5
}

func (c *Config) authk8sValid() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.authk8sValidLocked()
}

func (c *Config) authk8s() error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.authk8sValidLocked() {
		return nil
	}

	vault := c.vault.Logical()
	kconfig := c.config.GetStringMapString("vault.k8s")
	role := kconfig["role"]
	loginpath := kconfig["loginpath"]
	jwtpath := kconfig["jwtpath"]
	if role == "" {
		return NewError("No vault role set", http.StatusBadRequest, nil)
	}
	if loginpath == "" {
		return NewError("No vault k8s login path set", http.StatusBadRequest, nil)
	}
	if jwtpath == "" {
		return NewError("No path for vault k8s service account jwt auth", http.StatusBadRequest, nil)
	}
	jwtbytes, err := ioutil.ReadFile(jwtpath)
	if err != nil {
		return NewError("Failed to read vault k8s service account jwt", http.StatusInternalServerError, err)
	}
	jwt := string(jwtbytes)
	authsecret, err := vault.Write(loginpath, map[string]interface{}{
		"jwt":  jwt,
		"role": role,
	})
	if err != nil {
		return NewError("Failed to auth with vault k8s", http.StatusInternalServerError, err)
	}
	c.vaultExpire = time.Now().Round(0).Unix() + int64(authsecret.Auth.LeaseDuration)
	c.vault.SetToken(authsecret.Auth.ClientToken)
	return nil
}

// IsDebug returns if the configuration is in debug mode
func (c *Config) IsDebug() bool {
	return c.LogLevel == levelDebug
}

type (
	// ConfigRegistrar sets default values on the config parser
	ConfigRegistrar interface {
		SetDefault(key string, value interface{})
	}

	configRegistrar struct {
		prefix string
		v      *viper.Viper
	}
)

func (r *configRegistrar) SetDefault(key string, value interface{}) {
	r.v.SetDefault(r.prefix+"."+key, value)
}

func (c *Config) registrar(prefix string) ConfigRegistrar {
	return &configRegistrar{
		prefix: prefix,
		v:      c.config,
	}
}

type (
	// ConfigReader gets values from the config parser
	ConfigReader interface {
		Name() string
		URL() string
		GetStrMap(key string) map[string]string
		GetBool(key string) bool
		GetInt(key string) int
		GetStr(key string) string
		GetStrSlice(key string) []string
		GetSecret(key string) (vaultSecretVal, error)
		InvalidateSecret(key string)
	}

	vaultSecretVal map[string]interface{}

	vaultSecret struct {
		key    string
		value  vaultSecretVal
		expire int64
	}

	configReader struct {
		serviceOpt
		c     *Config
		cache map[string]vaultSecret
		mu    sync.RWMutex
	}
)

func (r *configReader) Name() string {
	return r.name
}

func (r *configReader) URL() string {
	return r.url
}

func (r *configReader) GetStrMap(key string) map[string]string {
	if key == "" {
		key = r.name
	} else {
		key = r.name + "." + key
	}
	return r.c.config.GetStringMapString(key)
}

func (r *configReader) GetBool(key string) bool {
	return r.c.config.GetBool(r.name + "." + key)
}

func (r *configReader) GetInt(key string) int {
	return r.c.config.GetInt(r.name + "." + key)
}

func (r *configReader) GetStr(key string) string {
	return r.c.config.GetString(r.name + "." + key)
}

func (r *configReader) GetStrSlice(key string) []string {
	return r.c.config.GetStringSlice(r.name + "." + key)
}

func (s *vaultSecret) isValid() bool {
	return s.expire == 0 || s.expire-time.Now().Round(0).Unix() > 5
}

func (r *configReader) GetSecret(key string) (vaultSecretVal, error) {
	kvpath := r.GetStr(key)
	if kvpath == "" {
		return nil, NewError("Invalid secret key", http.StatusInternalServerError, nil)
	}

	if v, ok := r.getCacheSecret(key); ok {
		return v, nil
	}

	if err := r.c.ensureValidAuth(); err != nil {
		return nil, err
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if v, ok := r.getCacheSecretLocked(key); ok {
		return v, nil
	}

	vault := r.c.vault.Logical()
	s, err := vault.Read(kvpath)
	if err != nil {
		return nil, NewError("Failed to read vault secret", http.StatusInternalServerError, err)
	}

	var expire int64
	if s.LeaseDuration > 0 {
		expire = time.Now().Round(0).Unix() + int64(s.LeaseDuration)
	}
	r.setCacheSecretLocked(key, s.Data, expire)

	return s.Data, nil
}

func (r *configReader) InvalidateSecret(key string) {
	r.mu.Lock()
	defer r.mu.Unlock()
	delete(r.cache, key)
}

func (r *configReader) getCacheSecretLocked(key string) (vaultSecretVal, bool) {
	s, ok := r.cache[key]
	if !ok {
		return nil, false
	}
	if !s.isValid() {
		return nil, false
	}
	return s.value, true
}

func (r *configReader) getCacheSecret(key string) (map[string]interface{}, bool) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.getCacheSecretLocked(key)
}

func (r *configReader) setCacheSecretLocked(key string, value vaultSecretVal, expire int64) {
	r.cache[key] = vaultSecret{
		key:    key,
		value:  value,
		expire: expire,
	}
}

func (c *Config) reader(opt serviceOpt) ConfigReader {
	return &configReader{
		serviceOpt: opt,
		c:          c,
		cache:      map[string]vaultSecret{},
	}
}
<commit_msg>Refactor SecretReader as its own interface<commit_after>package governor

import (
	"context"
	vaultapi "github.com/hashicorp/vault/api"
	"github.com/spf13/viper"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"sync"
	"time"
)

type (
	// ConfigOpts is the server base configuration
	ConfigOpts struct {
		DefaultFile string
		Appname     string
		Description string
		Version     string
		VersionHash string
		EnvPrefix   string
	}

	// Config is the complete server configuration including the dynamic
	// (runtime) options
	Config struct {
		config        *viper.Viper
		vault         *vaultapi.Client
		vaultK8sAuth  bool
		vaultExpire   int64
		mu            sync.RWMutex
		Appname       string
		Version       string
		VersionHash   string
		LogLevel      int
		LogOutput     io.Writer
		Port          string
		BaseURL       string
		PublicDir     string
		MaxReqSize    string
		FrontendProxy []string
		Origins       []string
		RouteRewrite  map[string]string
	}
)

func newConfig(conf ConfigOpts) *Config {
	v := viper.New()
	v.SetDefault("mode", "INFO")
	v.SetDefault("logoutput", "STDOUT")
	v.SetDefault("port", "8080")
	v.SetDefault("baseurl", "/")
	v.SetDefault("publicdir", "public")
	v.SetDefault("templatedir", "templates")
	v.SetDefault("maxreqsize", "2M")
	v.SetDefault("frontendproxy", []string{})
	v.SetDefault("alloworigins", []string{})
	v.SetDefault("vault.addr", "")
	v.SetDefault("vault.k8s.auth", false)
	v.SetDefault("vault.k8s.role", "")
	v.SetDefault("vault.k8s.loginpath", "/auth/kubernetes/login")
	v.SetDefault("vault.k8s.jwtpath", "/var/run/secrets/kubernetes.io/serviceaccount/token")

	v.SetConfigName(conf.DefaultFile)
	v.AddConfigPath(".")
	v.AddConfigPath(path.Join(".", "config"))
	if cfgdir, err := os.UserConfigDir(); err == nil {
		v.AddConfigPath(path.Join(cfgdir, conf.Appname))
	}
	v.SetConfigType("yaml")

	v.SetEnvPrefix(conf.EnvPrefix)
	v.AutomaticEnv()

	return &Config{
		config:      v,
		Appname:     conf.Appname,
		Version:     conf.Version,
		VersionHash: conf.VersionHash,
	}
}

func (c *Config) setConfigFile(file string) {
	c.config.SetConfigFile(file)
}

func (c *Config) init() error {
	if err := c.config.ReadInConfig(); err != nil {
		return NewError("Failed to read in config", http.StatusInternalServerError, err)
	}
	c.LogLevel = envToLevel(c.config.GetString("mode"))
	c.LogOutput = envToLogOutput(c.config.GetString("logoutput"))
	c.Port = c.config.GetString("port")
	c.BaseURL = c.config.GetString("baseurl")
	c.PublicDir = c.config.GetString("publicdir")
	c.MaxReqSize = c.config.GetString("maxreqsize")
	c.FrontendProxy = c.config.GetStringSlice("frontendproxy")
	c.Origins = c.config.GetStringSlice("alloworigins")
	c.RouteRewrite = c.config.GetStringMapString("routerewrite")
	return nil
}

func (c *Config) initvault(ctx context.Context, l Logger) error {
	vconfig := c.config.GetStringMapString("vault")
	vaultconfig := vaultapi.DefaultConfig()
	if err := vaultconfig.Error; err != nil {
		l.Warn("error creating vault config", map[string]string{
			"phase":      "init",
			"error":      err.Error(),
			"actiontype": "vaultdefaultconfig",
		})
	}
	if vaddr := vconfig["addr"]; vaddr != "" {
		vaultconfig.Address = vaddr
	}
	vault, err := vaultapi.NewClient(vaultconfig)
	if err != nil {
		return NewError("Failed to create vault client", http.StatusInternalServerError, err)
	}
	c.vault = vault
	if c.config.GetBool("vault.k8s.auth") {
		c.vaultK8sAuth = true
		if err := c.authk8s(); err != nil {
			return err
		}
	}
	return nil
}

func (c *Config) ensureValidAuth() error {
	if !c.vaultK8sAuth {
		return nil
	}
	if c.authk8sValid() {
		return nil
	}
	return c.authk8s()
}

func (c *Config) authk8sValidLocked() bool {
	return c.vaultExpire-time.Now().Round(0).Unix() > 5
}

func (c *Config) authk8sValid() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.authk8sValidLocked()
}

func (c *Config) authk8s() error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.authk8sValidLocked() {
		return nil
	}

	vault := c.vault.Logical()
	kconfig := c.config.GetStringMapString("vault.k8s")
	role := kconfig["role"]
	loginpath := kconfig["loginpath"]
	jwtpath := kconfig["jwtpath"]
	if role == "" {
		return NewError("No vault role set", http.StatusBadRequest, nil)
	}
	if loginpath == "" {
		return NewError("No vault k8s login path set", http.StatusBadRequest, nil)
	}
	if jwtpath == "" {
		return NewError("No path for vault k8s service account jwt auth", http.StatusBadRequest, nil)
	}
	jwtbytes, err := ioutil.ReadFile(jwtpath)
	if err != nil {
		return NewError("Failed to read vault k8s service account jwt", http.StatusInternalServerError, err)
	}
	jwt := string(jwtbytes)
	authsecret, err := vault.Write(loginpath, map[string]interface{}{
		"jwt":  jwt,
		"role": role,
	})
	if err != nil {
		return NewError("Failed to auth with vault k8s", http.StatusInternalServerError, err)
	}
	c.vaultExpire = time.Now().Round(0).Unix() + int64(authsecret.Auth.LeaseDuration)
	c.vault.SetToken(authsecret.Auth.ClientToken)
	return nil
}

// IsDebug returns if the configuration is in debug mode
func (c *Config) IsDebug() bool {
	return c.LogLevel == levelDebug
}

type (
	// ConfigRegistrar sets default values on the config parser
	ConfigRegistrar interface {
		SetDefault(key string, value interface{})
	}

	configRegistrar struct {
		prefix string
		v      *viper.Viper
	}
)

func (r *configRegistrar) SetDefault(key string, value interface{}) {
	r.v.SetDefault(r.prefix+"."+key, value)
}

func (c *Config) registrar(prefix string) ConfigRegistrar {
	return &configRegistrar{
		prefix: prefix,
		v:      c.config,
	}
}

type (
	// ConfigReader gets values from the config parser
	ConfigReader interface {
		Name() string
		URL() string
		GetStrMap(key string) map[string]string
		GetBool(key string) bool
		GetInt(key string) int
		GetStr(key string) string
		GetStrSlice(key string) []string
		SecretReader
	}

	// SecretReader gets values from a secret engine
	SecretReader interface {
		GetSecret(key string) (vaultSecretVal, error)
		InvalidateSecret(key string)
	}

	vaultSecretVal map[string]interface{}

	vaultSecret struct {
		key    string
		value  vaultSecretVal
		expire int64
	}

	configReader struct {
		serviceOpt
		c     *Config
		cache map[string]vaultSecret
		mu    sync.RWMutex
	}
)

func (r *configReader) Name() string {
	return r.name
}

func (r *configReader) URL() string {
	return r.url
}

func (r *configReader) GetStrMap(key string) map[string]string {
	if key == "" {
		key = r.name
	} else {
		key = r.name + "." + key
	}
	return r.c.config.GetStringMapString(key)
}

func (r *configReader) GetBool(key string) bool {
	return r.c.config.GetBool(r.name + "." + key)
}

func (r *configReader) GetInt(key string) int {
	return r.c.config.GetInt(r.name + "." + key)
}

func (r *configReader) GetStr(key string) string {
	return r.c.config.GetString(r.name + "." + key)
}

func (r *configReader) GetStrSlice(key string) []string {
	return r.c.config.GetStringSlice(r.name + "." + key)
}

func (s *vaultSecret) isValid() bool {
	return s.expire == 0 || s.expire-time.Now().Round(0).Unix() > 5
}

func (r *configReader) GetSecret(key string) (vaultSecretVal, error) {
	kvpath := r.GetStr(key)
	if kvpath == "" {
		return nil, NewError("Invalid secret key", http.StatusInternalServerError, nil)
	}

	if v, ok := r.getCacheSecret(key); ok {
		return v, nil
	}

	if err := r.c.ensureValidAuth(); err != nil {
		return nil, err
	}

	r.mu.Lock()
	defer r.mu.Unlock()

	if v, ok := r.getCacheSecretLocked(key); ok {
		return v, nil
	}

	vault := r.c.vault.Logical()
	s, err := vault.Read(kvpath)
	if err != nil {
		return nil, NewError("Failed to read vault secret", http.StatusInternalServerError, err)
	}

	var expire int64
	if s.LeaseDuration > 0 {
		expire = time.Now().Round(0).Unix() + int64(s.LeaseDuration)
	}
	r.setCacheSecretLocked(key, s.Data, expire)

	return s.Data, nil
}

func (r *configReader) InvalidateSecret(key string) {
	r.mu.Lock()
	defer r.mu.Unlock()
	delete(r.cache, key)
}

func (r *configReader) getCacheSecretLocked(key string) (vaultSecretVal, bool) {
	s, ok := r.cache[key]
	if !ok {
		return nil, false
	}
	if !s.isValid() {
		return nil, false
	}
	return s.value, true
}

func (r *configReader) getCacheSecret(key string) (map[string]interface{}, bool) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.getCacheSecretLocked(key)
}

func (r *configReader) setCacheSecretLocked(key string, value vaultSecretVal, expire int64) {
	r.cache[key] = vaultSecret{
		key:    key,
		value:  value,
		expire: expire,
	}
}

func (c *Config) reader(opt serviceOpt) ConfigReader {
	return &configReader{
		serviceOpt: opt,
		c:          c,
		cache:      map[string]vaultSecret{},
	}
}
<|endoftext|>