<commit_before>// Package mpd implements parsing and generating of MPEG-DASH Media Presentation Description (MPD) files.
package mpd

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"regexp"
	"strconv"
)

// http://mpeg.chiariglione.org/standards/mpeg-dash
// https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
// http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd

var emptyElementRE = regexp.MustCompile(`></[A-Za-z]+>`)

// ConditionalUint (ConditionalUintType) defined in XSD as a union of unsignedInt and boolean.
type ConditionalUint struct {
	u *uint64
	b *bool
}

// MarshalXMLAttr encodes ConditionalUint.
func (c ConditionalUint) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
	if c.u != nil {
		return xml.Attr{Name: name, Value: strconv.FormatUint(*c.u, 10)}, nil
	}

	if c.b != nil {
		return xml.Attr{Name: name, Value: strconv.FormatBool(*c.b)}, nil
	}

	// both are nil - no attribute, client will threat it like "false"
	return xml.Attr{}, nil
}

// UnmarshalXMLAttr decodes ConditionalUint.
func (c *ConditionalUint) UnmarshalXMLAttr(attr xml.Attr) error {
	u, err := strconv.ParseUint(attr.Value, 10, 64)
	if err == nil {
		c.u = &u
		return nil
	}

	b, err := strconv.ParseBool(attr.Value)
	if err == nil {
		c.b = &b
		return nil
	}

	return fmt.Errorf("ConditionalUint: can't UnmarshalXMLAttr %#v", attr)
}

// check interfaces
var (
	_ xml.MarshalerAttr   = ConditionalUint{}
	_ xml.UnmarshalerAttr = &ConditionalUint{}
)

// MPD represents root XML element.
type MPD struct {
	XMLNS                      *string `xml:"xmlns,attr"`
	Type                       *string `xml:"type,attr"`
	MinimumUpdatePeriod        *string `xml:"minimumUpdatePeriod,attr"`
	AvailabilityStartTime      *string `xml:"availabilityStartTime,attr"`
	MediaPresentationDuration  *string `xml:"mediaPresentationDuration,attr"`
	MinBufferTime              *string `xml:"minBufferTime,attr"`
	SuggestedPresentationDelay *string `xml:"suggestedPresentationDelay,attr"`
	TimeShiftBufferDepth       *string `xml:"timeShiftBufferDepth,attr"`
	PublishTime                *string `xml:"publishTime,attr"`
	Profiles                   string  `xml:"profiles,attr"`
	Period                     *Period `xml:"Period,omitempty"`
}

// Do not try to use encoding.TextMarshaler and encoding.TextUnmarshaler:
// https://github.com/golang/go/issues/6859#issuecomment-118890463

// Encode generates MPD XML.
func (m *MPD) Encode() ([]byte, error) {
	x := new(bytes.Buffer)
	e := xml.NewEncoder(x)
	e.Indent("", "  ")
	err := e.Encode(m)
	if err != nil {
		return nil, err
	}

	// hacks for self-closing tags
	res := new(bytes.Buffer)
	res.WriteString(`<?xml version="1.0" encoding="utf-8"?>`)
	res.WriteByte('\n')
	for {
		s, err := x.ReadString('\n')
		if s != "" {
			s = emptyElementRE.ReplaceAllString(s, `/>`)
			res.WriteString(s)
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
	}
	res.WriteByte('\n')
	return res.Bytes(), err
}

// Decode parses MPD XML.
func (m *MPD) Decode(b []byte) error {
	return xml.Unmarshal(b, m)
}

// Period represents XSD's PeriodType.
type Period struct {
	Start          *string          `xml:"start,attr"`
	ID             *string          `xml:"id,attr"`
	Duration       *string          `xml:"duration,attr"`
	AdaptationSets []*AdaptationSet `xml:"AdaptationSet,omitempty"`
}

// AdaptationSet represents XSD's AdaptationSetType.
type AdaptationSet struct {
	MimeType                string           `xml:"mimeType,attr"`
	SegmentAlignment        ConditionalUint  `xml:"segmentAlignment,attr"`
	SubsegmentAlignment     ConditionalUint  `xml:"subsegmentAlignment,attr"`
	StartWithSAP            *uint64          `xml:"startWithSAP,attr"`
	SubsegmentStartsWithSAP *uint64          `xml:"subsegmentStartsWithSAP,attr"`
	BitstreamSwitching      *bool            `xml:"bitstreamSwitching,attr"`
	Lang                    *string          `xml:"lang,attr"`
	ContentProtections      []Descriptor     `xml:"ContentProtection,omitempty"`
	Representations         []Representation `xml:"Representation,omitempty"`
}

// Representation represents XSD's RepresentationType.
type Representation struct {
	ID                 *string          `xml:"id,attr"`
	Width              *uint64          `xml:"width,attr"`
	Height             *uint64          `xml:"height,attr"`
	FrameRate          *string          `xml:"frameRate,attr"`
	Bandwidth          *uint64          `xml:"bandwidth,attr"`
	AudioSamplingRate  *string          `xml:"audioSamplingRate,attr"`
	Codecs             *string          `xml:"codecs,attr"`
	ContentProtections []Descriptor     `xml:"ContentProtection,omitempty"`
	SegmentTemplate    *SegmentTemplate `xml:"SegmentTemplate,omitempty"`
}

// Descriptor represents XSD's DescriptorType.
type Descriptor struct {
	SchemeIDURI *string `xml:"schemeIdUri,attr"`
	Value       *string `xml:"value,attr"`
}

// SegmentTemplate represents XSD's SegmentTemplateType.
type SegmentTemplate struct {
	Timescale              *uint64            `xml:"timescale,attr"`
	Media                  *string            `xml:"media,attr"`
	Initialization         *string            `xml:"initialization,attr"`
	StartNumber            *uint64            `xml:"startNumber,attr"`
	PresentationTimeOffset *uint64            `xml:"presentationTimeOffset,attr"`
	SegmentTimelineS       []SegmentTimelineS `xml:"SegmentTimeline>S,omitempty"`
}

// SegmentTimelineS represents XSD's SegmentTimelineType's inner S elements.
type SegmentTimelineS struct {
	T *uint64 `xml:"t,attr"`
	D uint64  `xml:"d,attr"`
	R *int64  `xml:"r,attr"`
}
<commit_msg>add codecs attr on AdaptationSet struct<commit_after>// Package mpd implements parsing and generating of MPEG-DASH Media Presentation Description (MPD) files.
package mpd

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"regexp"
	"strconv"
)

// http://mpeg.chiariglione.org/standards/mpeg-dash
// https://www.brendanlong.com/the-structure-of-an-mpeg-dash-mpd.html
// http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd

var emptyElementRE = regexp.MustCompile(`></[A-Za-z]+>`)

// ConditionalUint (ConditionalUintType) defined in XSD as a union of unsignedInt and boolean.
type ConditionalUint struct {
	u *uint64
	b *bool
}

// MarshalXMLAttr encodes ConditionalUint.
func (c ConditionalUint) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
	if c.u != nil {
		return xml.Attr{Name: name, Value: strconv.FormatUint(*c.u, 10)}, nil
	}

	if c.b != nil {
		return xml.Attr{Name: name, Value: strconv.FormatBool(*c.b)}, nil
	}

	// both are nil - no attribute, client will threat it like "false"
	return xml.Attr{}, nil
}

// UnmarshalXMLAttr decodes ConditionalUint.
func (c *ConditionalUint) UnmarshalXMLAttr(attr xml.Attr) error {
	u, err := strconv.ParseUint(attr.Value, 10, 64)
	if err == nil {
		c.u = &u
		return nil
	}

	b, err := strconv.ParseBool(attr.Value)
	if err == nil {
		c.b = &b
		return nil
	}

	return fmt.Errorf("ConditionalUint: can't UnmarshalXMLAttr %#v", attr)
}

// check interfaces
var (
	_ xml.MarshalerAttr   = ConditionalUint{}
	_ xml.UnmarshalerAttr = &ConditionalUint{}
)

// MPD represents root XML element.
type MPD struct {
	XMLNS                      *string `xml:"xmlns,attr"`
	Type                       *string `xml:"type,attr"`
	MinimumUpdatePeriod        *string `xml:"minimumUpdatePeriod,attr"`
	AvailabilityStartTime      *string `xml:"availabilityStartTime,attr"`
	MediaPresentationDuration  *string `xml:"mediaPresentationDuration,attr"`
	MinBufferTime              *string `xml:"minBufferTime,attr"`
	SuggestedPresentationDelay *string `xml:"suggestedPresentationDelay,attr"`
	TimeShiftBufferDepth       *string `xml:"timeShiftBufferDepth,attr"`
	PublishTime                *string `xml:"publishTime,attr"`
	Profiles                   string  `xml:"profiles,attr"`
	Period                     *Period `xml:"Period,omitempty"`
}

// Do not try to use encoding.TextMarshaler and encoding.TextUnmarshaler:
// https://github.com/golang/go/issues/6859#issuecomment-118890463

// Encode generates MPD XML.
func (m *MPD) Encode() ([]byte, error) {
	x := new(bytes.Buffer)
	e := xml.NewEncoder(x)
	e.Indent("", "  ")
	err := e.Encode(m)
	if err != nil {
		return nil, err
	}

	// hacks for self-closing tags
	res := new(bytes.Buffer)
	res.WriteString(`<?xml version="1.0" encoding="utf-8"?>`)
	res.WriteByte('\n')
	for {
		s, err := x.ReadString('\n')
		if s != "" {
			s = emptyElementRE.ReplaceAllString(s, `/>`)
			res.WriteString(s)
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
	}
	res.WriteByte('\n')
	return res.Bytes(), err
}

// Decode parses MPD XML.
func (m *MPD) Decode(b []byte) error {
	return xml.Unmarshal(b, m)
}

// Period represents XSD's PeriodType.
type Period struct {
	Start          *string          `xml:"start,attr"`
	ID             *string          `xml:"id,attr"`
	Duration       *string          `xml:"duration,attr"`
	AdaptationSets []*AdaptationSet `xml:"AdaptationSet,omitempty"`
}

// AdaptationSet represents XSD's AdaptationSetType.
type AdaptationSet struct {
	MimeType                string           `xml:"mimeType,attr"`
	SegmentAlignment        ConditionalUint  `xml:"segmentAlignment,attr"`
	SubsegmentAlignment     ConditionalUint  `xml:"subsegmentAlignment,attr"`
	StartWithSAP            *uint64          `xml:"startWithSAP,attr"`
	SubsegmentStartsWithSAP *uint64          `xml:"subsegmentStartsWithSAP,attr"`
	BitstreamSwitching      *bool            `xml:"bitstreamSwitching,attr"`
	Lang                    *string          `xml:"lang,attr"`
	ContentProtections      []Descriptor     `xml:"ContentProtection,omitempty"`
	Representations         []Representation `xml:"Representation,omitempty"`
	Codecs                  *string          `xml:"codecs,attr"`
}

// Representation represents XSD's RepresentationType.
type Representation struct {
	ID                 *string          `xml:"id,attr"`
	Width              *uint64          `xml:"width,attr"`
	Height             *uint64          `xml:"height,attr"`
	FrameRate          *string          `xml:"frameRate,attr"`
	Bandwidth          *uint64          `xml:"bandwidth,attr"`
	AudioSamplingRate  *string          `xml:"audioSamplingRate,attr"`
	Codecs             *string          `xml:"codecs,attr"`
	ContentProtections []Descriptor     `xml:"ContentProtection,omitempty"`
	SegmentTemplate    *SegmentTemplate `xml:"SegmentTemplate,omitempty"`
}

// Descriptor represents XSD's DescriptorType.
type Descriptor struct {
	SchemeIDURI *string `xml:"schemeIdUri,attr"`
	Value       *string `xml:"value,attr"`
}

// SegmentTemplate represents XSD's SegmentTemplateType.
type SegmentTemplate struct {
	Timescale              *uint64            `xml:"timescale,attr"`
	Media                  *string            `xml:"media,attr"`
	Initialization         *string            `xml:"initialization,attr"`
	StartNumber            *uint64            `xml:"startNumber,attr"`
	PresentationTimeOffset *uint64            `xml:"presentationTimeOffset,attr"`
	SegmentTimelineS       []SegmentTimelineS `xml:"SegmentTimeline>S,omitempty"`
}

// SegmentTimelineS represents XSD's SegmentTimelineType's inner S elements.
type SegmentTimelineS struct {
	T *uint64 `xml:"t,attr"`
	D uint64  `xml:"d,attr"`
	R *int64  `xml:"r,attr"`
}
<|endoftext|>


<commit_before>// mtr is a package that processes and saves
// an `mtr --raw` call's output.
package mtr

import (
	"bytes"
	"fmt"
	"math"
	"net"
	"os/exec"
	"strconv"
)

type Host struct {
	IP              net.IP  `json:"ip"`
	Name            string  `json:"hostname"`
	PacketMicrosecs []int   `json:"packet-times"`
	Sent            int     `json:"sent"`
	Received        int     `json:"received"`
	Dropped         int     `json:"dropped"`
	LostPercent     float64 `json:"lost-percent"`
	// All packet units are in microseconds
	Mean               float64 `json:"mean"`
	Best               int     `json:"best"`
	Worst              int     `json:"worst"`
	StandardDev        float64 `json:"standard-dev"`
	GeometricMean      float64 `json:"geometric-mean"`
	MeanJitter         float64 `json:"mean-jitter"`
	WorstJitter        int     `json:"worst-jitter"`
	InterarrivalJitter int     `json:"interarrival-jitter"` // calculated with rfc3550 A.8 shortcut
}

type MTR struct {
	Done        chan struct{}
	OutputRaw   []byte
	Error       error
	PacketsSent int
	Hosts       []*Host `json:"hosts"`
}

// New runs mtr --raw -c reportCycles hostname args... Thus, you can add more arguments
// to the default required hostname and report cycles. The MTR call is signified as done
// when the MTR.Done chan closes. First wait for this, then check the MTR.Error field before
// looking at the output. Other than that, the fields and json tags document what everything
// means.
func New(reportCycles int, host string, args ...string) *MTR {
	m := &MTR{Done: make(chan struct{})}
	args = append([]string{"--raw", "-c", strconv.Itoa(reportCycles), host}, args...)
	go func() {
		defer close(m.Done)
		m.OutputRaw, m.Error = exec.Command("mtr", args...).Output()
		if m.Error == nil {
			m.processOutput()
		}
	}()
	return m
}

func parseByteNum(input []byte) int {
	i := 0
	for _, v := range input {
		i = 10*i + int(v) - 48 // ascii 48 is `0`
	}
	return i
}

func parseHostnum(line []byte) (num int, finalFieldIdx int) {
	finalFieldIdx = bytes.IndexByte(line[2:], ' ') + 2
	return parseByteNum(line[2:finalFieldIdx]), finalFieldIdx + 1 // `c ### <content>`
}

func (m *MTR) processOutput() {
	// h (host): host #, ip address
	// d (dns): host #, resolved dns name
	// p (packet): host #, microseconds
	defer func() {
		if x := recover(); x != nil {
			m.Error = fmt.Errorf("Unable to process output, error %v, meaning the mtr output is WHACK! Check the output to see what's up!", x)
		}
	}()
	output := m.OutputRaw
	output = append(output, ' ') // tack on a space at the end so that `output = output[lineIdx+1:] doesn't panic on last newline
	for {
		lineIdx := bytes.IndexByte(output, '\n')
		if lineIdx == -1 {
			break
		}
		line := output[:lineIdx]
		output = output[lineIdx+1:]

		hostnum, finalFieldIdx := parseHostnum(line)

		switch line[0] {
		case 'h':
			for len(m.Hosts) < hostnum+1 {
				m.PacketsSent++
				m.Hosts = append(m.Hosts, &Host{})
			}
			m.Hosts[hostnum].IP = net.ParseIP(string(line[finalFieldIdx:]))
		case 'd':
			m.Hosts[hostnum].Name = string(line[finalFieldIdx:])
		case 'p':
			m.Hosts[hostnum].PacketMicrosecs = append(m.Hosts[hostnum].PacketMicrosecs, parseByteNum(line[finalFieldIdx:]))
		}
	}

	m.processHosts()
}

func (m *MTR) processHosts() {
	for _, host := range m.Hosts {
		host.Sent = m.PacketsSent
		host.Received = len(host.PacketMicrosecs)
		host.Dropped = host.Sent - host.Received
		host.LostPercent = float64(host.Dropped) / float64(host.Sent)
		totalPacketTime := 0
		best := 1<<31 - 1
		worst := 0
		jitters := make([]int, host.Received)
		worstJitter := 0
		geoMult := float64(1)
		for i, packet := range host.PacketMicrosecs {
			if i > 0 {
				newJitter := packet - host.PacketMicrosecs[i-1]
				if newJitter > worstJitter {
					worstJitter = newJitter
				}
				jitters[i] = newJitter
				host.InterarrivalJitter += newJitter - ((host.InterarrivalJitter + 8) >> 4) // rfc3550 A.8
			}
			totalPacketTime += packet
			if packet > worst {
				worst = packet
			}
			if packet < best {
				best = packet
			}
			geoMult *= float64(packet)
		}
		// mtr keeps a running average, so values may be different than a true average
		host.Mean = float64(totalPacketTime) / float64(host.Received)
		host.Best = best
		host.Worst = worst
		sqrDiff := float64(0)
		jitterSum := 0
		for i, packet := range host.PacketMicrosecs {
			diff := float64(packet) - host.Mean
			sqrDiff += diff * diff
			jitterSum += jitters[i]
		}
		host.MeanJitter = float64(jitterSum) / float64(host.Received)
		host.StandardDev = math.Sqrt(sqrDiff / float64(host.Mean))
		host.GeometricMean = math.Pow(geoMult, 1/float64(host.Received))
	}
}
<commit_msg>mtr: math fixes, remove geometric mean<commit_after>// mtr is a package that processes and saves
// an `mtr --raw` call's output.
package mtr

import (
	"bytes"
	"fmt"
	"math"
	"net"
	"os/exec"
	"strconv"
)

type Host struct {
	IP              net.IP  `json:"ip"`
	Name            string  `json:"hostname"`
	Hop             int     `json:"hop-number"`
	PacketMicrosecs []int   `json:"packet-times"`
	Sent            int     `json:"sent"`
	Received        int     `json:"received"`
	Dropped         int     `json:"dropped"`
	LostPercent     float64 `json:"lost-percent"`
	// All packet units are in microseconds
	Mean               float64 `json:"mean"`
	Best               int     `json:"best"`
	Worst              int     `json:"worst"`
	StandardDev        float64 `json:"standard-dev"`
	MeanJitter         float64 `json:"mean-jitter"`
	WorstJitter        int     `json:"worst-jitter"`
	InterarrivalJitter int     `json:"interarrival-jitter"` // calculated with rfc3550 A.8 shortcut
}

type MTR struct {
	Done        chan struct{}
	OutputRaw   []byte
	Error       error
	PacketsSent int
	Hosts       []*Host `json:"hosts"`
}

// New runs mtr --raw -c reportCycles hostname args... Thus, you can add more arguments
// to the default required hostname and report cycles. The MTR call is signified as done
// when the MTR.Done chan closes. First wait for this, then check the MTR.Error field before
// looking at the output. Other than that, the fields and json tags document what everything
// means.
func New(reportCycles int, host string, args ...string) *MTR {
	m := &MTR{Done: make(chan struct{}), PacketsSent: reportCycles}
	args = append([]string{"--raw", "-c", strconv.Itoa(reportCycles), host}, args...)
	go func() {
		defer close(m.Done)
		m.OutputRaw, m.Error = exec.Command("mtr", args...).Output()
		if m.Error == nil {
			m.processOutput()
		}
	}()
	return m
}

func parseByteNum(input []byte) int {
	i := 0
	for _, v := range input {
		i = 10*i + int(v) - 48 // ascii 48 is `0`
	}
	return i
}

func parseHostnum(line []byte) (num int, finalFieldIdx int) {
	finalFieldIdx = bytes.IndexByte(line[2:], ' ') + 2
	return parseByteNum(line[2:finalFieldIdx]), finalFieldIdx + 1 // `c ### <content>`
}

func (m *MTR) processOutput() {
	// h (host): host #, ip address
	// d (dns): host #, resolved dns name
	// p (packet): host #, microseconds
	defer func() {
		if x := recover(); x != nil {
			m.Error = fmt.Errorf("Unable to process output, error %v, meaning the mtr output is WHACK! Check the output to see what's up!", x)
		}
	}()
	output := m.OutputRaw
	output = append(output, ' ') // tack on a space at the end so that `output = output[lineIdx+1:] doesn't panic on last newline
	for {
		lineIdx := bytes.IndexByte(output, '\n')
		if lineIdx == -1 {
			break
		}
		line := output[:lineIdx]
		output = output[lineIdx+1:]

		hostnum, finalFieldIdx := parseHostnum(line)

		switch line[0] {
		case 'h':
			if len(m.Hosts) < hostnum+1 {
				m.Hosts = append(m.Hosts, &Host{Hop: hostnum})
			}
			m.Hosts[hostnum].IP = net.ParseIP(string(line[finalFieldIdx:]))
		case 'd':
			m.Hosts[hostnum].Name = string(line[finalFieldIdx:])
		case 'p':
			m.Hosts[hostnum].PacketMicrosecs = append(m.Hosts[hostnum].PacketMicrosecs, parseByteNum(line[finalFieldIdx:]))
		}
	}

	m.processHosts()
}

func (m *MTR) processHosts() {
	for _, host := range m.Hosts {
		host.Sent = m.PacketsSent
		host.Received = len(host.PacketMicrosecs)
		host.Dropped = host.Sent - host.Received
		host.LostPercent = float64(host.Dropped) / float64(host.Sent)
		totalPacketTime := 0
		best := 1<<31 - 1
		worst := 0
		jitters := make([]int, host.Received)
		worstJitter := 0
		for i, packet := range host.PacketMicrosecs {
			if i > 0 {
				newJitter := packet - host.PacketMicrosecs[i-1]
				if newJitter < 0 {
					newJitter = -newJitter
				}
				if newJitter > worstJitter {
					worstJitter = newJitter
				}
				jitters[i] = newJitter
				host.InterarrivalJitter += newJitter - ((host.InterarrivalJitter + 8) >> 4) // rfc3550 A.8
			}
			totalPacketTime += packet
			if packet > worst {
				worst = packet
			}
			if packet < best {
				best = packet
			}
		}
		// mtr keeps a running average, so values may be different than a true average
		host.Mean = float64(totalPacketTime) / float64(host.Received)
		host.WorstJitter = worstJitter
		host.Best = best
		host.Worst = worst
		sqrDiff := float64(0)
		jitterSum := 0
		for i, packet := range host.PacketMicrosecs {
			diff := float64(packet) - host.Mean
			sqrDiff += diff * diff
			jitterSum += jitters[i]
		}
		host.MeanJitter = float64(jitterSum) / float64(host.Received)
		host.StandardDev = math.Sqrt(sqrDiff / float64(host.Mean))
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route)}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if matched := route.Match(req, match); matched {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	defer context.Clear(req)
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the slash behavior for new routes.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa.
//
// Special case: when a route sets a path prefix, strict slash is
// automatically set to false for that route because the redirect behavior
// can't be determined for prefixes.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request),) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairs(pairs ...string) (map[string]string, error) {
	length := len(pairs)
	if length%2 != 0 {
		return nil, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

// matchMap returns true if the given key/value pairs exist in a given map.
func matchMap(toCheck map[string]string, toMatch map[string][]string,
	canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<commit_msg>Update mux.go<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route)}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if matched := route.Match(req, match); matched {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	defer context.Clear(req)
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the slash behavior for new routes.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa.
//
// Special case: when a route sets a path prefix, strict slash is
// automatically set to false for that route because the redirect behavior
// can't be determined for prefixes.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request)) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairs(pairs ...string) (map[string]string, error) {
	length := len(pairs)
	if length%2 != 0 {
		return nil, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

// matchMap returns true if the given key/value pairs exist in a given map.
func matchMap(toCheck map[string]string, toMatch map[string][]string,
	canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"
	"regexp"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route), KeepContext: false}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
	// If true, do not clear the request context after handling the request
	KeepContext bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if route.Match(req, match) {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {

		// Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query.
		// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
		// http://code.google.com/p/go/issues/detail?id=5252
		url := *req.URL
		url.Path = p
		p = url.String()

		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if r.Match(req, &match) {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		handler = r.NotFoundHandler
		if handler == nil {
			handler = http.NotFoundHandler()
		}
	}
	if !r.KeepContext {
		defer context.Clear(req)
	}
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the trailing slash behavior for new routes. The initial
// value is false.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa. In other words, your application will always
// see the path as specified in the route.
//
// When false, if the route path is "/path", accessing "/path/" will not match
// this route and vice versa.
//
// Special case: when a route sets a path prefix using the PathPrefix() method,
// strict slash is ignored for that route because the redirect behavior can't
// be determined from a prefix alone. However, any subrouters created from that
// route inherit the original StrictSlash setting.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

func (r *Router) buildVars(m map[string]string) map[string]string {
	if r.parent != nil {
		m = r.parent.buildVars(m)
	}
	return m
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request)) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// BuildVars registers a new route with a custom function for modifying
// route variables before building a URL.
func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route {
	return r.NewRoute().BuildVarsFunc(f)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

func checkPairs(pairs ...string) (int, error) {
	length := len(pairs)
	if length%2 != 0 {
		return length, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	return length, nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairsToString(pairs ...string) (map[string]string, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]*regexp.Regexp, length/2)
	for i := 0; i < length; i += 2 {
		regex, _ := regexp.Compile(pairs[i+1])
		m[pairs[i]] = regex
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

type equals func(interface{}, interface{}) bool

// matchMap returns true if the given key/value pairs exist in a given map.
func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}

// matchMap returns true if the given key/value pairs exist in a given map.
func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != nil {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v.MatchString(value) {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<commit_msg>Fixed up commenting and removed unused code<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mux

import (
	"fmt"
	"net/http"
	"path"
	"regexp"

	"github.com/gorilla/context"
)

// NewRouter returns a new router instance.
func NewRouter() *Router {
	return &Router{namedRoutes: make(map[string]*Route), KeepContext: false}
}

// Router registers routes to be matched and dispatches a handler.
//
// It implements the http.Handler interface, so it can be registered to serve
// requests:
//
//     var router = mux.NewRouter()
//
//     func main() {
//         http.Handle("/", router)
//     }
//
// Or, for Google App Engine, register it in a init() function:
//
//     func init() {
//         http.Handle("/", router)
//     }
//
// This will send all incoming requests to the router.
type Router struct {
	// Configurable Handler to be used when no route matches.
	NotFoundHandler http.Handler
	// Parent route, if this is a subrouter.
	parent parentRoute
	// Routes to be matched, in order.
	routes []*Route
	// Routes by name for URL building.
	namedRoutes map[string]*Route
	// See Router.StrictSlash(). This defines the flag for new routes.
	strictSlash bool
	// If true, do not clear the request context after handling the request
	KeepContext bool
}

// Match matches registered routes against the request.
func (r *Router) Match(req *http.Request, match *RouteMatch) bool {
	for _, route := range r.routes {
		if route.Match(req, match) {
			return true
		}
	}
	return false
}

// ServeHTTP dispatches the handler registered in the matched route.
//
// When there is a match, the route variables can be retrieved calling
// mux.Vars(request).
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {

		// Added 3 lines (Philip Schlump) - It was droping the query string and #whatever from query.
		// This matches with fix in go 1.2 r.c. 4 for same problem.  Go Issue:
		// http://code.google.com/p/go/issues/detail?id=5252
		url := *req.URL
		url.Path = p
		p = url.String()

		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match RouteMatch
	var handler http.Handler
	if r.Match(req, &match) {
		handler = match.Handler
		setVars(req, match.Vars)
		setCurrentRoute(req, match.Route)
	}
	if handler == nil {
		handler = r.NotFoundHandler
		if handler == nil {
			handler = http.NotFoundHandler()
		}
	}
	if !r.KeepContext {
		defer context.Clear(req)
	}
	handler.ServeHTTP(w, req)
}

// Get returns a route registered with the given name.
func (r *Router) Get(name string) *Route {
	return r.getNamedRoutes()[name]
}

// GetRoute returns a route registered with the given name. This method
// was renamed to Get() and remains here for backwards compatibility.
func (r *Router) GetRoute(name string) *Route {
	return r.getNamedRoutes()[name]
}

// StrictSlash defines the trailing slash behavior for new routes. The initial
// value is false.
//
// When true, if the route path is "/path/", accessing "/path" will redirect
// to the former and vice versa. In other words, your application will always
// see the path as specified in the route.
//
// When false, if the route path is "/path", accessing "/path/" will not match
// this route and vice versa.
//
// Special case: when a route sets a path prefix using the PathPrefix() method,
// strict slash is ignored for that route because the redirect behavior can't
// be determined from a prefix alone. However, any subrouters created from that
// route inherit the original StrictSlash setting.
func (r *Router) StrictSlash(value bool) *Router {
	r.strictSlash = value
	return r
}

// ----------------------------------------------------------------------------
// parentRoute
// ----------------------------------------------------------------------------

// getNamedRoutes returns the map where named routes are registered.
func (r *Router) getNamedRoutes() map[string]*Route {
	if r.namedRoutes == nil {
		if r.parent != nil {
			r.namedRoutes = r.parent.getNamedRoutes()
		} else {
			r.namedRoutes = make(map[string]*Route)
		}
	}
	return r.namedRoutes
}

// getRegexpGroup returns regexp definitions from the parent route, if any.
func (r *Router) getRegexpGroup() *routeRegexpGroup {
	if r.parent != nil {
		return r.parent.getRegexpGroup()
	}
	return nil
}

func (r *Router) buildVars(m map[string]string) map[string]string {
	if r.parent != nil {
		m = r.parent.buildVars(m)
	}
	return m
}

// ----------------------------------------------------------------------------
// Route factories
// ----------------------------------------------------------------------------

// NewRoute registers an empty route.
func (r *Router) NewRoute() *Route {
	route := &Route{parent: r, strictSlash: r.strictSlash}
	r.routes = append(r.routes, route)
	return route
}

// Handle registers a new route with a matcher for the URL path.
// See Route.Path() and Route.Handler().
func (r *Router) Handle(path string, handler http.Handler) *Route {
	return r.NewRoute().Path(path).Handler(handler)
}

// HandleFunc registers a new route with a matcher for the URL path.
// See Route.Path() and Route.HandlerFunc().
func (r *Router) HandleFunc(path string, f func(http.ResponseWriter,
	*http.Request)) *Route {
	return r.NewRoute().Path(path).HandlerFunc(f)
}

// Headers registers a new route with a matcher for request header values.
// See Route.Headers().
func (r *Router) Headers(pairs ...string) *Route {
	return r.NewRoute().Headers(pairs...)
}

// Host registers a new route with a matcher for the URL host.
// See Route.Host().
func (r *Router) Host(tpl string) *Route {
	return r.NewRoute().Host(tpl)
}

// MatcherFunc registers a new route with a custom matcher function.
// See Route.MatcherFunc().
func (r *Router) MatcherFunc(f MatcherFunc) *Route {
	return r.NewRoute().MatcherFunc(f)
}

// Methods registers a new route with a matcher for HTTP methods.
// See Route.Methods().
func (r *Router) Methods(methods ...string) *Route {
	return r.NewRoute().Methods(methods...)
}

// Path registers a new route with a matcher for the URL path.
// See Route.Path().
func (r *Router) Path(tpl string) *Route {
	return r.NewRoute().Path(tpl)
}

// PathPrefix registers a new route with a matcher for the URL path prefix.
// See Route.PathPrefix().
func (r *Router) PathPrefix(tpl string) *Route {
	return r.NewRoute().PathPrefix(tpl)
}

// Queries registers a new route with a matcher for URL query values.
// See Route.Queries().
func (r *Router) Queries(pairs ...string) *Route {
	return r.NewRoute().Queries(pairs...)
}

// Schemes registers a new route with a matcher for URL schemes.
// See Route.Schemes().
func (r *Router) Schemes(schemes ...string) *Route {
	return r.NewRoute().Schemes(schemes...)
}

// BuildVars registers a new route with a custom function for modifying
// route variables before building a URL.
func (r *Router) BuildVarsFunc(f BuildVarsFunc) *Route {
	return r.NewRoute().BuildVarsFunc(f)
}

// ----------------------------------------------------------------------------
// Context
// ----------------------------------------------------------------------------

// RouteMatch stores information about a matched route.
type RouteMatch struct {
	Route   *Route
	Handler http.Handler
	Vars    map[string]string
}

type contextKey int

const (
	varsKey contextKey = iota
	routeKey
)

// Vars returns the route variables for the current request, if any.
func Vars(r *http.Request) map[string]string {
	if rv := context.Get(r, varsKey); rv != nil {
		return rv.(map[string]string)
	}
	return nil
}

// CurrentRoute returns the matched route for the current request, if any.
func CurrentRoute(r *http.Request) *Route {
	if rv := context.Get(r, routeKey); rv != nil {
		return rv.(*Route)
	}
	return nil
}

func setVars(r *http.Request, val interface{}) {
	context.Set(r, varsKey, val)
}

func setCurrentRoute(r *http.Request, val interface{}) {
	context.Set(r, routeKey, val)
}

// ----------------------------------------------------------------------------
// Helpers
// ----------------------------------------------------------------------------

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}

// uniqueVars returns an error if two slices contain duplicated strings.
func uniqueVars(s1, s2 []string) error {
	for _, v1 := range s1 {
		for _, v2 := range s2 {
			if v1 == v2 {
				return fmt.Errorf("mux: duplicated route variable %q", v2)
			}
		}
	}
	return nil
}

func checkPairs(pairs ...string) (int, error) {
	length := len(pairs)
	if length%2 != 0 {
		return length, fmt.Errorf(
			"mux: number of parameters must be multiple of 2, got %v", pairs)
	}
	return length, nil
}

// mapFromPairs converts variadic string parameters to a string map.
func mapFromPairsToString(pairs ...string) (map[string]string, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]string, length/2)
	for i := 0; i < length; i += 2 {
		m[pairs[i]] = pairs[i+1]
	}
	return m, nil
}

func mapFromPairsToRegex(pairs ...string) (map[string]*regexp.Regexp, error) {
	length, err := checkPairs(pairs...)
	if err != nil {
		return nil, err
	}
	m := make(map[string]*regexp.Regexp, length/2)
	for i := 0; i < length; i += 2 {
		regex, _ := regexp.Compile(pairs[i+1])
		m[pairs[i]] = regex
	}
	return m, nil
}

// matchInArray returns true if the given string value is in the array.
func matchInArray(arr []string, value string) bool {
	for _, v := range arr {
		if v == value {
			return true
		}
	}
	return false
}

// matchMapWithString returns true if the given key/value pairs exist in a given map.
func matchMapWithString(toCheck map[string]string, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != "" {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v == value {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}

// matchMapWithRegex returns true if the given key/value pairs exist in a given map compiled against
// the given regex
func matchMapWithRegex(toCheck map[string]*regexp.Regexp, toMatch map[string][]string, canonicalKey bool) bool {
	for k, v := range toCheck {
		// Check if key exists.
		if canonicalKey {
			k = http.CanonicalHeaderKey(k)
		}
		if values := toMatch[k]; values == nil {
			return false
		} else if v != nil {
			// If value was defined as an empty string we only check that the
			// key exists. Otherwise we also check for equality.
			valueExists := false
			for _, value := range values {
				if v.MatchString(value) {
					valueExists = true
					break
				}
			}
			if !valueExists {
				return false
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"sync"
)

// Accept incoming connections and spawn Go routines to handle them.
func accept(listener net.Listener, wg *sync.WaitGroup, stopper chan bool, leaf *x509.Certificate) {
	defer wg.Done()
	defer listener.Close()

	for {
		// Wait for new connection
		conn, err := listener.Accept()

		// Check if we're supposed to stop
		select {
		case _ = <-stopper:
			logger.Printf("closing socket with cert serial no. %d (expiring %s)", leaf.SerialNumber, leaf.NotAfter.String())
			return
		default:
		}

		if err != nil {
			logger.Printf("error accepting connection: %s", err)
			continue
		}

		tlsConn, ok := conn.(*tls.Conn)
		if !ok {
			logger.Printf("received non-TLS connection from %s? Ignoring", conn.RemoteAddr())
			conn.Close()
			continue
		}

		// Force handshake. Handshake usually happens on first read/write, but
		// we want to authenticate before reading/writing so we need to force
		// the handshake to get the client cert.
		err = tlsConn.Handshake()
		if err != nil {
			logger.Printf("failed TLS handshake on %s: %s", conn.RemoteAddr(), err)
			conn.Close()
			continue
		}

		if !authorized(tlsConn.ConnectionState()) {
			logger.Printf("rejecting connection from %s: bad client certificate", conn.RemoteAddr())
			conn.Close()
			continue
		}

		wg.Add(1)
		go handle(conn, wg)
	}
}

// Handle incoming connection by opening new connection to our backend service
// and fusing them together.
func handle(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()
	defer conn.Close()

	logger.Printf("incoming connection: %s", conn.RemoteAddr())

	backend, err := net.Dial((*forwardAddress).Network(), (*forwardAddress).String())

	if err != nil {
		logger.Printf("failed to dial backend: %s", err)
		return
	}

	defer backend.Close()
	fuse(conn, backend)
}

// Fuse connections together
func fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	go func() { copyData(client, backend) }()
	copyData(backend, client)
}

// Copy data between two connections
func copyData(dst net.Conn, src net.Conn) {
	defer logger.Printf("closed pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())
	logger.Printf("opening pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())

	_, err := io.Copy(dst, src)

	if err != nil {
		logger.Printf("%s", err)
	}
}

// Helper function to decode a *net.TCPAddr into a tuple of network and
// address. Must use this since kavu/so_reuseport does not currently
// support passing "tcp" to support for IPv4 and IPv6. We must pass "tcp4"
// or "tcp6" explicitly.
func decodeAddress(tuple *net.TCPAddr) (network, address string) {
	if tuple.IP.To4() != nil {
		network = "tcp4"
	} else {
		network = "tcp6"
	}

	address = tuple.String()
	return
}
<commit_msg>Close client connection is server connection closes.<commit_after>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"crypto/tls"
	"crypto/x509"
	"io"
	"net"
	"sync"
)

// Accept incoming connections and spawn Go routines to handle them.
func accept(listener net.Listener, wg *sync.WaitGroup, stopper chan bool, leaf *x509.Certificate) {
	defer wg.Done()
	defer listener.Close()

	for {
		// Wait for new connection
		conn, err := listener.Accept()

		// Check if we're supposed to stop
		select {
		case _ = <-stopper:
			logger.Printf("closing socket with cert serial no. %d (expiring %s)", leaf.SerialNumber, leaf.NotAfter.String())
			return
		default:
		}

		if err != nil {
			logger.Printf("error accepting connection: %s", err)
			continue
		}

		tlsConn, ok := conn.(*tls.Conn)
		if !ok {
			logger.Printf("received non-TLS connection from %s? Ignoring", conn.RemoteAddr())
			conn.Close()
			continue
		}

		// Force handshake. Handshake usually happens on first read/write, but
		// we want to authenticate before reading/writing so we need to force
		// the handshake to get the client cert.
		err = tlsConn.Handshake()
		if err != nil {
			logger.Printf("failed TLS handshake on %s: %s", conn.RemoteAddr(), err)
			conn.Close()
			continue
		}

		if !authorized(tlsConn.ConnectionState()) {
			logger.Printf("rejecting connection from %s: bad client certificate", conn.RemoteAddr())
			conn.Close()
			continue
		}

		wg.Add(1)
		go handle(conn, wg)
	}
}

// Handle incoming connection by opening new connection to our backend service
// and fusing them together.
func handle(conn net.Conn, wg *sync.WaitGroup) {
	defer wg.Done()

	logger.Printf("incoming connection: %s", conn.RemoteAddr())

	backend, err := net.Dial((*forwardAddress).Network(), (*forwardAddress).String())

	if err != nil {
		logger.Printf("failed to dial backend: %s", err)
		return
	}

	fuse(conn, backend)
}

// Fuse connections together
func fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	go func() { copyData(client, backend) }()
	copyData(backend, client)
}

// Copy data between two connections
func copyData(dst net.Conn, src net.Conn) {
	defer dst.Close()
	defer src.Close()
	defer logger.Printf("closed pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())
	logger.Printf("opening pipe: %s <- %s", dst.RemoteAddr(), src.RemoteAddr())

	_, err := io.Copy(dst, src)

	if err != nil {
		logger.Printf("%s", err)
	}
}

// Helper function to decode a *net.TCPAddr into a tuple of network and
// address. Must use this since kavu/so_reuseport does not currently
// support passing "tcp" to support for IPv4 and IPv6. We must pass "tcp4"
// or "tcp6" explicitly.
func decodeAddress(tuple *net.TCPAddr) (network, address string) {
	if tuple.IP.To4() != nil {
		network = "tcp4"
	} else {
		network = "tcp6"
	}

	address = tuple.String()
	return
}
<|endoftext|>


<commit_before>// +build ignore
package main

import "testing"

// START OMIT

var testCases = []struct {
	in  string
	out string
}{
	{"a", "A"},
	{"b", "B"},
	{"c", "C"},
	{"d", "D"},
	{"e", "E"},
}

func TestToUpper(t *testing.T) {
	for _, tc := range testCases {
		if result := ToUpper(tc.in); result != out {
			t.Errorf("ToUpper(%q) => %q, want %q", tc.in, result, tt.out)
		}
	}
}

// END OMIT
<commit_msg>Update 2015/0220-good-package/table_driven_tests_test.go<commit_after>// +build ignore
package main

import "testing"

// START OMIT

var testCases = []struct {
	in  string
	out string
}{
	{"a", "A"},
	{"b", "B"},
	{"c", "C"},
	{"d", "D"},
	{"e", "E"},
}

func TestToUpper(t *testing.T) {
	for _, tc := range testCases {
		if result := ToUpper(tc.in); result != tc.out {
			t.Errorf("ToUpper(%q) => %q, want %q", tc.in, result, tc.out)
		}
	}
}

// END OMIT
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"errors"
	"log"
	"os"
	"path"
	"strings"

	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jbrukh/bayesian"
	"github.com/avabot/ava/shared/datatypes"
)

const (
	Command bayesian.Class = "Command"
	Actor   bayesian.Class = "Actor"
	Object  bayesian.Class = "Object"
	Time    bayesian.Class = "Time"
	Place   bayesian.Class = "Place"
	None    bayesian.Class = "None"
)

var (
	ErrMissingFlexIdType = errors.New("missing flexidtype")
	ErrSentenceTooShort  = errors.New("sentence too short to classify")
)

func train(c *bayesian.Classifier, s string) error {
	log.Println("training classifier")
	if err := trainClassifier(c, s); err != nil {
		return err
	}
	buf := bytes.NewBuffer([]byte{})
	if err := c.WriteTo(buf); err != nil {
		return err
	}
	q := `UPDATE ml SET data=$1 WHERE name='ner'`
	_, err := db.Exec(q, buf.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func loadClassifier(c *bayesian.Classifier) (*bayesian.Classifier, error) {
	log.Println("loading classifier")
	var err error
	q := `SELECT data FROM ml WHERE name='ner' LIMIT 1`
	row := db.QueryRowx(q)
	var tmp []byte
	err = row.Scan(&tmp)
	buf := bytes.NewBuffer(tmp)
	if err == sql.ErrNoRows {
		c, err = buildClassifier(c)
		if err != nil {
			log.Println("err building classifier")
			return c, err
		}
	} else if err != nil {
		log.Println("err getting classifier from DB")
		return c, err
	}
	c, err = bayesian.NewClassifierFromReader(buf)
	if err != nil {
		log.Println("err building new classifier from reader", err)
		return c, err
	}
	return c, nil
}

func buildClassifier(c *bayesian.Classifier) (*bayesian.Classifier, error) {
	c = bayesian.NewClassifier(Command, Actor, Object, Time, Place, None)
	filename := path.Join("data", "training", "imperative.txt")
	fi, err := os.Open(filename)
	if err != nil {
		return c, err
	}
	defer fi.Close()
	scanner := bufio.NewScanner(fi)
	line := 1
	for scanner.Scan() {
		if err := trainClassifier(c, scanner.Text()); err != nil {
			log.Println("err: line", line, "::", err)
		}
		line++
	}
	if err := scanner.Err(); err != nil {
		return c, err
	}
	buf := bytes.NewBuffer([]byte{})
	if err := c.WriteTo(buf); err != nil {
		return c, err
	}
	q := `INSERT INTO ml (name, data) VALUES ('ner', $1)`
	_, err = db.Exec(q, buf.Bytes())
	if err != nil {
		log.Println("err updating ml.ner", err)
		return c, err
	}
	log.Println("new classifier trained")
	return c, nil
}

func trainClassifier(c *bayesian.Classifier, s string) error {
	if len(s) == 0 || s[0] == '/' {
		return nil
	}
	ws := strings.Fields(s)
	l := len(ws)
	for i := 0; i < l; i++ {
		var word2 string
		var word3 string
		word1, entity, err := extractEntity(ws[i])
		if err != nil {
			return err
		}
		if entity == "" {
			continue
		}
		trigram := word1
		if i+1 < l {
			word2, _, err = extractEntity(ws[i+1])
			if err != nil {
				return err
			}
			trigram += " " + word2
		}
		if i+2 < l {
			word3, _, err = extractEntity(ws[i+2])
			if err != nil {
				return err
			}
			trigram += " " + word3
		}
		c.Learn([]string{word1}, entity)
		if word2 != "" {
			c.Learn([]string{word1 + " " + word2}, entity)
		}
		if word3 != "" {
			c.Learn([]string{trigram}, entity)
		}
	}
	return nil
}

// classify builds a StructuredInput from a sentence. The bool specifies whether
// additional training is needed for that sentence.
func classify(c *bayesian.Classifier, s string) (*datatypes.StructuredInput, bool, error) {
	si := &datatypes.StructuredInput{}
	if len(s) == 0 {
		return si, false, ErrSentenceTooShort
	}
	ws := strings.Fields(s)
	var wc []datatypes.WordClass
	var needsTraining bool
	for i := range ws {
		var err error
		var tmp datatypes.WordClass
		tmp, needsTraining, err = classifyTrigram(c, s, ws, i)
		if err != nil {
			return si, false, err
		}
		wc = append(wc, tmp)
	}
	if err := si.Add(wc); err != nil {
		return si, false, err
	}
	return si, needsTraining, nil
}

// addContext to a StructuredInput, replacing pronouns with the nouns to which
// they refer. TODO refactor
func addContext(m *datatypes.Message) (*datatypes.Message, bool, error) {
	ctxAdded := false
	for _, w := range m.Input.StructuredInput.Pronouns() {
		var ctx string
		var err error
		switch datatypes.Pronouns[w] {
		case datatypes.ObjectI:
			ctx, err = getContextObject(m.User,
				m.Input.StructuredInput,
				"objects")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Objects {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Objects[i] = ctx
				ctxAdded = true
			}
		case datatypes.ActorI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"actors")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Actors {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Actors[i] = ctx
				ctxAdded = true
			}
		case datatypes.TimeI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"times")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Times {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Times[i] = ctx
				ctxAdded = true
			}
		case datatypes.PlaceI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"places")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Places {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Places[i] = ctx
				ctxAdded = true
			}
		default:
			return m, false,
				errors.New("unknown type found for pronoun")
		}
		log.Println("ctx: ", ctx)
	}
	return m, ctxAdded, nil
}

// extractEntity from a word. If a Command, strip any contraction. For example,
// "where's" -> where. Since Ava ignores linking verbs, there's no need to
// add "is" back into the sentence.
func extractEntity(w string) (string, bayesian.Class, error) {
	w = strings.TrimRight(w, ").,;?!:")
	if w[0] != '_' {
		return w, "", nil
	}
	switch w[1] {
	case 'C':
		return w[3:], Command, nil
	case 'O':
		return w[3:], Object, nil
	case 'A':
		return w[3:], Actor, nil
	case 'T':
		return w[3:], Time, nil
	case 'P':
		return w[3:], Place, nil
	case 'N':
		return w[3:], None, nil
	}
	return w, "", errors.New("syntax error in entity")
}

// classifyTrigram determines the best classification for a word in a sentence
// given its surrounding context (i, i+1, i+2). Underflow on the returned
// probabilities is possible, but ignored, since classifyTrigram prefers a >=70%
// confidence level. The bool returned specified whether additional training is
// needed.
func classifyTrigram(c *bayesian.Classifier, s string, ws []string, i int) (
	datatypes.WordClass, bool, error) {
	// TODO: Given the last 2 words of a sentence, construct the trigram
	// including prior words.
	var wc datatypes.WordClass
	l := len(ws)
	word1, _, err := extractEntity(ws[i])
	if err != nil {
		return wc, false, err
	}
	bigram := word1
	trigram := word1
	var word2, word3 string
	if i+1 < l {
		word2, _, err = extractEntity(ws[i+1])
		if err != nil {
			return wc, false, err
		}
		bigram += " " + word2
		trigram += " " + word2
	}
	if i+2 < l {
		word3, _, err = extractEntity(ws[i+2])
		if err != nil {
			return wc, false, err
		}
		trigram += " " + word3
	}
	var needsTraining bool
	probs, likely, _ := c.ProbScores([]string{trigram})
	if max(probs) <= 0.7 {
		probs, likely, _ = c.ProbScores([]string{bigram})
		needsTraining = true
	}
	m := max(probs)
	if m <= 0.7 {
		probs, likely, _ = c.ProbScores([]string{word1})
		needsTraining = true
	}
	m = max(probs)
	if m <= 0.7 {
		log.Println(word1, " || ", datatypes.String[likely+1], " || ", m)
		needsTraining = true
	}
	return datatypes.WordClass{word1, likely + 1}, needsTraining, nil
}

func max(slice []float64) float64 {
	m := slice[0]
	for index := 1; index < len(slice); index++ {
		if slice[index] > m {
			m = slice[index]
		}
	}
	return m
}
<commit_msg>Fix index out of range error<commit_after>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"errors"
	"log"
	"os"
	"path"
	"strings"

	"github.com/avabot/ava/Godeps/_workspace/src/github.com/jbrukh/bayesian"
	"github.com/avabot/ava/shared/datatypes"
)

const (
	Command bayesian.Class = "Command"
	Actor   bayesian.Class = "Actor"
	Object  bayesian.Class = "Object"
	Time    bayesian.Class = "Time"
	Place   bayesian.Class = "Place"
	None    bayesian.Class = "None"
)

var (
	ErrMissingFlexIdType = errors.New("missing flexidtype")
	ErrSentenceTooShort  = errors.New("sentence too short to classify")
)

func train(c *bayesian.Classifier, s string) error {
	log.Println("training classifier")
	if err := trainClassifier(c, s); err != nil {
		return err
	}
	buf := bytes.NewBuffer([]byte{})
	if err := c.WriteTo(buf); err != nil {
		return err
	}
	q := `UPDATE ml SET data=$1 WHERE name='ner'`
	_, err := db.Exec(q, buf.Bytes())
	if err != nil {
		return err
	}
	return nil
}

func loadClassifier(c *bayesian.Classifier) (*bayesian.Classifier, error) {
	log.Println("loading classifier")
	var err error
	q := `SELECT data FROM ml WHERE name='ner' LIMIT 1`
	row := db.QueryRowx(q)
	var tmp []byte
	err = row.Scan(&tmp)
	buf := bytes.NewBuffer(tmp)
	if err == sql.ErrNoRows {
		c, err = buildClassifier(c)
		if err != nil {
			log.Println("err building classifier")
			return c, err
		}
	} else if err != nil {
		log.Println("err getting classifier from DB")
		return c, err
	}
	c, err = bayesian.NewClassifierFromReader(buf)
	if err != nil {
		log.Println("err building new classifier from reader", err)
		return c, err
	}
	return c, nil
}

func buildClassifier(c *bayesian.Classifier) (*bayesian.Classifier, error) {
	c = bayesian.NewClassifier(Command, Actor, Object, Time, Place, None)
	filename := path.Join("data", "training", "imperative.txt")
	fi, err := os.Open(filename)
	if err != nil {
		return c, err
	}
	defer fi.Close()
	scanner := bufio.NewScanner(fi)
	line := 1
	for scanner.Scan() {
		if err := trainClassifier(c, scanner.Text()); err != nil {
			log.Println("err: line", line, "::", err)
		}
		line++
	}
	if err := scanner.Err(); err != nil {
		return c, err
	}
	buf := bytes.NewBuffer([]byte{})
	if err := c.WriteTo(buf); err != nil {
		return c, err
	}
	q := `INSERT INTO ml (name, data) VALUES ('ner', $1)`
	_, err = db.Exec(q, buf.Bytes())
	if err != nil {
		log.Println("err updating ml.ner", err)
		return c, err
	}
	log.Println("new classifier trained")
	return c, nil
}

func trainClassifier(c *bayesian.Classifier, s string) error {
	if len(s) == 0 || s[0] == '/' {
		return nil
	}
	ws := strings.Fields(s)
	l := len(ws)
	for i := 0; i < l; i++ {
		var word2 string
		var word3 string
		word1, entity, err := extractEntity(ws[i])
		if err != nil {
			return err
		}
		if entity == "" {
			continue
		}
		trigram := word1
		if i+1 < l {
			word2, _, err = extractEntity(ws[i+1])
			if err != nil {
				return err
			}
			trigram += " " + word2
		}
		if i+2 < l {
			word3, _, err = extractEntity(ws[i+2])
			if err != nil {
				return err
			}
			trigram += " " + word3
		}
		c.Learn([]string{word1}, entity)
		if word2 != "" {
			c.Learn([]string{word1 + " " + word2}, entity)
		}
		if word3 != "" {
			c.Learn([]string{trigram}, entity)
		}
	}
	return nil
}

// classify builds a StructuredInput from a sentence. The bool specifies whether
// additional training is needed for that sentence.
func classify(c *bayesian.Classifier, s string) (*datatypes.StructuredInput, bool, error) {
	si := &datatypes.StructuredInput{}
	if len(s) == 0 {
		return si, false, ErrSentenceTooShort
	}
	ws := strings.Fields(s)
	var wc []datatypes.WordClass
	var needsTraining bool
	for i := range ws {
		var err error
		var tmp datatypes.WordClass
		tmp, needsTraining, err = classifyTrigram(c, s, ws, i)
		if err != nil {
			return si, false, err
		}
		wc = append(wc, tmp)
	}
	if err := si.Add(wc); err != nil {
		return si, false, err
	}
	return si, needsTraining, nil
}

// addContext to a StructuredInput, replacing pronouns with the nouns to which
// they refer. TODO refactor
func addContext(m *datatypes.Message) (*datatypes.Message, bool, error) {
	ctxAdded := false
	for _, w := range m.Input.StructuredInput.Pronouns() {
		var ctx string
		var err error
		switch datatypes.Pronouns[w] {
		case datatypes.ObjectI:
			ctx, err = getContextObject(m.User,
				m.Input.StructuredInput,
				"objects")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Objects {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Objects[i] = ctx
				ctxAdded = true
			}
		case datatypes.ActorI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"actors")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Actors {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Actors[i] = ctx
				ctxAdded = true
			}
		case datatypes.TimeI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"times")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Times {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Times[i] = ctx
				ctxAdded = true
			}
		case datatypes.PlaceI:
			ctx, err = getContextObject(m.User, m.Input.StructuredInput,
				"places")
			if err != nil {
				return m, false, err
			}
			if ctx == "" {
				return m, false, nil
			}
			for i, o := range m.Input.StructuredInput.Places {
				if o != w {
					continue
				}
				m.Input.StructuredInput.Places[i] = ctx
				ctxAdded = true
			}
		default:
			return m, false,
				errors.New("unknown type found for pronoun")
		}
		log.Println("ctx: ", ctx)
	}
	return m, ctxAdded, nil
}

// extractEntity from a word. If a Command, strip any contraction. For example,
// "where's" -> where. Since Ava ignores linking verbs, there's no need to
// add "is" back into the sentence.
func extractEntity(w string) (string, bayesian.Class, error) {
	w = strings.TrimRight(w, ").,;?!:")
	if w[0] != '_' {
		return w, "", nil
	}
	switch w[1] {
	case 'C':
		return w[3:], Command, nil
	case 'O':
		return w[3:], Object, nil
	case 'A':
		return w[3:], Actor, nil
	case 'T':
		return w[3:], Time, nil
	case 'P':
		return w[3:], Place, nil
	case 'N':
		return w[3:], None, nil
	}
	return w, "", errors.New("syntax error in entity")
}

// classifyTrigram determines the best classification for a word in a sentence
// given its surrounding context (i, i+1, i+2). Underflow on the returned
// probabilities is possible, but ignored, since classifyTrigram prefers a >=70%
// confidence level. The bool returned specified whether additional training is
// needed.
func classifyTrigram(c *bayesian.Classifier, s string, ws []string, i int) (
	datatypes.WordClass, bool, error) {
	// TODO: Given the last 2 words of a sentence, construct the trigram
	// including prior words.
	var wc datatypes.WordClass
	l := len(ws)
	word1, _, err := extractEntity(ws[i])
	if err != nil {
		return wc, false, err
	}
	bigram := word1
	trigram := word1
	var word2, word3 string
	if i+1 < l {
		word2, _, err = extractEntity(ws[i+1])
		if err != nil {
			return wc, false, err
		}
		bigram += " " + word2
		trigram += " " + word2
	}
	if i+2 < l {
		word3, _, err = extractEntity(ws[i+2])
		if err != nil {
			return wc, false, err
		}
		trigram += " " + word3
	}
	var needsTraining bool
	probs, likely, _ := c.ProbScores([]string{trigram})
	if max(probs) <= 0.7 {
		probs, likely, _ = c.ProbScores([]string{bigram})
		needsTraining = true
	}
	m := max(probs)
	if m <= 0.7 {
		probs, likely, _ = c.ProbScores([]string{word1})
		needsTraining = true
	}
	m = max(probs)
	if m <= 0.7 {
		log.Println(word1, " || ", datatypes.String[likely+1], " || ", m)
		needsTraining = true
	}
	return datatypes.WordClass{word1, likely + 1}, needsTraining, nil
}

func max(slice []float64) float64 {
	if len(slice) == 0 {
		return 0.0
	}
	m := slice[0]
	for index := 1; index < len(slice); index++ {
		if slice[index] > m {
			m = slice[index]
		}
	}
	return m
}
<|endoftext|>


<commit_before>package nth

import "sort"

// Element finds the nth rank ordered element and ensures it is at the nth
// position.
func Element(data sort.Interface, n int) {
	l := data.Len()
	if n < 0 || n >= l {
		return
	}
	quickSelect(data, n, 0, l)
}

func quickSelect(data sort.Interface, n, a, b int) {
	pivot := a
	for i := a + 1; i < b; i++ {
		if data.Less(i, pivot) { // data[i] < data[pivot]
			// data[i] needs to be before data[pivot]
			data.Swap(pivot, i)
			pivot++
			data.Swap(pivot, i)
		}
	}
	if a+n < pivot {
		quickSelect(data, n, a, pivot)
	} else if pivot < a+n {
		quickSelect(data, a+n-pivot-1, pivot+1, b)
	}
}
<commit_msg>implementation of quickSelectAdaptive with stubbed out pivot functions<commit_after>package nth

import "sort"

// Element finds the nth rank ordered element and ensures it is at the nth
// position.
func Element(data sort.Interface, n int) {
	l := data.Len()
	if n < 0 || n >= l {
		return
	}
	quickSelectAdaptive(data, n, 0, l)
}

// quickSelectAdaptive is from "Fast Deterministic Selection" by Andrei
// Alexandrescu https://arxiv.org/abs/1606.00484
//
// It is deterministic O(n) selection algorithm tuned for real world
// workloads.
func quickSelectAdaptive(data sort.Interface, k, a, b int) {
	var (
		l int // |A| from the paper
		p int // pivot position
	)
	for {
		l = b - a
		r := float64(k) / float64(l) // r <- real(k) / real(|A|)
		if l < 12 {
			p = hoarePartition(data, a+l/2, a, b) // HoarePartition(A, |A| / 2)
		} else if r < 7.0/16.0 {
			if r < 1.0/12.0 {
				p = repeatedStepFarLeft(data, k, a, b)
			} else {
				p = repeatedStepLeft(data, k, a, b)
			}
		} else if r >= 1.0-7.0/16.0 {
			if r >= 1.0-1.0/12.0 {
				p = repeatedStepFarRight(data, k, a, b)
			} else {
				p = repeatedStepRight(data, k, a, b)
			}
		} else {
			p = repeatedStepImproved(data, k, a, b)
		}
		if p == k {
			return
		}
		if p > k {
			b = p // A <- A[0:p]
		} else {
			// i <- k - p - 1  // TODO what is i?
			a = p + 1 // A <- A[p+1:|A|]
		}
	}
}

func simplePartition(data sort.Interface, k, a, b int) int {
	p := a
	for i := a + 1; i < b; i++ {
		if data.Less(i, p) {
			data.Swap(p, i)
			p++
			data.Swap(p, i)
		}
	}
	return p
}

// TODO implement
var hoarePartition = simplePartition
var repeatedStepFarLeft = simplePartition
var repeatedStepLeft = simplePartition
var repeatedStepFarRight = simplePartition
var repeatedStepRight = simplePartition
var repeatedStepImproved = simplePartition
<|endoftext|>