<commit_before>package goreq

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	cookies           []*http.Cookie
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	RedirectHeaders   bool
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
	CookieJar         http.CookieJar
	ShowDebug         bool
	OnBeforeRequest   func(goreq *Request, httpreq *http.Request)
}

type compression struct {
	writer          func(buffer io.Writer) (io.WriteCloser, error)
	reader          func(buffer io.Reader) (io.ReadCloser, error)
	ContentEncoding string
}

type Response struct {
	*http.Response
	Uri  string
	Body *Body
	req  *http.Request
}

func (r Response) CancelRequest() {
	cancelRequest(r.req)

}

func cancelRequest(r *http.Request) {
	if transport, ok := DefaultTransport.(transportRequestCanceler); ok {
		transport.CancelRequest(r)
	}
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

type transportRequestCanceler interface {
	CancelRequest(*http.Request)
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func (b *Body) FromJsonTo(o interface{}) error {
	return json.NewDecoder(b).Decode(o)
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func Gzip() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return gzip.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return gzip.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "gzip"}
}

func Deflate() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return zlib.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return zlib.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func Zlib() *compression {
	return Deflate()
}

func paramParse(query interface{}) (string, error) {
	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	case *url.Values:
		return query.(*url.Values).Encode(), nil
	default:
		var v = &url.Values{}
		err := paramParseStruct(v, query)
		return v.Encode(), err
	}
}

func paramParseStruct(v *url.Values, query interface{}) error {
	var (
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)
	for t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface {
		s = s.Elem()
		t = s.Type()
	}
	if t.Kind() != reflect.Struct {
		return errors.New("Can not parse QueryString.")
	}

	for i := 0; i < t.NumField(); i++ {
		var name string

		field := s.Field(i)
		typeField := t.Field(i)

		if !field.CanInterface() {
			continue
		}

		urlTag := typeField.Tag.Get("url")
		if urlTag == "-" {
			continue
		}

		name, opts := parseTag(urlTag)

		var omitEmpty, squash bool
		omitEmpty = opts.Contains("omitempty")
		squash = opts.Contains("squash")

		if squash {
			err := paramParseStruct(v, field.Interface())
			if err != nil {
				return err
			}
			continue
		}

		if urlTag == "" {
			name = strings.ToLower(typeField.Name)
		}

		if val := fmt.Sprintf("%v", field.Interface()); !(omitEmpty && len(val) == 0) {
			v.Add(name, val)
		}
	}
	return nil
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

var DefaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var DefaultTransport http.RoundTripper = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyFromEnvironment}
var DefaultClient = &http.Client{Transport: DefaultTransport}

var proxyTransport http.RoundTripper
var proxyClient *http.Client

func SetConnectTimeout(duration time.Duration) {
	DefaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) WithHeader(name string, value string) Request {
	r.AddHeader(name, value)
	return r
}

func (r *Request) AddCookie(c *http.Cookie) {
	r.cookies = append(r.cookies, c)
}

func (r Request) WithCookie(c *http.Cookie) Request {
	r.AddCookie(c)
	return r

}

func (r Request) Do() (*Response, error) {
	var client = DefaultClient
	var transport = DefaultTransport
	var resUri string
	var redirectFailed bool

	r.Method = valueOrDefault(r.Method, "GET")

	// use a client with a cookie jar if necessary. We create a new client not
	// to modify the default one.
	if r.CookieJar != nil {
		client = &http.Client{
			Transport: transport,
			Jar:       r.CookieJar,
		}
	}

	if r.Proxy != "" {
		proxyUrl, err := url.Parse(r.Proxy)
		if err != nil {
			// proxy address is in a wrong format
			return nil, &Error{Err: err}
		}

		//If jar is specified new client needs to be built
		if proxyTransport == nil || client.Jar != nil {
			proxyTransport = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyURL(proxyUrl)}
			proxyClient = &http.Client{Transport: proxyTransport, Jar: client.Jar}
		} else if proxyTransport, ok := proxyTransport.(*http.Transport); ok {
			proxyTransport.Proxy = http.ProxyURL(proxyUrl)
		}
		transport = proxyTransport
		client = proxyClient
	}

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {

		if len(via) > r.MaxRedirects {
			redirectFailed = true
			return errors.New("Error redirecting. MaxRedirects reached")
		}

		resUri = req.URL.String()

		//By default Golang will not redirect request headers
		// https://code.google.com/p/go/issues/detail?id=4800&q=request%20header
		if r.RedirectHeaders {
			for key, val := range via[0].Header {
				req.Header[key] = val
			}
		}
		return nil
	}

	if transport, ok := transport.(*http.Transport); ok {
		if r.Insecure {
			if transport.TLSClientConfig != nil {
				transport.TLSClientConfig.InsecureSkipVerify = true
			} else {
				transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
			}
		} else if transport.TLSClientConfig != nil {
			// the default TLS client (when transport.TLSClientConfig==nil) is
			// already set to verify, so do nothing in that case
			transport.TLSClientConfig.InsecureSkipVerify = false
		}
	}

	req, err := r.NewRequest()

	if err != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: err}
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			cancelRequest(req)
			timeout = true
		})
	}

	if r.ShowDebug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			log.Println(err)
		}
		log.Println(string(dump))
	}

	if r.OnBeforeRequest != nil {
		r.OnBeforeRequest(&r, req)
	}
	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		var response *Response
		//If redirect fails we still want to return response data
		if redirectFailed {
			if res != nil {
				response = &Response{res, resUri, &Body{reader: res.Body}, req}
			} else {
				response = &Response{res, resUri, nil, req}
			}
		}

		//If redirect fails and we haven't set a redirect count we shouldn't return an error
		if redirectFailed && r.MaxRedirects == 0 {
			return response, nil
		}

		return response, &Error{timeout: timeout, Err: err}
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{res, resUri, &Body{reader: res.Body, compressedReader: compressedReader}, req}, nil
	}

	return &Response{res, resUri, &Body{reader: res.Body}, req}, nil
}

func (r Request) addHeaders(headersMap http.Header) {
	if len(r.UserAgent) > 0 {
		headersMap.Add("User-Agent", r.UserAgent)
	}
	if r.Accept != "" {
		headersMap.Add("Accept", r.Accept)
	}
	if r.ContentType != "" {
		headersMap.Add("Content-Type", r.ContentType)
	}
}

func (r Request) NewRequest() (*http.Request, error) {

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}

	req, err := http.NewRequest(r.Method, r.Uri, bodyReader)
	if err != nil {
		return nil, err
	}
	// add headers to the request
	req.Host = r.Host

	r.addHeaders(req.Header)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	//use basic auth if required
	if r.BasicAuthUsername != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	for _, c := range r.cookies {
		req.AddCookie(c)
	}
	return req, nil
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}
<commit_msg>closing http connection by default<commit_after>package goreq

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"compress/zlib"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"reflect"
	"strings"
	"time"
)

type Request struct {
	headers           []headerTuple
	cookies           []*http.Cookie
	Method            string
	Uri               string
	Body              interface{}
	QueryString       interface{}
	Timeout           time.Duration
	ContentType       string
	Accept            string
	Host              string
	UserAgent         string
	Insecure          bool
	MaxRedirects      int
	RedirectHeaders   bool
	Proxy             string
	Compression       *compression
	BasicAuthUsername string
	BasicAuthPassword string
	CookieJar         http.CookieJar
	ShowDebug         bool
	OnBeforeRequest   func(goreq *Request, httpreq *http.Request)
}

type compression struct {
	writer          func(buffer io.Writer) (io.WriteCloser, error)
	reader          func(buffer io.Reader) (io.ReadCloser, error)
	ContentEncoding string
}

type Response struct {
	*http.Response
	Uri  string
	Body *Body
	req  *http.Request
}

func (r Response) CancelRequest() {
	cancelRequest(r.req)

}

func cancelRequest(r *http.Request) {
	if transport, ok := DefaultTransport.(transportRequestCanceler); ok {
		transport.CancelRequest(r)
	}
}

type headerTuple struct {
	name  string
	value string
}

type Body struct {
	reader           io.ReadCloser
	compressedReader io.ReadCloser
}

type Error struct {
	timeout bool
	Err     error
}

type transportRequestCanceler interface {
	CancelRequest(*http.Request)
}

func (e *Error) Timeout() bool {
	return e.timeout
}

func (e *Error) Error() string {
	return e.Err.Error()
}

func (b *Body) Read(p []byte) (int, error) {
	if b.compressedReader != nil {
		return b.compressedReader.Read(p)
	}
	return b.reader.Read(p)
}

func (b *Body) Close() error {
	err := b.reader.Close()
	if b.compressedReader != nil {
		return b.compressedReader.Close()
	}
	return err
}

func (b *Body) FromJsonTo(o interface{}) error {
	return json.NewDecoder(b).Decode(o)
}

func (b *Body) ToString() (string, error) {
	body, err := ioutil.ReadAll(b)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

func Gzip() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return gzip.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return gzip.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "gzip"}
}

func Deflate() *compression {
	reader := func(buffer io.Reader) (io.ReadCloser, error) {
		return zlib.NewReader(buffer)
	}
	writer := func(buffer io.Writer) (io.WriteCloser, error) {
		return zlib.NewWriter(buffer), nil
	}
	return &compression{writer: writer, reader: reader, ContentEncoding: "deflate"}
}

func Zlib() *compression {
	return Deflate()
}

func paramParse(query interface{}) (string, error) {
	switch query.(type) {
	case url.Values:
		return query.(url.Values).Encode(), nil
	case *url.Values:
		return query.(*url.Values).Encode(), nil
	default:
		var v = &url.Values{}
		err := paramParseStruct(v, query)
		return v.Encode(), err
	}
}

func paramParseStruct(v *url.Values, query interface{}) error {
	var (
		s = reflect.ValueOf(query)
		t = reflect.TypeOf(query)
	)
	for t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface {
		s = s.Elem()
		t = s.Type()
	}
	if t.Kind() != reflect.Struct {
		return errors.New("Can not parse QueryString.")
	}

	for i := 0; i < t.NumField(); i++ {
		var name string

		field := s.Field(i)
		typeField := t.Field(i)

		if !field.CanInterface() {
			continue
		}

		urlTag := typeField.Tag.Get("url")
		if urlTag == "-" {
			continue
		}

		name, opts := parseTag(urlTag)

		var omitEmpty, squash bool
		omitEmpty = opts.Contains("omitempty")
		squash = opts.Contains("squash")

		if squash {
			err := paramParseStruct(v, field.Interface())
			if err != nil {
				return err
			}
			continue
		}

		if urlTag == "" {
			name = strings.ToLower(typeField.Name)
		}

		if val := fmt.Sprintf("%v", field.Interface()); !(omitEmpty && len(val) == 0) {
			v.Add(name, val)
		}
	}
	return nil
}

func prepareRequestBody(b interface{}) (io.Reader, error) {
	switch b.(type) {
	case string:
		// treat is as text
		return strings.NewReader(b.(string)), nil
	case io.Reader:
		// treat is as text
		return b.(io.Reader), nil
	case []byte:
		//treat as byte array
		return bytes.NewReader(b.([]byte)), nil
	case nil:
		return nil, nil
	default:
		// try to jsonify it
		j, err := json.Marshal(b)
		if err == nil {
			return bytes.NewReader(j), nil
		}
		return nil, err
	}
}

var DefaultDialer = &net.Dialer{Timeout: 1000 * time.Millisecond}
var DefaultTransport http.RoundTripper = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyFromEnvironment}
var DefaultClient = &http.Client{Transport: DefaultTransport}

var proxyTransport http.RoundTripper
var proxyClient *http.Client

func SetConnectTimeout(duration time.Duration) {
	DefaultDialer.Timeout = duration
}

func (r *Request) AddHeader(name string, value string) {
	if r.headers == nil {
		r.headers = []headerTuple{}
	}
	r.headers = append(r.headers, headerTuple{name: name, value: value})
}

func (r Request) WithHeader(name string, value string) Request {
	r.AddHeader(name, value)
	return r
}

func (r *Request) AddCookie(c *http.Cookie) {
	r.cookies = append(r.cookies, c)
}

func (r Request) WithCookie(c *http.Cookie) Request {
	r.AddCookie(c)
	return r

}

func (r Request) Do() (*Response, error) {
	var client = DefaultClient
	var transport = DefaultTransport
	var resUri string
	var redirectFailed bool

	r.Method = valueOrDefault(r.Method, "GET")

	// use a client with a cookie jar if necessary. We create a new client not
	// to modify the default one.
	if r.CookieJar != nil {
		client = &http.Client{
			Transport: transport,
			Jar:       r.CookieJar,
		}
	}

	if r.Proxy != "" {
		proxyUrl, err := url.Parse(r.Proxy)
		if err != nil {
			// proxy address is in a wrong format
			return nil, &Error{Err: err}
		}

		//If jar is specified new client needs to be built
		if proxyTransport == nil || client.Jar != nil {
			proxyTransport = &http.Transport{Dial: DefaultDialer.Dial, Proxy: http.ProxyURL(proxyUrl)}
			proxyClient = &http.Client{Transport: proxyTransport, Jar: client.Jar}
		} else if proxyTransport, ok := proxyTransport.(*http.Transport); ok {
			proxyTransport.Proxy = http.ProxyURL(proxyUrl)
		}
		transport = proxyTransport
		client = proxyClient
	}

	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {

		if len(via) > r.MaxRedirects {
			redirectFailed = true
			return errors.New("Error redirecting. MaxRedirects reached")
		}

		resUri = req.URL.String()

		//By default Golang will not redirect request headers
		// https://code.google.com/p/go/issues/detail?id=4800&q=request%20header
		if r.RedirectHeaders {
			for key, val := range via[0].Header {
				req.Header[key] = val
			}
		}
		return nil
	}

	if transport, ok := transport.(*http.Transport); ok {
		if r.Insecure {
			if transport.TLSClientConfig != nil {
				transport.TLSClientConfig.InsecureSkipVerify = true
			} else {
				transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
			}
		} else if transport.TLSClientConfig != nil {
			// the default TLS client (when transport.TLSClientConfig==nil) is
			// already set to verify, so do nothing in that case
			transport.TLSClientConfig.InsecureSkipVerify = false
		}
	}

	req, err := r.NewRequest()

	if err != nil {
		// we couldn't parse the URL.
		return nil, &Error{Err: err}
	}

	timeout := false
	var timer *time.Timer
	if r.Timeout > 0 {
		timer = time.AfterFunc(r.Timeout, func() {
			cancelRequest(req)
			timeout = true
		})
	}

	if r.ShowDebug {
		dump, err := httputil.DumpRequest(req, true)
		if err != nil {
			log.Println(err)
		}
		log.Println(string(dump))
	}

	if r.OnBeforeRequest != nil {
		r.OnBeforeRequest(&r, req)
	}
	res, err := client.Do(req)
	if timer != nil {
		timer.Stop()
	}

	if err != nil {
		if !timeout {
			switch err := err.(type) {
			case *net.OpError:
				timeout = err.Timeout()
			case *url.Error:
				if op, ok := err.Err.(*net.OpError); ok {
					timeout = op.Timeout()
				}
			}
		}

		var response *Response
		//If redirect fails we still want to return response data
		if redirectFailed {
			if res != nil {
				response = &Response{res, resUri, &Body{reader: res.Body}, req}
			} else {
				response = &Response{res, resUri, nil, req}
			}
		}

		//If redirect fails and we haven't set a redirect count we shouldn't return an error
		if redirectFailed && r.MaxRedirects == 0 {
			return response, nil
		}

		return response, &Error{timeout: timeout, Err: err}
	}

	if r.Compression != nil && strings.Contains(res.Header.Get("Content-Encoding"), r.Compression.ContentEncoding) {
		compressedReader, err := r.Compression.reader(res.Body)
		if err != nil {
			return nil, &Error{Err: err}
		}
		return &Response{res, resUri, &Body{reader: res.Body, compressedReader: compressedReader}, req}, nil
	}

	return &Response{res, resUri, &Body{reader: res.Body}, req}, nil
}

func (r Request) addHeaders(headersMap http.Header) {
	if len(r.UserAgent) > 0 {
		headersMap.Add("User-Agent", r.UserAgent)
	}
	if r.Accept != "" {
		headersMap.Add("Accept", r.Accept)
	}
	if r.ContentType != "" {
		headersMap.Add("Content-Type", r.ContentType)
	}
}

func (r Request) NewRequest() (*http.Request, error) {

	b, e := prepareRequestBody(r.Body)
	if e != nil {
		// there was a problem marshaling the body
		return nil, &Error{Err: e}
	}

	if r.QueryString != nil {
		param, e := paramParse(r.QueryString)
		if e != nil {
			return nil, &Error{Err: e}
		}
		r.Uri = r.Uri + "?" + param
	}

	var bodyReader io.Reader
	if b != nil && r.Compression != nil {
		buffer := bytes.NewBuffer([]byte{})
		readBuffer := bufio.NewReader(b)
		writer, err := r.Compression.writer(buffer)
		if err != nil {
			return nil, &Error{Err: err}
		}
		_, e = readBuffer.WriteTo(writer)
		writer.Close()
		if e != nil {
			return nil, &Error{Err: e}
		}
		bodyReader = buffer
	} else {
		bodyReader = b
	}

	req, err := http.NewRequest(r.Method, r.Uri, bodyReader)
	if err != nil {
		return nil, err
	}
	// add headers to the request
	req.Host = r.Host
	req.Close = true

	r.addHeaders(req.Header)
	if r.Compression != nil {
		req.Header.Add("Content-Encoding", r.Compression.ContentEncoding)
		req.Header.Add("Accept-Encoding", r.Compression.ContentEncoding)
	}
	if r.headers != nil {
		for _, header := range r.headers {
			req.Header.Add(header.name, header.value)
		}
	}

	//use basic auth if required
	if r.BasicAuthUsername != "" {
		req.SetBasicAuth(r.BasicAuthUsername, r.BasicAuthPassword)
	}

	for _, c := range r.cookies {
		req.AddCookie(c)
	}
	return req, nil
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}
<|endoftext|>


<commit_before>package annotations

//Annotations represents a collection of Annotation instances
type annotations []annotation

//Annotation is the main struct used to create and return structures
type annotation struct {
	Thing       thing        `json:"thing,omitempty"`
	Provenances []provenance `json:"provenances,omitempty"`
}

//Thing represents a concept being linked to
type thing struct {
	ID        string   `json:"id,omitempty"`
	PrefLabel string   `json:"prefLabel,omitempty"`
	Types     []string `json:"types,omitempty"`
	Predicate string   `json:"predicate,omitempty"`
}

//Provenance indicates the scores and where they came from
type provenance struct {
	Scores    []score `json:"scores,omitempty"`
	AgentRole string  `json:"agentRole,omitempty"`
	AtTime    string  `json:"atTime,omitempty"`
}

//Score represents one of our scores for the annotation
type score struct {
	ScoringSystem string  `json:"scoringSystem,omitempty"`
	Value         float64 `json:"value,omitempty"`
}

const (
	mentionsPred            = "http://www.ft.com/ontology/annotation/mentions"
	mentionsRel             = "MENTIONS"
	relevanceScoringSystem  = "http://api.ft.com/scoringsystem/FT-RELEVANCE-SYSTEM"
	confidenceScoringSystem = "http://api.ft.com/scoringsystem/FT-CONFIDENCE-SYSTEM"
)

var relations = map[string]string{
	"mentions":                                            "MENTIONS",
	"isClassifiedBy":                                      "IS_CLASSIFIED_BY",
	"about":                                               "ABOUT",
	"isPrimarilyClassifiedBy":                             "IS_PRIMARILY_CLASSIFIED_BY",
	"http://www.ft.com/ontology/annotation/majorMentions": "MAJOR_MENTIONS",
}
<commit_msg>URI not name of the predicate defined<commit_after>package annotations

//Annotations represents a collection of Annotation instances
type annotations []annotation

//Annotation is the main struct used to create and return structures
type annotation struct {
	Thing       thing        `json:"thing,omitempty"`
	Provenances []provenance `json:"provenances,omitempty"`
}

//Thing represents a concept being linked to
type thing struct {
	ID        string   `json:"id,omitempty"`
	PrefLabel string   `json:"prefLabel,omitempty"`
	Types     []string `json:"types,omitempty"`
	Predicate string   `json:"predicate,omitempty"`
}

//Provenance indicates the scores and where they came from
type provenance struct {
	Scores    []score `json:"scores,omitempty"`
	AgentRole string  `json:"agentRole,omitempty"`
	AtTime    string  `json:"atTime,omitempty"`
}

//Score represents one of our scores for the annotation
type score struct {
	ScoringSystem string  `json:"scoringSystem,omitempty"`
	Value         float64 `json:"value,omitempty"`
}

const (
	mentionsPred            = "http://www.ft.com/ontology/annotation/mentions"
	mentionsRel             = "MENTIONS"
	relevanceScoringSystem  = "http://api.ft.com/scoringsystem/FT-RELEVANCE-SYSTEM"
	confidenceScoringSystem = "http://api.ft.com/scoringsystem/FT-CONFIDENCE-SYSTEM"
)

var relations = map[string]string{
	"mentions":                "MENTIONS",
	"isClassifiedBy":          "IS_CLASSIFIED_BY",
	"about":                   "ABOUT",
	"isPrimarilyClassifiedBy": "IS_PRIMARILY_CLASSIFIED_BY",
	"majorMentions":           "MAJOR_MENTIONS",
}
<|endoftext|>


<commit_before>//
// Package report provides basic report rendering for Simple Timesheet Notation.
//
// report_test.go test the stn/report package.
// @author R. S. Doiel, <rsdoiel@gmail.com>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license
// See: http://opensource.org/licenses/BSD-2-Clause
//
package report

import (
	"fmt"
	"log"
	"strings"
	"testing"

	"github.com/rsdoiel/ok"
	"github.com/rsodiel/stngo/stn"
)

func TestAggregator(t *testing.T) {
	text := `2015-07-06T08:00:00-07:00	2015-07-06T08:30:00-07:00	misc	email, update basecamp
2015-07-06T08:30:00-07:00	2015-07-06T11:00:00-07:00	ArchivesSpace	running through migration process, updating notes, testing
2015-07-06T11:00:00-07:00	2015-07-06T11:45:00-07:00	misc	update Mac
2015-07-03T08:00:00-07:00	2015-07-03T15:30:00-07:00	Holiday	4th of July observed
2015-07-02T07:45:00-07:00	2015-07-02T09:30:00-07:00	misc	email, review stuff
2015-07-02T09:30:00-07:00	2015-07-02T10:30:00-07:00	DLD meeting
2015-07-02T10:30:00-07:00	2015-07-02T12:00:00-07:00	ArchivesSpace	running through migration process
2015-07-02T03:00:00-07:00	2015-07-02T03:30:00-07:00	ArchivesSpace	Hangouts with Tommy to upgrade cls-arch.library.caltech.edu to v1.3.0, go over migration questions
2015-07-01T07:45:00-07:00	2015-07-01T09:30:00-07:00	ArchivesSpace	continue reading docs, articles about approach and what problems are being solved.
2015-07-01T09:30:00-07:00	2015-07-01T11:00:00-07:00	ArchivesSpace	meeting in SFL's MCR (3rd floot Multi-media Conference Room)`
	aggregation := new(EntryAggregation)
	entry := new(stn.Entry)
	lines := strings.Split(text, "\n")
	linesTotal := 0
	for i, line := range lines {
		if entry.FromString(line) == true {
			if aggregation.Aggregate(entry) != true {
				log.Fatalf("Can't aggregate entry %d: %v", i, entry)
			}
			linesTotal++
		} else {
			log.Fatalf("Can't read line no. %d: [%s]\n", i, line)
		}
	}
	outText := aggregation.Summarize()
	outLines := strings.Split(outText, "\n")
	ok.Ok(t, len(outLines) == 8, fmt.Sprintf("lines %d: [%s]\n", linesTotal, outText))
}
<commit_msg>Added comment<commit_after>//
// Package report provides basic report rendering for Simple Timesheet Notation.
//
// report_test.go test the stn/report package.
// @author R. S. Doiel, <rsdoiel@gmail.com>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license
// See: http://opensource.org/licenses/BSD-2-Clause
//
package report

import (
	"fmt"
	"log"
	"strings"
	"testing"

    // Local packages
	"github.com/rsdoiel/ok"
	"github.com/rsdoiel/stngo/stn"
)

func TestAggregator(t *testing.T) {
	text := `2015-07-06T08:00:00-07:00	2015-07-06T08:30:00-07:00	misc	email, update basecamp
2015-07-06T08:30:00-07:00	2015-07-06T11:00:00-07:00	ArchivesSpace	running through migration process, updating notes, testing
2015-07-06T11:00:00-07:00	2015-07-06T11:45:00-07:00	misc	update Mac
2015-07-03T08:00:00-07:00	2015-07-03T15:30:00-07:00	Holiday	4th of July observed
2015-07-02T07:45:00-07:00	2015-07-02T09:30:00-07:00	misc	email, review stuff
2015-07-02T09:30:00-07:00	2015-07-02T10:30:00-07:00	DLD meeting
2015-07-02T10:30:00-07:00	2015-07-02T12:00:00-07:00	ArchivesSpace	running through migration process
2015-07-02T03:00:00-07:00	2015-07-02T03:30:00-07:00	ArchivesSpace	Hangouts with Tommy to upgrade cls-arch.library.caltech.edu to v1.3.0, go over migration questions
2015-07-01T07:45:00-07:00	2015-07-01T09:30:00-07:00	ArchivesSpace	continue reading docs, articles about approach and what problems are being solved.
2015-07-01T09:30:00-07:00	2015-07-01T11:00:00-07:00	ArchivesSpace	meeting in SFL's MCR (3rd floot Multi-media Conference Room)`
	aggregation := new(EntryAggregation)
	entry := new(stn.Entry)
	lines := strings.Split(text, "\n")
	linesTotal := 0
	for i, line := range lines {
		if entry.FromString(line) == true {
			if aggregation.Aggregate(entry) != true {
				log.Fatalf("Can't aggregate entry %d: %v", i, entry)
			}
			linesTotal++
		} else {
			log.Fatalf("Can't read line no. %d: [%s]\n", i, line)
		}
	}
	outText := aggregation.Summarize()
	outLines := strings.Split(outText, "\n")
	ok.Ok(t, len(outLines) == 8, fmt.Sprintf("lines %d: [%s]\n", linesTotal, outText))
}
<|endoftext|>


<commit_before>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/log"
)

const OldAccessLogPath = "/var/log/traffic_ops/access.log"
const NewLogPath = "/var/log/traffic_ops/traffic_ops_golang.log"
const DefaultMaxDBConnections = 50

func GetPerlConfigs(cdnConfPath string, dbConfPath string) (Config, error) {
	configBytes, err := ioutil.ReadFile(cdnConfPath)
	if err != nil {
		return Config{}, fmt.Errorf("reading CDN conf '%v': %v", cdnConfPath, err)
	}
	dbConfBytes, err := ioutil.ReadFile(dbConfPath)
	if err != nil {
		return Config{}, fmt.Errorf("reading db conf '%v': %v", dbConfPath, err)
	}
	return getPerlConfigsFromStrs(string(configBytes), string(dbConfBytes))
}

func getPerlConfigsFromStrs(cdnConfBytes string, dbConfBytes string) (Config, error) {
	cfg, err := getCDNConf(cdnConfBytes)
	if err != nil {
		return Config{}, fmt.Errorf("parsing CDN conf '%v': %v", cdnConfBytes, err)
	}

	dbconf, err := getDbConf(string(dbConfBytes))
	if err != nil {
		return Config{}, fmt.Errorf("parsing db conf '%v': %v", dbConfBytes, err)
	}
	cfg.DBUser = dbconf.User
	cfg.DBPass = dbconf.Password
	cfg.DBServer = dbconf.Hostname
	cfg.DBDB = dbconf.DBName
	cfg.DBSSL = false // TODO fix
	if dbconf.Port != "" {
		cfg.DBServer += ":" + dbconf.Port
	}

	cfg.LogLocationInfo = NewLogPath
	cfg.LogLocationError = NewLogPath
	cfg.LogLocationWarning = NewLogPath
	cfg.LogLocationEvent = OldAccessLogPath
	cfg.LogLocationDebug = log.LogLocationNull

	if dbconf.MaxConnections != nil {
		cfg.MaxDBConnections = *dbconf.MaxConnections
	} else {
		cfg.MaxDBConnections = DefaultMaxDBConnections
	}

	return cfg, nil
}

func getCDNConf(s string) (Config, error) {
	cfg := Config{}
	obj, err := ParsePerlObj(s)
	if err != nil {
		return Config{}, fmt.Errorf("parsing Perl object: %v", err)
	}

	if cfg.HTTPPort, err = getPort(obj); err != nil {
		return Config{}, err
	}

	if cfg.TOSecret, err = getSecret(obj); err != nil {
		return Config{}, err
	}

	oldPort, err := getOldPort(obj)
	if err != nil {
		return Config{}, err
	}
	cfg.TOURLStr = "https://127.0.0.1:" + oldPort
	if cfg.TOURL, err = url.Parse(cfg.TOURLStr); err != nil {
		return Config{}, fmt.Errorf("Invalid Traffic Ops URL '%v': err", cfg.TOURL, err)
	}

	cfg.CertPath, err = getConfigCert(obj)
	if err != nil {
		return Config{}, err
	}

	cfg.KeyPath, err = getConfigKey(obj)
	if err != nil {
		return Config{}, err
	}

	return cfg, nil
}

func getPort(obj map[string]interface{}) (string, error) {
	portStrI, ok := obj["traffic_ops_golang_port"]
	if !ok {
		return "", fmt.Errorf("missing traffic_ops_golang_port key")
	}
	portStr, ok := portStrI.(string)
	if !ok {
		return "", fmt.Errorf("traffic_ops_golang_port key '%v' not a string", portStrI)
	}

	port, err := strconv.Atoi(portStr)
	if err != nil || port < 0 || port > 65535 {
		return "", fmt.Errorf("invalid port '%s'", portStr)
	}
	return strconv.Itoa(port), nil
}

func getOldPort(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	listenRe := regexp.MustCompile(`:(\d+)`)
	portMatch := listenRe.FindStringSubmatch(listen)
	if len(portMatch) < 2 {
		return "", fmt.Errorf("failed to find port in listen '%s'", listen)
	}
	portStr := portMatch[1]

	port, err := strconv.Atoi(portStr)
	if err != nil || port < 0 || port > 65535 {
		return "", fmt.Errorf("invalid port in listen '%s'", listen)
	}
	return strconv.Itoa(port), nil
}

func getConfigCert(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	keyStr := "cert="
	start := strings.Index(listen, keyStr)
	if start < 0 {
		return "", fmt.Errorf("failed to find key in listen '%s'", listen)
	}
	listen = listen[start+len(keyStr):]
	end := strings.Index(listen, "&")
	if end < 0 {
		return listen[start:], nil
	}
	return listen[:end], nil
}

func getConfigKey(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	keyStr := "key="
	start := strings.Index(listen, keyStr)
	if start < 0 {
		return "", fmt.Errorf("failed to find key in listen '%s'", listen)
	}
	listen = listen[start+len(keyStr):]
	end := strings.Index(listen, "&")
	if end < 0 {
		return listen[start:], nil
	}
	return listen[:end], nil
}

func getSecret(obj map[string]interface{}) (string, error) {
	secretsI, ok := obj["secrets"]
	if !ok {
		return "", fmt.Errorf("missing secrets key")
	}
	secrets, ok := secretsI.([]interface{})
	if !ok {
		return "", fmt.Errorf("secrets key '%v' not an array", secretsI)
	}

	if len(secrets) < 1 {
		return "", fmt.Errorf("empty secrets key")
	}
	secretI := secrets[0]
	secret, ok := secretI.(string)
	if !ok {
		return "", fmt.Errorf("secret '%v' not a string", secretI)
	}

	return secret, nil
}

type DatabaseConf struct {
	Description    string `json:"description"`
	DBName         string `json:"dbname"`
	Hostname       string `json:"hostname"`
	User           string `json:"user"`
	Password       string `json:"password"`
	Port           string `json:"port"`
	Type           string `json:"type"`
	MaxConnections *int   `json:"max_connections"`
}

func getDbConf(s string) (DatabaseConf, error) {
	dbc := DatabaseConf{}
	err := json.Unmarshal([]byte(s), &dbc)
	return dbc, err
}
<commit_msg>Move TO Golang DBMaxConns from db.conf to cdn.conf<commit_after>package main

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"github.com/apache/incubator-trafficcontrol/traffic_monitor_golang/common/log"
)

const OldAccessLogPath = "/var/log/traffic_ops/access.log"
const NewLogPath = "/var/log/traffic_ops/traffic_ops_golang.log"
const DefaultMaxDBConnections = 50

func GetPerlConfigs(cdnConfPath string, dbConfPath string) (Config, error) {
	configBytes, err := ioutil.ReadFile(cdnConfPath)
	if err != nil {
		return Config{}, fmt.Errorf("reading CDN conf '%v': %v", cdnConfPath, err)
	}
	dbConfBytes, err := ioutil.ReadFile(dbConfPath)
	if err != nil {
		return Config{}, fmt.Errorf("reading db conf '%v': %v", dbConfPath, err)
	}
	return getPerlConfigsFromStrs(string(configBytes), string(dbConfBytes))
}

func getPerlConfigsFromStrs(cdnConfBytes string, dbConfBytes string) (Config, error) {
	cfg, err := getCDNConf(cdnConfBytes)
	if err != nil {
		return Config{}, fmt.Errorf("parsing CDN conf '%v': %v", cdnConfBytes, err)
	}

	dbconf, err := getDbConf(string(dbConfBytes))
	if err != nil {
		return Config{}, fmt.Errorf("parsing db conf '%v': %v", dbConfBytes, err)
	}
	cfg.DBUser = dbconf.User
	cfg.DBPass = dbconf.Password
	cfg.DBServer = dbconf.Hostname
	cfg.DBDB = dbconf.DBName
	cfg.DBSSL = false // TODO fix
	if dbconf.Port != "" {
		cfg.DBServer += ":" + dbconf.Port
	}

	cfg.LogLocationInfo = NewLogPath
	cfg.LogLocationError = NewLogPath
	cfg.LogLocationWarning = NewLogPath
	cfg.LogLocationEvent = OldAccessLogPath
	cfg.LogLocationDebug = log.LogLocationNull

	return cfg, nil
}

func getCDNConf(s string) (Config, error) {
	cfg := Config{}
	obj, err := ParsePerlObj(s)
	if err != nil {
		return Config{}, fmt.Errorf("parsing Perl object: %v", err)
	}

	if cfg.HTTPPort, err = getPort(obj); err != nil {
		return Config{}, err
	}

	if cfg.TOSecret, err = getSecret(obj); err != nil {
		return Config{}, err
	}

	oldPort, err := getOldPort(obj)
	if err != nil {
		return Config{}, err
	}
	cfg.TOURLStr = "https://127.0.0.1:" + oldPort
	if cfg.TOURL, err = url.Parse(cfg.TOURLStr); err != nil {
		return Config{}, fmt.Errorf("Invalid Traffic Ops URL '%v': err", cfg.TOURL, err)
	}

	cfg.CertPath, err = getConfigCert(obj)
	if err != nil {
		return Config{}, err
	}

	cfg.KeyPath, err = getConfigKey(obj)
	if err != nil {
		return Config{}, err
	}

	if dbMaxConns, err := getDBMaxConns(obj); err != nil {
		log.Warnf("failed to get Max DB Connections from cdn.conf (%v), using default %v\n", err, DefaultMaxDBConnections)
		cfg.MaxDBConnections = DefaultMaxDBConnections
	} else {
		cfg.MaxDBConnections = dbMaxConns
	}

	return cfg, nil
}

func getPort(obj map[string]interface{}) (string, error) {
	portStrI, ok := obj["traffic_ops_golang_port"]
	if !ok {
		return "", fmt.Errorf("missing traffic_ops_golang_port key")
	}
	portStr, ok := portStrI.(string)
	if !ok {
		return "", fmt.Errorf("traffic_ops_golang_port key '%v' not a string", portStrI)
	}

	port, err := strconv.Atoi(portStr)
	if err != nil || port < 0 || port > 65535 {
		return "", fmt.Errorf("invalid port '%s'", portStr)
	}
	return strconv.Itoa(port), nil
}

func getDBMaxConns(obj map[string]interface{}) (int, error) {
	inum, ok := obj["traffic_ops_golang_max_db_connections"]
	if !ok {
		return 0, fmt.Errorf("missing traffic_ops_golang_max_db_connections key")
	}
	num, ok := inum.(float64)
	if !ok {
		return 0, fmt.Errorf("traffic_ops_golang_max_db_connections key '%v' type %T not a number", inum, inum)
	}
	return int(num), nil
}

func getOldPort(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	listenRe := regexp.MustCompile(`:(\d+)`)
	portMatch := listenRe.FindStringSubmatch(listen)
	if len(portMatch) < 2 {
		return "", fmt.Errorf("failed to find port in listen '%s'", listen)
	}
	portStr := portMatch[1]

	port, err := strconv.Atoi(portStr)
	if err != nil || port < 0 || port > 65535 {
		return "", fmt.Errorf("invalid port in listen '%s'", listen)
	}
	return strconv.Itoa(port), nil
}

func getConfigCert(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	keyStr := "cert="
	start := strings.Index(listen, keyStr)
	if start < 0 {
		return "", fmt.Errorf("failed to find key in listen '%s'", listen)
	}
	listen = listen[start+len(keyStr):]
	end := strings.Index(listen, "&")
	if end < 0 {
		return listen[start:], nil
	}
	return listen[:end], nil
}

func getConfigKey(obj map[string]interface{}) (string, error) {
	hypnotoadI, ok := obj["hypnotoad"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad key")
	}
	hypnotoad, ok := hypnotoadI.(map[string]interface{})
	if !ok {
		return "", fmt.Errorf("hypnotoad key '%v' not an object", hypnotoadI)
	}

	listenArrI, ok := hypnotoad["listen"]
	if !ok {
		return "", fmt.Errorf("missing hypnotoad.listen key")
	}
	listenArr, ok := listenArrI.([]interface{})
	if !ok {
		return "", fmt.Errorf("listen key '%v' type %T not an array", listenArrI, listenArrI)
	}
	if len(listenArr) < 1 {
		return "", fmt.Errorf("empty hypnotoad.listen key")
	}
	listenI := listenArr[0]
	listen, ok := listenI.(string)
	if !ok {
		return "", fmt.Errorf("listen[0] key '%v' type %T not a string", listenI, listenI)
	}

	keyStr := "key="
	start := strings.Index(listen, keyStr)
	if start < 0 {
		return "", fmt.Errorf("failed to find key in listen '%s'", listen)
	}
	listen = listen[start+len(keyStr):]
	end := strings.Index(listen, "&")
	if end < 0 {
		return listen[start:], nil
	}
	return listen[:end], nil
}

func getSecret(obj map[string]interface{}) (string, error) {
	secretsI, ok := obj["secrets"]
	if !ok {
		return "", fmt.Errorf("missing secrets key")
	}
	secrets, ok := secretsI.([]interface{})
	if !ok {
		return "", fmt.Errorf("secrets key '%v' not an array", secretsI)
	}

	if len(secrets) < 1 {
		return "", fmt.Errorf("empty secrets key")
	}
	secretI := secrets[0]
	secret, ok := secretI.(string)
	if !ok {
		return "", fmt.Errorf("secret '%v' not a string", secretI)
	}

	return secret, nil
}

type DatabaseConf struct {
	Description string `json:"description"`
	DBName      string `json:"dbname"`
	Hostname    string `json:"hostname"`
	User        string `json:"user"`
	Password    string `json:"password"`
	Port        string `json:"port"`
	Type        string `json:"type"`
}

func getDbConf(s string) (DatabaseConf, error) {
	dbc := DatabaseConf{}
	err := json.Unmarshal([]byte(s), &dbc)
	return dbc, err
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io"
	"strconv"
)

func init() {
	RegisterFeedType("primary", &FeedType{
		Start: func(mgr *Manager, feedName, indexName, indexUUID,
			sourceType, sourceName, sourceUUID, params string,
			dests map[string]Dest) error {
			return mgr.registerFeed(NewPrimaryFeed(feedName, indexName,
				BasicPartitionFunc, dests))
		},
		Partitions:  PrimaryFeedPartitions,
		Public:      true,
		Description: "primary - a primary data source",
		StartSample: &DestSourceParams{},
	})
}

// A PrimaryFeed implements both the Feed and Dest interfaces, for
// chainability; and is also useful for testing.
type PrimaryFeed struct {
	name      string
	indexName string
	pf        DestPartitionFunc
	dests     map[string]Dest
	closeCh   chan bool
	doneCh    chan bool
	doneErr   error
	doneMsg   string
}

func NewPrimaryFeed(name, indexName string, pf DestPartitionFunc,
	dests map[string]Dest) *PrimaryFeed {
	return &PrimaryFeed{
		name:      name,
		indexName: indexName,
		pf:        pf,
		dests:     dests,
		closeCh:   make(chan bool),
		doneCh:    make(chan bool),
		doneErr:   nil,
		doneMsg:   "",
	}
}

func (t *PrimaryFeed) Name() string {
	return t.name
}

func (t *PrimaryFeed) IndexName() string {
	return t.indexName
}

func (t *PrimaryFeed) Start() error {
	return nil
}

func (t *PrimaryFeed) Close() error {
	return nil
}

func (t *PrimaryFeed) Dests() map[string]Dest {
	return t.dests
}

func (t *PrimaryFeed) Stats(w io.Writer) error {
	_, err := w.Write([]byte("{}"))
	return err
}

// -----------------------------------------------------

type DestSourceParams struct {
	NumPartitions int `json:"numPartitions"`
}

func PrimaryFeedPartitions(sourceType, sourceName, sourceUUID, sourceParams,
	server string) ([]string, error) {
	dsp := &DestSourceParams{}
	if sourceParams != "" {
		err := json.Unmarshal([]byte(sourceParams), dsp)
		if err != nil {
			return nil, fmt.Errorf("feed_primary: DataSourcePartitions/dest"+
				" could not parse sourceParams: %s, err: %v", sourceParams, err)
		}
	}
	numPartitions := dsp.NumPartitions
	rv := make([]string, numPartitions)
	for i := 0; i < numPartitions; i++ {
		rv[i] = strconv.Itoa(i)
	}
	return rv, nil
}

// -----------------------------------------------------

func (t *PrimaryFeed) OnDataUpdate(partition string,
	key []byte, seq uint64, val []byte) error {
	dest, err := t.pf(partition, key, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnDataUpdate(partition, key, seq, val)
}

func (t *PrimaryFeed) OnDataDelete(partition string,
	key []byte, seq uint64) error {
	dest, err := t.pf(partition, key, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnDataDelete(partition, key, seq)
}

func (t *PrimaryFeed) OnSnapshotStart(partition string,
	snapStart, snapEnd uint64) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnSnapshotStart(partition, snapStart, snapEnd)
}

func (t *PrimaryFeed) SetOpaque(partition string,
	value []byte) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.SetOpaque(partition, value)
}

func (t *PrimaryFeed) GetOpaque(partition string) (
	value []byte, lastSeq uint64, err error) {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return nil, 0, fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.GetOpaque(partition)
}

func (t *PrimaryFeed) Rollback(partition string,
	rollbackSeq uint64) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.Rollback(partition, rollbackSeq)
}

func (t *PrimaryFeed) ConsistencyWait(partition, partitionUUID string,
	consistencyLevel string,
	consistencySeq uint64,
	cancelCh <-chan bool) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.ConsistencyWait(partition, partitionUUID,
		consistencyLevel, consistencySeq, cancelCh)
}

func (t *PrimaryFeed) Count(pindex *PIndex, cancelCh <-chan bool) (
	uint64, error) {
	return 0, fmt.Errorf("feed_primary: PrimaryFeed.Count unimplemented")
}

func (t *PrimaryFeed) Query(pindex *PIndex, req []byte, w io.Writer,
	cancelCh <-chan bool) error {
	return fmt.Errorf("feed_primary: PrimaryFeed.Query unimplemented")
}
<commit_msg>mark primary feed type as non-public<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io"
	"strconv"
)

func init() {
	RegisterFeedType("primary", &FeedType{
		Start: func(mgr *Manager, feedName, indexName, indexUUID,
			sourceType, sourceName, sourceUUID, params string,
			dests map[string]Dest) error {
			return mgr.registerFeed(NewPrimaryFeed(feedName, indexName,
				BasicPartitionFunc, dests))
		},
		Partitions:  PrimaryFeedPartitions,
		Public:      false,
		Description: "primary - a primary data source",
		StartSample: &DestSourceParams{},
	})
}

// A PrimaryFeed implements both the Feed and Dest interfaces, for
// chainability; and is also useful for testing.
type PrimaryFeed struct {
	name      string
	indexName string
	pf        DestPartitionFunc
	dests     map[string]Dest
	closeCh   chan bool
	doneCh    chan bool
	doneErr   error
	doneMsg   string
}

func NewPrimaryFeed(name, indexName string, pf DestPartitionFunc,
	dests map[string]Dest) *PrimaryFeed {
	return &PrimaryFeed{
		name:      name,
		indexName: indexName,
		pf:        pf,
		dests:     dests,
		closeCh:   make(chan bool),
		doneCh:    make(chan bool),
		doneErr:   nil,
		doneMsg:   "",
	}
}

func (t *PrimaryFeed) Name() string {
	return t.name
}

func (t *PrimaryFeed) IndexName() string {
	return t.indexName
}

func (t *PrimaryFeed) Start() error {
	return nil
}

func (t *PrimaryFeed) Close() error {
	return nil
}

func (t *PrimaryFeed) Dests() map[string]Dest {
	return t.dests
}

func (t *PrimaryFeed) Stats(w io.Writer) error {
	_, err := w.Write([]byte("{}"))
	return err
}

// -----------------------------------------------------

type DestSourceParams struct {
	NumPartitions int `json:"numPartitions"`
}

func PrimaryFeedPartitions(sourceType, sourceName, sourceUUID, sourceParams,
	server string) ([]string, error) {
	dsp := &DestSourceParams{}
	if sourceParams != "" {
		err := json.Unmarshal([]byte(sourceParams), dsp)
		if err != nil {
			return nil, fmt.Errorf("feed_primary: DataSourcePartitions/dest"+
				" could not parse sourceParams: %s, err: %v", sourceParams, err)
		}
	}
	numPartitions := dsp.NumPartitions
	rv := make([]string, numPartitions)
	for i := 0; i < numPartitions; i++ {
		rv[i] = strconv.Itoa(i)
	}
	return rv, nil
}

// -----------------------------------------------------

func (t *PrimaryFeed) OnDataUpdate(partition string,
	key []byte, seq uint64, val []byte) error {
	dest, err := t.pf(partition, key, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnDataUpdate(partition, key, seq, val)
}

func (t *PrimaryFeed) OnDataDelete(partition string,
	key []byte, seq uint64) error {
	dest, err := t.pf(partition, key, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnDataDelete(partition, key, seq)
}

func (t *PrimaryFeed) OnSnapshotStart(partition string,
	snapStart, snapEnd uint64) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.OnSnapshotStart(partition, snapStart, snapEnd)
}

func (t *PrimaryFeed) SetOpaque(partition string,
	value []byte) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.SetOpaque(partition, value)
}

func (t *PrimaryFeed) GetOpaque(partition string) (
	value []byte, lastSeq uint64, err error) {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return nil, 0, fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.GetOpaque(partition)
}

func (t *PrimaryFeed) Rollback(partition string,
	rollbackSeq uint64) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.Rollback(partition, rollbackSeq)
}

func (t *PrimaryFeed) ConsistencyWait(partition, partitionUUID string,
	consistencyLevel string,
	consistencySeq uint64,
	cancelCh <-chan bool) error {
	dest, err := t.pf(partition, nil, t.dests)
	if err != nil {
		return fmt.Errorf("feed_primary: PrimaryFeed pf, err: %v", err)
	}
	return dest.ConsistencyWait(partition, partitionUUID,
		consistencyLevel, consistencySeq, cancelCh)
}

func (t *PrimaryFeed) Count(pindex *PIndex, cancelCh <-chan bool) (
	uint64, error) {
	return 0, fmt.Errorf("feed_primary: PrimaryFeed.Count unimplemented")
}

func (t *PrimaryFeed) Query(pindex *PIndex, req []byte, w io.Writer,
	cancelCh <-chan bool) error {
	return fmt.Errorf("feed_primary: PrimaryFeed.Query unimplemented")
}
<|endoftext|>


<commit_before>package graph

// All a node needs to do is identify itself. This allows the user to pass in nodes more interesting than an int,
// but also allow us to reap the benefits of having a map-storable, ==able type.
type Node interface {
	ID() int
}

// Allows edges to do something more interesting that just be a group of nodes. Head and Tail are always directed, that is
// it always goes FROM Head TO Tail.
type Edge interface {
	Head() Node
	Tail() Node
}

// A Graph implements all methods necessary to run graph-specific algorithms on it. 90% of the time you want to actually implement DirectedGraph or UndirectedGraph, since the
// default adjacency functions are (somewhat deliberately) slow.
//
// The Graph interface is directed. This means that EdgeList() should return an edge where Head always goes towards Tail. If your graph is undirected and you only maintain edges for one direction,
// simply return two edges for each one of your edges, with the Head and Tail swapped in each one.
type Graph interface {
	NodeExists(node Node) bool // Returns whether a node with the given Node is currently in the graph
	Degree(node Node) int      // Degree is equivalent to len(Successors(node)) + len(Predecessors(node)); this means that reflexive edges are counted twice
	EdgeList() []Edge          // Returns a list of all edges in the graph. Edges in EdgeList() are always directed, even when only implementing UndirectedGraph.
	NodeList() []Node          // Returns a list of all node IDs in no particular order, useful for determining things like if a graph is fully connected. The caller is free to modify this list (so don't pass a reference to your own list)
}

// Despite its name, a graph implementing UndirectedGraph is not required to be undirected, however, the functions Neighbors and IsNeighbor
// are built to treat all edges as if they were undirected edges. Directed graphs implement this interface too, because sometimes knowing all
// adjacent nodes is useful even in a directed graph.
type UndirectedGraph interface {
	Graph
	Neighbors(node Node) []Node          // Returns all nodes connected by any edge to this node
	IsNeighbor(node, neighbor Node) bool // Returns whether neighbor is connected by an edge to node
}

// Directed graphs are characterized by having seperable Heads and Tails in their edges. That is, if node1 goes to node2, that does not necessarily imply that node2 goes to node1.
//
// While it's possible for a directed graph to have fully reciprocal edges (i.e. the graph is symmetric) -- it is not required to be. The graph is also required to implement UndirectedGraph
// because it can be useful to know all neighbors regardless of direction; not because this graph treats directed graphs as special cases of undirected ones (the truth is, in fact, the opposite)
type DirectedGraph interface {
	UndirectedGraph
	Successors(node Node) []Node               // Gives the nodes connected by OUTBOUND edges, if the graph is an undirected graph, this set is equal to Predecessors
	IsSuccessor(node, successor Node) bool     // If successor shows up in the list returned by Successors(node), then it's a successor. If node doesn't exist, this should always return false
	Predecessors(node Node) []Node             // Gives the nodes connected by INBOUND edges, if the graph is an undirected graph, this set is equal to Successors
	IsPredecessor(node, predecessor Node) bool // If predecessor shows up in the list returned by Predecessors(node), then it's a predecessor. If node doesn't exist, this should always return false
}

// A crunch graph forces a sparse graph to become a dense graph. That is, if the node IDs are [1,4,9,7] it would "crunch" the ids into the contiguous block [0,1,2,3]
//
// All dense graphs should have the first ID at 0
type CrunchGraph interface {
	Graph
	Crunch()
}

// A Graph that implements Coster has an actual cost between adjacent nodes, also known as a weighted graph. If a graph implements coster and a function needs to read cost (e.g. A*), this function will
// take precedence over the Uniform Cost function (all weights are 1) if "nil" is passed in for the function argument
//
// If no edge exists between node1 and node2, the cost should be taken to be +inf (can be gotten by math.Inf(1))
type Coster interface {
	Cost(node1, node2 Node) float64
}

// Guarantees that something implementing Coster is also a Graph
type CostGraph interface {
	Coster
	Graph
}

// A graph that implements HeuristicCoster implements a heuristic between any two given nodes. Like Coster, if a graph implements this and a function needs a heuristic cost (e.g. A*), this function will
// take precedence over the Null Heuristic (always returns 0) if "nil" is passed in for the function argument
type HeuristicCoster interface {
	HeuristicCost(node1, node2 Node) float64 // If HeuristicCost is not intended to be used, it can be implemented as the null heuristic (always returns 0)
}

// A Mutable Graph is a graph that can be changed in an arbitrary way. It is useful for several algorithms; for instance, Johnson's Algorithm requires adding a temporary node and changing edge weights.
// Another case where this is used is computing minimum spanning trees. Since trees are graphs, a minimum spanning tree can be created using this interface.
//
// Note that just because a graph does not implement MutableGraph does not mean that this package expects it to be invariant (though even a MutableGraph should be treated as invariant while an algorithm
// is operating on it), it simply means that without this interface this package can not properly handle the graph in order to, say, fill it with a minimum spanning tree.
//
// In functions that take a MutableGraph as an argument, it should not be the same as the Graph argument as concurrent modification will likely cause problems in most cases.
//
// Mutable graphs should always record the IDs as they are represented -- which means they are sparse by nature.
type MutableGraph interface {
	CostGraph
	NewNode(successors []Node) Node       // Adds a node with an arbitrary ID, and returns the new, unique ID used
	AddNode(node Node, successors []Node) // The graph itself is responsible for adding reciprocal edges if it's undirected. Likewise, the graph itself must add any non-existant nodes listed in successors.
	AddEdge(e Edge)                       // For a digraph, adds node1->node2; the graph is free to initialize this to any value it wishes. Node1 must exist, or it will result in undefined behavior, node2 must be created by the function if absent
	SetEdgeCost(e Edge, cost float64)     // The behavior is undefined if the edge has not been created with AddEdge (or the edge was removed before this function was called). For a directed graph only sets node1->node2
	RemoveNode(node Node)                 // The graph is reponsible for removing edges to a node that is removed
	RemoveEdge(e Edge)                    // The graph is responsible for removing reciprocal edges if it's undirected
	EmptyGraph()                          // Clears the graph of all nodes and edges
	SetDirected(bool)                     // This package will only call SetDirected on an empty graph, so there's no need to worry about the case where a graph suddenly becomes (un)directed
}

// A DStarGraph is a special interface that allows the DStarLite function to be used on a graph
//
// D*-lite is an algorithm that allows for the graph representation to change when actions are taken, whether this be from actions taken by the agent or simply new information gathered.
// As such, there's a Move function, that allows the graph to take into account an agent moving to the next node. This is always followed by a call to ChangedEdges.
//
// Traditionally in D*-lite, the algorithm would scan every edge to see if the cost changed, and then update its information if it detected any changes. This slightly remixed step
// allows the graph to provide notification of any changes, and even provide an alternate cost function if it needs to. This can be used to speed up the algorithm significantly
// since the graph no longer has to scan for changes, and only updates when told to. If changedEdges is nil or of len 0, no updates will be performed. If changedEdges is not nil, it
// will update the internal representation. If newCostFunc is non-nil it will be swapped with dStar's current cost function if and only if changedEdges is non-nil/len>0, however,
// newCostFunc is not required to be non-nil if updates are present. DStar will continue using the current cost function if that is the case.
type DStarGraph interface {
	Graph
	Move(target Node)
	ChangedEdges() (newCostFunc func(Node, Node) float64, changedEdges []Edge)
}
<commit_msg>Initial interface change<commit_after>package graph

// All a node needs to do is identify itself. This allows the user to pass in nodes more interesting than an int,
// but also allow us to reap the benefits of having a map-storable, ==able type.
type Node interface {
	ID() int
}

// Allows edges to do something more interesting that just be a group of nodes. While the methods are called Head and Tail,
// they are not considered directed unless the given interface specifies otherwise
type Edge interface {
	Head() Node
	Tail() Node
}

// A Graph ensures the behavior of an undirected graph, necessary to run certain algorithms on it.
//
// The Graph interface is directed. This means that EdgeList() should return an edge where Head always goes towards Tail. If your graph is undirected and you only maintain edges for one direction,
// simply return two edges for each one of your edges, with the Head and Tail swapped in each one.
type Graph interface {
	NodeExists(node Node) bool           // Returns whether a node with the given Node is currently in the graph
	Degree(node Node) int                // Degree is equivalent to len(Successors(node)) + len(Predecessors(node)); this means that reflexive edges are counted twice
	NodeList() []Node                    // Returns a list of all node IDs in no particular order, useful for determining things like if a graph is fully connected. The caller is free to modify this list (so don't pass a reference to your own list)
	Neighbors(node Node) []Node          // Returns all nodes connected by any edge to this node
	IsNeighbor(node, neighbor Node) bool // Returns whether neighbor is connected by an edge to node
}

// Directed graphs are characterized by having seperable Heads and Tails in their edges. That is, if node1 goes to node2, that does not necessarily imply that node2 goes to node1.
//
// While it's possible for a directed graph to have fully reciprocal edges (i.e. the graph is symmetric) -- it is not required to be. The graph is also required to implement UndirectedGraph
// because it can be useful to know all neighbors regardless of direction; not because this graph treats directed graphs as special cases of undirected ones (the truth is, in fact, the opposite)
type DirectedGraph interface {
	Graph
	Successors(node Node) []Node               // Gives the nodes connected by OUTBOUND edges, if the graph is an undirected graph, this set is equal to Predecessors
	IsSuccessor(node, successor Node) bool     // If successor shows up in the list returned by Successors(node), then it's a successor. If node doesn't exist, this should always return false
	Predecessors(node Node) []Node             // Gives the nodes connected by INBOUND edges, if the graph is an undirected graph, this set is equal to Successors
	IsPredecessor(node, predecessor Node) bool // If predecessor shows up in the list returned by Predecessors(node), then it's a predecessor. If node doesn't exist, this should always return false
}

// Returns all undirected edges in the graph
type EdgeLister interface {
	EdgeList() []Edge
}

// A crunch graph forces a sparse graph to become a dense graph. That is, if the node IDs are [1,4,9,7] it would "crunch" the ids into the contiguous block [0,1,2,3]
//
// All dense graphs should have the first ID at 0
type CrunchGraph interface {
	Graph
	Crunch()
}

// A Graph that implements Coster has an actual cost between adjacent nodes, also known as a weighted graph. If a graph implements coster and a function needs to read cost (e.g. A*), this function will
// take precedence over the Uniform Cost function (all weights are 1) if "nil" is passed in for the function argument
//
// If no edge exists between node1 and node2, the cost should be taken to be +inf (can be gotten by math.Inf(1))
type Coster interface {
	Cost(node1, node2 Node) float64
}

// Guarantees that something implementing Coster is also a Graph
type CostGraph interface {
	Coster
	Graph
}

// A graph that implements HeuristicCoster implements a heuristic between any two given nodes. Like Coster, if a graph implements this and a function needs a heuristic cost (e.g. A*), this function will
// take precedence over the Null Heuristic (always returns 0) if "nil" is passed in for the function argument
type HeuristicCoster interface {
	HeuristicCost(node1, node2 Node) float64 // If HeuristicCost is not intended to be used, it can be implemented as the null heuristic (always returns 0)
}

// A Mutable Graph is a graph that can be changed in an arbitrary way. It is useful for several algorithms; for instance, Johnson's Algorithm requires adding a temporary node and changing edge weights.
// Another case where this is used is computing minimum spanning trees. Since trees are graphs, a minimum spanning tree can be created using this interface.
//
// Note that just because a graph does not implement MutableGraph does not mean that this package expects it to be invariant (though even a MutableGraph should be treated as invariant while an algorithm
// is operating on it), it simply means that without this interface this package can not properly handle the graph in order to, say, fill it with a minimum spanning tree.
//
// In functions that take a MutableGraph as an argument, it should not be the same as the Graph argument as concurrent modification will likely cause problems in most cases.
//
// Mutable graphs should always record the IDs as they are represented -- which means they are sparse by nature.
type MutableGraph interface {
	CostGraph
	NewNode(successors []Node) Node       // Adds a node with an arbitrary ID, and returns the new, unique ID used
	AddNode(node Node, successors []Node) // The graph itself is responsible for adding reciprocal edges if it's undirected. Likewise, the graph itself must add any non-existant nodes listed in successors.
	AddEdge(e Edge)                       // For a digraph, adds node1->node2; the graph is free to initialize this to any value it wishes. Node1 must exist, or it will result in undefined behavior, node2 must be created by the function if absent
	SetEdgeCost(e Edge, cost float64)     // The behavior is undefined if the edge has not been created with AddEdge (or the edge was removed before this function was called). For a directed graph only sets node1->node2
	RemoveNode(node Node)                 // The graph is reponsible for removing edges to a node that is removed
	RemoveEdge(e Edge)                    // The graph is responsible for removing reciprocal edges if it's undirected
	EmptyGraph()                          // Clears the graph of all nodes and edges
	SetDirected(bool)                     // This package will only call SetDirected on an empty graph, so there's no need to worry about the case where a graph suddenly becomes (un)directed
}

// A DStarGraph is a special interface that allows the DStarLite function to be used on a graph
//
// D*-lite is an algorithm that allows for the graph representation to change when actions are taken, whether this be from actions taken by the agent or simply new information gathered.
// As such, there's a Move function, that allows the graph to take into account an agent moving to the next node. This is always followed by a call to ChangedEdges.
//
// Traditionally in D*-lite, the algorithm would scan every edge to see if the cost changed, and then update its information if it detected any changes. This slightly remixed step
// allows the graph to provide notification of any changes, and even provide an alternate cost function if it needs to. This can be used to speed up the algorithm significantly
// since the graph no longer has to scan for changes, and only updates when told to. If changedEdges is nil or of len 0, no updates will be performed. If changedEdges is not nil, it
// will update the internal representation. If newCostFunc is non-nil it will be swapped with dStar's current cost function if and only if changedEdges is non-nil/len>0, however,
// newCostFunc is not required to be non-nil if updates are present. DStar will continue using the current cost function if that is the case.
type DStarGraph interface {
	Graph
	Move(target Node)
	ChangedEdges() (newCostFunc func(Node, Node) float64, changedEdges []Edge)
}
<|endoftext|>


<commit_before>package ca

import (
	"archive/tar"
	"compress/gzip"
	"crypto/x509"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

const (
	appleTarballCertPage = "https://opensource.apple.com/tarballs/security_certificates"
	appleLocalTarballName = "data.tar.gz"
)

// This returns the list of tarball names (so we can find the latest one)
// and returns nil if an error occurs
func getTarballNames() []string {
	resp, err := http.DefaultClient.Get(appleTarballCertPage)
	if err != nil {
		return nil
	}
	defer func() {
		if resp.Body != nil {
			resp.Body.Close()
		}
	}()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil
	}

	// Pull out the tarball names and links
	// e.g. href="security_certificates-55070.30.7.tar.gz"
	r := regexp.MustCompile(`href="([\w\_\-.]+\.tar\.gz)`)
	finds := r.FindAll(b, -1)

	out := make([]string, 0)
	for i := range finds {
		s := strings.TrimPrefix(string(finds[i]), `href="`)
		out = append(out, s)
	}

	return out
}

// Sort the names in a-z order, which since they're all named basically
// 'security_certificates-[\d]+.tar.gz' the largest tarball should be the newest.
func findLatestTarball(names []string) string {
	if len(names) == 0 {
		return ""
	}
	sort.Strings(names)
	return names[len(names)-1]
}

// Download the tarball and return abs local fs path to tarball
func downloadTarball(u string) (string, error) {
	dir, err := ioutil.TempDir("", "cert-manage-fetcher")
	if err != nil {
		return "", err
	}
	// Deleting dir and file happens wihen `cleanup` is called
	tmp := filepath.Join(dir, appleLocalTarballName)
	out, err := os.Create(tmp)
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Download the file
	resp, err := http.DefaultClient.Get(u)
	if err != nil {
		return "", err
	}
	defer func() {
		if resp.Body != nil {
			resp.Body.Close()
		}
	}()

	// Gracefully copy file to temp location
	_, err = io.Copy(out, resp.Body)

	return tmp, nil
}

// Pick out which files from the tarball are cert files and
// extract those out into our tempdir
func extractTarball(p string) error {
	dir, _ := path.Split(p)
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	// gzip and tar
	g, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	r := tar.NewReader(g)

	// Read each file out
	for {
		hdr, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		// Only process files we care about
		if hdr.Typeflag == tar.TypeReg && isCertFile(hdr.Name) {
			_, name := path.Split(hdr.Name)
			f, err := os.Create(path.Join(dir, name))
			// TODO(adam): Does this cause a big slew of fd.Close() calls at the end?
			//             Instead should we `go f.Close()` after? What if err != nil
			defer f.Close()
			if err != nil {
				return nil
			}
			io.Copy(f, r)
		}
	}

	return nil
}

func isCertFile(p string) bool {
	// Grab all files in certs/* and roots/*
	// We do this a bit strangely because we parse out the tar header info and
	// split apart the path. e.g. security-certificates-16/certs/Visa.crt
	//
	// This split will turn into 'security-certificates-16/certs', 'Visa.crt'
	// of which we match on the first half if we're looking at a file in the `certs/`
	// directory.
	paths := []string{"certs/", "roots/"}
	dir, _ := path.Split(p)
	for i := range paths {
		if strings.HasSuffix(dir, paths[i]) {
			return true
		}
	}
	return false
}

// Read out all certs from the extracted files in our tempdir
func collectCerts(p string) ([]*x509.Certificate, error) {
	dir, _ := path.Split(p)

	certs := make([]*x509.Certificate, 0)
	walker := func(p string, info os.FileInfo, err error) error {
		// Ignore directories and symlinks (non-regular files)
		if info.IsDir() || !info.Mode().IsRegular() {
			return nil
		}
		// Ignore skipdirs
		if err != nil && err != filepath.SkipDir {
			return nil
		}
		// Ignore the archive
		if _, name := path.Split(p); name == appleLocalTarballName {
			return nil
		}

		// Read the cert as DER encoded
		b, err := ioutil.ReadFile(p)
		if err != nil {
			return err
		}
		cs, err := x509.ParseCertificates(b)
		if err != nil {
			return err
		}
		certs = append(certs, cs...)
		return nil
	}

	err := filepath.Walk(dir, walker)
	return certs, err
}

// Cleanup the parent dir for a given path
func cleanup(p string) error {
	return os.RemoveAll(filepath.Dir(p))
}

// Pull the latest set of certs from Apple's page and
// extract them.
func Apple() ([]*x509.Certificate, error) {
	latest := findLatestTarball(getTarballNames())
	u := appleTarballCertPage + "/" + latest
	p, err := downloadTarball(u)
	if err != nil {
		return nil, err
	}
	defer cleanup(p)

	err = extractTarball(p)
	if err != nil {
		return nil, err
	}

	return collectCerts(p)
}
<commit_msg>fetch/apple: better error messages on individual cert file errors<commit_after>package ca

import (
	"archive/tar"
	"compress/gzip"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

const (
	appleTarballCertPage = "https://opensource.apple.com/tarballs/security_certificates"
	appleLocalTarballName = "data.tar.gz"
)

// This returns the list of tarball names (so we can find the latest one)
// and returns nil if an error occurs
func getTarballNames() []string {
	resp, err := http.DefaultClient.Get(appleTarballCertPage)
	if err != nil {
		return nil
	}
	defer func() {
		if resp.Body != nil {
			resp.Body.Close()
		}
	}()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil
	}

	// Pull out the tarball names and links
	// e.g. href="security_certificates-55070.30.7.tar.gz"
	r := regexp.MustCompile(`href="([\w\_\-.]+\.tar\.gz)`)
	finds := r.FindAll(b, -1)

	out := make([]string, 0)
	for i := range finds {
		s := strings.TrimPrefix(string(finds[i]), `href="`)
		out = append(out, s)
	}

	return out
}

// Sort the names in a-z order, which since they're all named basically
// 'security_certificates-[\d]+.tar.gz' the largest tarball should be the newest.
func findLatestTarball(names []string) string {
	if len(names) == 0 {
		return ""
	}
	sort.Strings(names)
	return names[len(names)-1]
}

// Download the tarball and return abs local fs path to tarball
func downloadTarball(u string) (string, error) {
	dir, err := ioutil.TempDir("", "cert-manage-fetcher")
	if err != nil {
		return "", err
	}
	// Deleting dir and file happens wihen `cleanup` is called
	tmp := filepath.Join(dir, appleLocalTarballName)
	out, err := os.Create(tmp)
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Download the file
	resp, err := http.DefaultClient.Get(u)
	if err != nil {
		return "", err
	}
	defer func() {
		if resp.Body != nil {
			resp.Body.Close()
		}
	}()

	// Gracefully copy file to temp location
	_, err = io.Copy(out, resp.Body)

	return tmp, nil
}

// Pick out which files from the tarball are cert files and
// extract those out into our tempdir
func extractTarball(p string) error {
	dir, _ := path.Split(p)
	f, err := os.Open(p)
	if err != nil {
		return err
	}
	defer f.Close()

	// gzip and tar
	g, err := gzip.NewReader(f)
	if err != nil {
		return err
	}
	r := tar.NewReader(g)

	// Read each file out
	for {
		hdr, err := r.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		// Only process files we care about
		if hdr.Typeflag == tar.TypeReg && isCertFile(hdr.Name) {
			_, name := path.Split(hdr.Name)
			f, err := os.Create(path.Join(dir, name))
			// TODO(adam): Does this cause a big slew of fd.Close() calls at the end?
			//             Instead should we `go f.Close()` after? What if err != nil
			defer f.Close()
			if err != nil {
				return nil
			}
			io.Copy(f, r)
		}
	}

	return nil
}

func isCertFile(p string) bool {
	// Grab all files in certs/* and roots/*
	// We do this a bit strangely because we parse out the tar header info and
	// split apart the path. e.g. security-certificates-16/certs/Visa.crt
	//
	// This split will turn into 'security-certificates-16/certs', 'Visa.crt'
	// of which we match on the first half if we're looking at a file in the `certs/`
	// directory.
	paths := []string{"certs/", "roots/"}
	dir, _ := path.Split(p)
	for i := range paths {
		if strings.HasSuffix(dir, paths[i]) {
			return true
		}
	}
	return false
}

// Read out all certs from the extracted files in our tempdir
func collectCerts(p string) ([]*x509.Certificate, error) {
	dir, _ := path.Split(p)

	certs := make([]*x509.Certificate, 0)
	walker := func(p string, info os.FileInfo, err error) error {
		// Ignore directories and symlinks (non-regular files)
		if info.IsDir() || !info.Mode().IsRegular() {
			return nil
		}
		// Ignore skipdirs
		if err != nil && err != filepath.SkipDir {
			return nil
		}
		// Ignore the archive
		_, name := path.Split(p)
		if name == appleLocalTarballName {
			return nil
		}

		// Read the cert as DER encoded
		b, err := ioutil.ReadFile(p)
		if err != nil {
			return fmt.Errorf("error reading %s -- %s", name, err.Error())
		}
		cs, err := x509.ParseCertificates(b)
		if err != nil {
			return fmt.Errorf("error parsing cert %s -- %s", name, err.Error())
		}
		certs = append(certs, cs...)
		return nil
	}

	err := filepath.Walk(dir, walker)
	return certs, err
}

// Cleanup the parent dir for a given path
func cleanup(p string) error {
	return os.RemoveAll(filepath.Dir(p))
}

// Pull the latest set of certs from Apple's page and
// extract them.
func Apple() ([]*x509.Certificate, error) {
	latest := findLatestTarball(getTarballNames())
	u := appleTarballCertPage + "/" + latest
	p, err := downloadTarball(u)
	if err != nil {
		return nil, err
	}
	defer cleanup(p)

	err = extractTarball(p)
	if err != nil {
		return nil, err
	}

	return collectCerts(p)
}
<|endoftext|>


<commit_before>package gitmedia

import (
	_ "../gitconfig"
	"bufio"
	"fmt"
	"os"
	"strings"
)

type TypesCommand struct {
	*Command
}

func (c *TypesCommand) Run() {
	var sub string
	if len(c.SubCommands) > 0 {
		sub = c.SubCommands[0]
	}

	switch sub {
	case "add":
		fmt.Println("Adding type")
	case "remove":
		fmt.Println("Removing type")
	default:
		fmt.Println("Listing types")
		listTypes()
	}

}

func listTypes() {
	attributes, err := os.Open(".gitattributes")
	if err != nil {
		return // No .gitattibtues == no file types
	}

	scanner := bufio.NewScanner(attributes)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}

		if strings.Contains(line, "filter=media") {
			fields := strings.Fields(line)
			fmt.Println("   ", fields[0])
		}
	}

}

func init() {
	registerCommand("types", func(c *Command) RunnableCommand {
		return &TypesCommand{Command: c}
	})
}
<commit_msg>ンンー ンンンン ンーンン<commit_after>package gitmedia

import (
	_ "../gitconfig"
	"bufio"
	"fmt"
	"os"
	"strings"
)

type TypesCommand struct {
	*Command
}

func (c *TypesCommand) Run() {
	var sub string
	if len(c.SubCommands) > 0 {
		sub = c.SubCommands[0]
	}

	switch sub {
	case "add":
		c.addType()
	case "remove":
		fmt.Println("Removing type")
	default:
		c.listTypes()
	}

}

func (c *TypesCommand) addType() {
	if len(c.SubCommands) < 2 {
		fmt.Println("git media types add <type> [type]*")
		return
	}

	knownTypes := findTypes()
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		fmt.Println("Error opening .gitattributes file")
		return
	}

	for _, t := range c.SubCommands[1:] {
		isKnownType := false
		for _, k := range knownTypes {
			if t == k {
				isKnownType = true
			}
		}

		if isKnownType {
			fmt.Println(t, "already supported")
			continue
		}

		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=media -crlf", t))
		if err != nil {
			fmt.Println("Error adding type", t)
			continue
		}
		fmt.Println("Adding type", t)
	}

	attributesFile.Close()
}

func (c *TypesCommand) listTypes() {
	fmt.Println("Listing types")
	knownTypes := findTypes()
	for _, t := range knownTypes {
		fmt.Println("    ", t)
	}
}

func findTypes() []string {
	types := make([]string, 0)

	attributes, err := os.Open(".gitattributes")
	if err != nil {
		return types // No .gitattibtues == no file types
	}

	scanner := bufio.NewScanner(attributes)
	for scanner.Scan() {
		line := scanner.Text()
		if line == "" {
			continue
		}

		if strings.Contains(line, "filter=media") {
			fields := strings.Fields(line)
			types = append(types, fields[0])
		}
	}

	return types
}

func init() {
	registerCommand("types", func(c *Command) RunnableCommand {
		return &TypesCommand{Command: c}
	})
}
<|endoftext|>


<commit_before>package pg

import (
	"time"

	"github.com/go-pg/pg/orm"
)

type QueryProcessedEvent struct {
	StartTime time.Time
	Query     interface{}
	Params    []interface{}
	Result    orm.Result
	Error     error
}

type queryProcessedHook func(orm.DB, *QueryProcessedEvent)

// OnQueryProcessed calls the fn with QueryProcessedEvent
// when query is processed.
func (db *DB) OnQueryProcessed(fn queryProcessedHook) {
	db.queryProcessedHooks = append(db.queryProcessedHooks, fn)
}

func (db *DB) queryProcessed(
	ormDB orm.DB,
	start time.Time,
	query interface{},
	params []interface{},
	res orm.Result,
	err error,
) {
	if len(db.queryProcessedHooks) == 0 {
		return
	}

	event := &QueryProcessedEvent{
		StartTime: start,
		Query:     query,
		Params:    params,
		Result:    res,
		Error:     err,
	}
	for _, hook := range db.queryProcessedHooks {
		hook(ormDB, event)
	}
}
<commit_msg>Doc-friendly func signature<commit_after>package pg

import (
	"time"

	"github.com/go-pg/pg/orm"
)

type QueryProcessedEvent struct {
	StartTime time.Time
	Query     interface{}
	Params    []interface{}
	Result    orm.Result
	Error     error
}

type queryProcessedHook func(orm.DB, *QueryProcessedEvent)

// OnQueryProcessed calls the fn with QueryProcessedEvent
// when query is processed.
func (db *DB) OnQueryProcessed(fn func(orm.DB, *QueryProcessedEvent)) {
	db.queryProcessedHooks = append(db.queryProcessedHooks, fn)
}

func (db *DB) queryProcessed(
	ormDB orm.DB,
	start time.Time,
	query interface{},
	params []interface{},
	res orm.Result,
	err error,
) {
	if len(db.queryProcessedHooks) == 0 {
		return
	}

	event := &QueryProcessedEvent{
		StartTime: start,
		Query:     query,
		Params:    params,
		Result:    res,
		Error:     err,
	}
	for _, hook := range db.queryProcessedHooks {
		hook(ormDB, event)
	}
}
<|endoftext|>


<commit_before>package buf

import (
	"sync"
)

const (
	// Size of a regular buffer.
	Size = 2 * 1024
)

func createAllocFunc(size uint32) func() interface{} {
	return func() interface{} {
		return make([]byte, size)
	}
}

const (
	numPools  = 5
	sizeMulti = 4
)

var (
	pool     [numPools]sync.Pool
	poolSize [numPools]uint32
)

func init() {
	size := uint32(Size)
	for i := 0; i < numPools; i++ {
		pool[i] = sync.Pool{
			New: createAllocFunc(size),
		}
		poolSize[i] = size
		size *= sizeMulti
	}
}

func newBytes(size uint32) []byte {
	for idx, ps := range poolSize {
		if size <= ps {
			return pool[idx].Get().([]byte)
		}
	}
	return make([]byte, size)
}

func freeBytes(b []byte) {
	size := uint32(cap(b))
	b = b[0:cap(b)]
	for i := numPools - 1; i >= 0; i-- {
		ps := poolSize[i]
		if size >= ps {
			pool[i].Put(b)
			return
		}
	}
}
<commit_msg>simplify code<commit_after>package buf

import (
	"sync"
)

const (
	// Size of a regular buffer.
	Size = 2 * 1024
)

func createAllocFunc(size uint32) func() interface{} {
	return func() interface{} {
		return make([]byte, size)
	}
}

const (
	numPools  = 5
	sizeMulti = 4
)

var (
	pool     [numPools]sync.Pool
	poolSize [numPools]uint32
)

func init() {
	size := uint32(Size)
	for i := 0; i < numPools; i++ {
		pool[i] = sync.Pool{
			New: createAllocFunc(size),
		}
		poolSize[i] = size
		size *= sizeMulti
	}
}

func newBytes(size uint32) []byte {
	for idx, ps := range poolSize {
		if size <= ps {
			return pool[idx].Get().([]byte)
		}
	}
	return make([]byte, size)
}

func freeBytes(b []byte) {
	size := uint32(cap(b))
	b = b[0:cap(b)]
	for i := numPools - 1; i >= 0; i-- {
		if size >= poolSize[i] {
			pool[i].Put(b)
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io"
	"log"
	"net"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
)

func hookHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	id := params["id"]
	log.Printf("Received hook for id '%s' from %s\n", id, r.RemoteAddr)
	rb, err := NewRunBook(id)
	if err != nil {
		log.Println(err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	remoteIP := net.ParseIP(strings.Split(r.RemoteAddr, ":")[0])
	if !rb.AddrIsAllowed(remoteIP) {
		log.Printf("Hook id '%s' is not allowed from %v\n", id, r.RemoteAddr)
		http.Error(w, "Not authorized.", http.StatusUnauthorized)
		return
	}
	interoplatePOSTData(rb, r)
	response, err := rb.execute()
	if err != nil {
		log.Println(err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	if echo {
		data, err := json.MarshalIndent(response, "", "  ")
		if err != nil {
			log.Println(err.Error())
		}
		w.Write(data)
	}
}

func interoplatePOSTData(rb *runBook, r *http.Request) {
	if r.ContentLength == 0 {
		return
	}
	data := make([]byte, r.ContentLength)
	_, err := r.Body.Read(data)
	if err != nil && err != io.EOF {
		log.Fatal(err)
		return
	}
	defer r.Body.Close()
	stringData := string(data[:r.ContentLength])
	for i := range rb.Scripts {
		for j := range rb.Scripts[i].Args {
			rb.Scripts[i].Args[j] = strings.Replace(rb.Scripts[i].Args[j], "{{POST}}", stringData, -1)
		}
	}
}
<commit_msg>Use ioutil.ReadAll to guarantee entire request body read in.<commit_after>package main

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
)

func hookHandler(w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	id := params["id"]
	log.Printf("Received hook for id '%s' from %s\n", id, r.RemoteAddr)
	rb, err := NewRunBook(id)
	if err != nil {
		log.Println(err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	remoteIP := net.ParseIP(strings.Split(r.RemoteAddr, ":")[0])
	if !rb.AddrIsAllowed(remoteIP) {
		log.Printf("Hook id '%s' is not allowed from %v\n", id, r.RemoteAddr)
		http.Error(w, "Not authorized.", http.StatusUnauthorized)
		return
	}
	interoplatePOSTData(rb, r)
	response, err := rb.execute()
	if err != nil {
		log.Println(err.Error())
		http.Error(w, err.Error(), 500)
		return
	}
	if echo {
		data, err := json.MarshalIndent(response, "", "  ")
		if err != nil {
			log.Println(err.Error())
		}
		w.Write(data)
	}
}

func interoplatePOSTData(rb *runBook, r *http.Request) {
	if r.ContentLength == 0 {
		return
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil && err != io.EOF {
		log.Fatal(err)
		return
	}
	defer r.Body.Close()
	stringData := string(data[:r.ContentLength])
	for i := range rb.Scripts {
		for j := range rb.Scripts[i].Args {
			rb.Scripts[i].Args[j] = strings.Replace(rb.Scripts[i].Args[j], "{{POST}}", stringData, -1)
		}
	}
}
<|endoftext|>


<commit_before>// bench contains benchmarks and common utilities useful to benchmarks
//
// In order to write new benchmarks, one must satisfy the Benchmark and Command
// interfaces in bench.go. In order to use the benchmark from pilosactl, it
// needs to be wired in in two places. The first is BagentCommand.ParseFlags,
// where a case statement needs to be added, and the second is just adding the
// benchmark to the BagentCommand.Usage usage string.
//
// When writing a new benchmark, there are a few things to keep in mind other
// than just implementing the interface:
//
// 1. The benchmark should modify it's own configuration in its Init method based
// on the agentNum it is given. How it modifies is specific to the benchmark,
// but the idea is that it should make sense to call the benchmark with the same
// configuration, but multiple different agent numbers, and it should do useful
// work each time (i.e. not just setting the same bits, or running the same
// queries).
//
// 2. The Init method should do everything that needs to be done to get the
// benchmark to a runnable state - all code in run should be the stuff that we
// actually want to time.
//
// 3. The Run method does not need to report the total runtime - that is collected
// by calling code.
//
// 4. Usage should follow the format in other benchmarks, and explain how the
// benchmark uses agentNum to modify its behavior
//
//
// Files:
//
// 1. client.go contains pilosa client code which is shared by many benchmarks
//
// 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/
// so as not to pull in a bunch of useless deps.
//
// 3. stats.go contains useful code for gathering stats about a series of timed
// operations.
package bench
<commit_msg>it's -> its<commit_after>// bench contains benchmarks and common utilities useful to benchmarks
//
// In order to write new benchmarks, one must satisfy the Benchmark and Command
// interfaces in bench.go. In order to use the benchmark from pilosactl, it
// needs to be wired in in two places. The first is BagentCommand.ParseFlags,
// where a case statement needs to be added, and the second is just adding the
// benchmark to the BagentCommand.Usage usage string.
//
// When writing a new benchmark, there are a few things to keep in mind other
// than just implementing the interface:
//
// 1. The benchmark should modify its own configuration in its Init method based
// on the agentNum it is given. How it modifies is specific to the benchmark,
// but the idea is that it should make sense to call the benchmark with the same
// configuration, but multiple different agent numbers, and it should do useful
// work each time (i.e. not just setting the same bits, or running the same
// queries).
//
// 2. The Init method should do everything that needs to be done to get the
// benchmark to a runnable state - all code in run should be the stuff that we
// actually want to time.
//
// 3. The Run method does not need to report the total runtime - that is collected
// by calling code.
//
// 4. Usage should follow the format in other benchmarks, and explain how the
// benchmark uses agentNum to modify its behavior
//
//
// Files:
//
// 1. client.go contains pilosa client code which is shared by many benchmarks
//
// 2. errgroup.go contains the ErrGroup implementation copied from golang.org/x/
// so as not to pull in a bunch of useless deps.
//
// 3. stats.go contains useful code for gathering stats about a series of timed
// operations.
package bench
<|endoftext|>


<commit_before><commit_msg>Add gobot.go main package<commit_after><|endoftext|>