<commit_before>package gocd

import (
	"context"
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"strings"
)

// APIClientRequest helper struct to reduce amount of code.
type APIClientRequest struct {
	Method       string
	Path         string
	APIVersion   string
	RequestBody  interface{}
	ResponseType string
	ResponseBody interface{}
	Headers      map[string]string
}

// Handles any call to HEAD by returning whether or not we got a 2xx code.
func (c *Client) genericHeadAction(ctx context.Context, path string, apiversion string) (bool, *APIResponse, error) {
	_, resp, err := c.httpAction(ctx, &APIClientRequest{
		Method:       "HEAD",
		Path:         path,
		APIVersion:   apiversion,
		ResponseType: responseTypeJSON,
	})

	exists := resp.HTTP.StatusCode >= 300 || resp.HTTP.StatusCode < 200

	return exists, resp, err

}

func (c *Client) patchAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "PATCH"
	return c.httpAction(ctx, r)
}

func (c *Client) getAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "GET"
	return c.httpAction(ctx, r)
}

func (c *Client) postAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "POST"
	return c.httpAction(ctx, r)
}

func (c *Client) putAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "PUT"
	return c.httpAction(ctx, r)
}

// Returns a message from the DELETE action on the provided HTTP resource.
func (c *Client) deleteAction(ctx context.Context, path string, apiversion string) (string, *APIResponse, error) {
	a := StringResponse{}
	_, resp, err := c.httpAction(ctx, &APIClientRequest{
		Method:       "DELETE",
		Path:         path,
		APIVersion:   apiversion,
		ResponseType: responseTypeJSON,
		ResponseBody: &a,
	})

	return a.Message, resp, err
}

func (c *Client) httpAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {

	log.Debugf("HTTP Request")
	log.Debugf("%s %s", r.Method, r.Path)

	if r.ResponseType == "" {
		r.ResponseType = responseTypeJSON
	}

	var isVersioned bool
	var ver Versioned
	if ver, isVersioned = (r.RequestBody).(Versioned); isVersioned {
		if r.Headers == nil {
			r.Headers = map[string]string{}
		}
		r.Headers["If-Match"] = fmt.Sprintf("\"%s\"", ver.GetVersion())
	}

	// Build the request
	var reqBody interface{}
	if r.RequestBody != nil {
		reqBody = r.RequestBody
	} else {
		reqBody = nil
	}

	req, err := c.NewRequest(r.Method, r.Path, reqBody, r.APIVersion)
	if err != nil {
		return false, nil, err
	}

	if len(r.Headers) > 0 {
		for key, value := range r.Headers {
			req.HTTP.Header.Set(key, value)
		}
	}

	for header, values := range req.HTTP.Header {
		for _, value := range values {
			log.Debugf("%s: %s", header, value)
		}
	}
	if r.RequestBody != nil {
		log.Debug()
		log.Debug(r.RequestBody)
		log.Debug()
		log.Debug()
	}

	resp, err := c.Do(ctx, req, r.ResponseBody, r.ResponseType)
	if err != nil {
		log.Error(err)
		return r.ResponseBody, resp, err
	}

	if ver, isVersioned = (r.ResponseBody).(Versioned); isVersioned {
		parseVersions(resp.HTTP, ver)
	}

	if r.ResponseType == responseTypeJSON {
		log.Debug()
		log.Debug("Response")
		log.Debugf("%s %s", resp.HTTP.Proto, resp.HTTP.Status)
		for header, values := range resp.HTTP.Header {
			for _, value := range values {
				log.Debugf("%s: %s", header, value)
			}
		}
		log.Debug()
		b, _ := json.Marshal(r.ResponseBody)
		log.Debugf("%s", b)
	}

	return r.ResponseBody, resp, err
}

func parseVersions(response *http.Response, versioned Versioned) {
	etag := response.Header.Get("Etag")
	versioned.SetVersion(
		strings.Replace(etag, "\"", "", -1),
	)
}
<commit_msg>Removed extra debug statements<commit_after>package gocd

import (
	"context"
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"strings"
)

// APIClientRequest helper struct to reduce amount of code.
type APIClientRequest struct {
	Method       string
	Path         string
	APIVersion   string
	RequestBody  interface{}
	ResponseType string
	ResponseBody interface{}
	Headers      map[string]string
}

// Handles any call to HEAD by returning whether or not we got a 2xx code.
func (c *Client) genericHeadAction(ctx context.Context, path string, apiversion string) (bool, *APIResponse, error) {
	_, resp, err := c.httpAction(ctx, &APIClientRequest{
		Method:       "HEAD",
		Path:         path,
		APIVersion:   apiversion,
		ResponseType: responseTypeJSON,
	})

	exists := resp.HTTP.StatusCode >= 300 || resp.HTTP.StatusCode < 200

	return exists, resp, err

}

func (c *Client) patchAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "PATCH"
	return c.httpAction(ctx, r)
}

func (c *Client) getAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "GET"
	return c.httpAction(ctx, r)
}

func (c *Client) postAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "POST"
	return c.httpAction(ctx, r)
}

func (c *Client) putAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {
	r.Method = "PUT"
	return c.httpAction(ctx, r)
}

// Returns a message from the DELETE action on the provided HTTP resource.
func (c *Client) deleteAction(ctx context.Context, path string, apiversion string) (string, *APIResponse, error) {
	a := StringResponse{}
	_, resp, err := c.httpAction(ctx, &APIClientRequest{
		Method:       "DELETE",
		Path:         path,
		APIVersion:   apiversion,
		ResponseType: responseTypeJSON,
		ResponseBody: &a,
	})

	return a.Message, resp, err
}

func (c *Client) httpAction(ctx context.Context, r *APIClientRequest) (interface{}, *APIResponse, error) {

	log.Debugf("HTTP Request")
	log.Debugf("%s %s", r.Method, r.Path)

	if r.ResponseType == "" {
		r.ResponseType = responseTypeJSON
	}

	if ver, isVersioned := (r.RequestBody).(Versioned); isVersioned {
		if r.Headers == nil {
			r.Headers = map[string]string{}
		}
		r.Headers["If-Match"] = fmt.Sprintf("\"%s\"", ver.GetVersion())
	}

	// Build the request
	req, err := c.NewRequest(r.Method, r.Path, r.RequestBody, r.APIVersion)
	if err != nil {
		return false, nil, err
	}

	if len(r.Headers) > 0 {
		for key, value := range r.Headers {
			req.HTTP.Header.Set(key, value)
		}
	}

	for header, values := range req.HTTP.Header {
		for _, value := range values {
			log.Debugf("%s: %s", header, value)
		}
	}
	if r.RequestBody != nil {
		log.Debugf("\n%s\n\n", r.RequestBody)
	}

	resp, err := c.Do(ctx, req, r.ResponseBody, r.ResponseType)
	if err != nil {
		log.Error(err)
		return r.ResponseBody, resp, err
	}

	if ver, isVersioned := (r.ResponseBody).(Versioned); isVersioned {
		parseVersions(resp.HTTP, ver)
	}

	if r.ResponseType == responseTypeJSON {
		log.Debugf("\nResponse\n%s %s\n", resp.HTTP.Proto, resp.HTTP.Status)
		for header, values := range resp.HTTP.Header {
			for _, value := range values {
				log.Debugf("%s: %s", header, value)
			}
		}
		b, _ := json.Marshal(r.ResponseBody)
		log.Debugf("\n%s", b)
	}

	return r.ResponseBody, resp, err
}

func parseVersions(response *http.Response, versioned Versioned) {
	etag := response.Header.Get("Etag")
	versioned.SetVersion(
		strings.Replace(etag, "\"", "", -1),
	)
}
<|endoftext|>


<commit_before>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return errors.New("stopped after first redirect")
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<commit_msg>not deliberately failed redirects are errors now<commit_after>package gofetcher

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"time"

	"github.com/cocaine/cocaine-framework-go/cocaine"
	"github.com/ugorji/go/codec"
)

const (
	DefaultTimeout         = 5000
	DefaultFollowRedirects = true
	KeepAliveTimeout       = 30
)

// took from httputil/reverseproxy.go
// Hop-by-hop headers. These are removed when sent to the backend.
// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = []string{
	"Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te", // canonicalized version of "TE"
	"Trailers",
	"Transfer-Encoding",
	"Upgrade",
}

type WarnError struct {
	err error
}

func (s *WarnError) Error() string { return s.err.Error() }

func NewWarn(err error) *WarnError {
	return &WarnError{err: err}
}

type Gofetcher struct {
	Logger    *cocaine.Logger
	Transport http.RoundTripper

	UserAgent string
}

type Cookies map[string]string

type Request struct {
	Method          string
	URL             string
	Body            io.Reader
	Timeout         int64
	Cookies         Cookies
	Headers         http.Header
	FollowRedirects bool
}

type responseAndError struct {
	res *http.Response
	err error
}

type Response struct {
	httpResponse *http.Response
	body         []byte
	header       http.Header
	runtime      time.Duration
}

func NewGofetcher() *Gofetcher {
	logger, err := cocaine.NewLogger()
	if err != nil {
		fmt.Printf("Could not initialize logger due to error: %v", err)
		return nil
	}
	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: KeepAliveTimeout * time.Second,
			DualStack: true,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	gofetcher := Gofetcher{logger, transport, ""}
	return &gofetcher
}

func (gofetcher *Gofetcher) SetUserAgent(userAgent string) {
	gofetcher.UserAgent = userAgent
}

type noRedirectError struct{}

func (nr noRedirectError) Error() string {
	return "stopped after first redirect"
}

func noRedirect(_ *http.Request, via []*http.Request) error {
	if len(via) > 0 {
		return noRedirectError{}
	}
	return nil
}

func (gofetcher *Gofetcher) PrepareRequest(request *Request) (*http.Request, *http.Client, error) {
	var (
		err            error
		httpRequest    *http.Request
		requestTimeout time.Duration = time.Duration(request.Timeout) * time.Millisecond
	)

	httpClient := &http.Client{
		Transport: gofetcher.Transport,
		Timeout:   requestTimeout,
	}
	if request.FollowRedirects == false {
		httpClient.CheckRedirect = noRedirect
	}

	httpRequest, err = http.NewRequest(request.Method, request.URL, request.Body)
	if err != nil {
		return nil, nil, err
	}
	for name, value := range request.Cookies {
		httpRequest.AddCookie(&http.Cookie{Name: name, Value: value})
	}
	httpRequest.Header = request.Headers

	// Remove hop-by-hop headers to the backend.  Especially
	// important is "Connection" because we want a persistent
	// connection, regardless of what the client sent to us.  This
	// is modifying the same underlying map from req (shallow
	// copied above) so we only copy it if necessary.
	for _, h := range hopHeaders {
		httpRequest.Header.Del(h)
	}
	httpRequest.Header.Add("Connection", "keep-alive")
	httpRequest.Header.Add("Keep-Alive", fmt.Sprintf("%d", KeepAliveTimeout))

	if gofetcher.UserAgent != "" && len(httpRequest.Header["User-Agent"]) == 0 {
		httpRequest.Header.Set("User-Agent", gofetcher.UserAgent)
	}

	return httpRequest, httpClient, nil
}

func (gofetcher *Gofetcher) ExecuteRequest(req *http.Request, client *http.Client, attempt int) (*http.Response, error) {
	var (
		httpResponse *http.Response
		err          error
	)

	gofetcher.Logger.Infof("Requested url: %s, method: %s, timeout: %d, headers: %v, attempt: %d",
		req.URL.String(), req.Method, client.Timeout, req.Header, attempt)

	resultChan := make(chan responseAndError)
	started := time.Now()
	go func() {
		res, err := client.Do(req)
		resultChan <- responseAndError{res, err}
	}()
	// http connection stay active after timeout exceeded in go <1.3, cause we can't close it using current client api.
	// Read more about timeouts: https://code.google.com/p/go/issues/detail?id=3362
	//
	select {
	case result := <-resultChan:
		httpResponse, err = result.res, result.err
	case <-time.After(client.Timeout):
		err = errors.New(fmt.Sprintf("Request timeout[%s] exceeded", client.Timeout.String()))
		go func() {
			// close httpResponse when it ready
			result := <-resultChan
			if result.res != nil {
				result.res.Body.Close()
			}
		}()
	}
	if err != nil {
		// special case for redirect failure (returns both response and error)
		// read more https://code.google.com/p/go/issues/detail?id=3795
		if httpResponse == nil {
			if urlError, ok := err.(*url.Error); ok {
				// golang bug: golang.org/issue/3514
				if urlError.Err == io.EOF {
					gofetcher.Logger.Infof("Got EOF error while loading %s, attempt(%d)", req.URL.String(), attempt)
					if attempt == 1 {
						return gofetcher.ExecuteRequest(req, client, attempt+1)
					}
				}
			}
			return nil, NewWarn(err)
		}

		if _, ok := err.(noRedirectError); !ok {
			// http client failed to redirect and this is not because we requested it to
			// usually it means that server sent response with redirect status code but omitted "Location" header
			return nil, NewWarn(err)
		}
	}

	for _, h := range hopHeaders {
		httpResponse.Header.Del(h)
	}

	runtime := time.Since(started)
	gofetcher.Logger.Info(fmt.Sprintf("Response code: %d, url: %s, runtime: %v",
		httpResponse.StatusCode, req.URL.String(), runtime))
	return httpResponse, nil

}

// Normal methods

func parseHeaders(rawHeaders map[string]interface{}) http.Header {
	headers := make(http.Header)
	for name, values := range rawHeaders {
		for _, value := range values.([]interface{}) {
			headers.Add(name, string(value.([]uint8))) // to transform in canonical form
		}
	}
	return headers
}

func parseCookies(rawCookie map[string]interface{}) Cookies {
	cookies := Cookies{}
	for key, value := range rawCookie {
		cookies[key] = string(value.([]uint8))
	}
	return cookies
}

func parseTimeout(rawTimeout interface{}) (timeout int64) {
	// is it possible to got timeout in int64 instead of uint64?
	switch rawTimeout.(type) {
	case uint64:
		timeout = int64(rawTimeout.(uint64))
	case int64:
		timeout = rawTimeout.(int64)
	}
	return timeout
}

func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}

func (gofetcher *Gofetcher) WriteError(response *cocaine.Response, request *Request, err error) {
	if _, casted := err.(*WarnError); casted {
		gofetcher.Logger.Warnf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	} else {
		gofetcher.Logger.Errf("Error occured: %v, while downloading %s",
			err.Error(), request.URL)
	}
	response.Write([]interface{}{false, err.Error(), 0, http.Header{}})
}

func (gofetcher *Gofetcher) WriteResponse(response *cocaine.Response, request *Request, resp *http.Response, body []byte) {
	response.Write([]interface{}{true, body, resp.StatusCode, resp.Header})
}

func (gofetcher *Gofetcher) handler(method string, request *cocaine.Request, response *cocaine.Response) {
	defer response.Close()

	requestBody := <-request.Read()
	httpRequest := gofetcher.ParseRequest(method, requestBody)

	req, client, err := gofetcher.PrepareRequest(httpRequest)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	resp, err := gofetcher.ExecuteRequest(req, client, 1)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		gofetcher.WriteError(response, httpRequest, err)
		return
	}

	gofetcher.WriteResponse(response, httpRequest, resp, body)
}

func (gofetcher *Gofetcher) GetHandler(method string) func(request *cocaine.Request, response *cocaine.Response) {
	return func(request *cocaine.Request, response *cocaine.Response) {
		gofetcher.handler(method, request, response)
	}
}

// Http methods

func (gofetcher *Gofetcher) HttpProxy(res http.ResponseWriter, req *http.Request) {
	var (
		timeout int64 = DefaultTimeout
		resp    *http.Response
	)
	url := req.FormValue("url")
	timeoutArg := req.FormValue("timeout")
	if timeoutArg != "" {
		tout, _ := strconv.Atoi(timeoutArg)
		timeout = int64(tout)
	}
	httpRequest := &Request{Method: req.Method, URL: url, Timeout: timeout,
		FollowRedirects: DefaultFollowRedirects, Headers: req.Header, Body: req.Body}
	prepReq, prepClient, err := gofetcher.PrepareRequest(httpRequest)
	if err == nil {
		resp, err = gofetcher.ExecuteRequest(prepReq, prepClient, 1)
	}

	if err != nil {
		res.Header().Set("Content-Type", "text/html")
		res.WriteHeader(500)
		res.Write([]byte(err.Error()))
		if _, casted := err.(*WarnError); casted {
			gofetcher.Logger.Warnf("Gofetcher error: %v", err)
		} else {
			gofetcher.Logger.Errf("Gofetcher error: %v", err)
		}

	} else {
		for key, values := range resp.Header {
			for _, value := range values {
				res.Header().Add(key, value)
			}
		}
		res.WriteHeader(200)
		if _, err := io.Copy(res, resp.Body); err != nil {
			gofetcher.Logger.Errf("Error: %v", err)
		}
	}
}

func (gofetcher *Gofetcher) HttpEcho(res http.ResponseWriter, req *http.Request) {
	gofetcher.Logger.Info("Http echo handler requested")
	text := req.FormValue("text")
	res.Header().Set("Content-Type", "text/html")
	res.WriteHeader(200)
	res.Write([]byte(text))
}
<|endoftext|>


<commit_before>package stats

import(
	"github.com/trendrr/goshire/dynmap"
	"github.com/trendrr/goshire/timeamount"
	"time"
	"log"

)

// A simple stats collector

type Stats struct {
	itemChan chan statsItem
	getChan chan getRequest
	Persister StatsPersister
	items map[timeamount.TimeAmount]*StatsSave

}

type StatsPersister interface {
	Persist(t timeamount.TimeAmount, val dynmap.DynMap )

}

type statsItemType int
const (
	SET statsItemType = 0
	INC                = 1
)

type getRequest struct {
	resultChan chan map[timeamount.TimeAmount]*StatsSave
}

type statsItem struct {
	Key string
	Val int64
	typ statsItemType
}

type StatsSave struct {
	Epoch int64
	TimeAmount timeamount.TimeAmount
	Values *dynmap.DynMap
}

// Creates a new Stats tracker. caller must still envoke the Start function
// timeamounts should be in the form "{num} {timeframe}"
// example
// NewStats("1 minute", "30 minute", "1 day")
func New(timeamounts ...string) (*Stats, error) {
	s := &Stats{
		itemChan : make(chan statsItem, 500),
		items : make(map[timeamount.TimeAmount]*StatsSave),
	}
	for _,ta := range(timeamounts) {
		t, err := timeamount.Parse(ta)
		if err != nil {
			return nil, err
		}
		s.items[t] = &StatsSave{
				Epoch : t.ToTrendrrEpoch(time.Now()),
				TimeAmount : t,
				Values : dynmap.New(),
		}
	}
	return s, nil
}

// Sets the given key with the given value.
func (this *Stats) Set(key string, val int64) {
	select {
	case this.itemChan <- statsItem{Key : key, Val : int64(val), typ : SET} :
	default :
		log.Printf( "Could not Inc Key: %s Val: %d", key, val)
	}
}

func (this *Stats) Inc(key string, val int) {
	select {
	case this.itemChan <- statsItem{Key : key, Val : int64(val), typ : INC} :
	default :
		log.Printf( "Could not Inc Key: %s Val: %d", key, val)
	}
}

func (this *Stats) Get() map[timeamount.TimeAmount]*StatsSave {
	greq := getRequest{
		resultChan : make(chan map[timeamount.TimeAmount]*StatsSave),
	}
	this.getChan <- greq
	return <- greq.resultChan
}

//starts the event loop.
//this is necessary for it to do anything!
func (this *Stats) Start() {
	go this.eventLoop()
}

func (this *Stats) Close() error {
	//TODO: cleanly exit
	return nil
}

func (this *Stats) eventLoop() {
	//TODO: add kill chan
	for {
		select {
		case item := <- this.itemChan:
			this.add(item)
		case greq := <- this.getChan:
			greq.resultChan <- this.get()

		}
	}
}

// Gets a clone of the current state
func (this *Stats) get() map[timeamount.TimeAmount]*StatsSave{
	mp := make(map[timeamount.TimeAmount]*StatsSave)
	for ta,ss := range(this.items) {
		res := &StatsSave{
			Values : ss.Values.Clone(),
			TimeAmount: ss.TimeAmount,
			Epoch : ss.Epoch,
		}
		mp[ta] = res
	}
	return mp
}

func (this *Stats) add(item statsItem) {
	for ta, sts := range(this.items) {
		epoch := ta.ToTrendrrEpoch(time.Now())

		if epoch != sts.Epoch {
			// need to persist this..
			this.persist(sts)
			sts = &StatsSave{
				Epoch : epoch,
				TimeAmount : ta,
				Values : dynmap.New(),
			}
			this.items[ta] = sts
		}

		if item.typ == INC {
			val := sts.Values.MustInt64(item.Key, int64(0))
			sts.Values.PutWithDot(item.Key, int64(val+item.Val))
		} else if item.typ == SET {
			sts.Values.PutWithDot(item.Key, int64(item.Val))
		}

	}
}

func (this *Stats) persist(item *StatsSave) {
	//Do something
	json, _ := item.Values.MarshalJSON()

	log.Printf("TODO PERSISTING %s %s", item.TimeAmount.String(), string(json))
}<commit_msg>more on stats<commit_after>package stats

import(
	"github.com/trendrr/goshire/dynmap"
	"github.com/trendrr/goshire/timeamount"
	"time"
	"log"

)

// A simple stats collector

type Stats struct {
	itemChan chan statsItem
	getChan chan getRequest
	Persister StatsPersister
	items map[timeamount.TimeAmount]*StatsSave

}

type StatsPersister interface {
	Persist(t timeamount.TimeAmount, val dynmap.DynMap )

}

type statsItemType int
const (
	SET statsItemType = 0
	INC                = 1
)

type getRequest struct {
	resultChan chan map[timeamount.TimeAmount]*StatsSave
}

type statsItem struct {
	Key string
	Val int64
	typ statsItemType
}

type StatsSave struct {
	Epoch int64
	TimeAmount timeamount.TimeAmount
	Values *dynmap.DynMap
}

// Creates a new Stats tracker. caller must still envoke the Start function
// timeamounts should be in the form "{num} {timeframe}"
// example
// NewStats("1 minute", "30 minute", "1 day")
func New(timeamounts ...string) (*Stats, error) {
	s := &Stats{
		itemChan : make(chan statsItem, 5),
		getChan : make(chan getRequest, 10),
		items : make(map[timeamount.TimeAmount]*StatsSave),
	}
	for _,ta := range(timeamounts) {
		t, err := timeamount.Parse(ta)
		if err != nil {
			return nil, err
		}
		s.items[t] = &StatsSave{
				Epoch : t.ToTrendrrEpoch(time.Now()),
				TimeAmount : t,
				Values : dynmap.New(),
		}
	}
	return s, nil
}

// Sets the given key with the given value.
func (this *Stats) Set(key string, val int64) {
	select {
	case this.itemChan <- statsItem{Key : key, Val : int64(val), typ : SET} :
	default :
		log.Printf( "Could not Inc Key: %s Val: %d", key, val)
	}
}

func (this *Stats) Inc(key string, val int) {
	select {
	case this.itemChan <- statsItem{Key : key, Val : int64(val), typ : INC} :
	default :
		log.Printf( "Could not Inc Key: %s Val: %d", key, val)
	}

}

func (this *Stats) Get() map[timeamount.TimeAmount]*StatsSave {
	greq := getRequest{
		resultChan : make(chan map[timeamount.TimeAmount]*StatsSave),
	}
	this.getChan <- greq
	return <- greq.resultChan
}

//starts the event loop.
//this is necessary for it to do anything!
func (this *Stats) Start() {
	go this.eventLoop()
}

func (this *Stats) Close() error {
	//TODO: cleanly exit
	return nil
}

func (this *Stats) eventLoop() {
	//TODO: add kill chan
	for {
		select {
		case item := <- this.itemChan:
			this.add(item)
		case greq := <- this.getChan:
			greq.resultChan <- this.get()

		}
	}
}

// Gets a clone of the current state
func (this *Stats) get() map[timeamount.TimeAmount]*StatsSave{
	mp := make(map[timeamount.TimeAmount]*StatsSave)
	for ta,ss := range(this.items) {
		res := &StatsSave{
			Values : ss.Values.Clone(),
			TimeAmount: ss.TimeAmount,
			Epoch : ss.Epoch,
		}
		mp[ta] = res
	}
	return mp
}

func (this *Stats) add(item statsItem) {
	for ta, sts := range(this.items) {
		epoch := ta.ToTrendrrEpoch(time.Now())

		if epoch != sts.Epoch {
			// need to persist this..
			this.persist(sts)
			sts = &StatsSave{
				Epoch : epoch,
				TimeAmount : ta,
				Values : dynmap.New(),
			}
			this.items[ta] = sts
		}

		if item.typ == INC {
			val := sts.Values.MustInt64(item.Key, int64(0))
			sts.Values.PutWithDot(item.Key, int64(val+item.Val))
		} else if item.typ == SET {
			sts.Values.PutWithDot(item.Key, int64(item.Val))
		}

	}
}

func (this *Stats) persist(item *StatsSave) {
	//Do something
	json, _ := item.Values.MarshalJSON()

	log.Printf("TODO PERSISTING %s %s", item.TimeAmount.String(), string(json))
}<|endoftext|>


<commit_before>package client

import (
	"code.google.com/p/go.crypto/curve25519"
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	//"github.com/agl/ed25519"
	"github.com/andres-erbsen/dename/client"
	//"github.com/andres-erbsen/dename/protocol"
	"bytes"
	"code.google.com/p/go.crypto/nacl/box"
	protobuf "code.google.com/p/gogoprotobuf/proto"
	"crypto/subtle"
	"errors"
	"github.com/andres-erbsen/chatterbox/proto"
	"github.com/andres-erbsen/chatterbox/ratchet"
	testutil2 "github.com/andres-erbsen/dename/server/testutil" //TODO: Move MakeToken to TestUtil
	"github.com/andres-erbsen/dename/testutil"
	//"io"
	"testing"
	"time"
)

const PROFILE_FIELD_ID = 1984

func TestAliceTalksToBob(t *testing.T) {
	dir, err := ioutil.TempDir("", "testdb")
	handleError(err, t)

	defer os.RemoveAll(dir)
	db, err := leveldb.OpenFile(dir, nil)
	handleError(err, t)

	defer db.Close()

	server, pks := setUpServerTest(db, t)

	connA, inBufA, outBufA, pkpA := setUpClientTest(server, t)
	connB, inBufB, outBufB, pkpB := setUpClientTest(server, t)

	config, f := testutil.SingleServer(t)
	defer f()
	time.Sleep(100)

	//func createNewUser(name []byte, t *testing.T, config *client.Config, serverAddr string, pkTransport *[32]byte, idServer *[32]byte, signingKey *[32]byte, authKey *[32]byte) (*[32]byte, *client.Client) {
	pkTransportA, skTransportA, err := box.GenerateKey(rand.Reader)
	handleError(err)

	pkTransportB, skTransportA, err := box.GenerateKey(rand.Reader)
	handleError(err)

	ska, dnmca := createNewUser([]byte("Alice"), t, config, server.listener.Addr().String(), pkTransportA, pks)
	skb, dnmcb := createNewUser([]byte("Bob"), t, config)

	profileA
	profileB

	msg = []byte{"Envelope1"}
	ratch, err := encryptAuthFirst(msg, []byte("Bob"), ska, config)
	handleError(err)

	clientA := StartClient([]byte("Alice"))
	clientB := StartClient([]byte("Bob"), server.listener.Addr().String(), skb, connB, config)

	//clientB.decryptAuthFirst(

	//clientB := StartClient([]byte("Bob),
}

func setUpClientTest(server *Server, t *testing.T) (*transport.Conn, []byte, []byte, *[32]byte) {
	oldConn, err := net.Dial("tcp", server.listener.Addr().String())
	handleError(err, t)

	pkp, skp, err := box.GenerateKey(rand.Reader)
	handleError(err, t)

	conn, _, err := transport.Handshake(oldConn, pkp, skp, nil, MAX_MESSAGE_SIZE)
	handleError(err, t)

	inBuf := make([]byte, MAX_MESSAGE_SIZE)
	outBuf := make([]byte, MAX_MESSAGE_SIZE)

	return conn, inBuf, outBuf, pkp
}

func setUpServerTest(db *leveldb.DB, t *testing.T) (*Server, *[32]byte) {
	shutdown := make(chan struct{})

	pks, sks, err := box.GenerateKey(rand.Reader)
	handleError(err, t)

	server, err := StartServer(db, shutdown, pks, sks)
	handleError(err, t)

	return server, nil
}

func handleError(err error, t *testing.T) {
	if err != nil {
		t.Error(err)
	}
}

func TestMessageEncryptionAuthentication(t *testing.T) {
	config, f := testutil.SingleServer(t)
	defer f()
	time.Sleep(100)

	ska, dnmca := createNewUser([]byte("Alice"), t, config)
	skb, dnmcb := createNewUser([]byte("Bob"), t, config)

	ratchA := &ratchet.Ratchet{
		FillAuth:  FillAuthWith(ska),
		CheckAuth: CheckAuthWith(dnmca),
		Rand:      nil,
		Now:       nil,
	}
	ratchB := &ratchet.Ratchet{
		FillAuth:  FillAuthWith(skb),
		CheckAuth: CheckAuthWith(dnmcb),
		Rand:      nil,
		Now:       nil,
	}

	//pka0, ska0, err := box.GenerateKey(rand.Reader)
	//handleError(err, t)
	pkb0, skb0, err := box.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}

	msg, err := protobuf.Marshal(&proto.Message{
		Subject:  nil,
		Contents: []byte("Message"),
		Dename:   []byte("Alice"),
	})
	if err != nil {
		t.Fatal(err)
	}
	out := append([]byte{}, (*pkb0)[:]...)

	out = ratchA.EncryptFirst(out, msg, pkb0)
	msg2, err := ratchB.DecryptFirst(out[32:], skb0)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(msg, msg2) {
		t.Error("Original and decrypted message not the same.")
	}
}

func createNewUser(name []byte, t *testing.T, config *client.Config, serverAddr string, pkTransport *[32]byte, idServer *[32]byte, signingKey *[32]byte, authKey *[32]byte) (*[32]byte, *client.Client) {
	newClient, err := client.NewClient(config, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	//TODO: All these names are horrible, please change them
	pkAuth, skAuth, err := box.GenerateKey(rand.Reader)

	chatProfile := &proto.Profile{
		ServerAddressTCP:  []byte(serverAddr),
		ServerTransportPK: (proto.Byte32)(pkTransport),
		UserIDAtServer:    (proto.Byte32)(idServer),
		KeySigningKey:     (proto.Byte32)(signingKey),
		MessageAuthKey:    (proto.Byte32)(authKey),
	}

	chatProfileBytes, err := protobuf.Marshal(chatProfile)
	if err != nil {
		t.Fatal(err)
	}

	profile, sk, err := client.NewProfile(nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	client.SetProfileField(profile, PROFILE_FIELD_ID, chatProfileBytes)

	err = newClient.Register(sk, name, profile, testutil2.MakeToken())
	if err != nil {
		t.Fatal(err)
	}

	//Remove this outside of the test
	profile2, err := newClient.Lookup(name)
	if !profile.Equal(profile2) {
		t.Errorf("Correct profile not added to server:\nprofile: %v\nprofile2 %v\n", profile, profile2)
	}

	return skAuth, newClient
}

//func FillAuthWith(ourAuthPrivate *[32]byte) func([]byte, []byte, *[32]byte) {
//return func(tag, data []byte, theirAuthPublic *[32]byte) {
//var sharedAuthKey [32]byte
//curve25519.ScalarMult(&sharedAuthKey, ourAuthPrivate, theirAuthPublic)
//h := hmac.New(sha256.New, sharedAuthKey[:])
//h.Write(data)
//h.Sum(nil)
//copy(tag, h.Sum(nil))
//}
//}

//func CheckAuthWith(dnmc *client.Client) func([]byte, []byte, []byte, *[32]byte) error {
//return func(tag, data, msg []byte, ourAuthPrivate *[32]byte) error {
//var sharedAuthKey [32]byte
//message := new(proto.Message)
//if err := message.Unmarshal(msg); err != nil {
//return err
//}
//profile, err := dnmc.Lookup(message.Dename)
//if err != nil {
//return err
//}

//chatProfileBytes, err := client.GetProfileField(profile, PROFILE_FIELD_ID)
//if err != nil {
//return err
//}

//chatProfile := new(proto.Profile)
//if err := chatProfile.Unmarshal(chatProfileBytes); err != nil {
//return err
//}

//theirAuthPublic := (*[32]byte)(&chatProfile.MessageAuthKey)

//curve25519.ScalarMult(&sharedAuthKey, ourAuthPrivate, theirAuthPublic)
//h := hmac.New(sha256.New, sharedAuthKey[:])
//h.Write(data)
//if subtle.ConstantTimeCompare(tag, h.Sum(nil)[:len(tag)]) == 0 {

//return errors.New("Authentication failed: failed to reproduce envelope auth tag using the current auth pubkey from dename")
//}
//return nil
//}
//}
<commit_msg>Nothing is broken<commit_after>package client

import (
	"bytes"
	"code.google.com/p/go.crypto/nacl/box"
	protobuf "code.google.com/p/gogoprotobuf/proto"
	"crypto/rand"
	"fmt"
	"github.com/andres-erbsen/chatterbox/proto"
	"github.com/andres-erbsen/chatterbox/ratchet"
	"github.com/andres-erbsen/dename/client"
	testutil2 "github.com/andres-erbsen/dename/server/testutil" //TODO: Move MakeToken to TestUtil
	"github.com/andres-erbsen/dename/testutil"
	"testing"
	"time"
)

func handleError(err error, t *testing.T) {
	if err != nil {
		t.Error(err)
	}
}

func TestMessageEncryptionAuthentication(t *testing.T) {
	config, f := testutil.SingleServer(t)
	defer f()
	time.Sleep(100)

	ska, dnmca := createNewUser([]byte("Alice"), t, config)
	skb, dnmcb := createNewUser([]byte("Bob"), t, config)

	ratchA := &ratchet.Ratchet{
		FillAuth:  FillAuthWith(ska),
		CheckAuth: CheckAuthWith(dnmca),
		Rand:      nil,
		Now:       nil,
	}
	ratchB := &ratchet.Ratchet{
		FillAuth:  FillAuthWith(skb),
		CheckAuth: CheckAuthWith(dnmcb),
		Rand:      nil,
		Now:       nil,
	}

	//pka0, ska0, err := box.GenerateKey(rand.Reader)
	//handleError(err, t)
	pkb0, skb0, err := box.GenerateKey(rand.Reader)
	if err != nil {
		t.Fatal(err)
	}

	msg, err := protobuf.Marshal(&proto.Message{
		Subject:  nil,
		Contents: []byte("Message"),
		Dename:   []byte("Alice"),
	})
	if err != nil {
		t.Fatal(err)
	}
	out := append([]byte{}, (*pkb0)[:]...)

	out = ratchA.EncryptFirst(out, msg, pkb0)
	msg2, err := ratchB.DecryptFirst(out[32:], skb0)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(msg, msg2) {
		t.Error("Original and decrypted message not the same.")
	}
}

func createNewUser(name []byte, t *testing.T, config *client.Config) (*[32]byte, *client.Client) {
	newClient, err := client.NewClient(config, nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	//TODO: All these names are horrible, please change them
	pkAuth, skAuth, err := box.GenerateKey(rand.Reader)

	chatProfile := &proto.Profile{
		ServerAddressTCP:  "",
		ServerTransportPK: (proto.Byte32)([32]byte{}),
		UserIDAtServer:    (proto.Byte32)([32]byte{}),
		KeySigningKey:     (proto.Byte32)([32]byte{}),
		MessageAuthKey:    (proto.Byte32)(*pkAuth),
	}

	chatProfileBytes, err := protobuf.Marshal(chatProfile)
	if err != nil {
		t.Fatal(err)
	}

	profile, sk, err := client.NewProfile(nil, nil)
	if err != nil {
		t.Fatal(err)
	}

	client.SetProfileField(profile, PROFILE_FIELD_ID, chatProfileBytes)

	err = newClient.Register(sk, name, profile, testutil2.MakeToken())
	if err != nil {
		t.Fatal(err)
	}

	//Remove this outside of the test
	profile2, err := newClient.Lookup(name)
	if !profile.Equal(profile2) {
		t.Error("Correct profile not added to server.")
		fmt.Printf("profile: %v\n", profile)
		fmt.Printf("profile2: %v\n", profile2)
	}

	return skAuth, newClient
}
<|endoftext|>


<commit_before>package main

import "github.com/nsf/termbox-go"
import "time"
import "flag"

func main() {
	loops := flag.Int("loops", 0, "number of times to loop (default: infinite)")
	delay := flag.Int("delay", 75, "frame delay in ms")
	orientation := flag.String("orientation", "regular", "regular or aussie")
	flag.Parse()

	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()

	event_queue := make(chan termbox.Event)
	go func() {
		for {
			event_queue <- termbox.PollEvent()
		}
	}()

	termbox.SetOutputMode(termbox.Output256)

	loop_index := 0
	draw(*orientation)

loop:
	for {
		select {
		case ev := <-event_queue:
			if ev.Type == termbox.EventKey && (ev.Key == termbox.KeyEsc || ev.Key == termbox.KeyCtrlC) {
				break loop
			}
		default:
			loop_index++
			if *loops > 0 && (loop_index/9) >= *loops {
				break loop
			}
			draw(*orientation)
			time.Sleep(time.Duration(*delay) * time.Millisecond)
		}
	}
}
<commit_msg>Exit on EventInterrupt rather on hardcoded Ctrl+C<commit_after>package main

import "github.com/nsf/termbox-go"
import "time"
import "flag"

func main() {
	loops := flag.Int("loops", 0, "number of times to loop (default: infinite)")
	delay := flag.Int("delay", 75, "frame delay in ms")
	orientation := flag.String("orientation", "regular", "regular or aussie")
	flag.Parse()

	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()

	event_queue := make(chan termbox.Event)
	go func() {
		for {
			event_queue <- termbox.PollEvent()
		}
	}()

	termbox.SetOutputMode(termbox.Output256)

	loop_index := 0
	draw(*orientation)

loop:
	for {
		select {
		case ev := <-event_queue:
			if (ev.Type == termbox.EventKey && ev.Key == termbox.KeyEsc) || ev.Type == termbox.EventInterrupt {
				break loop
			}
		default:
			loop_index++
			if *loops > 0 && (loop_index/9) >= *loops {
				break loop
			}
			draw(*orientation)
			time.Sleep(time.Duration(*delay) * time.Millisecond)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
)

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy stack
	stack []*Node

	// edge in stack
	node *Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	isStringFlag bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool

}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.node = p.newNode()
	p.node.Type = NodeRoot
	p.stack = make([]*Node, 0)
	p.stack = append(p.stack, p.node)

	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node.Debug())
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) trimOutput() {
	p.output = strings.TrimSpace(p.output)
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}

func (p *Parser) lastNode() *Node {
	if len(p.stack) > 0 {
		return p.stack[len(p.stack) - 1]
	} else {
		return nil
	}
}

func (p *Parser) popNode() *Node {
	n := p.lastNode()
	if n != nil {
		p.stack = p.stack[0:len(p.stack)-1]
	}
	return n
}

func (p *Parser) pushNode(n *Node) {
	p.stack = append(p.stack, n)
}

func (p *Parser) dedentFromStack() {
	n := p.lastNode()
	if n != nil && n.Type != NodeRoot {
		p.output += n.OpenString()
		p.trimOutput()
		p.output += n.CloseString()
		p.popNode()
	}
}

func (p *Parser) processToken(tok rune, text string) {
	// indent/dedent/nodent is the first place to look to see if new nodes need
	// to be made. checks are for parent node being certain types like NodeText
	// or NodeComment
	switch tok {
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
		if p.node.Type != NodeText {
			p.output += p.node.OpenString()
			p.node = p.newNode()
			p.pushNode(p.node)
		}
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentFromStack()
		if p.node.Type != NodeText {
			p.dedentFromStack()
		}
		p.node = p.newNode()
		p.pushNode(p.node)
	case TokNodent:
		p.isIndent = false
		p.isDedent = false
		switch p.node.Type {
		case NodeRoot:
			p.node = p.newNode()
			p.pushNode(p.node)
		case NodeText:
		default:
			// replace top of stack with new node (pop then push)
			n := p.popNode()
			p.output += n.OpenString()
			p.trimOutput()
			p.output += n.CloseString()
			p.node = p.newNode()
			p.pushNode(p.node)
		}
	}

	if p.node.Type == NodeText && tok != TokDedent {
		if tok != TokIndent {
			if tok == TokNewLine {
				p.node.text += " "
			} else if tok != TokWhitespace || p.node.text != "" {
				p.node.text += text
			}
		}
		return
	}

	switch tok {
	case TokWord, TokComma:
		switch p.node.Type {
		case NodeNil:
			p.node.Type = NodeTag
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
			p.attrAssigned = false
		case NodeTag:
			if p.isAttr {
				if p.attrAssigned {
					if _, found := p.node.attrs[p.attrName]; found {
						p.node.attrs[p.attrName] += " "
						p.node.attrs[p.attrName] += text
					} else {
						p.node.attrs[p.attrName] = text
					}

					// reset to look for a new attribute assignment
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
					p.node.attrString = ""
				} else if p.attrName == "" {
					p.node.attrString += text
					p.attrName = text
				} else {
					p.node.text += p.node.attrString
					p.node.text += text
					p.node.attrString = ""
					p.isAttr = false
					p.attrAssigned = false
					p.attrName = ""
					p.attrValue = ""
				}
			} else {
				p.node.text += text
			}
		}
	case TokString:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr {
				if p.attrAssigned {
					if _, found := p.node.attrs[p.attrName]; found {
						p.node.attrs[p.attrName] += " "
						p.node.attrs[p.attrName] += text[1:len(text)-1]
					} else {
						p.node.attrs[p.attrName] = text[1:len(text)-1]
					}

					// reset to look for a new attribute assignment
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
					p.node.attrString = ""
				} else if p.attrName != "" {
					// TODO using string in attrName shouldn't be allowed
					p.node.text += p.node.attrString
					p.node.text += text[1:len(text)-1]
					p.node.attrString = ""
					p.isAttr = false
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
				}
			} else {
				p.node.text += text
			}
		case NodeText:
			p.node.text += text
		}
	case TokAssign:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr {
				if p.attrName != "" && p.attrValue == "" {
					p.attrAssigned = true
					p.node.attrString += text
				}
				if p.attrName == "" || p.attrValue != "" {
					p.attrAssigned = false
					p.isAttr = false
					p.node.text += p.node.attrString
					p.node.attrString = ""
				}
			} else {
				p.node.text += text
			}
		case NodeText:
			p.node.text += text
		}
	case TokWhitespace:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr && p.node.attrString != "" {
				p.node.attrString += text

			// skip initial whitespace for text
			} else if p.node.text != "" {
				p.node.text += text
			}
		}
	case TokStringFlag:
		switch p.node.Type {
		case NodeNil:
			// new node, but indent indicates continue with text of previous node
			p.node.Type = NodeText
		case NodeTag:
			if p.isAttr {
				p.node.text += p.node.attrString
				p.node.attrString = ""
			} else {
				p.node.text += text
			}
			p.isAttr = false
		}
	case TokNewLine:
		// reset
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
		p.attrAssigned = false
	case TokEOF:
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			lastNode := p.popNode()
			p.trimOutput()
			p.output += lastNode.CloseString()
		}
	// nop
	case TokIndent, TokDedent, TokNodent:
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

<commit_msg>dedenting from Node stack adds a space if it is a NodeText<commit_after>package main

import (
	"fmt"
	"strings"
)

type Parser struct {

	debug bool

	Scanner Scanner
	output string

	// node heirarchy stack
	stack []*Node

	// edge in stack
	node *Node

	// states as the Parser object figures out how to output the tokens

	// new line has started
	isNewLine bool

	isIndent bool
	isDedent bool

	// flag indicates we are still checking for attribute assignments
	isAttr bool

	isStringFlag bool

	// attrName and attrValue are buffers to determine if TokWord for a tag are
	// potentially for an attribute, or if they are just normal text
	attrName string
	attrValue string
	attrAssigned bool

}

func (p *Parser) Output() string {
	var tokens []rune
	var text string

	p.node = p.newNode()
	p.node.Type = NodeRoot
	p.stack = make([]*Node, 0)
	p.stack = append(p.stack, p.node)

	p.output = ""
	p.isNewLine = true
	p.isIndent = false
	p.isDedent = false
	p.isAttr = false
	p.attrName = ""
	p.attrValue = ""
	p.attrAssigned = false

	for {
		tokens = p.Scanner.Scan()
		for _, token := range tokens {
			text = p.Scanner.TokenText()
			if p.debug {
				fmt.Printf("[debug] token: [%s]\n", TokenString(token))
				fmt.Printf("[debug] text:  [%s]\n", text)
			}
			p.processToken(token, text)
			if p.debug {
				fmt.Printf("[debug] node:  [%s]\n\n", p.node.Debug())
			}
		}
		if tokens[0] == TokEOF {
			break
		}
	}
	return p.output
}

func (p *Parser) trimOutput() {
	p.output = strings.TrimSpace(p.output)
}

func (p *Parser) newNode() *Node {
	n := new(Node)
	n.attrs = make(map[string]string)
	return n
}

func (p *Parser) lastNode() *Node {
	if len(p.stack) > 0 {
		return p.stack[len(p.stack) - 1]
	} else {
		return nil
	}
}

func (p *Parser) popNode() *Node {
	n := p.lastNode()
	if n != nil {
		p.stack = p.stack[0:len(p.stack)-1]
	}
	return n
}

func (p *Parser) pushNode(n *Node) {
	p.stack = append(p.stack, n)
}

func (p *Parser) dedentFromStack() {
	n := p.lastNode()
	if n != nil && n.Type != NodeRoot {
		p.output += n.OpenString()
		p.trimOutput()
		// newlines in NodeText have spaces. Adding one here for consistency
		if n.Type == NodeText {
			p.output += " "
		}
		p.output += n.CloseString()
		p.popNode()
	}
}

func (p *Parser) processToken(tok rune, text string) {
	// indent/dedent/nodent is the first place to look to see if new nodes need
	// to be made. checks are for parent node being certain types like NodeText
	// or NodeComment
	switch tok {
	case TokIndent:
		p.isIndent = true
		p.isDedent = false
		if p.node.Type != NodeText {
			p.output += p.node.OpenString()
			p.node = p.newNode()
			p.pushNode(p.node)
		}
	case TokDedent:
		p.isIndent = false
		p.isDedent = true
		p.dedentFromStack()
		if p.node.Type != NodeText {
			p.dedentFromStack()
		}
		p.node = p.newNode()
		p.pushNode(p.node)
	case TokNodent:
		p.isIndent = false
		p.isDedent = false
		switch p.node.Type {
		case NodeRoot:
			p.node = p.newNode()
			p.pushNode(p.node)
		case NodeText:
		default:
			// replace top of stack with new node (pop then push)
			n := p.popNode()
			p.output += n.OpenString()
			p.trimOutput()
			p.output += n.CloseString()
			p.node = p.newNode()
			p.pushNode(p.node)
		}
	}

	if p.node.Type == NodeText && tok != TokDedent {
		if tok != TokIndent {
			if tok == TokNewLine {
				p.node.text += " "
			} else if tok != TokWhitespace || p.node.text != "" {
				p.node.text += text
			}
		}
		return
	}

	switch tok {
	case TokWord, TokComma:
		switch p.node.Type {
		case NodeNil:
			p.node.Type = NodeTag
			p.node.tag = text
			// found the tag, now check for attributes
			p.isAttr = true
			p.attrAssigned = false
		case NodeTag:
			if p.isAttr {
				if p.attrAssigned {
					if _, found := p.node.attrs[p.attrName]; found {
						p.node.attrs[p.attrName] += " "
						p.node.attrs[p.attrName] += text
					} else {
						p.node.attrs[p.attrName] = text
					}

					// reset to look for a new attribute assignment
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
					p.node.attrString = ""
				} else if p.attrName == "" {
					p.node.attrString += text
					p.attrName = text
				} else {
					p.node.text += p.node.attrString
					p.node.text += text
					p.node.attrString = ""
					p.isAttr = false
					p.attrAssigned = false
					p.attrName = ""
					p.attrValue = ""
				}
			} else {
				p.node.text += text
			}
		}
	case TokString:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr {
				if p.attrAssigned {
					if _, found := p.node.attrs[p.attrName]; found {
						p.node.attrs[p.attrName] += " "
						p.node.attrs[p.attrName] += text[1:len(text)-1]
					} else {
						p.node.attrs[p.attrName] = text[1:len(text)-1]
					}

					// reset to look for a new attribute assignment
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
					p.node.attrString = ""
				} else if p.attrName != "" {
					// TODO using string in attrName shouldn't be allowed
					p.node.text += p.node.attrString
					p.node.text += text[1:len(text)-1]
					p.node.attrString = ""
					p.isAttr = false
					p.attrName = ""
					p.attrValue = ""
					p.attrAssigned = false
				}
			} else {
				p.node.text += text
			}
		case NodeText:
			p.node.text += text
		}
	case TokAssign:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr {
				if p.attrName != "" && p.attrValue == "" {
					p.attrAssigned = true
					p.node.attrString += text
				}
				if p.attrName == "" || p.attrValue != "" {
					p.attrAssigned = false
					p.isAttr = false
					p.node.text += p.node.attrString
					p.node.attrString = ""
				}
			} else {
				p.node.text += text
			}
		case NodeText:
			p.node.text += text
		}
	case TokWhitespace:
		switch p.node.Type {
		case NodeTag:
			if p.isAttr && p.node.attrString != "" {
				p.node.attrString += text

			// skip initial whitespace for text
			} else if p.node.text != "" {
				p.node.text += text
			}
		}
	case TokStringFlag:
		switch p.node.Type {
		case NodeNil:
			// new node, but indent indicates continue with text of previous node
			p.node.Type = NodeText
		case NodeTag:
			if p.isAttr {
				p.node.text += p.node.attrString
				p.node.attrString = ""
			} else {
				p.node.text += text
			}
			p.isAttr = false
		}
	case TokNewLine:
		// reset
		p.isIndent = false
		p.isDedent = false
		p.isAttr = false
		p.attrName = ""
		p.attrValue = ""
		p.attrAssigned = false
	case TokEOF:
		// close the remaining nodes in the stack
		for len(p.stack) > 0 {
			lastNode := p.popNode()
			p.trimOutput()
			p.output += lastNode.CloseString()
		}
	// nop
	case TokIndent, TokDedent, TokNodent:
	default:
		if !p.isAttr {
			p.node.text += text
		}
	}
}

<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

type clippingHandler func(*rawClipping)

type parser struct {
	handler clippingHandler
}

func (p *parser) parseClippingFile(clippingFile string) {
	f, err := os.Open(clippingFile)
	if err != nil {
		log.Fatalln("Cannot open file", err.Error())
	}
	defer f.Close()

	p.parse(f)
}

type clippingType int

const (
	undefined clippingType = iota
	highlight
	note
	bookmark
)

type rawClipping struct {
	baseClipping
	cType clippingType
}

func (i *parser) parse(r io.Reader) {
	c := new(rawClipping)

	lineNo := 0

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		lineNo++

		if line == "==========" {
			lineNo = 0
			if c != nil {
				i.handler(c)
			}
			c = new(rawClipping)
			continue
		}

		switch lineNo {
		case 1:
			i.extractBook(line, c)
		case 2:
			i.extractLocationAndDate(line, c)
		case 3:
			continue
		default:
			if c.Content != "" {
				c.Content += "\n"
			}
			c.Content += line
		}

	}
}

func (i *parser) extractBook(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " (")
	if ix < 0 {
		c.Book.Title = str
	} else {
		c.Book.Title = str[:ix]
		c.Book.Authors = str[ix+2 : len(str)-1]
	}
}

func (i *parser) extractLocationAndDate(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " | ")

	i.extractLocation(str[:ix], c)
	i.extractAddDate(str[ix+3:], c)
	c.cType = extractType(str)
}

func extractType(str string) clippingType {
	if strings.Contains(str, "Your Highlight") {
		return highlight
	} else if strings.Contains(str, "Your Note") {
		return note
	} else if strings.Contains(str, "Your Bookmark") {
		return bookmark
	} else {
		log.Fatalln("Cannot deternime clipping type:", str)
		return undefined
	}
}

func (i *parser) extractAddDate(str string, c *rawClipping) {
	ix := strings.Index(str, ",")
	dateStr := str[ix+2:]
	t, err := time.Parse("January 2, 2006 3:04:05 PM", dateStr)
	if err != nil {
		log.Fatalln("Cannot parse date:", dateStr)
	}
	c.CreationTime = t.Unix()
}

func (i *parser) extractLocation(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " ")
	pageStr := strings.Split(str[ix+1:], "-")
	ii, err := strconv.Atoi(pageStr[0])
	if err != nil {
		log.Fatalln("Cannot parse start page", pageStr[0], "in", str)
	}
	c.Loc.Start = ii

	if len(pageStr) == 2 {
		ii, err = strconv.Atoi(pageStr[1])
		if err != nil {
			log.Fatalln("Cannot pares end page", pageStr[1], "in", str)
		}
		c.Loc.End = ii
	} else {
		c.Loc.End = ii
	}
}
<commit_msg>Remove BOM from book titles<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os"
	"strconv"
	"strings"
	"time"
)

type clippingHandler func(*rawClipping)

type parser struct {
	handler clippingHandler
}

var bom = []byte{0xef, 0xbb, 0xbf}

func (p *parser) parseClippingFile(clippingFile string) {
	f, err := os.Open(clippingFile)
	if err != nil {
		log.Fatalln("Cannot open file", err.Error())
	}
	defer f.Close()

	p.parse(f)
}

type clippingType int

const (
	undefined clippingType = iota
	highlight
	note
	bookmark
)

type rawClipping struct {
	baseClipping
	cType clippingType
}

func (i *parser) parse(r io.Reader) {
	c := new(rawClipping)

	lineNo := 0

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		lineNo++

		if line == "==========" {
			lineNo = 0
			if c != nil {
				i.handler(c)
			}
			c = new(rawClipping)
			continue
		}

		switch lineNo {
		case 1:
			i.extractBook(line, c)
		case 2:
			i.extractLocationAndDate(line, c)
		case 3:
			continue
		default:
			if c.Content != "" {
				c.Content += "\n"
			}
			c.Content += line
		}

	}
}

func (i *parser) extractBook(str string, c *rawClipping) {
	if bytes.HasPrefix([]byte(str), bom) {
		str = str[3:]
	}
	ix := strings.LastIndex(str, " (")
	if ix < 0 {
		c.Book.Title = str
	} else {
		c.Book.Title = str[:ix]
		c.Book.Authors = str[ix+2 : len(str)-1]
	}
}

func (i *parser) extractLocationAndDate(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " | ")

	i.extractLocation(str[:ix], c)
	i.extractAddDate(str[ix+3:], c)
	c.cType = extractType(str)
}

func extractType(str string) clippingType {
	if strings.Contains(str, "Your Highlight") {
		return highlight
	} else if strings.Contains(str, "Your Note") {
		return note
	} else if strings.Contains(str, "Your Bookmark") {
		return bookmark
	} else {
		log.Fatalln("Cannot deternime clipping type:", str)
		return undefined
	}
}

func (i *parser) extractAddDate(str string, c *rawClipping) {
	ix := strings.Index(str, ",")
	dateStr := str[ix+2:]
	t, err := time.Parse("January 2, 2006 3:04:05 PM", dateStr)
	if err != nil {
		log.Fatalln("Cannot parse date:", dateStr)
	}
	c.CreationTime = t.Unix()
}

func (i *parser) extractLocation(str string, c *rawClipping) {
	ix := strings.LastIndex(str, " ")
	pageStr := strings.Split(str[ix+1:], "-")
	ii, err := strconv.Atoi(pageStr[0])
	if err != nil {
		log.Fatalln("Cannot parse start page", pageStr[0], "in", str)
	}
	c.Loc.Start = ii

	if len(pageStr) == 2 {
		ii, err = strconv.Atoi(pageStr[1])
		if err != nil {
			log.Fatalln("Cannot pares end page", pageStr[1], "in", str)
		}
		c.Loc.End = ii
	} else {
		c.Loc.End = ii
	}
}
<|endoftext|>


<commit_before>package tql

/*
  Tql, Simple SQL-Like query language

  BNF:
  select <property> [, <property> ...] | *]
    [from <from>]
    [where <condition> [and <condition> ...]] [limit <num> offset <num> | limit <num>, <num>]

  <condition> := <property> {< | <= | > | >= | = | != | in} <value>
*/

import (
	"regexp"
	"strconv"
	"strings"
)

var tokenize_regex = `(?:"[^"\n\r]*")+|(?:'[^'\n\r]*')+|<=|>=|!=|=|<|>|,|\*|-?\d+(?:\.\d+)?|\w+(?:\.\w+)*|(?:"[^"\s]+")+|\(|\)|\S+`

type Tql struct {
	tokens  []string
	query   string
	pos     int
	props   []string
	from    string
	conds   []Cond
	limit   int64
	offset  int64
	orderBy string // TODO
	order   int // TODO
}

const (
	ValString = iota
	ValInt
	ValFloat
	ValQuoteString
	ValBool
	ValNull
	ValReference
	ValList
)

type Val struct {
	valType int
	val     interface{}
}

type Cond struct {
	identifier string
	op         string
	val        Val
}

func NewTql(query string) *Tql {
	t := new(Tql)
	t.pos = 0
	t.query = strings.ToLower(query)
	t.limit = -1
	t.offset = -1
    t.order = -1
	if re, err := regexp.Compile(tokenize_regex); err != nil {
		return nil
	} else {
		t.tokens = re.FindAllString(t.query, -1)
		if len(t.tokens) <= 0 {
			return nil
		}
	}
	t.__Select()
	return t
}

func (t *Tql) __Expect(expect string) {
	if t.__Consume(expect) == false {
		panic("token error")
	}
}

func (t *Tql) __Consume(expect string) bool {
	if t.pos < len(t.tokens) {
		if t.tokens[t.pos] == expect {
			t.pos += 1
			return true
		}
	}
	return false
}

func (t *Tql) __ConsumeRegexp(regex string) (bool, string) {
	if t.pos < len(t.tokens) {
		token := t.tokens[t.pos]
		re, _ := regexp.Compile(regex)
		if re.MatchString(token) {
			t.pos += 1
			return true, token
		}
	}
	return false, ""
}

// consume a identifier an return
var identifier_regex = `(\w+(?:\.\w+)*)$`

func (t *Tql) __Identifier() (bool, string) {
	if b, ident := t.__ConsumeRegexp(identifier_regex); b {
		return true, ident
	}
	return false, ""
}

func (t *Tql) __ExpectIdentifier() string {
	if b, ident := t.__Identifier(); b {
		return ident
	}
	panic("identifier error")
}

func (t *Tql) __Select() bool {
	t.__Expect("select")
	if !t.__Consume("*") {
		t.props = append(t.props, t.__ExpectIdentifier())
		for t.__Consume(",") {
			t.props = append(t.props, t.__ExpectIdentifier())
		}
	} else {
		t.props = append(t.props, "*")
	}
	return t.__From()
}

func (t *Tql) __From() bool {
	if t.__Consume("from") {
		t.from = t.__ExpectIdentifier()
	} else {
		return false
	}
	return t.__Where()
}

func (t *Tql) __Where() bool {
	if t.__Consume("where") {
		return t.__ParseFilterList()
	}
	return t.__orderBy()
}

var num_regex = `(\d+)$`

func (t *Tql) __Limit() bool {
	if t.__Consume("limit") {
		_, limit := t.__ConsumeRegexp(num_regex)
		n, err := strconv.ParseInt(limit, 10, 64)
		if err != nil {
			return false
		}
		if t.__Consume(",") {
			t.offset = n
			_, limit := t.__ConsumeRegexp(num_regex)
			n, err = strconv.ParseInt(limit, 10, 64)
			if err != nil {
				return false
			}
		}
		t.limit = n
	}
	return t.__Offset()
}

func (t *Tql) __Offset() bool {
	if t.__Consume("offset") {
		b, offset := t.__ConsumeRegexp(num_regex)
		if !b {
			return false
		}
		n, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			return false
		}
		t.offset = n
		return true
	}
	return true
}

var quoted_string_regex = `((?:\'[^\'\n\r]*\')+)|((?:"[^"\n\r]*")+)`

func (t *Tql) __Value() (bool, Val) {
	if t.pos < len(t.tokens) {
		token := t.tokens[t.pos]
		// try int
		i, err := strconv.ParseInt(token, 10, 64)
		if err == nil {
			t.pos += 1
			return true, Val{ValInt, i}
		}
		// try float
		f, err := strconv.ParseFloat(token, 64)
		if err == nil {
			t.pos += 1
			return true, Val{ValFloat, f}
		}
		// try quote string
		b, val := t.__ConsumeRegexp(quoted_string_regex)
		if b {
			if t.tokens[t.pos-1][0] == '\'' {
				val = strings.Replace(val, "''", "'", -1)
			} else {
				val = strings.Replace(val, `""`, `"`, -1)
			}
			return true, Val{ValQuoteString, val}
		}
		// try bool
		b = t.__Consume("true")
		if b {
			return true, Val{ValBool, true}
		}
		b = t.__Consume("false")
		if b {
			return true, Val{ValBool, false}
		}
		// try null
		b = t.__Consume("null")
		if b {
			return true, Val{ValNull, nil}
		}
	}
	return false, Val{}
}

func (t *Tql) __ValueList() (bool, Val) {
	var vals []Val
	t.__Expect("(")
	for {
		if b, val := t.__Value(); b {
			vals = append(vals, val)
		} else {
			return false, Val{}
		}
		if !t.__Consume(",") {
			break
		}
	}
	t.__Expect(")")
	return true, Val{ValList, vals}
}

func (t *Tql) __orderBy() bool {
    if t.__Consume("order") {
        if t.__Consume("by") {
            t.orderBy = t.__ExpectIdentifier()
            if t.__Consume("asc") {
                t.order = 1
            } else if t.__Consume("desc") {
                t.order = -1
            }
        } else {
            panic("parsing order error")
        }
    }
    return t.__Limit()
}

var condition_regex = `(<=|>=|!=|=|<|>|in)$`

func (t *Tql) __ParseFilterList() bool {
	b, ident := t.__Identifier()
	if !b {
		return b
	}
	b, op := t.__ConsumeRegexp(condition_regex)
	if !b {
		return b
	}
	b, val := t.__Value()
	if !b && op == "in" {
		b, val = t.__ValueList()
	}
	if !b {
		return b
	}
	t.conds = append(t.conds, Cond{ident, op, val})
	if t.__Consume("and") {
		return t.__ParseFilterList()
	}
	return t.__orderBy()
}
<commit_msg>modify doc<commit_after>package tql

/*
  Tql, Simple SQL-Like query language

  BNF:
  select <property> [, <property> ...] | *]
    [from <from>]
    [where <condition> [and <condition> ...]]
    [order by <property> [asc|desc]]
    [limit <num> offset <num> | limit <num>, <num>]

  <condition> := <property> {< | <= | > | >= | = | != | in} <value>
*/

import (
	"regexp"
	"strconv"
	"strings"
)

var tokenize_regex = `(?:"[^"\n\r]*")+|(?:'[^'\n\r]*')+|<=|>=|!=|=|<|>|,|\*|-?\d+(?:\.\d+)?|\w+(?:\.\w+)*|(?:"[^"\s]+")+|\(|\)|\S+`

type Tql struct {
	tokens  []string
	query   string
	pos     int
	props   []string
	from    string
	conds   []Cond
	limit   int64
	offset  int64
	orderBy string
	order   int
}

const (
	ValString = iota
	ValInt
	ValFloat
	ValQuoteString
	ValBool
	ValNull
	ValReference
	ValList
)

type Val struct {
	valType int
	val     interface{}
}

type Cond struct {
	identifier string
	op         string
	val        Val
}

func NewTql(query string) *Tql {
	t := new(Tql)
	t.pos = 0
	t.query = strings.ToLower(query)
	t.limit = -1
	t.offset = -1
    t.order = -1
	if re, err := regexp.Compile(tokenize_regex); err != nil {
		return nil
	} else {
		t.tokens = re.FindAllString(t.query, -1)
		if len(t.tokens) <= 0 {
			return nil
		}
	}
	t.__Select()
	return t
}

func (t *Tql) __Expect(expect string) {
	if t.__Consume(expect) == false {
		panic("token error")
	}
}

func (t *Tql) __Consume(expect string) bool {
	if t.pos < len(t.tokens) {
		if t.tokens[t.pos] == expect {
			t.pos += 1
			return true
		}
	}
	return false
}

func (t *Tql) __ConsumeRegexp(regex string) (bool, string) {
	if t.pos < len(t.tokens) {
		token := t.tokens[t.pos]
		re, _ := regexp.Compile(regex)
		if re.MatchString(token) {
			t.pos += 1
			return true, token
		}
	}
	return false, ""
}

// consume a identifier an return
var identifier_regex = `(\w+(?:\.\w+)*)$`

func (t *Tql) __Identifier() (bool, string) {
	if b, ident := t.__ConsumeRegexp(identifier_regex); b {
		return true, ident
	}
	return false, ""
}

func (t *Tql) __ExpectIdentifier() string {
	if b, ident := t.__Identifier(); b {
		return ident
	}
	panic("identifier error")
}

func (t *Tql) __Select() bool {
	t.__Expect("select")
	if !t.__Consume("*") {
		t.props = append(t.props, t.__ExpectIdentifier())
		for t.__Consume(",") {
			t.props = append(t.props, t.__ExpectIdentifier())
		}
	} else {
		t.props = append(t.props, "*")
	}
	return t.__From()
}

func (t *Tql) __From() bool {
	if t.__Consume("from") {
		t.from = t.__ExpectIdentifier()
	} else {
		return false
	}
	return t.__Where()
}

func (t *Tql) __Where() bool {
	if t.__Consume("where") {
		return t.__ParseFilterList()
	}
	return t.__orderBy()
}

var num_regex = `(\d+)$`

func (t *Tql) __Limit() bool {
	if t.__Consume("limit") {
		_, limit := t.__ConsumeRegexp(num_regex)
		n, err := strconv.ParseInt(limit, 10, 64)
		if err != nil {
			return false
		}
		if t.__Consume(",") {
			t.offset = n
			_, limit := t.__ConsumeRegexp(num_regex)
			n, err = strconv.ParseInt(limit, 10, 64)
			if err != nil {
				return false
			}
		}
		t.limit = n
	}
	return t.__Offset()
}

func (t *Tql) __Offset() bool {
	if t.__Consume("offset") {
		b, offset := t.__ConsumeRegexp(num_regex)
		if !b {
			return false
		}
		n, err := strconv.ParseInt(offset, 10, 64)
		if err != nil {
			return false
		}
		t.offset = n
		return true
	}
	return true
}

var quoted_string_regex = `((?:\'[^\'\n\r]*\')+)|((?:"[^"\n\r]*")+)`

func (t *Tql) __Value() (bool, Val) {
	if t.pos < len(t.tokens) {
		token := t.tokens[t.pos]
		// try int
		i, err := strconv.ParseInt(token, 10, 64)
		if err == nil {
			t.pos += 1
			return true, Val{ValInt, i}
		}
		// try float
		f, err := strconv.ParseFloat(token, 64)
		if err == nil {
			t.pos += 1
			return true, Val{ValFloat, f}
		}
		// try quote string
		b, val := t.__ConsumeRegexp(quoted_string_regex)
		if b {
			if t.tokens[t.pos-1][0] == '\'' {
				val = strings.Replace(val, "''", "'", -1)
			} else {
				val = strings.Replace(val, `""`, `"`, -1)
			}
			return true, Val{ValQuoteString, val}
		}
		// try bool
		b = t.__Consume("true")
		if b {
			return true, Val{ValBool, true}
		}
		b = t.__Consume("false")
		if b {
			return true, Val{ValBool, false}
		}
		// try null
		b = t.__Consume("null")
		if b {
			return true, Val{ValNull, nil}
		}
	}
	return false, Val{}
}

func (t *Tql) __ValueList() (bool, Val) {
	var vals []Val
	t.__Expect("(")
	for {
		if b, val := t.__Value(); b {
			vals = append(vals, val)
		} else {
			return false, Val{}
		}
		if !t.__Consume(",") {
			break
		}
	}
	t.__Expect(")")
	return true, Val{ValList, vals}
}

func (t *Tql) __orderBy() bool {
    if t.__Consume("order") {
        if t.__Consume("by") {
            t.orderBy = t.__ExpectIdentifier()
            if t.__Consume("asc") {
                t.order = 1
            } else if t.__Consume("desc") {
                t.order = -1
            }
        } else {
            panic("parsing order error")
        }
    }
    return t.__Limit()
}

var condition_regex = `(<=|>=|!=|=|<|>|in)$`

func (t *Tql) __ParseFilterList() bool {
	b, ident := t.__Identifier()
	if !b {
		return b
	}
	b, op := t.__ConsumeRegexp(condition_regex)
	if !b {
		return b
	}
	b, val := t.__Value()
	if !b && op == "in" {
		b, val = t.__ValueList()
	}
	if !b {
		return b
	}
	t.conds = append(t.conds, Cond{ident, op, val})
	if t.__Consume("and") {
		return t.__ParseFilterList()
	}
	return t.__orderBy()
}
<|endoftext|>


<commit_before>package store

import (
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/adamdecaf/cert-manage/tools/file"
	"github.com/adamdecaf/cert-manage/whitelist"
)

var (
	ErrNoBackupMade = errors.New("unable to make backup of store")

	// internal options
	debug = len(os.Getenv("TRAVIS_OS_NAME")) > 0 ||
		len(os.Getenv("DEBUG")) > 0 ||
		strings.Contains(os.Getenv("GODEBUG"), "x509roots=1")

	backupDirPerms os.FileMode = 0744

	// Define a mapping between -app and the Store instance
	appStores = map[string]Store{
		"chrome":  ChromeStore(),
		"firefox": FirefoxStore(),
		"java":    JavaStore(),
	}
)

// Store represents a certificate store (often called 'pool') and has
// operations on it which mutate the underlying state (e.g. a file or
// directory).
type Store interface {
	// Backup will attempt to save a backup of the certificate store
	// on the local system
	Backup() error

	// List returns the currently trusted X509 certificates contained
	// within the cert store
	List() ([]*x509.Certificate, error)

	// Remove will distrust the certificate in the store
	//
	// Note: This may not actually delete the certificate, but modify
	// the store such that the certificate is no longer trusted.
	// This is done when possible to limit the actual deletions to
	// preserve restore capabilities
	Remove(whitelist.Whitelist) error

	// Restore will bring the system back to it's previous state
	// if a backup exists, otherwise it will attempt to bring the
	// cert trust status to the system's default state
	//
	// Optionally, this can take a specific filepath to use as the
	// restore point. This may not be supported on all stores.
	//
	// Note: It is strongly advised that any additional certs installed
	// be verified are still properly installed and working after
	// Restore() is called.
	Restore(where string) error
}

// Platform returns a new instance of Store for the running os/platform
func Platform() Store {
	return platform()
}

// ForApp returns a `Store` instance for the given app
func ForApp(app string) (Store, error) {
	s, ok := appStores[strings.ToLower(app)]
	if !ok {
		return nil, fmt.Errorf("application '%s' not found", app)
	}
	return s, nil
}

// getCertManageDir returns the fs location (always creating first) where a specific
// store can save files into. This path is recommended for backups
//
// If `name` is an absolute fs reference then just ensure that directory is created
// and has permissions setup properly.
func getCertManageDir(name string) (string, error) {
	parent := getCertManageParentDir()
	err := os.MkdirAll(parent, os.ModeDir)
	if err != nil {
		return "", err
	}
	err = os.Chmod(parent, backupDirPerms)
	if err != nil {
		return "", err
	}

	dir := filepath.Join(parent, name)
	// If `name` is actually an absolute fs reference then just ensure
	// it's created and owned properly, otherwise append whatever was
	// provided onto the parent dir.
	if filepath.IsAbs(name) {
		s, err := os.Stat(name)
		if err != nil {
			return "", err
		}
		if !s.IsDir() {
			return "", fmt.Errorf("since %s exists and cannot be a file, should be a dir", name)
		}
		dir = name
	}

	// Create the dir and set ownership
	err = os.MkdirAll(dir, os.ModeDir)
	if err != nil {
		return "", err
	}
	err = os.Chmod(dir, backupDirPerms)
	if err != nil {
		return "", err
	}

	return dir, nil
}

func getCertManageParentDir() string {
	uhome := file.HomeDir()
	if uhome != "" {
		// Setup parent dir
		if runtime.GOOS == "darwin" {
			return filepath.Join(uhome, "/Library/cert-manage")
		}
		if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
			return filepath.Join(uhome, ".cert-manage")
		}
	}
	return ""
}

// getLatestBackup returns the "biggest" file or dir at a given path
//
// This sorting is done by assuming filenames follow a pattern like
// file-%d.ext where %d is a sortable timestamp and the filename follows
// lexigraphical sorting. Results are sorted in descending order and the
// first element (if exists) is returned
func getLatestBackup(dir string) (string, error) {
	fis, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	if len(fis) == 0 {
		return "", nil
	}

	// get largest
	file.SortFileInfos(fis)
	latest := fis[len(fis)-1]
	return filepath.Join(dir, latest.Name()), nil
}
<commit_msg>store: allow getCertManageDir to create child dirs<commit_after>package store

import (
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/adamdecaf/cert-manage/tools/file"
	"github.com/adamdecaf/cert-manage/whitelist"
)

var (
	ErrNoBackupMade = errors.New("unable to make backup of store")

	// internal options
	debug = len(os.Getenv("TRAVIS_OS_NAME")) > 0 ||
		len(os.Getenv("DEBUG")) > 0 ||
		strings.Contains(os.Getenv("GODEBUG"), "x509roots=1")

	backupDirPerms os.FileMode = 0744

	// Define a mapping between -app and the Store instance
	appStores = map[string]Store{
		"chrome":  ChromeStore(),
		"firefox": FirefoxStore(),
		"java":    JavaStore(),
	}
)

// Store represents a certificate store (often called 'pool') and has
// operations on it which mutate the underlying state (e.g. a file or
// directory).
type Store interface {
	// Backup will attempt to save a backup of the certificate store
	// on the local system
	Backup() error

	// List returns the currently trusted X509 certificates contained
	// within the cert store
	List() ([]*x509.Certificate, error)

	// Remove will distrust the certificate in the store
	//
	// Note: This may not actually delete the certificate, but modify
	// the store such that the certificate is no longer trusted.
	// This is done when possible to limit the actual deletions to
	// preserve restore capabilities
	Remove(whitelist.Whitelist) error

	// Restore will bring the system back to it's previous state
	// if a backup exists, otherwise it will attempt to bring the
	// cert trust status to the system's default state
	//
	// Optionally, this can take a specific filepath to use as the
	// restore point. This may not be supported on all stores.
	//
	// Note: It is strongly advised that any additional certs installed
	// be verified are still properly installed and working after
	// Restore() is called.
	Restore(where string) error
}

// Platform returns a new instance of Store for the running os/platform
func Platform() Store {
	return platform()
}

// ForApp returns a `Store` instance for the given app
func ForApp(app string) (Store, error) {
	s, ok := appStores[strings.ToLower(app)]
	if !ok {
		return nil, fmt.Errorf("application '%s' not found", app)
	}
	return s, nil
}

// getCertManageDir returns the fs location (always creating first) where a specific
// store can save files into. This path is recommended for backups
//
// If `name` is an absolute fs reference then just ensure that directory is created
// and has permissions setup properly.
func getCertManageDir(name string) (string, error) {
	parent := getCertManageParentDir()
	err := os.MkdirAll(parent, os.ModeDir)
	if err != nil {
		return "", err
	}
	err = os.Chmod(parent, backupDirPerms)
	if err != nil {
		return "", err
	}

	dir := filepath.Join(parent, name)
	// If `name` is actually an absolute fs reference then just ensure
	// it's created and owned properly, otherwise append whatever was
	// provided onto the parent dir.
	if filepath.IsAbs(name) {
		s, err := os.Stat(name)
		if err != nil && !os.IsNotExist(err) {
			return "", err
		}
		if s != nil && !s.IsDir() {
			return "", fmt.Errorf("since %s exists and cannot be a file, should be a dir", name)
		}
		dir = name
	}

	// Create the dir and set ownership
	err = os.MkdirAll(dir, os.ModeDir)
	if err != nil {
		return "", err
	}
	err = os.Chmod(dir, backupDirPerms)
	if err != nil {
		return "", err
	}

	return dir, nil
}

func getCertManageParentDir() string {
	uhome := file.HomeDir()
	if uhome != "" {
		// Setup parent dir
		if runtime.GOOS == "darwin" {
			return filepath.Join(uhome, "/Library/cert-manage")
		}
		if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
			return filepath.Join(uhome, ".cert-manage")
		}
	}
	return ""
}

// getLatestBackup returns the "biggest" file or dir at a given path
//
// This sorting is done by assuming filenames follow a pattern like
// file-%d.ext where %d is a sortable timestamp and the filename follows
// lexigraphical sorting. Results are sorted in descending order and the
// first element (if exists) is returned
func getLatestBackup(dir string) (string, error) {
	fis, err := ioutil.ReadDir(dir)
	if err != nil {
		return "", err
	}
	if len(fis) == 0 {
		return "", nil
	}

	// get largest
	file.SortFileInfos(fis)
	latest := fis[len(fis)-1]
	return filepath.Join(dir, latest.Name()), nil
}
<|endoftext|>


<commit_before>// Package store provides a distributed SQLite instance.
//
// Distributed consensus is provided via the Raft algorithm.
package store

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	sql "github.com/otoolep/rqlite/db"
)

const (
	retainSnapshotCount = 2
	raftTimeout         = 10 * time.Second
)

type command struct {
	Tx      bool     `json:"tx,omitempty"`
	Queries []string `json:"queries,omitempty"`
}

// Store is a SQLite database, where all changes are made via Raft consensus.
type Store struct {
	raftDir  string
	raftBind string

	mu sync.Mutex

	raft *raft.Raft // The consensus mechanism
	db   *sql.DB    // The underlying SQLite store

	logger *log.Logger

	SQLiteDB string
}

// New returns a new Store.
func New(db *sql.DB, dir, bind string) *Store {
	return &Store{
		raftDir:  dir,
		raftBind: bind,
		db:       db,
		logger:   log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomesthe first node, and therefore leader, of the cluster.
func (s *Store) Open(enableSingle bool) error {
	// Setup Raft configuration.
	config := raft.DefaultConfig()

	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(s.raftDir, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if enableSingle && len(peers) <= 1 {
		s.logger.Println("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Setup Raft communication.
	addr, err := net.ResolveTCPAddr("tcp", s.raftBind)
	if err != nil {
		return err
	}
	transport, err := raft.NewTCPTransport(s.raftBind, addr, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}

	// Create peer storage.
	peerStore := raft.NewJSONPeers(s.raftDir, transport)

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(s.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(s.raftDir, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}

	// Instantiate the Raft systems.
	ra, err := raft.NewRaft(config, (*fsm)(s), logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	s.raft = ra

	return nil
}

func (s *Store) Execute(queries []string, tx bool) ([]*sql.Result, error) {
	if s.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	c := &command{
		Tx:      tx,
		Queries: queries,
	}
	b, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	f := s.raft.Apply(b, raftTimeout)
	if err, ok := f.(error); ok {
		return nil, err
	}

	return nil, nil
}

func (s *Store) Query(queries []string, tx bool) ([]*sql.Rows, error) {
	// Go straight to the local database. Optionally check if leader. Hard way?
	return nil, nil
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (s *Store) Join(addr string) error {
	s.logger.Printf("received join request for remote node as %s", addr)

	f := s.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	s.logger.Printf("node at %s joined successfully", addr)
	return nil
}

type fsm Store

// Apply applies a Raft log entry to the database.
func (f *fsm) Apply(l *raft.Log) interface{} {
	var c command
	if err := json.Unmarshal(l.Data, &c); err != nil {
		panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
	}

	r, err := f.db.Execute(c.Queries, c.Tx)
	if err != nil {
		return err
	}
	return r
}

// Snapshot returns a snapshot of the database.
func (f *fsm) Snapshot() (raft.FSMSnapshot, error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	return nil, nil
}

// Restore restores the database to a previous state.
func (f *fsm) Restore(rc io.ReadCloser) error {
	return nil
}

type fsmSnapshot struct {
}

func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	return nil
}

func (f *fsmSnapshot) Release() {}

func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<commit_msg>Start threading ApplyFuture for Execute response<commit_after>// Package store provides a distributed SQLite instance.
//
// Distributed consensus is provided via the Raft algorithm.
package store

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/hashicorp/raft"
	"github.com/hashicorp/raft-boltdb"
	sql "github.com/otoolep/rqlite/db"
)

const (
	retainSnapshotCount = 2
	raftTimeout         = 10 * time.Second
)

type command struct {
	Tx      bool     `json:"tx,omitempty"`
	Queries []string `json:"queries,omitempty"`
}

// Store is a SQLite database, where all changes are made via Raft consensus.
type Store struct {
	raftDir  string
	raftBind string

	mu sync.Mutex

	raft *raft.Raft // The consensus mechanism
	db   *sql.DB    // The underlying SQLite store

	logger *log.Logger

	SQLiteDB string
}

// New returns a new Store.
func New(db *sql.DB, dir, bind string) *Store {
	return &Store{
		raftDir:  dir,
		raftBind: bind,
		db:       db,
		logger:   log.New(os.Stderr, "[store] ", log.LstdFlags),
	}
}

// Open opens the store. If enableSingle is set, and there are no existing peers,
// then this node becomesthe first node, and therefore leader, of the cluster.
func (s *Store) Open(enableSingle bool) error {
	// Setup Raft configuration.
	config := raft.DefaultConfig()

	// Check for any existing peers.
	peers, err := readPeersJSON(filepath.Join(s.raftDir, "peers.json"))
	if err != nil {
		return err
	}

	// Allow the node to entry single-mode, potentially electing itself, if
	// explicitly enabled and there is only 1 node in the cluster already.
	if enableSingle && len(peers) <= 1 {
		s.logger.Println("enabling single-node mode")
		config.EnableSingleNode = true
		config.DisableBootstrapAfterElect = false
	}

	// Setup Raft communication.
	addr, err := net.ResolveTCPAddr("tcp", s.raftBind)
	if err != nil {
		return err
	}
	transport, err := raft.NewTCPTransport(s.raftBind, addr, 3, 10*time.Second, os.Stderr)
	if err != nil {
		return err
	}

	// Create peer storage.
	peerStore := raft.NewJSONPeers(s.raftDir, transport)

	// Create the snapshot store. This allows the Raft to truncate the log.
	snapshots, err := raft.NewFileSnapshotStore(s.raftDir, retainSnapshotCount, os.Stderr)
	if err != nil {
		return fmt.Errorf("file snapshot store: %s", err)
	}

	// Create the log store and stable store.
	logStore, err := raftboltdb.NewBoltStore(filepath.Join(s.raftDir, "raft.db"))
	if err != nil {
		return fmt.Errorf("new bolt store: %s", err)
	}

	// Instantiate the Raft systems.
	ra, err := raft.NewRaft(config, (*fsm)(s), logStore, logStore, snapshots, peerStore, transport)
	if err != nil {
		return fmt.Errorf("new raft: %s", err)
	}
	s.raft = ra

	return nil
}

func (s *Store) Execute(queries []string, tx bool) ([]*sql.Result, error) {
	if s.raft.State() != raft.Leader {
		return nil, fmt.Errorf("not leader")
	}

	c := &command{
		Tx:      tx,
		Queries: queries,
	}
	b, err := json.Marshal(c)
	if err != nil {
		return nil, err
	}

	r := s.raft.Apply(b, raftTimeout).(*fsmResponse)
	return r.Response().([]*sql.Result), r.Error()
}

func (s *Store) Query(queries []string, tx bool) ([]*sql.Rows, error) {
	// Go straight to the local database. Optionally check if leader. Hard way?
	return nil, nil
}

// Join joins a node, located at addr, to this store. The node must be ready to
// respond to Raft communications at that address.
func (s *Store) Join(addr string) error {
	s.logger.Printf("received join request for remote node as %s", addr)

	f := s.raft.AddPeer(addr)
	if f.Error() != nil {
		return f.Error()
	}
	s.logger.Printf("node at %s joined successfully", addr)
	return nil
}

type fsm Store

type fsmResponse struct {
	results []*sql.Result
	error error
	index uint64
}

func (f *fsmResponse) Response() interface{} {
	return f.results
}

func (f *fsmResponse) Index() uint64 {
	return f.index
}

func (f *fsmResponse) Error() error {
	return f.error
}

// Apply applies a Raft log entry to the database.
func (f *fsm) Apply(l *raft.Log) interface{} {
	var c command
	if err := json.Unmarshal(l.Data, &c); err != nil {
		panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error()))
	}

	r, err := f.db.Execute(c.Queries, c.Tx)
	return &fsmResponse{
		results: r,
		error: err,
		index: 1,
	}
}

// Snapshot returns a snapshot of the database.
func (f *fsm) Snapshot() (raft.FSMSnapshot, error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	return nil, nil
}

// Restore restores the database to a previous state.
func (f *fsm) Restore(rc io.ReadCloser) error {
	return nil
}

type fsmSnapshot struct {
}

func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	return nil
}

func (f *fsmSnapshot) Release() {}

func readPeersJSON(path string) ([]string, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if len(b) == 0 {
		return nil, nil
	}

	var peers []string
	dec := json.NewDecoder(bytes.NewReader(b))
	if err := dec.Decode(&peers); err != nil {
		return nil, err
	}

	return peers, nil
}
<|endoftext|>