<commit_before>package web

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	nanosPerMilli = 1000000

	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

type query struct {
	sqlString string
	parsed    *sql.Query
	ce        cacheEntry
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Expires", "0")
	resp.Header().Set("Cache-control", "no-cache, no-store, must-revalidate")
	resp.Header().Set("Content-Encoding", "gzip")
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string) (ce cacheEntry, err error) {
	parsed, parseErr := sql.Parse(sqlString)
	if parseErr != nil {
		return nil, parseErr
	}

	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Request query to run in background
	h.queries <- &query{sqlString, parsed, ce}

	return
}

func (h *handler) coalesceQueries() {
	for q := range h.queries {
		coalescedQueries := append([]*query(nil), q)
		var remainingQueries []*query
		table := q.parsed.From
	coalesceLoop:
		for {
			select {
			case query := <-h.queries:
				if table == query.parsed.From {
					coalescedQueries = append(coalescedQueries)
				} else {
					remainingQueries = append(remainingQueries, query)
				}
			case <-time.After(15 * time.Second):
				break coalesceLoop
			}
		}

		log.Debugf("Coalesced %d queries to %v", len(coalescedQueries), table)
		// re-queue queries that weren't included in this run
		for _, query := range remainingQueries {
			h.queries <- query
		}
		h.coalescedQueries <- coalescedQueries
	}
}

func (h *handler) processQueries() {
	for queries := range h.coalescedQueries {
		for _, query := range queries {
			sqlString := query.sqlString
			ce := query.ce
			result, err := h.doQuery(sqlString, ce.permalink())
			if err != nil {
				err = fmt.Errorf("Unable to query: %v", err)
				log.Error(err)
				ce = ce.fail(err)
			} else {
				resultBytes, err := compress(json.Marshal(result))
				if err != nil {
					err = fmt.Errorf("Unable to marshal result: %v", err)
					log.Error(err)
					ce = ce.fail(err)
				} else if len(resultBytes) > h.MaxResponseBytes {
					err = fmt.Errorf("Query result size %v exceeded limit of %v", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(h.MaxResponseBytes)))
					log.Error(err)
					ce = ce.fail(err)
				} else {
					ce = ce.succeed(resultBytes)
				}
			}
			h.cache.put(sqlString, ce)
			log.Debugf("Cached results for %v", sqlString)
		}
	}
}

func compress(resultBytes []byte, err error) ([]byte, error) {
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(make([]byte, 0, len(resultBytes)))
	gw, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = gw.Write(resultBytes)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	compressed := buf.Bytes()
	log.Debugf("Compressed result from %v down to %v using gzip", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(len(compressed))))
	return compressed, nil
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        time.Now().UnixNano() / nanosPerMilli,
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	estimatedResultBytes := 0
	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			estimatedResultBytes += len(dim) + len(valueBytes)
			return true
		})

		estimatedResultBytes += 8 * len(row.Values)
		if estimatedResultBytes > h.MaxResponseBytes {
			mx.Unlock()
			// Note - the estimated size here is always an underestimate of the final
			// JSON size, so this is a conservative way to check. The final check
			// after generating the JSON may sometimes catch things that slipped
			// through here.
			return false, fmt.Errorf("Estimated query result size %v exceeded limit of %v", humanize.Bytes(uint64(estimatedResultBytes)), humanize.Bytes(uint64(h.MaxResponseBytes)))
		}

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   row.TS / nanosPerMilli,
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<commit_msg>Cleaned up web coalescing<commit_after>package web

import (
	"bytes"
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"math"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/getlantern/zenodb/core"
	"github.com/getlantern/zenodb/encoding"
	"github.com/getlantern/zenodb/sql"
	"github.com/gorilla/mux"
	"github.com/retailnext/hllpp"
)

const (
	nanosPerMilli = 1000000

	pauseTime    = 250 * time.Millisecond
	shortTimeout = 5 * time.Second
	longTimeout  = 1000 * time.Hour
)

type QueryResult struct {
	SQL                string
	Permalink          string
	TS                 int64
	TSCardinality      uint64
	Fields             []string
	FieldCardinalities []uint64
	Dims               []string
	DimCardinalities   []uint64
	Rows               []*ResultRow
}

type ResultRow struct {
	TS   int64
	Key  map[string]interface{}
	Vals []float64
}

type query struct {
	sqlString string
	parsed    *sql.Query
	ce        cacheEntry
}

func (h *handler) runQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, longTimeout)
}

func (h *handler) asyncQuery(resp http.ResponseWriter, req *http.Request) {
	h.sqlQuery(resp, req, shortTimeout)
}

func (h *handler) cachedQuery(resp http.ResponseWriter, req *http.Request) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	permalink := mux.Vars(req)["permalink"]
	ce, err := h.cache.getByPermalink(permalink)
	if ce == nil {
		http.NotFound(resp, req)
		return
	}
	h.respondWithCacheEntry(resp, req, ce, err, shortTimeout)
}

func (h *handler) sqlQuery(resp http.ResponseWriter, req *http.Request, timeout time.Duration) {
	if !h.authenticate(resp, req) {
		resp.WriteHeader(http.StatusForbidden)
		return
	}

	log.Debug(req.URL)
	sqlString, _ := url.QueryUnescape(req.URL.RawQuery)

	ce, err := h.query(req, sqlString)
	h.respondWithCacheEntry(resp, req, ce, err, timeout)
}

func (h *handler) respondWithCacheEntry(resp http.ResponseWriter, req *http.Request, ce cacheEntry, err error, timeout time.Duration) {
	limit := int(timeout / pauseTime)
	for i := 0; i < limit; i++ {
		if err != nil {
			log.Error(err)
			resp.WriteHeader(http.StatusInternalServerError)
			fmt.Fprint(resp, err.Error())
			return
		}
		switch ce.status() {
		case statusSuccess:
			h.respondSuccess(resp, req, ce)
			return
		case statusError:
			h.respondError(resp, req, ce)
			return
		case statusPending:
			// Pause a little bit and try again
			time.Sleep(pauseTime)
			ce, err = h.cache.getByPermalink(ce.permalink())
		}
	}
	// Let the client know that we're still working on it
	resp.WriteHeader(http.StatusAccepted)
	fmt.Fprintf(resp, "/cached/%v", ce.permalink())
}

func (h *handler) respondSuccess(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.Header().Set("Content-Type", "application/json")
	resp.Header().Set("Expires", "0")
	resp.Header().Set("Cache-control", "no-cache, no-store, must-revalidate")
	resp.Header().Set("Content-Encoding", "gzip")
	resp.WriteHeader(http.StatusOK)
	resp.Write(ce.data())
}

func (h *handler) respondError(resp http.ResponseWriter, req *http.Request, ce cacheEntry) {
	resp.WriteHeader(http.StatusInternalServerError)
	resp.Write(ce.error())
}

func (h *handler) query(req *http.Request, sqlString string) (ce cacheEntry, err error) {
	parsed, parseErr := sql.Parse(sqlString)
	if parseErr != nil {
		return nil, parseErr
	}

	if req.Header.Get("Cache-control") == "no-cache" {
		ce, err = h.cache.begin(sqlString)
		if err != nil {
			return
		}
	} else {
		var created bool
		ce, created, err = h.cache.getOrBegin(sqlString)
		if err != nil || !created {
			return
		}
		if ce.status() != statusPending {
			log.Debugf("Found results for %v in cache", sqlString)
			return
		}
	}

	// Request query to run in background
	h.queries <- &query{sqlString, parsed, ce}

	return
}

func (h *handler) coalesceQueries() {
	for q := range h.queries {
		coalescedQueries := append([]*query(nil), q)
		var remainingQueries []*query
		table := q.parsed.From
	coalesceLoop:
		for {
			select {
			case query := <-h.queries:
				if table == query.parsed.From {
					coalescedQueries = append(coalescedQueries, query)
				} else {
					remainingQueries = append(remainingQueries, query)
				}
			case <-time.After(15 * time.Second):
				break coalesceLoop
			}
		}

		log.Debugf("Coalesced %d queries to %v", len(coalescedQueries), table)
		// re-queue queries that weren't included in this run
		for _, query := range remainingQueries {
			h.queries <- query
		}
		h.coalescedQueries <- coalescedQueries
	}
}

func (h *handler) processQueries() {
	for queries := range h.coalescedQueries {
		for _, query := range queries {
			sqlString := query.sqlString
			ce := query.ce
			result, err := h.doQuery(sqlString, ce.permalink())
			if err != nil {
				err = fmt.Errorf("Unable to query: %v", err)
				log.Error(err)
				ce = ce.fail(err)
			} else {
				resultBytes, err := compress(json.Marshal(result))
				if err != nil {
					err = fmt.Errorf("Unable to marshal result: %v", err)
					log.Error(err)
					ce = ce.fail(err)
				} else if len(resultBytes) > h.MaxResponseBytes {
					err = fmt.Errorf("Query result size %v exceeded limit of %v", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(h.MaxResponseBytes)))
					log.Error(err)
					ce = ce.fail(err)
				} else {
					ce = ce.succeed(resultBytes)
				}
			}
			h.cache.put(sqlString, ce)
			log.Debugf("Cached results for %v", sqlString)
		}
	}
}

func compress(resultBytes []byte, err error) ([]byte, error) {
	if err != nil {
		return nil, err
	}
	buf := bytes.NewBuffer(make([]byte, 0, len(resultBytes)))
	gw, err := gzip.NewWriterLevel(buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = gw.Write(resultBytes)
	if err != nil {
		return nil, err
	}
	err = gw.Close()
	if err != nil {
		return nil, err
	}
	compressed := buf.Bytes()
	log.Debugf("Compressed result from %v down to %v using gzip", humanize.Bytes(uint64(len(resultBytes))), humanize.Bytes(uint64(len(compressed))))
	return compressed, nil
}

func (h *handler) doQuery(sqlString string, permalink string) (*QueryResult, error) {
	rs, err := h.db.Query(sqlString, false, nil, false)
	if err != nil {
		return nil, err
	}

	var addDim func(dim string)

	result := &QueryResult{
		SQL:       sqlString,
		Permalink: permalink,
		TS:        time.Now().UnixNano() / nanosPerMilli,
	}
	groupBy := rs.GetGroupBy()
	if len(groupBy) > 0 {
		addDim = func(dim string) {
			// noop
		}
		for _, gb := range groupBy {
			result.Dims = append(result.Dims, gb.Name)
		}
	} else {
		addDim = func(dim string) {
			found := false
			for _, existing := range result.Dims {
				if existing == dim {
					found = true
					break
				}
			}
			if !found {
				result.Dims = append(result.Dims, dim)
			}
		}
	}

	var fields core.Fields
	var fieldCardinalities []*hllpp.HLLPP
	dimCardinalities := make(map[string]*hllpp.HLLPP)
	tsCardinality := hllpp.New()
	cbytes := make([]byte, 8)

	estimatedResultBytes := 0
	var mx sync.Mutex
	ctx, cancel := context.WithTimeout(context.Background(), h.QueryTimeout)
	defer cancel()
	rs.Iterate(ctx, func(inFields core.Fields) error {
		fields = inFields
		for _, field := range fields {
			result.Fields = append(result.Fields, field.Name)
			fieldCardinalities = append(fieldCardinalities, hllpp.New())
		}
		return nil
	}, func(row *core.FlatRow) (bool, error) {
		mx.Lock()
		key := make(map[string]interface{}, 10)
		row.Key.Iterate(true, true, func(dim string, value interface{}, valueBytes []byte) bool {
			key[dim] = value
			addDim(dim)
			hlp := dimCardinalities[dim]
			if hlp == nil {
				hlp = hllpp.New()
				dimCardinalities[dim] = hlp
			}
			hlp.Add(valueBytes)
			estimatedResultBytes += len(dim) + len(valueBytes)
			return true
		})

		estimatedResultBytes += 8 * len(row.Values)
		if estimatedResultBytes > h.MaxResponseBytes {
			mx.Unlock()
			// Note - the estimated size here is always an underestimate of the final
			// JSON size, so this is a conservative way to check. The final check
			// after generating the JSON may sometimes catch things that slipped
			// through here.
			return false, fmt.Errorf("Estimated query result size %v exceeded limit of %v", humanize.Bytes(uint64(estimatedResultBytes)), humanize.Bytes(uint64(h.MaxResponseBytes)))
		}

		encoding.Binary.PutUint64(cbytes, uint64(row.TS))
		tsCardinality.Add(cbytes)

		resultRow := &ResultRow{
			TS:   row.TS / nanosPerMilli,
			Key:  key,
			Vals: make([]float64, 0, len(row.Values)),
		}

		for i, value := range row.Values {
			resultRow.Vals = append(resultRow.Vals, value)
			encoding.Binary.PutUint64(cbytes, math.Float64bits(value))
			fieldCardinalities[i].Add(cbytes)
		}
		result.Rows = append(result.Rows, resultRow)
		mx.Unlock()
		return true, nil
	})

	result.TSCardinality = tsCardinality.Count()
	result.Dims = make([]string, 0, len(dimCardinalities))
	for dim := range dimCardinalities {
		result.Dims = append(result.Dims, dim)
	}
	sort.Strings(result.Dims)
	for _, dim := range result.Dims {
		result.DimCardinalities = append(result.DimCardinalities, dimCardinalities[dim].Count())
	}

	result.FieldCardinalities = make([]uint64, 0, len(fieldCardinalities))
	for _, fieldCardinality := range fieldCardinalities {
		result.FieldCardinalities = append(result.FieldCardinalities, fieldCardinality.Count())
	}

	return result, nil
}

func intToBytes(i uint64) []byte {
	b := make([]byte, 8)
	encoding.Binary.PutUint64(b, i)
	return b
}
<|endoftext|>


<commit_before>// Serve webpages with onthefly and http
package webhandle

import (
	"bytes"
	"fmt"
	. "github.com/xyproto/onthefly"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
)

type HandleFunc func(http.ResponseWriter, *http.Request)

type (
	// Various function signatures for handling requests
	WebHandle              (func(w http.ResponseWriter, r *http.Request, val string) string)
	SimpleContextHandle    (func(w http.ResponseWriter, r *http.Request) string)
	TemplateValueGenerator func(w http.ResponseWriter, r *http.Request) TemplateValues
)

// Create a web.go compatible function that returns a string that is the HTML for this page
func GenerateHTML(page *Page) HandleFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", page.GetXML(true))
	}
}

// Create a web.go compatible function that returns a string that is the CSS for this page
func GenerateCSS(page *Page) HandleFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Content-Type", "text/css")
		fmt.Fprintf(w, "%s", page.GetCSS())
	}
}

// Create a web.go compatible function that returns a string that is the XML for this page
func GenerateXML(page *Page) HandleFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Content-Type", "text/xml")
		fmt.Fprintf(w, "%s", page.GetXML(false))
	}
}

// Creates a page based on the contents of "error.log". Useful for showing compile errors while creating an application.
func GenerateErrorHandle(errorfilename string) HandleFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		data, err := ioutil.ReadFile(errorfilename)
		if err != nil {
			errors := strings.Replace(string(data), "\n", "
", -1)
			fmt.Fprintf(w, "%s", Message("Errors", errors))
			return
		}
		fmt.Fprintf(w, "%s", Message("Good", "No errors"))
	}
}

// Handles pages that are not found
func NotFound(val string) HandleFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", Message("No", "Page not found"))
	}
}

// Takes a filename and returns a function that can handle the request
func File(filename string) HandleFunc {
	var extension string
	if strings.Contains(filename, ".") {
		extension = filepath.Ext(filename)
	}
	return func(w http.ResponseWriter, r *http.Request) {
		if extension != "" {
			w.Header().Add("Content-Type", mime.TypeByExtension("."+extension))
		}
		imagebytes, _ := ioutil.ReadFile(filename)
		buf := bytes.NewBuffer(imagebytes)
		// TODO: Write bytes directly
		fmt.Fprintf(w, "%s", buf.String())
	}
}

// Takes an url and a filename and offers that file at the given url
func PublishFile(mux *http.ServeMux, url, filename string) {
	mux.HandleFunc(url, File(filename))
}

// Takes a filename and offers that file at the root url
func PublishRootFile(mux *http.ServeMux, filename string) {
	mux.HandleFunc("/"+filename, File(filename))
}

// Expose the HTML and CSS generated by a page building function to the two given urls
func PublishPage(mux *http.ServeMux, htmlurl, cssurl string, buildfunction func(string) *Page) {
	page := buildfunction(cssurl)
	mux.HandleFunc(htmlurl, GenerateHTML(page))
	mux.HandleFunc(cssurl, GenerateCSS(page))
}

// Serve a static file
func Publish(mux *http.ServeMux, url, filename string) {
	mux.HandleFunc(url, File(filename))
}
<commit_msg>Use gorilla/mux<commit_after>// Serve webpages with onthefly and http
package webhandle

import (
	"bytes"
	"fmt"
	"github.com/gorilla/mux"
	. "github.com/xyproto/onthefly"
	"io/ioutil"
	"mime"
	"net/http"
	"path/filepath"
	"strings"
)

type HTTPHandler func(http.ResponseWriter, *http.Request)

type (
	// Various function signatures for handling requests
	WebHandle              (func(w http.ResponseWriter, r *http.Request, val string) string)
	SimpleContextHandle    (func(w http.ResponseWriter, r *http.Request) string)
	TemplateValueGenerator func(w http.ResponseWriter, r *http.Request) TemplateValues
)

// Create a web.go compatible function that returns a string that is the HTML for this page
func GenerateHTML(page *Page) HTTPHandler {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", page.GetXML(true))
	}
}

// Create a web.go compatible function that returns a string that is the CSS for this page
func GenerateCSS(page *Page) HTTPHandler {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Content-Type", "text/css")
		fmt.Fprintf(w, "%s", page.GetCSS())
	}
}

// Create a web.go compatible function that returns a string that is the XML for this page
func GenerateXML(page *Page) HTTPHandler {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Content-Type", "text/xml")
		fmt.Fprintf(w, "%s", page.GetXML(false))
	}
}

// Creates a page based on the contents of "error.log". Useful for showing compile errors while creating an application.
func GenerateErrorHandle(errorfilename string) HTTPHandler {
	return func(w http.ResponseWriter, r *http.Request) {
		data, err := ioutil.ReadFile(errorfilename)
		if err != nil {
			errors := strings.Replace(string(data), "\n", "
", -1)
			fmt.Fprintf(w, "%s", Message("Errors", errors))
			return
		}
		fmt.Fprintf(w, "%s", Message("Good", "No errors"))
	}
}

// Handles pages that are not found
func NotFound(val string) HTTPHandler {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%s", Message("No", "Page not found"))
	}
}

// Takes a filename and returns a function that can handle the request
func File(filename string) HTTPHandler {
	var extension string
	if strings.Contains(filename, ".") {
		extension = filepath.Ext(filename)
	}
	return func(w http.ResponseWriter, r *http.Request) {
		if extension != "" {
			w.Header().Add("Content-Type", mime.TypeByExtension("."+extension))
		}
		imagebytes, _ := ioutil.ReadFile(filename)
		buf := bytes.NewBuffer(imagebytes)
		// TODO: Write bytes directly
		fmt.Fprintf(w, "%s", buf.String())
	}
}

// Takes an url and a filename and offers that file at the given url
func PublishFile(r *mux.Router, url, filename string) {
	r.HandleFunc(url, File(filename))
}

// Takes a filename and offers that file at the root url
func PublishRootFile(r *mux.Router, filename string) {
	r.HandleFunc("/"+filename, File(filename))
}

// Expose the HTML and CSS generated by a page building function to the two given urls
func PublishPage(r *mux.Router, htmlurl, cssurl string, buildfunction func(string) *Page) {
	page := buildfunction(cssurl)
	r.HandleFunc(htmlurl, GenerateHTML(page))
	r.HandleFunc(cssurl, GenerateCSS(page))
}

// Serve a static file
func Publish(r *mux.Router, url, filename string) {
	r.HandleFunc(url, File(filename))
}
<|endoftext|>


<commit_before>package tracker

import (
	"appengine"
	"appengine/mail"
	"appengine/urlfetch"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"strconv"
	"strings"
)

const ITEM_URL string = "<ITEM_URL_HERE>"
const SENDER string = "<SENDER_EMAIL_ADDRESS>"
const THRESHOLD float64 = 15.00 //THRESHOLD
const MESSAGE string = `
Your favorite item has its price dropped below your threshold!

Item URL: %s
Threshold Set: %f
Current Price: %f

Good luck!
`

func init() {
	http.HandleFunc("/trigger", handleTrigger)
}

func handleTrigger(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	client := urlfetch.Client(c)

	resp, err := client.Get(ITEM_URL)

	if err != nil {
		c.Infof(err.Error())
		return
	}

	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)

	if err != nil {
		c.Infof(err.Error())
		return
	}

	var price float64 = 0.0

	doc.Find("span.market_listing_price_with_fee").Each(func(i int, s *goquery.Selection) {

		priceVal := strings.TrimSpace(s.Text())

		if price != 0 {
			return
		}

		if strings.HasSuffix(priceVal, "USD") || strings.HasPrefix(priceVal, "$") {
			price, _ = strconv.ParseFloat(strings.TrimPrefix(strings.TrimSuffix(priceVal, " USD"), "$"), 64)
		}
	})

	if price < THRESHOLD {
		sendEmail(price, c)
	}

	fmt.Fprintf(rw, "Triggered: %f", price)
}

func sendEmail(price float64, c appengine.Context) {
	msg := &mail.Message{
		Sender:  SENDER,
		To:      []string{"<RECIPIENT_EMAIL_ADDRESSES"}, //comma-delimited email addresses
		Subject: "Steam Item price dropped below threshold",
		Body:    fmt.Sprintf(MESSAGE, ITEM_URL, THRESHOLD, price),
	}

	if err := mail.Send(c, msg); err != nil {
		c.Errorf("Couldn't send email: %v", err)
	}
}
<commit_msg>send email when price equals or less than threshold, previously only when < threshold<commit_after>package tracker

import (
	"appengine"
	"appengine/mail"
	"appengine/urlfetch"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"net/http"
	"strconv"
	"strings"
)

const ITEM_URL string = "<ITEM_URL_HERE>"
const SENDER string = "<SENDER_EMAIL_ADDRESS>"
const THRESHOLD float64 = 15.00 //THRESHOLD
const MESSAGE string = `
Your favorite item has its price dropped below your threshold!

Item URL: %s
Threshold Set: %f
Current Price: %f

Good luck!
`

func init() {
	http.HandleFunc("/trigger", handleTrigger)
}

func handleTrigger(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	client := urlfetch.Client(c)

	resp, err := client.Get(ITEM_URL)

	if err != nil {
		c.Infof(err.Error())
		return
	}

	defer resp.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(resp)

	if err != nil {
		c.Infof(err.Error())
		return
	}

	var price float64 = 0.0

	doc.Find("span.market_listing_price_with_fee").Each(func(i int, s *goquery.Selection) {

		priceVal := strings.TrimSpace(s.Text())

		if price != 0 {
			return
		}

		if strings.HasSuffix(priceVal, "USD") || strings.HasPrefix(priceVal, "$") {
			price, _ = strconv.ParseFloat(strings.TrimPrefix(strings.TrimSuffix(priceVal, " USD"), "$"), 64)
		}
	})

	if price <= THRESHOLD {
		sendEmail(price, c)
	}

	fmt.Fprintf(rw, "Triggered: %f", price)
}

func sendEmail(price float64, c appengine.Context) {
	msg := &mail.Message{
		Sender:  SENDER,
		To:      []string{"<RECIPIENT_EMAIL_ADDRESSES"}, //comma-delimited email addresses
		Subject: "Steam Item price dropped below threshold",
		Body:    fmt.Sprintf(MESSAGE, ITEM_URL, THRESHOLD, price),
	}

	if err := mail.Send(c, msg); err != nil {
		c.Errorf("Couldn't send email: %v", err)
	}
}
<|endoftext|>


<commit_before>package toml

import (
	"bytes"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

type parser struct {
	mappings []*mapping
	lx       *lexer
	context  []string // the.current.key.group
}

type mapping struct {
	key   []string
	value interface{}
}

func newMapping() *mapping {
	return &mapping{
		key:   make([]string, 0, 1),
		value: nil,
	}
}

func toMap(ms []*mapping) (map[string]interface{}, error) {
	themap := make(map[string]interface{}, 5)
	implicits := make(map[string]bool)
	getMap := func(key []string) (map[string]interface{}, error) {
		// This is where we make sure that duplicate keys cannot be created.
		// Note that something like:
		//
		//	[x.y.z]
		//	[x]
		//
		// Is allowed, but this is not:
		//
		//	[x]
		//	[x.y.z]
		//	[x]
		//
		// In the former case, `x` is created implicitly by `[x.y.z]` while
		// in the latter, it is created explicitly and therefore should not
		// be allowed to be duplicated.
		var ok bool

		m := themap
		accum := make([]string, 0)
		for _, name := range key[0 : len(key)-1] {
			accum = append(accum, name)
			if _, ok = m[name]; !ok {
				implicits[strings.Join(accum, ".")] = true
				m[name] = make(map[string]interface{}, 5)
			}
			m, ok = m[name].(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("The key group '%s' is duplicated "+
					"elsewhere as a regular key.", strings.Join(accum, "."))
			}
		}

		// If the last part of the key already exists and wasn't created
		// implicitly, we've got a dupe.
		last := key[len(key)-1]
		implicitKey := strings.Join(append(accum, last), ".")
		if _, ok := m[last]; ok && !implicits[implicitKey] {
			return nil, fmt.Errorf("Key '%s' is a duplicate.", implicitKey)
		}
		return m, nil
	}
	for _, m := range ms {
		submap, err := getMap(m.key)
		if err != nil {
			return nil, err
		}
		base := m.key[len(m.key)-1]

		// At this point, maps have been created explicitly.
		// But if this is just a key group create an empty map and move on.
		if m.value == nil {
			submap[base] = make(map[string]interface{}, 5)
			continue
		}

		// We now expect that `submap[base]` is empty. Otherwise, we've
		// got a duplicate on our hands.
		if _, ok := submap[base]; ok {
			return nil, fmt.Errorf("Key '%s' is a duplicate.",
				strings.Join(m.key, "."))
		}
		submap[base] = m.value
	}

	return themap, nil
}

func parse(data string) (ms map[string]interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err, _ = r.(error)
			return
		}
	}()

	p := &parser{
		mappings: make([]*mapping, 0, 50),
		lx:       lex(data),
	}
	for {
		item := p.next()
		if item.typ == itemEOF {
			break
		}
		p.topLevel(item)
	}

	return toMap(p.mappings)
}

func (p *parser) next() item {
	it := p.lx.nextItem()
	if it.typ == itemError {
		p.errorf("Near line %d: %s", it.line, it.val)
	}
	return it
}

func (p *parser) errorf(format string, v ...interface{}) {
	panic(fmt.Errorf(format, v...))
}

func (p *parser) bug(format string, v ...interface{}) {
	log.Fatalf("BUG: %s\n\n", fmt.Sprintf(format, v...))
}

func (p *parser) expect(typ itemType) item {
	it := p.next()
	p.assertEqual(typ, it.typ)
	return it
}

func (p *parser) assertEqual(expected, got itemType) {
	if expected != got {
		p.bug("Expected '%s' but got '%s'.", expected, got)
	}
}

func (p *parser) topLevel(item item) {
	switch item.typ {
	case itemCommentStart:
		p.expect(itemText)
	case itemKeyGroupStart:
		m := newMapping()
		kg := p.expect(itemText)
		for ; kg.typ == itemText; kg = p.next() {
			m.key = append(m.key, kg.val)
		}
		p.assertEqual(itemKeyGroupEnd, kg.typ)
		p.mappings = append(p.mappings, m)
		p.context = m.key
	case itemKeyStart:
		kname := p.expect(itemText)
		m := newMapping()
		for _, k := range p.context {
			m.key = append(m.key, k)
		}
		m.key = append(m.key, kname.val)
		m.value = p.value(p.next())
		p.mappings = append(p.mappings, m)
	default:
		p.bug("Unexpected type at top level: %s", item.typ)
	}
}

func (p *parser) value(it item) interface{} {
	switch it.typ {
	case itemString:
		return replaceEscapes(it.val)
	case itemBool:
		switch it.val {
		case "true":
			return true
		case "false":
			return false
		}
		p.bug("Expected boolean value, but got '%s'.", it.val)
	case itemInteger:
		num, err := strconv.ParseInt(it.val, 10, 64)
		if err != nil {
			if e, ok := err.(*strconv.NumError); ok &&
				e.Err == strconv.ErrRange {

				p.errorf("Integer '%s' is out of the range of 64-bit "+
					"signed integers.", it.val)
			} else {
				p.bug("Expected integer value, but got '%s'.", it.val)
			}
		}
		return num
	case itemFloat:
		num, err := strconv.ParseFloat(it.val, 64)
		if err != nil {
			if e, ok := err.(*strconv.NumError); ok &&
				e.Err == strconv.ErrRange {

				p.errorf("Float '%s' is out of the range of 64-bit "+
					"IEEE-754 floating-point numbers.", it.val)
			} else {
				p.bug("Expected float value, but got '%s'.", it.val)
			}
		}
		return num
	case itemDatetime:
		t, err := time.Parse("2006-01-02T15:04:05Z", it.val)
		if err != nil {
			p.bug("Expected Zulu formatted DateTime, but got '%s'.", it.val)
		}
		return t
	case itemArrayStart:
		theType := itemNIL
		array := make([]interface{}, 0)
		for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
			if it.typ == itemCommentStart {
				p.expect(itemText)
				continue
			}

			if theType == itemNIL {
				theType = it.typ
				array = append(array, p.value(it))
				continue
			}
			if theType != it.typ {
				p.errorf("Array has values of type '%s' and '%s'.",
					theType, it.typ)
			}
			array = append(array, p.value(it))
		}
		return array
	}
	p.bug("Unexpected value type: %s", it.typ)
	panic("unreachable")
}

func replaceEscapes(s string) string {
	return strings.NewReplacer(
		"\\0", string(byte(0)),
		"\\t", "\t",
		"\\n", "\n",
		"\\r", "\r",
		"\\\"", "\"",
		"\\\\", "\\",
	).Replace(s)
}

type mappingsNice []*mapping

func (ms mappingsNice) String() string {
	buf := new(bytes.Buffer)
	for _, m := range ms {
		fmt.Fprintln(buf, strings.Join(m.key, "."))
		fmt.Fprintln(buf, m.value)
		fmt.Fprintln(buf, strings.Repeat("-", 45))
	}
	return buf.String()
}
<commit_msg>Fix a bug caught by test suite: defining key groups after they were already implicitly defined destroys any previous definition in the hash.<commit_after>package toml

import (
	"bytes"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
)

type parser struct {
	mappings []*mapping
	lx       *lexer
	context  []string // the.current.key.group
}

type mapping struct {
	key   []string
	value interface{}
}

func newMapping() *mapping {
	return &mapping{
		key:   make([]string, 0, 1),
		value: nil,
	}
}

func toMap(ms []*mapping) (map[string]interface{}, error) {
	themap := make(map[string]interface{}, 5)
	implicits := make(map[string]bool)
	getMap := func(key []string) (map[string]interface{}, error) {
		// This is where we make sure that duplicate keys cannot be created.
		// Note that something like:
		//
		//	[x.y.z]
		//	[x]
		//
		// Is allowed, but this is not:
		//
		//	[x]
		//	[x.y.z]
		//	[x]
		//
		// In the former case, `x` is created implicitly by `[x.y.z]` while
		// in the latter, it is created explicitly and therefore should not
		// be allowed to be duplicated.
		var ok bool

		m := themap
		accum := make([]string, 0)
		for _, name := range key[0 : len(key)-1] {
			accum = append(accum, name)
			if _, ok = m[name]; !ok {
				implicits[strings.Join(accum, ".")] = true
				m[name] = make(map[string]interface{}, 5)
			}
			m, ok = m[name].(map[string]interface{})
			if !ok {
				return nil, fmt.Errorf("The key group '%s' is duplicated "+
					"elsewhere as a regular key.", strings.Join(accum, "."))
			}
		}

		// If the last part of the key already exists and wasn't created
		// implicitly, we've got a dupe.
		last := key[len(key)-1]
		implicitKey := strings.Join(append(accum, last), ".")
		if _, ok := m[last]; ok && !implicits[implicitKey] {
			return nil, fmt.Errorf("Key '%s' is a duplicate.", implicitKey)
		}
		return m, nil
	}
	for _, m := range ms {
		submap, err := getMap(m.key)
		if err != nil {
			return nil, err
		}
		base := m.key[len(m.key)-1]

		// At this point, maps have been created explicitly.
		// But if this is just a key group create an empty map if
		// one doesn't exist and move on.
		if m.value == nil {
			if _, ok := submap[base]; !ok {
				submap[base] = make(map[string]interface{}, 5)
			}
			continue
		}

		// We now expect that `submap[base]` is empty. Otherwise, we've
		// got a duplicate on our hands.
		if _, ok := submap[base]; ok {
			return nil, fmt.Errorf("Key '%s' is a duplicate.",
				strings.Join(m.key, "."))
		}
		submap[base] = m.value
	}

	return themap, nil
}

func parse(data string) (ms map[string]interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err, _ = r.(error)
			return
		}
	}()

	p := &parser{
		mappings: make([]*mapping, 0, 50),
		lx:       lex(data),
	}
	for {
		item := p.next()
		if item.typ == itemEOF {
			break
		}
		p.topLevel(item)
	}

	return toMap(p.mappings)
}

func (p *parser) next() item {
	it := p.lx.nextItem()
	if it.typ == itemError {
		p.errorf("Near line %d: %s", it.line, it.val)
	}
	return it
}

func (p *parser) errorf(format string, v ...interface{}) {
	panic(fmt.Errorf(format, v...))
}

func (p *parser) bug(format string, v ...interface{}) {
	log.Fatalf("BUG: %s\n\n", fmt.Sprintf(format, v...))
}

func (p *parser) expect(typ itemType) item {
	it := p.next()
	p.assertEqual(typ, it.typ)
	return it
}

func (p *parser) assertEqual(expected, got itemType) {
	if expected != got {
		p.bug("Expected '%s' but got '%s'.", expected, got)
	}
}

func (p *parser) topLevel(item item) {
	switch item.typ {
	case itemCommentStart:
		p.expect(itemText)
	case itemKeyGroupStart:
		m := newMapping()
		kg := p.expect(itemText)
		for ; kg.typ == itemText; kg = p.next() {
			m.key = append(m.key, kg.val)
		}
		p.assertEqual(itemKeyGroupEnd, kg.typ)
		p.mappings = append(p.mappings, m)
		p.context = m.key
	case itemKeyStart:
		kname := p.expect(itemText)
		m := newMapping()
		for _, k := range p.context {
			m.key = append(m.key, k)
		}
		m.key = append(m.key, kname.val)
		m.value = p.value(p.next())
		p.mappings = append(p.mappings, m)
	default:
		p.bug("Unexpected type at top level: %s", item.typ)
	}
}

func (p *parser) value(it item) interface{} {
	switch it.typ {
	case itemString:
		return replaceEscapes(it.val)
	case itemBool:
		switch it.val {
		case "true":
			return true
		case "false":
			return false
		}
		p.bug("Expected boolean value, but got '%s'.", it.val)
	case itemInteger:
		num, err := strconv.ParseInt(it.val, 10, 64)
		if err != nil {
			if e, ok := err.(*strconv.NumError); ok &&
				e.Err == strconv.ErrRange {

				p.errorf("Integer '%s' is out of the range of 64-bit "+
					"signed integers.", it.val)
			} else {
				p.bug("Expected integer value, but got '%s'.", it.val)
			}
		}
		return num
	case itemFloat:
		num, err := strconv.ParseFloat(it.val, 64)
		if err != nil {
			if e, ok := err.(*strconv.NumError); ok &&
				e.Err == strconv.ErrRange {

				p.errorf("Float '%s' is out of the range of 64-bit "+
					"IEEE-754 floating-point numbers.", it.val)
			} else {
				p.bug("Expected float value, but got '%s'.", it.val)
			}
		}
		return num
	case itemDatetime:
		t, err := time.Parse("2006-01-02T15:04:05Z", it.val)
		if err != nil {
			p.bug("Expected Zulu formatted DateTime, but got '%s'.", it.val)
		}
		return t
	case itemArrayStart:
		theType := itemNIL
		array := make([]interface{}, 0)
		for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
			if it.typ == itemCommentStart {
				p.expect(itemText)
				continue
			}

			if theType == itemNIL {
				theType = it.typ
				array = append(array, p.value(it))
				continue
			}
			if theType != it.typ {
				p.errorf("Array has values of type '%s' and '%s'.",
					theType, it.typ)
			}
			array = append(array, p.value(it))
		}
		return array
	}
	p.bug("Unexpected value type: %s", it.typ)
	panic("unreachable")
}

func replaceEscapes(s string) string {
	return strings.NewReplacer(
		"\\0", string(byte(0)),
		"\\t", "\t",
		"\\n", "\n",
		"\\r", "\r",
		"\\\"", "\"",
		"\\\\", "\\",
	).Replace(s)
}

type mappingsNice []*mapping

func (ms mappingsNice) String() string {
	buf := new(bytes.Buffer)
	for _, m := range ms {
		fmt.Fprintln(buf, strings.Join(m.key, "."))
		fmt.Fprintln(buf, m.value)
		fmt.Fprintln(buf, strings.Repeat("-", 45))
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package ledger

import (
	"bufio"
	"fmt"
	"io"
	"math/big"
	"regexp"
	"sort"
	"strings"

	date "github.com/howeyc/ledger/internal/github.com/joyt/godate"
	"github.com/marcmak/calc/calc"
)

const (
	whitespace = " \t"
)

// ParseLedger parses a ledger file and returns a list of Transactions.
//
// Transactions are sorted by date.
func ParseLedger(ledgerReader io.Reader) (generalLedger []*Transaction, err error) {
	c, e := ParseLedgerAsync(ledgerReader)
	for {
		select {
		case trans := <-c:
			generalLedger = append(generalLedger, trans)
		case err := <-e:
			sort.Sort(sortTransactionsByDate{generalLedger})
			return generalLedger, err
		}
	}
}

var accountToAmountSpace = regexp.MustCompile(" {2,}|\t+")

// ParseLedgerAsync parses a ledger file and returns a Transaction and error channels .
//
func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error) {
	c = make(chan *Transaction)
	e = make(chan error)

	go func() {

		var trans *Transaction
		scanner := bufio.NewScanner(ledgerReader)
		var line string
		var filename string
		var lineCount int

		errorMsg := func(msg string) {
			e <- fmt.Errorf("%s:%d: %s", filename, lineCount, msg)
		}

		for scanner.Scan() {
			line = scanner.Text()

			// update filename/line if sentinel comment is found
			if strings.HasPrefix(line, markerPrefix) {
				filename, lineCount = parseMarker(line)
				continue
			}

			// remove heading and tailing space from the line
			trimmedLine := strings.Trim(line, whitespace)
			lineCount++

			// handle comments
			if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >= 0 {
				trimmedLine = trimmedLine[:commentIdx]
				if len(trimmedLine) == 0 {
					continue
				}
			}

			if len(trimmedLine) == 0 {
				if trans != nil {
					transErr := balanceTransaction(trans)
					if transErr != nil {
						errorMsg("Unable to balance transaction, " + transErr.Error())
					}
					c <- trans
					trans = nil
				}
			} else if trans == nil {
				lineSplit := strings.SplitN(line, " ", 2)
				if len(lineSplit) != 2 {
					errorMsg("Unable to parse payee line: " + line)
					continue
				}
				dateString := lineSplit[0]
				transDate, dateErr := date.Parse(dateString)
				if dateErr != nil {
					errorMsg("Unable to parse date: " + dateString)
				}
				payeeString := lineSplit[1]
				trans = &Transaction{Payee: payeeString, Date: transDate}
			} else {
				var accChange Account
				lineSplit := accountToAmountSpace.Split(trimmedLine, -1)
				nonEmptyWords := []string{}
				for _, word := range lineSplit {
					if len(word) > 0 {
						nonEmptyWords = append(nonEmptyWords, word)
					}
				}
				lastIndex := len(nonEmptyWords) - 1
				balErr, rationalNum := getBalance(strings.Trim(nonEmptyWords[lastIndex], whitespace))
				if !balErr {
					// Assuming no balance and whole line is account name
					accChange.Name = strings.Join(nonEmptyWords, " ")
				} else {
					accChange.Name = strings.Join(nonEmptyWords[:lastIndex], " ")
					accChange.Balance = rationalNum
				}
				trans.AccountChanges = append(trans.AccountChanges, accChange)
			}
		}
		// If the file does not end on empty line, we must attempt to balance last
		// transaction of the file.
		if trans != nil {
			transErr := balanceTransaction(trans)
			if transErr != nil {
				errorMsg("Unable to balance transaction, " + transErr.Error())
			}
			c <- trans
			trans = nil
		}
		e <- nil
	}()
	return c, e
}

func getBalance(balance string) (bool, *big.Rat) {
	rationalNum := new(big.Rat)
	if strings.Contains(balance, "(") {
		rationalNum.SetFloat64(calc.Solve(balance))
		return true, rationalNum
	}
	_, isValid := rationalNum.SetString(balance)
	return isValid, rationalNum
}

// Takes a transaction and balances it. This is mainly to fill in the empty part
// with the remaining balance.
func balanceTransaction(input *Transaction) error {
	balance := new(big.Rat)
	var emptyFound bool
	var emptyAccIndex int
	for accIndex, accChange := range input.AccountChanges {
		if accChange.Balance == nil {
			if emptyFound {
				return fmt.Errorf("more than one account change empty")
			}
			emptyAccIndex = accIndex
			emptyFound = true
		} else {
			balance = balance.Add(balance, accChange.Balance)
		}
	}
	if balance.Sign() != 0 {
		if !emptyFound {
			return fmt.Errorf("no empty account change to place extra balance")
		}
	}
	if emptyFound {
		input.AccountChanges[emptyAccIndex].Balance = balance.Neg(balance)
	}
	return nil
}
<commit_msg>Moved ledger parsing logic into a helper function using a callback model similar to filepath.Walk so that ledger.ParseLedger doesn't have to incur memory and context switching overhead of channels and goroutines<commit_after>package ledger

import (
	"bufio"
	"fmt"
	"io"
	"math/big"
	"regexp"
	"sort"
	"strings"

	date "github.com/howeyc/ledger/internal/github.com/joyt/godate"
	"github.com/marcmak/calc/calc"
)

const (
	whitespace = " \t"
)

// ParseLedger parses a ledger file and returns a list of Transactions.
//
// Transactions are sorted by date.
func ParseLedger(ledgerReader io.Reader) (generalLedger []*Transaction, err error) {
	parseLedger(ledgerReader, func(t *Transaction, e error) (stop bool) {
		if e != nil {
			err = e
			stop = true
			return
		}

		generalLedger = append(generalLedger, t)
		return
	})

	if err != nil {
		sort.Sort(sortTransactionsByDate{generalLedger})
	}

	return
}

// ParseLedgerAsync parses a ledger file and returns a Transaction and error channels .
//
func ParseLedgerAsync(ledgerReader io.Reader) (c chan *Transaction, e chan error) {
	c = make(chan *Transaction)
	e = make(chan error)

	go func() {
		parseLedger(ledgerReader, func(t *Transaction, err error) (stop bool) {
			if err != nil {
				e <- err
			} else {
				c <- t
			}
			return
		})

		e <- nil
	}()
	return c, e
}

var accountToAmountSpace = regexp.MustCompile(" {2,}|\t+")

func parseLedger(ledgerReader io.Reader, callback func(t *Transaction, err error) (stop bool)) {
	var trans *Transaction
	scanner := bufio.NewScanner(ledgerReader)
	var line string
	var filename string
	var lineCount int

	errorMsg := func(msg string) (stop bool) {
		return callback(nil, fmt.Errorf("%s:%d: %s", filename, lineCount, msg))
	}

	for scanner.Scan() {
		line = scanner.Text()

		// update filename/line if sentinel comment is found
		if strings.HasPrefix(line, markerPrefix) {
			filename, lineCount = parseMarker(line)
			continue
		}

		// remove heading and tailing space from the line
		trimmedLine := strings.Trim(line, whitespace)
		lineCount++

		// handle comments
		if commentIdx := strings.Index(trimmedLine, ";"); commentIdx >= 0 {
			trimmedLine = trimmedLine[:commentIdx]
			if len(trimmedLine) == 0 {
				continue
			}
		}

		if len(trimmedLine) == 0 {
			if trans != nil {
				transErr := balanceTransaction(trans)
				if transErr != nil {
					errorMsg("Unable to balance transaction, " + transErr.Error())
				}
				callback(trans, nil)
				trans = nil
			}
		} else if trans == nil {
			lineSplit := strings.SplitN(line, " ", 2)
			if len(lineSplit) != 2 {
				if errorMsg("Unable to parse payee line: " + line) {
					return
				}
				continue
			}
			dateString := lineSplit[0]
			transDate, dateErr := date.Parse(dateString)
			if dateErr != nil {
				errorMsg("Unable to parse date: " + dateString)
			}
			payeeString := lineSplit[1]
			trans = &Transaction{Payee: payeeString, Date: transDate}
		} else {
			var accChange Account
			lineSplit := accountToAmountSpace.Split(trimmedLine, -1)
			var nonEmptyWords []string
			for _, word := range lineSplit {
				if len(word) > 0 {
					nonEmptyWords = append(nonEmptyWords, word)
				}
			}
			lastIndex := len(nonEmptyWords) - 1
			balErr, rationalNum := getBalance(strings.Trim(nonEmptyWords[lastIndex], whitespace))
			if !balErr {
				// Assuming no balance and whole line is account name
				accChange.Name = strings.Join(nonEmptyWords, " ")
			} else {
				accChange.Name = strings.Join(nonEmptyWords[:lastIndex], " ")
				accChange.Balance = rationalNum
			}
			trans.AccountChanges = append(trans.AccountChanges, accChange)
		}
	}
	// If the file does not end on empty line, we must attempt to balance last
	// transaction of the file.
	if trans != nil {
		transErr := balanceTransaction(trans)
		if transErr != nil {
			errorMsg("Unable to balance transaction, " + transErr.Error())
		}
		callback(trans, nil)
	}
}

func getBalance(balance string) (bool, *big.Rat) {
	rationalNum := new(big.Rat)
	if strings.Contains(balance, "(") {
		rationalNum.SetFloat64(calc.Solve(balance))
		return true, rationalNum
	}
	_, isValid := rationalNum.SetString(balance)
	return isValid, rationalNum
}

// Takes a transaction and balances it. This is mainly to fill in the empty part
// with the remaining balance.
func balanceTransaction(input *Transaction) error {
	balance := new(big.Rat)
	var emptyFound bool
	var emptyAccIndex int
	for accIndex, accChange := range input.AccountChanges {
		if accChange.Balance == nil {
			if emptyFound {
				return fmt.Errorf("more than one account change empty")
			}
			emptyAccIndex = accIndex
			emptyFound = true
		} else {
			balance = balance.Add(balance, accChange.Balance)
		}
	}
	if balance.Sign() != 0 {
		if !emptyFound {
			return fmt.Errorf("no empty account change to place extra balance")
		}
	}
	if emptyFound {
		input.AccountChanges[emptyAccIndex].Balance = balance.Neg(balance)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package parse provides a client for the Parse API.
package parse

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	userAgentHeader     = "User-Agent"
	userAgent           = "go-parse-1"
	masterKeyHeader     = "X-Parse-Master-Key"
	restAPIKeyHeader    = "X-Parse-REST-API-Key"
	sessionTokenHeader  = "X-Parse-Session-Token"
	applicationIDHeader = "X-Parse-Application-ID"
)

var (
	errEmptyApplicationID = errors.New("parse: cannot use empty ApplicationID")
	errEmptyMasterKey     = errors.New("parse: cannot use empty MasterKey")
	errEmptyRestAPIKey    = errors.New("parse: cannot use empty RestAPIKey")
	errEmptySessionToken  = errors.New("parse: cannot use empty SessionToken")

	// The default base URL for the API.
	defaultBaseURL = url.URL{
		Scheme: "https",
		Host:   "api.parse.com",
		Path:   "/1/",
	}
)

// Credentials allows for adding authentication information to a request.
type Credentials interface {
	Modify(r *http.Request) error
}

// MasterKey adds the Master Key to the request.
type MasterKey struct {
	ApplicationID string
	MasterKey     string
}

// Modify adds the Master Key header.
func (m MasterKey) Modify(r *http.Request) error {
	if m.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if m.MasterKey == "" {
		return errEmptyMasterKey
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(m.ApplicationID))
	r.Header.Set(masterKeyHeader, string(m.MasterKey))
	return nil
}

// RestAPIKey adds the Rest API Key to the request.
type RestAPIKey struct {
	ApplicationID string
	RestAPIKey    string
}

// Modify adds the Rest API Key header.
func (k RestAPIKey) Modify(r *http.Request) error {
	if k.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if k.RestAPIKey == "" {
		return errEmptyRestAPIKey
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(k.ApplicationID))
	r.Header.Set(restAPIKeyHeader, string(k.RestAPIKey))
	return nil
}

// SessionToken adds the Rest API Key and the Session Token to the request.
type SessionToken struct {
	ApplicationID string
	RestAPIKey    string
	SessionToken  string
}

// Modify adds the Session Token header.
func (t SessionToken) Modify(r *http.Request) error {
	if t.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if t.RestAPIKey == "" {
		return errEmptyRestAPIKey
	}
	if t.SessionToken == "" {
		return errEmptySessionToken
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(t.ApplicationID))
	r.Header.Set(restAPIKeyHeader, string(t.RestAPIKey))
	r.Header.Set(sessionTokenHeader, string(t.SessionToken))
	return nil
}

// An Error from the Parse API. When a valid Parse JSON error is found, the
// returned error will be of this type.
type Error struct {
	Message string `json:"error"`
	Code    int    `json:"code"`
}

func (e *Error) Error() string {
	var buf bytes.Buffer
	fmt.Fprint(&buf, "parse: api error with ")
	if e.Code != 0 {
		fmt.Fprintf(&buf, "code=%d", e.Code)
	}
	if e.Code != 0 && e.Message != "" {
		fmt.Fprint(&buf, " and ")
	}
	if e.Message != "" {
		fmt.Fprintf(&buf, "message=%q", e.Message)
	}
	return buf.String()
}

// A RawError with the HTTP StatusCode and Body. When a valid Parse JSON error
// is not found, the returned error will be of this type.
type RawError struct {
	StatusCode int
	Body       []byte
}

func (e *RawError) Error() string {
	return fmt.Sprintf("parse: error with status=%d and body=%q", e.StatusCode, e.Body)
}

// Client provides access to the Parse API.
type Client struct {
	// The underlying http.RoundTripper to perform the individual requests. When
	// nil http.DefaultTransport will be used.
	Transport http.RoundTripper

	// The base URL to parse relative URLs off. If you pass absolute URLs to
	// Client functions they are used as-is. When nil, the production Parse URL
	// will be used.
	BaseURL *url.URL

	// Credentials if set, will be included on every request.
	Credentials Credentials
}

func (c *Client) transport() http.RoundTripper {
	if c.Transport == nil {
		return http.DefaultTransport
	}
	return c.Transport
}

// Get performs a GET method call on the given url and unmarshal response into
// result.
func (c *Client) Get(u *url.URL, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "GET", URL: u}, nil, result)
}

// Post performs a POST method call on the given url with the given body and
// unmarshal response into result.
func (c *Client) Post(u *url.URL, body, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "POST", URL: u}, body, result)
}

// Put performs a PUT method call on the given url with the given body and
// unmarshal response into result.
func (c *Client) Put(u *url.URL, body, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "PUT", URL: u}, body, result)
}

// Delete performs a DELETE method call on the given url and unmarshal response
// into result.
func (c *Client) Delete(u *url.URL, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "DELETE", URL: u}, nil, result)
}

// RoundTrip performs a RoundTrip ignoring the request and response bodies. It
// is up to the caller to close them. This method modifies the request.
func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Proto = "HTTP/1.1"
	req.ProtoMajor = 1
	req.ProtoMinor = 1

	if req.URL == nil {
		if c.BaseURL == nil {
			req.URL = &defaultBaseURL
		} else {
			req.URL = c.BaseURL
		}
	} else {
		if !req.URL.IsAbs() {
			if c.BaseURL == nil {
				req.URL = defaultBaseURL.ResolveReference(req.URL)
			} else {
				req.URL = c.BaseURL.ResolveReference(req.URL)
			}
		}
	}

	if req.Host == "" {
		req.Host = req.URL.Host
	}

	if req.Header == nil {
		req.Header = make(http.Header)
	}

	req.Header.Add(userAgentHeader, userAgent)
	if c.Credentials != nil {
		if err := c.Credentials.Modify(req); err != nil {
			return nil, err
		}
	}

	res, err := c.transport().RoundTrip(req)
	if err != nil {
		return res, err
	}

	if res.StatusCode > 399 || res.StatusCode < 200 {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return res, err
		}

		if len(body) > 0 {
			var apiErr Error
			if json.Unmarshal(body, &apiErr) == nil {
				return res, &apiErr
			}
		}
		return res, &RawError{
			StatusCode: res.StatusCode,
			Body:       body,
		}
	}

	return res, nil
}

// Do performs a Parse API call. This method modifies the request and adds the
// Authentication headers. The body is JSON encoded and for responses in the
// 2xx or 3xx range the response will be JSON decoded into result, for others
// an error of type Error will be returned.
func (c *Client) Do(req *http.Request, body, result interface{}) (*http.Response, error) {
	// we need to buffer as Parse requires a Content-Length
	if body != nil {
		bd, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		if req.Header == nil {
			req.Header = make(http.Header)
		}
		req.Header.Set("Content-Type", "application/json")
		req.Body = ioutil.NopCloser(bytes.NewReader(bd))
		req.ContentLength = int64(len(bd))
	}

	res, err := c.RoundTrip(req)
	if err != nil {
		return res, err
	}
	defer res.Body.Close()

	if result != nil {
		if err := json.NewDecoder(res.Body).Decode(result); err != nil {
			return res, err
		}
	}
	return res, nil
}

// WithCredentials returns a new instance of the Client using the given
// Credentials. It discards the previous Credentials.
func (c *Client) WithCredentials(cr Credentials) *Client {
	var c2 Client
	c2 = *c
	c2.Credentials = cr
	return &c2
}
<commit_msg>Ensure response body is closed.<commit_after>// Package parse provides a client for the Parse API.
package parse

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
)

const (
	userAgentHeader     = "User-Agent"
	userAgent           = "go-parse-1"
	masterKeyHeader     = "X-Parse-Master-Key"
	restAPIKeyHeader    = "X-Parse-REST-API-Key"
	sessionTokenHeader  = "X-Parse-Session-Token"
	applicationIDHeader = "X-Parse-Application-ID"
)

var (
	errEmptyApplicationID = errors.New("parse: cannot use empty ApplicationID")
	errEmptyMasterKey     = errors.New("parse: cannot use empty MasterKey")
	errEmptyRestAPIKey    = errors.New("parse: cannot use empty RestAPIKey")
	errEmptySessionToken  = errors.New("parse: cannot use empty SessionToken")

	// The default base URL for the API.
	defaultBaseURL = url.URL{
		Scheme: "https",
		Host:   "api.parse.com",
		Path:   "/1/",
	}
)

// Credentials allows for adding authentication information to a request.
type Credentials interface {
	Modify(r *http.Request) error
}

// MasterKey adds the Master Key to the request.
type MasterKey struct {
	ApplicationID string
	MasterKey     string
}

// Modify adds the Master Key header.
func (m MasterKey) Modify(r *http.Request) error {
	if m.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if m.MasterKey == "" {
		return errEmptyMasterKey
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(m.ApplicationID))
	r.Header.Set(masterKeyHeader, string(m.MasterKey))
	return nil
}

// RestAPIKey adds the Rest API Key to the request.
type RestAPIKey struct {
	ApplicationID string
	RestAPIKey    string
}

// Modify adds the Rest API Key header.
func (k RestAPIKey) Modify(r *http.Request) error {
	if k.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if k.RestAPIKey == "" {
		return errEmptyRestAPIKey
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(k.ApplicationID))
	r.Header.Set(restAPIKeyHeader, string(k.RestAPIKey))
	return nil
}

// SessionToken adds the Rest API Key and the Session Token to the request.
type SessionToken struct {
	ApplicationID string
	RestAPIKey    string
	SessionToken  string
}

// Modify adds the Session Token header.
func (t SessionToken) Modify(r *http.Request) error {
	if t.ApplicationID == "" {
		return errEmptyApplicationID
	}
	if t.RestAPIKey == "" {
		return errEmptyRestAPIKey
	}
	if t.SessionToken == "" {
		return errEmptySessionToken
	}
	if r.Header == nil {
		r.Header = make(http.Header)
	}
	r.Header.Set(applicationIDHeader, string(t.ApplicationID))
	r.Header.Set(restAPIKeyHeader, string(t.RestAPIKey))
	r.Header.Set(sessionTokenHeader, string(t.SessionToken))
	return nil
}

// An Error from the Parse API. When a valid Parse JSON error is found, the
// returned error will be of this type.
type Error struct {
	Message string `json:"error"`
	Code    int    `json:"code"`
}

func (e *Error) Error() string {
	var buf bytes.Buffer
	fmt.Fprint(&buf, "parse: api error with ")
	if e.Code != 0 {
		fmt.Fprintf(&buf, "code=%d", e.Code)
	}
	if e.Code != 0 && e.Message != "" {
		fmt.Fprint(&buf, " and ")
	}
	if e.Message != "" {
		fmt.Fprintf(&buf, "message=%q", e.Message)
	}
	return buf.String()
}

// A RawError with the HTTP StatusCode and Body. When a valid Parse JSON error
// is not found, the returned error will be of this type.
type RawError struct {
	StatusCode int
	Body       []byte
}

func (e *RawError) Error() string {
	return fmt.Sprintf("parse: error with status=%d and body=%q", e.StatusCode, e.Body)
}

// Client provides access to the Parse API.
type Client struct {
	// The underlying http.RoundTripper to perform the individual requests. When
	// nil http.DefaultTransport will be used.
	Transport http.RoundTripper

	// The base URL to parse relative URLs off. If you pass absolute URLs to
	// Client functions they are used as-is. When nil, the production Parse URL
	// will be used.
	BaseURL *url.URL

	// Credentials if set, will be included on every request.
	Credentials Credentials
}

func (c *Client) transport() http.RoundTripper {
	if c.Transport == nil {
		return http.DefaultTransport
	}
	return c.Transport
}

// Get performs a GET method call on the given url and unmarshal response into
// result.
func (c *Client) Get(u *url.URL, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "GET", URL: u}, nil, result)
}

// Post performs a POST method call on the given url with the given body and
// unmarshal response into result.
func (c *Client) Post(u *url.URL, body, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "POST", URL: u}, body, result)
}

// Put performs a PUT method call on the given url with the given body and
// unmarshal response into result.
func (c *Client) Put(u *url.URL, body, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "PUT", URL: u}, body, result)
}

// Delete performs a DELETE method call on the given url and unmarshal response
// into result.
func (c *Client) Delete(u *url.URL, result interface{}) (*http.Response, error) {
	return c.Do(&http.Request{Method: "DELETE", URL: u}, nil, result)
}

// RoundTrip performs a RoundTrip ignoring the request and response bodies. It
// is up to the caller to close them. This method modifies the request.
func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Proto = "HTTP/1.1"
	req.ProtoMajor = 1
	req.ProtoMinor = 1

	if req.URL == nil {
		if c.BaseURL == nil {
			req.URL = &defaultBaseURL
		} else {
			req.URL = c.BaseURL
		}
	} else {
		if !req.URL.IsAbs() {
			if c.BaseURL == nil {
				req.URL = defaultBaseURL.ResolveReference(req.URL)
			} else {
				req.URL = c.BaseURL.ResolveReference(req.URL)
			}
		}
	}

	if req.Host == "" {
		req.Host = req.URL.Host
	}

	if req.Header == nil {
		req.Header = make(http.Header)
	}

	req.Header.Add(userAgentHeader, userAgent)
	if c.Credentials != nil {
		if err := c.Credentials.Modify(req); err != nil {
			return nil, err
		}
	}

	res, err := c.transport().RoundTrip(req)
	if res != nil && res.Body != nil {
		defer res.Body.Close()
	}
	if err != nil {
		return res, err
	}

	if res.StatusCode > 399 || res.StatusCode < 200 {
		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return res, err
		}

		if len(body) > 0 {
			var apiErr Error
			if json.Unmarshal(body, &apiErr) == nil {
				return res, &apiErr
			}
		}
		return res, &RawError{
			StatusCode: res.StatusCode,
			Body:       body,
		}
	}

	return res, nil
}

// Do performs a Parse API call. This method modifies the request and adds the
// Authentication headers. The body is JSON encoded and for responses in the
// 2xx or 3xx range the response will be JSON decoded into result, for others
// an error of type Error will be returned.
func (c *Client) Do(req *http.Request, body, result interface{}) (*http.Response, error) {
	// we need to buffer as Parse requires a Content-Length
	if body != nil {
		bd, err := json.Marshal(body)
		if err != nil {
			return nil, err
		}
		if req.Header == nil {
			req.Header = make(http.Header)
		}
		req.Header.Set("Content-Type", "application/json")
		req.Body = ioutil.NopCloser(bytes.NewReader(bd))
		req.ContentLength = int64(len(bd))
	}

	res, err := c.RoundTrip(req)
	if err != nil {
		return res, err
	}
	defer res.Body.Close()

	if result != nil {
		if err := json.NewDecoder(res.Body).Decode(result); err != nil {
			return res, err
		}
	}
	return res, nil
}

// WithCredentials returns a new instance of the Client using the given
// Credentials. It discards the previous Credentials.
func (c *Client) WithCredentials(cr Credentials) *Client {
	var c2 Client
	c2 = *c
	c2.Credentials = cr
	return &c2
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"bufio"
	"fmt"
	"io"
	"strconv"
)

const (
	_ = -iota
	EOF
	WORD
)

func parse(r io.Reader, name string) error {
	p := &parser{
		r:    bufio.NewReader(r),
		name: name,
		line: 1,
		col:  1,
	}
	p.program()
	return p.err
}

type parser struct {
	r    *bufio.Reader
	tok  int32
	err  error
	name string
	line int
	col  int
}

var reserved = map[rune]bool{
	'\n': true,
	'#':  true,
	'=':  true,
	'&':  true,
	'>':  true,
	'<':  true,
	'|':  true,
	';':  true,
	'(':  true,
	')':  true,
	'{':  true,
	'}':  true,
	'"':  true,
	'\'': true,
}

var space = map[rune]bool{
	' ':  true,
	'\t': true,
}

func (p *parser) next() {
	if p.err != nil {
		return
	}
	r := ' '
	var err error
	for space[r] {
		r, _, err = p.r.ReadRune()
		if err == io.EOF {
			p.tok = EOF
			return
		}
		if err != nil {
			p.errPass(err)
			return
		}
		p.col++
	}
	if reserved[r] {
		if r == '\n' {
			p.line++
			p.col = 1
		}
		p.tok = r
		return
	}
	read := false
	for !reserved[r] && !space[r] {
		r, _, err = p.r.ReadRune()
		if err == io.EOF {
			break
		}
		if err != nil {
			p.errPass(err)
			return
		}
		p.col++
		read = true
	}
	if read {
		if err := p.r.UnreadRune(); err != nil {
			p.errPass(err)
			return
		}
		p.col--
	}
	p.tok = WORD
	return
}

func (p *parser) discardUpTo(delim byte) {
	_, err := p.r.ReadBytes(delim)
	if err == io.EOF {
		p.tok = EOF
	} else if err != nil {
		p.errPass(err)
	}
}

func (p *parser) got(tok int32) bool {
	if p.tok == tok {
		p.next()
		return true
	}
	return false
}

func (p *parser) want(tok int32) {
	if p.tok != tok {
		p.errWanted(tok)
		return
	}
	p.next()
}

func tokStr(tok int32) string {
	switch tok {
	case EOF:
		return "EOF"
	case WORD:
		return "word"
	default:
		return strconv.QuoteRune(tok)
	}
}

func (p *parser) errPass(err error) {
	p.err = err
	p.tok = EOF
}

func (p *parser) lineErr(format string, v ...interface{}) {
	pos := fmt.Sprintf("%s:%d:%d: ", p.name, p.line, p.col)
	p.errPass(fmt.Errorf(pos + format, v...))
}

func (p *parser) errUnexpected() {
	p.lineErr("unexpected token %s", tokStr(p.tok))
}

func (p *parser) errWanted(tok int32) {
	p.lineErr("unexpected token %s, wanted %s", tokStr(p.tok), tokStr(tok))
}

func (p *parser) program() {
	p.next()
	for p.tok != EOF {
		p.command()
	}
}

func (p *parser) command() {
	switch {
	case p.got('\n'):
	case p.got('#'):
		p.discardUpTo('\n')
		p.next()
	case p.got('"'):
		p.strContent('"')
	case p.got('\''):
		p.strContent('\'')
	case p.got(WORD):
		switch {
		case p.got('='):
			p.got(WORD)
		case p.got('&'):
			if p.got('&') {
				p.command()
				return
			}
		case p.got('|'):
			p.got('|')
			p.command()
			return
		case p.got('('):
			p.want(')')
			p.want('{')
			for !p.got('}') {
				p.command()
			}
		default:
			for p.got(WORD) {
			}
		}
		for p.tok != EOF {
			switch {
			case p.got('>'):
				switch {
				case p.got('>'):
				case p.got('&'):
				}
				p.want(WORD)
			case p.got('<'):
				p.want(WORD)
			case p.got(';'):
				return
			case p.got('\n'):
				return
			default:
				p.errUnexpected()
			}
		}
	case p.got('{'):
		for !p.got('}') {
			p.command()
		}
		p.got(';')
		p.got('\n')
	default:
		p.errUnexpected()
	}
}

func (p *parser) strContent(delim byte) {
	_, err := p.r.ReadBytes(delim)
	if err != nil {
		p.errPass(err)
	}
}
<commit_msg>Fix last for that didn't check for EOF<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import (
	"bufio"
	"fmt"
	"io"
	"strconv"
)

const (
	_ = -iota
	EOF
	WORD
)

func parse(r io.Reader, name string) error {
	p := &parser{
		r:    bufio.NewReader(r),
		name: name,
		line: 1,
		col:  1,
	}
	p.program()
	return p.err
}

type parser struct {
	r    *bufio.Reader
	tok  int32
	err  error
	name string
	line int
	col  int
}

var reserved = map[rune]bool{
	'\n': true,
	'#':  true,
	'=':  true,
	'&':  true,
	'>':  true,
	'<':  true,
	'|':  true,
	';':  true,
	'(':  true,
	')':  true,
	'{':  true,
	'}':  true,
	'"':  true,
	'\'': true,
}

var space = map[rune]bool{
	' ':  true,
	'\t': true,
}

func (p *parser) next() {
	if p.err != nil {
		return
	}
	r := ' '
	var err error
	for space[r] {
		r, _, err = p.r.ReadRune()
		if err == io.EOF {
			p.tok = EOF
			return
		}
		if err != nil {
			p.errPass(err)
			return
		}
		p.col++
	}
	if reserved[r] {
		if r == '\n' {
			p.line++
			p.col = 1
		}
		p.tok = r
		return
	}
	read := false
	for !reserved[r] && !space[r] {
		r, _, err = p.r.ReadRune()
		if err == io.EOF {
			break
		}
		if err != nil {
			p.errPass(err)
			return
		}
		p.col++
		read = true
	}
	if read {
		if err := p.r.UnreadRune(); err != nil {
			p.errPass(err)
			return
		}
		p.col--
	}
	p.tok = WORD
	return
}

func (p *parser) discardUpTo(delim byte) {
	_, err := p.r.ReadBytes(delim)
	if err == io.EOF {
		p.tok = EOF
	} else if err != nil {
		p.errPass(err)
	}
}

func (p *parser) got(tok int32) bool {
	if p.tok == tok {
		p.next()
		return true
	}
	return false
}

func (p *parser) want(tok int32) {
	if p.tok != tok {
		p.errWanted(tok)
		return
	}
	p.next()
}

func tokStr(tok int32) string {
	switch tok {
	case EOF:
		return "EOF"
	case WORD:
		return "word"
	default:
		return strconv.QuoteRune(tok)
	}
}

func (p *parser) errPass(err error) {
	p.err = err
	p.tok = EOF
}

func (p *parser) lineErr(format string, v ...interface{}) {
	pos := fmt.Sprintf("%s:%d:%d: ", p.name, p.line, p.col)
	p.errPass(fmt.Errorf(pos+format, v...))
}

func (p *parser) errUnexpected() {
	p.lineErr("unexpected token %s", tokStr(p.tok))
}

func (p *parser) errWanted(tok int32) {
	p.lineErr("unexpected token %s, wanted %s", tokStr(p.tok), tokStr(tok))
}

func (p *parser) program() {
	p.next()
	for p.tok != EOF {
		p.command()
	}
}

func (p *parser) command() {
	switch {
	case p.got('\n'):
	case p.got('#'):
		p.discardUpTo('\n')
		p.next()
	case p.got('"'):
		p.strContent('"')
	case p.got('\''):
		p.strContent('\'')
	case p.got(WORD):
		switch {
		case p.got('='):
			p.got(WORD)
		case p.got('&'):
			if p.got('&') {
				p.command()
				return
			}
		case p.got('|'):
			p.got('|')
			p.command()
			return
		case p.got('('):
			p.want(')')
			p.want('{')
			for p.tok != EOF && !p.got('}') {
				p.command()
			}
		default:
			for p.got(WORD) {
			}
		}
		for p.tok != EOF {
			switch {
			case p.got('>'):
				switch {
				case p.got('>'):
				case p.got('&'):
				}
				p.want(WORD)
			case p.got('<'):
				p.want(WORD)
			case p.got(';'):
				return
			case p.got('\n'):
				return
			default:
				p.errUnexpected()
			}
		}
	case p.got('{'):
		for !p.got('}') {
			p.command()
		}
		p.got(';')
		p.got('\n')
	default:
		p.errUnexpected()
	}
}

func (p *parser) strContent(delim byte) {
	_, err := p.r.ReadBytes(delim)
	if err != nil {
		p.errPass(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/fatih/color"
)

func viewCommand(flags viewFlags, testname string) {
	if flags.test {
		color.Cyan("TEST...")
		path := ""
		testType := ""
		if flags.testSet == optimizerStandalone {
			path = buildPath(asmDir, testname+".asm")
			testType = "asm"
		} else {
			path = buildPath(pikaDir, testname+".pika")
			testType = "pika"
		}

		if !exists(path) {
			color.Magenta("the " + testType + " test " + testname + " could not be found")
			color.Magenta(path + " does not exist")
			os.Exit(1)
		}
		bytes, err := ioutil.ReadFile(path)
		crashOnError(err)
		fmt.Print(string(bytes))
	}
	if flags.asm {
		phase := asm
		color.Cyan("ASM...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.build {
		phase := build
		color.Cyan("BUILD...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.run {
		phase := run
		color.Cyan("RUN...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.testSet == codegenerator && (flags.asmo || flags.buildo) {
		color.Magenta("there is no reoptimize phase for the codegenerator")
		return
	}
	if flags.asmo {
		phase := asmo
		color.Cyan("ASMO...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.buildo {
		phase := buildo
		color.Cyan("ASMO...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
}

// TODO need to take into account asmo files
func viewOutput(phase, testSet, testname string, diff bool) {
	var resultPath, expectPath string
	if phase == asm || phase == asmo {
		if testSet == optimizer || testSet == optimizerStandalone ||
			phase == asmo {
			resultPath = buildPath(resultDir, phase, testSet, testname+asmoExt)
			expectPath = buildPath(expectDir, phase, testSet, testname+asmoExt)
		} else {
			resultPath = buildPath(resultDir, phase, testSet, testname+asmExt)
			expectPath = buildPath(expectDir, phase, testSet, testname+asmExt)
		}
	} else {
		resultPath = buildPath(resultDir, phase, testSet, testname+txtExt)
		expectPath = buildPath(expectDir, phase, testSet, testname+txtExt)
	}

	if diff {
		color.Yellow("diff...")
		if exists(resultPath) && exists(expectPath) {
			output := makeDiff(expectPath, resultPath)
			printDiff(output)
		}

		if !exists(expectPath) {
			color.Magenta("there is no expectation set for " + testname)
			color.Magenta(expectPath + " does not exist")
		}

		if !exists(resultPath) {
			color.Magenta("there is no result set for " + testname)
			color.Magenta(resultPath + " does not exist")
		}

	} else {
		color.Yellow("expect...")
		if !exists(expectPath) {
			color.Magenta("there is no expectation set for " + testname)
			color.Magenta(expectPath + " does not exist")
		} else {
			exp, err := ioutil.ReadFile(expectPath)
			crashOnError(err)

			fmt.Print(string(exp))
		}

		color.Yellow("result...")
		if !exists(resultPath) {
			color.Magenta("there is no result set for " + testname)
			color.Magenta(resultPath + " does not exist")
		} else {
			res, err := ioutil.ReadFile(resultPath)
			crashOnError(err)

			fmt.Print(string(res))

		}

	}
}

func makeDiff(expectPath, resultPath string) string {
	gitDiff := exec.Command("git", "diff", "--no-index", expectPath, resultPath)
	var stdOut bytes.Buffer
	gitDiff.Stdout = &stdOut
	gitDiff.Run()
	return string(stdOut.Bytes())
}

func printDiff(diff string) {
	lines := strings.Split(diff, "\n")
	if len(lines) < 4 {
		return
	}
	for _, line := range lines[2:] {
		if len(line) == 0 {
			continue
		}
		r := line[0]
		lightBlue := color.New(color.FgHiBlue)
		switch r {
		case '+':
			color.Green(line)
		case '-':
			color.Red(line)
		case '@':
			lightBlue.Println(line)
		default:
			fmt.Println(line)
		}
	}
}
<commit_msg>slightly refactor view command error handling<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/fatih/color"
)

func viewCommand(flags viewFlags, testname string) {
	if flags.test {
		color.Cyan("TEST...")
		path := ""
		if flags.testSet == optimizerStandalone {
			path = buildPath(asmDir, testname+".asm")
		} else {
			path = buildPath(pikaDir, testname+".pika")
		}

		if !exists(path) {
			color.Magenta(path + " does not exist")
			os.Exit(1)
			return
		}
		bytes, err := ioutil.ReadFile(path)
		crashOnError(err)
		fmt.Print(string(bytes))
	}
	if flags.asm {
		phase := asm
		color.Cyan("ASM...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.build {
		phase := build
		color.Cyan("BUILD...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.run {
		phase := run
		color.Cyan("RUN...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.testSet == codegenerator && (flags.asmo || flags.buildo) {
		color.Magenta("there is no reoptimize phase for the codegenerator")
		return
	}
	if flags.asmo {
		phase := asmo
		color.Cyan("ASMO...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
	if flags.buildo {
		phase := buildo
		color.Cyan("ASMO...")
		viewOutput(phase, flags.testSet, testname, flags.diff)
	}
}

// TODO need to take into account asmo files
func viewOutput(phase, testSet, testname string, diff bool) {
	var resultPath, expectPath string
	if phase == asm || phase == asmo {
		if testSet == optimizer || testSet == optimizerStandalone ||
			phase == asmo {
			resultPath = buildPath(resultDir, phase, testSet, testname+asmoExt)
			expectPath = buildPath(expectDir, phase, testSet, testname+asmoExt)
		} else {
			resultPath = buildPath(resultDir, phase, testSet, testname+asmExt)
			expectPath = buildPath(expectDir, phase, testSet, testname+asmExt)
		}
	} else {
		resultPath = buildPath(resultDir, phase, testSet, testname+txtExt)
		expectPath = buildPath(expectDir, phase, testSet, testname+txtExt)
	}

	if diff {
		color.Yellow("diff...")
		if exists(resultPath) && exists(expectPath) {
			output := makeDiff(expectPath, resultPath)
			printDiff(output)
		}

		if !exists(expectPath) {
			color.Magenta("there is no expectation set for " + testname)
			color.Magenta(expectPath + " does not exist")
		}

		if !exists(resultPath) {
			color.Magenta("there is no result set for " + testname)
			color.Magenta(resultPath + " does not exist")
		}

	} else {
		color.Yellow("expect...")
		if !exists(expectPath) {
			color.Magenta("there is no expectation set for " + testname)
			color.Magenta(expectPath + " does not exist")
		} else {
			exp, err := ioutil.ReadFile(expectPath)
			crashOnError(err)

			fmt.Print(string(exp))
		}

		color.Yellow("result...")
		if !exists(resultPath) {
			color.Magenta("there is no result set for " + testname)
			color.Magenta(resultPath + " does not exist")
		} else {
			res, err := ioutil.ReadFile(resultPath)
			crashOnError(err)

			fmt.Print(string(res))

		}

	}
}

func makeDiff(expectPath, resultPath string) string {
	gitDiff := exec.Command("git", "diff", "--no-index", expectPath, resultPath)
	var stdOut bytes.Buffer
	gitDiff.Stdout = &stdOut
	gitDiff.Run()
	return string(stdOut.Bytes())
}

func printDiff(diff string) {
	lines := strings.Split(diff, "\n")
	if len(lines) < 4 {
		return
	}
	for _, line := range lines[2:] {
		if len(line) == 0 {
			continue
		}
		r := line[0]
		lightBlue := color.New(color.FgHiBlue)
		switch r {
		case '+':
			color.Green(line)
		case '-':
			color.Red(line)
		case '@':
			lightBlue.Println(line)
		default:
			fmt.Println(line)
		}
	}
}
<|endoftext|>


<commit_before>package epochs

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"math/big"
	"time"
)

const secondsPerDay = 24 * 60 * 60
const nanosecondsPerDay = secondsPerDay * 1e9

// epoch2time gets a Unix time of the given x after dividing by q and
// adding s.
func epoch2time(x, q, s *big.Int) time.Time {
	z := new(big.Int)
	m := new(big.Int)
	z.DivMod(x, q, m)
	z.Add(z, s)
	r := m.Mul(m, big.NewInt(1e9)).Div(m, q)
	return time.Unix(z.Int64(), r.Int64()).UTC()
}

// time2epoch reverses epoch2time.
func time2epoch(t time.Time, m, s *big.Int) int64 {
	bf := new(big.Float).SetInt(big.NewInt(t.UnixNano()))
	bf.Quo(bf, big.NewFloat(1e9))
	bf.Sub(bf, new(big.Float).SetInt(s))
	bf.Mul(bf, new(big.Float).SetInt(m))

	r, acc := bf.Int64()
	if acc != big.Exact {
		fmt.Println(acc)
	}

	return r
}

// Chrome time is the number of microseconds since 1601-01-01, which
// is 11,644,473,600 seconds before the Unix epoch.
func Chrome(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(-11644473600),
	)
}

// ToChrome returns the Chrome time for the given time.Time.
func ToChrome(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(-11644473600),
	)
}

// Cocoa time is the number of seconds since 2001-01-01, which
// is 978,307,200 seconds after the Unix epoch.
func Cocoa(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1),
		big.NewInt(978307200),
	)
}

// ToCocoa returns the Cocoa time for the given time.Time.
func ToCocoa(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1),
		big.NewInt(978307200),
	)
}

// GoogleCalendar seems to count 32-day months from the day before the
// Unix epoch. @noppers worked out how to do this.
func GoogleCalendar(num int64) time.Time {

	n := int(num)

	totalDays := n / secondsPerDay
	seconds := n % secondsPerDay

	// A "Google month" has 32 days!
	months := totalDays / 32
	days := totalDays % 32

	// The "Google epoch" is apparently off by a day.
	t := time.Unix(-secondsPerDay, 0).UTC()

	// Add the days first...
	u := t.AddDate(0, 0, days)

	// ...then the months...
	v := u.AddDate(0, months, 0)

	// ...then the seconds.
	w := v.Add(time.Duration(seconds * 1e9))

	return w
}

// ToGoogleCalendar returns the GoogleCalendar time for the given time.Time.
func ToGoogleCalendar(t time.Time) int64 {
	y := t.Year() - 1970
	m := int(t.Month()) - 1
	r := ((((y*12+m)*32+t.Day())*24+t.Hour())*60+t.Minute())*60 + t.Second()
	return int64(r)
}

// ICQ time is the number of days since 1899-12-30, which is
// 2,209,161,600 seconds before the Unix epoch. Days can have a
// fractional part.
func ICQ(days float64) time.Time {

	t := time.Unix(-2209161600, 0).UTC()

	intdays := int(days)

	// Want the fractional part of the day in nanoseconds.
	fracday := int64((days - float64(intdays)) * nanosecondsPerDay)

	return t.AddDate(0, 0, intdays).Add(time.Duration(fracday))
}

// ToICQ returns the ICQ time for the given time.Time.
func ToICQ(t time.Time) float64 {
	t2 := time.Unix(-2209161600, 0)
	return float64(t.Sub(t2).Nanoseconds()) / nanosecondsPerDay
}

// Java time is the number of milliseconds since the Unix epoch.
func Java(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1000),
		big.NewInt(0),
	)
}

// ToJava returns the Java time for the given time.Time.
func ToJava(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1000),
		big.NewInt(0),
	)
}

// Mozilla time (e.g., formhistory.sqlite) is the number of
// microseconds since the Unix epoch.
func Mozilla(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(0),
	)
}

// ToMozilla returns the Mozilla time for the given time.Time.
func ToMozilla(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(0),
	)
}

// OLE time is the number of days since 1899-12-30, which is
// 2,209,161,600 seconds before the Unix epoch. Days can have a
// fractional part and is given as a string of hex characters
// representing an IEEE 8-byte floating-point number.
func OLE(days string) time.Time {
	var d [8]byte
	var f float64

	n, err := fmt.Sscanf(
		days,
		"%02x%02x%02x%02x%02x%02x%02x%02x",
		&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7],
	)
	if err != nil {
		fmt.Println("fmt.Sscanf failed:", err)
	}
	if n != 8 {
		fmt.Println("fmt.Sscanf did not scan 8 items:", n)
	}

	buf := bytes.NewReader(d[:])
	if err := binary.Read(buf, binary.LittleEndian, &f); err != nil {
		fmt.Println("binary.Read failed:", err)
	}

	return ICQ(f)
}

// ToOLE returns the OLE time for the given time.Time.
func ToOLE(t time.Time) string {
	icq := ToICQ(t)
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.LittleEndian, math.Float64bits(icq))
	if err != nil {
		fmt.Println("binary.Write failed:", err)
	}
	return fmt.Sprintf("%016x", buf.Bytes())
}

// Symbian time is the number of microseconds since the year 0, which
// is 62,167,219,200 seconds before the Unix epoch.
func Symbian(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(-62167219200),
	)
}

// ToSymbian returns the Symbian time for the given time.Time.
func ToSymbian(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(-62167219200),
	)
}

// Unix time is the number of seconds since 1970-01-01.
func Unix(num int64) time.Time {
	return time.Unix(num, 0).UTC()
}

// ToUnix returns the Unix time for the given time.Time.
func ToUnix(t time.Time) int64 {
	return t.Unix()
}

// UUIDv1 time (RFC 4122) is the number of hectonanoseconds (100 ns)
// since 1582-10-15, which is 12,219,292,800 seconds before the Unix
// epoch.
func UUIDv1(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-12219292800),
	)
}

// ToUUIDv1 returns the UUID version 1 time for the given time.Time.
func ToUUIDv1(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-12219292800),
	)
}

// WindowsDate time (e.g., .NET) is the number of hectonanoseconds
// (100 ns) since 0001-01-01, which is 62,135,596,800 seconds before
// the Unix epoch.
func WindowsDate(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-62135596800),
	)
}

// ToWindowsDate returns the WindowsDate time for the given time.Time.
func ToWindowsDate(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-62135596800),
	)
}

// WindowsFile time (e.g., NTFS) is the number of hectonanoseconds
// (100 ns) since 1601-01-01, which is 11,644,473,600 seconds before
// the Unix epoch.
func WindowsFile(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-11644473600),
	)
}

// ToWindowsFile returns the WindowsFile time for the given time.Time.
func ToWindowsFile(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-11644473600),
	)
}
<commit_msg>Add APFS time functions.<commit_after>package epochs

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"math"
	"math/big"
	"time"
)

const secondsPerDay = 24 * 60 * 60
const nanosecondsPerDay = secondsPerDay * 1e9

// epoch2time gets a Unix time of the given x after dividing by q and
// adding s.
func epoch2time(x, q, s *big.Int) time.Time {
	z := new(big.Int)
	m := new(big.Int)
	z.DivMod(x, q, m)
	z.Add(z, s)
	r := m.Mul(m, big.NewInt(1e9)).Div(m, q)
	return time.Unix(z.Int64(), r.Int64()).UTC()
}

// time2epoch reverses epoch2time.
func time2epoch(t time.Time, m, s *big.Int) int64 {
	bf := new(big.Float).SetInt(big.NewInt(t.UnixNano()))
	bf.Quo(bf, big.NewFloat(1e9))
	bf.Sub(bf, new(big.Float).SetInt(s))
	bf.Mul(bf, new(big.Float).SetInt(m))

	r, acc := bf.Int64()
	if acc != big.Exact {
		fmt.Println(acc)
	}

	return r
}

// APFS time is the number of nanoseconds since the Unix epoch.
// Cf., APFS filesystem format (https://blog.cugu.eu/post/apfs/).
func APFS(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e9),
		big.NewInt(0),
	)
}

// ToAPFS returns the APFS time for the given time.Time.
func ToAPFS(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e9),
		big.NewInt(0),
	)
}

// Chrome time is the number of microseconds since 1601-01-01, which
// is 11,644,473,600 seconds before the Unix epoch.
func Chrome(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(-11644473600),
	)
}

// ToChrome returns the Chrome time for the given time.Time.
func ToChrome(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(-11644473600),
	)
}

// Cocoa time is the number of seconds since 2001-01-01, which
// is 978,307,200 seconds after the Unix epoch.
func Cocoa(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1),
		big.NewInt(978307200),
	)
}

// ToCocoa returns the Cocoa time for the given time.Time.
func ToCocoa(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1),
		big.NewInt(978307200),
	)
}

// GoogleCalendar seems to count 32-day months from the day before the
// Unix epoch. @noppers worked out how to do this.
func GoogleCalendar(num int64) time.Time {

	n := int(num)

	totalDays := n / secondsPerDay
	seconds := n % secondsPerDay

	// A "Google month" has 32 days!
	months := totalDays / 32
	days := totalDays % 32

	// The "Google epoch" is apparently off by a day.
	t := time.Unix(-secondsPerDay, 0).UTC()

	// Add the days first...
	u := t.AddDate(0, 0, days)

	// ...then the months...
	v := u.AddDate(0, months, 0)

	// ...then the seconds.
	w := v.Add(time.Duration(seconds * 1e9))

	return w
}

// ToGoogleCalendar returns the GoogleCalendar time for the given time.Time.
func ToGoogleCalendar(t time.Time) int64 {
	y := t.Year() - 1970
	m := int(t.Month()) - 1
	r := ((((y*12+m)*32+t.Day())*24+t.Hour())*60+t.Minute())*60 + t.Second()
	return int64(r)
}

// ICQ time is the number of days since 1899-12-30, which is
// 2,209,161,600 seconds before the Unix epoch. Days can have a
// fractional part.
func ICQ(days float64) time.Time {

	t := time.Unix(-2209161600, 0).UTC()

	intdays := int(days)

	// Want the fractional part of the day in nanoseconds.
	fracday := int64((days - float64(intdays)) * nanosecondsPerDay)

	return t.AddDate(0, 0, intdays).Add(time.Duration(fracday))
}

// ToICQ returns the ICQ time for the given time.Time.
func ToICQ(t time.Time) float64 {
	t2 := time.Unix(-2209161600, 0)
	return float64(t.Sub(t2).Nanoseconds()) / nanosecondsPerDay
}

// Java time is the number of milliseconds since the Unix epoch.
func Java(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1000),
		big.NewInt(0),
	)
}

// ToJava returns the Java time for the given time.Time.
func ToJava(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1000),
		big.NewInt(0),
	)
}

// Mozilla time (e.g., formhistory.sqlite) is the number of
// microseconds since the Unix epoch.
func Mozilla(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(0),
	)
}

// ToMozilla returns the Mozilla time for the given time.Time.
func ToMozilla(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(0),
	)
}

// OLE time is the number of days since 1899-12-30, which is
// 2,209,161,600 seconds before the Unix epoch. Days can have a
// fractional part and is given as a string of hex characters
// representing an IEEE 8-byte floating-point number.
func OLE(days string) time.Time {
	var d [8]byte
	var f float64

	n, err := fmt.Sscanf(
		days,
		"%02x%02x%02x%02x%02x%02x%02x%02x",
		&d[0], &d[1], &d[2], &d[3], &d[4], &d[5], &d[6], &d[7],
	)
	if err != nil {
		fmt.Println("fmt.Sscanf failed:", err)
	}
	if n != 8 {
		fmt.Println("fmt.Sscanf did not scan 8 items:", n)
	}

	buf := bytes.NewReader(d[:])
	if err := binary.Read(buf, binary.LittleEndian, &f); err != nil {
		fmt.Println("binary.Read failed:", err)
	}

	return ICQ(f)
}

// ToOLE returns the OLE time for the given time.Time.
func ToOLE(t time.Time) string {
	icq := ToICQ(t)
	buf := new(bytes.Buffer)
	err := binary.Write(buf, binary.LittleEndian, math.Float64bits(icq))
	if err != nil {
		fmt.Println("binary.Write failed:", err)
	}
	return fmt.Sprintf("%016x", buf.Bytes())
}

// Symbian time is the number of microseconds since the year 0, which
// is 62,167,219,200 seconds before the Unix epoch.
func Symbian(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e6),
		big.NewInt(-62167219200),
	)
}

// ToSymbian returns the Symbian time for the given time.Time.
func ToSymbian(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e6),
		big.NewInt(-62167219200),
	)
}

// Unix time is the number of seconds since 1970-01-01.
func Unix(num int64) time.Time {
	return time.Unix(num, 0).UTC()
}

// ToUnix returns the Unix time for the given time.Time.
func ToUnix(t time.Time) int64 {
	return t.Unix()
}

// UUIDv1 time (RFC 4122) is the number of hectonanoseconds (100 ns)
// since 1582-10-15, which is 12,219,292,800 seconds before the Unix
// epoch.
func UUIDv1(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-12219292800),
	)
}

// ToUUIDv1 returns the UUID version 1 time for the given time.Time.
func ToUUIDv1(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-12219292800),
	)
}

// WindowsDate time (e.g., .NET) is the number of hectonanoseconds
// (100 ns) since 0001-01-01, which is 62,135,596,800 seconds before
// the Unix epoch.
func WindowsDate(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-62135596800),
	)
}

// ToWindowsDate returns the WindowsDate time for the given time.Time.
func ToWindowsDate(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-62135596800),
	)
}

// WindowsFile time (e.g., NTFS) is the number of hectonanoseconds
// (100 ns) since 1601-01-01, which is 11,644,473,600 seconds before
// the Unix epoch.
func WindowsFile(num int64) time.Time {
	return epoch2time(
		big.NewInt(num),
		big.NewInt(1e7),
		big.NewInt(-11644473600),
	)
}

// ToWindowsFile returns the WindowsFile time for the given time.Time.
func ToWindowsFile(t time.Time) int64 {
	return time2epoch(
		t,
		big.NewInt(1e7),
		big.NewInt(-11644473600),
	)
}
<|endoftext|>