<commit_before>package main

import (
	"bytes"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/kr/logfmt"
)

type logValue struct {
	Val  string
	Unit string // (e.g. ms, MB, etc)
}

type logMetrics struct {
	typ     int
	app     *string
	tags    *[]string
	prefix  *string
	metrics map[string]logValue
	events  []string
}

var dynoNumber *regexp.Regexp = regexp.MustCompile(`\.\d+$`)

func (lm *logMetrics) HandleLogfmt(key, val []byte) error {

	i := bytes.LastIndexFunc(val, isDigit)
	if i == -1 {
		lm.metrics[string(key)] = logValue{string(val), ""}
	} else {
		lm.metrics[string(key)] = logValue{string(val[:i+1]), string(val[i+1:])}
	}

	log.WithFields(log.Fields{
		"key":  string(key),
		"val":  lm.metrics[string(key)].Val,
		"unit": lm.metrics[string(key)].Unit,
	}).Debug("logMetric")

	return nil
}

// return true if r is an ASCII digit only, as opposed to unicode.IsDigit.
func isDigit(r rune) bool {
	return '0' <= r && r <= '9'
}

func parseMetrics(typ int, ld *logData, data *string, out chan *logMetrics) {
	var myslice []string
	lm := logMetrics{typ, ld.app, ld.tags, ld.prefix, make(map[string]logValue, 5), myslice}
;
	if typ == releaseMsg {
		events := append(lm.events, *data)
		lm.events = events
		out <- &lm
		return
	}

	if err := logfmt.Unmarshal([]byte(*data), &lm); err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Warn()
		return
	}
	if source, ok := lm.metrics["source"]; ok {
		tags := append(*lm.tags, "type:"+dynoNumber.ReplaceAllString(source.Val, ""))
		lm.tags = &tags
	}
	out <- &lm
}

var scalingRe = regexp.MustCompile("Scaled to (.*) by user .*")
var scaledDynoRe = regexp.MustCompile("([^@ ]*)@([^: ]*):([^ ]*)")

func parseScalingMessage(ld *logData, message *string, out chan *logMetrics) {
	if scalingInfo := scalingRe.FindStringSubmatch(*message); scalingInfo != nil {
		scaledDynoInfos := scaledDynoRe.FindAllStringSubmatch(scalingInfo[1], -1)
		logValues := make(map[string]logValue)
		for _, dynoInfo := range scaledDynoInfos {
			dynoName := dynoInfo[1]
			count := dynoInfo[2]
			dynoType := dynoInfo[3]
			log.WithFields(log.Fields{
				"dynoName": dynoName,
				"count": count,
				"dynoType": dynoType,
			}).Debug()
			logValues[dynoName] = logValue{count, dynoType}
		}
		events := []string{*message}
		lm := logMetrics{scalingMsg, ld.app, ld.tags, ld.prefix, logValues, events}
		out <- &lm
	} else {
		log.WithFields(log.Fields{
			"err": "Scaling message not matched",
			"message": *message,
		}).Warn()
	}
}

func logProcess(in chan *logData, out chan *logMetrics) {

	var data *logData
	var ok bool
	for {
		data, ok = <-in

		if !ok { //Exit, channel was closed
			return
		}

		log.Debugln(*data.line)
		var output []string
		output = strings.Split(*data.line, " - ")
		if len(output) < 2 {
			output = strings.Split(*data.line, " app[heroku-redis]: ")
		}
		if len(output) < 2 {
			continue
		}
		headers := strings.Split(strings.TrimSpace(output[0]), " ")
		if len(headers) >= 6 {
			headers = headers[3:6]
		}

		log.WithField("headers", headers).Debug("Line headers")
		if strings.Contains(output[1], "source=REDIS") {
			parseMetrics(sampleMsg, data, &output[1], out)
		} else if headers[1] == "heroku" {
			if headers[2] == "router" {
				parseMetrics(routerMsg, data, &output[1], out)
			} else {
				parseMetrics(sampleMsg, data, &output[1], out)
			}
		} else if headers[1] == "app" {
			if headers[2] == "api" {
				if strings.HasPrefix(output[1], "Release") {
					parseMetrics(releaseMsg, data, &output[1], out)
				} else {
					parseScalingMessage(data, &output[1], out)
				}
			} else {
				dynoType := dynoNumber.ReplaceAllString(headers[2], "")
				tags := append(*data.tags, "source:"+headers[2], "type:"+dynoType)
				data.tags = &tags
				parseMetrics(metricsTag, data, &output[1], out)
			}
		}
	}
}
<commit_msg>fix: code review comments (make code more readable)<commit_after>package main

import (
	"bytes"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/kr/logfmt"
)

type logValue struct {
	Val  string
	Unit string // (e.g. ms, MB, etc)
}

type logMetrics struct {
	typ     int
	app     *string
	tags    *[]string
	prefix  *string
	metrics map[string]logValue
	events  []string
}

var dynoNumber *regexp.Regexp = regexp.MustCompile(`\.\d+$`)

func (lm *logMetrics) HandleLogfmt(key, val []byte) error {

	i := bytes.LastIndexFunc(val, isDigit)
	if i == -1 {
		lm.metrics[string(key)] = logValue{string(val), ""}
	} else {
		lm.metrics[string(key)] = logValue{string(val[:i+1]), string(val[i+1:])}
	}

	log.WithFields(log.Fields{
		"key":  string(key),
		"val":  lm.metrics[string(key)].Val,
		"unit": lm.metrics[string(key)].Unit,
	}).Debug("logMetric")

	return nil
}

// return true if r is an ASCII digit only, as opposed to unicode.IsDigit.
func isDigit(r rune) bool {
	return '0' <= r && r <= '9'
}

func parseMetrics(typ int, ld *logData, data *string, out chan *logMetrics) {
	var myslice []string
	lm := logMetrics{typ, ld.app, ld.tags, ld.prefix, make(map[string]logValue, 5), myslice}

	if typ == releaseMsg {
		events := append(lm.events, *data)
		lm.events = events
		out <- &lm
		return
	}

	if err := logfmt.Unmarshal([]byte(*data), &lm); err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Warn()
		return
	}
	if source, ok := lm.metrics["source"]; ok {
		tags := append(*lm.tags, "type:"+dynoNumber.ReplaceAllString(source.Val, ""))
		lm.tags = &tags
	}
	out <- &lm
}

var scalingRe = regexp.MustCompile("Scaled to (.*) by user .*")
var scaledDynoRe = regexp.MustCompile("([^@ ]*)@([^: ]*):([^ ]*)")

func parseScalingMessage(ld *logData, message *string, out chan *logMetrics) {
	if scalingInfo := scalingRe.FindStringSubmatch(*message); scalingInfo != nil {
		scaledDynoInfos := scaledDynoRe.FindAllStringSubmatch(scalingInfo[1], -1)
		logValues := make(map[string]logValue)
		for _, dynoInfo := range scaledDynoInfos {
			dynoName := dynoInfo[1]
			count := dynoInfo[2]
			dynoType := dynoInfo[3]
			log.WithFields(log.Fields{
				"dynoName": dynoName,
				"count":    count,
				"dynoType": dynoType,
			}).Debug()
			logValues[dynoName] = logValue{count, dynoType}
		}
		events := []string{*message}
		lm := logMetrics{scalingMsg, ld.app, ld.tags, ld.prefix, logValues, events}
		out <- &lm
	} else {
		log.WithFields(log.Fields{
			"err":     "Scaling message not matched",
			"message": *message,
		}).Warn()
	}
}

func logProcess(in chan *logData, out chan *logMetrics) {

	var data *logData
	var ok bool
	for {
		data, ok = <-in

		if !ok { //Exit, channel was closed
			return
		}

		log.Debugln(*data.line)
		output := strings.Split(*data.line, " - ")
		redisMetrics := strings.Split(output[0], " app[heroku-redis]: source=REDIS")
		if len(output) < 2 && len(redisMetrics) == 1 {
			continue
		}
		headers := strings.Split(strings.TrimSpace(output[0]), " ")
		if len(headers) >= 6 {
			headers = headers[3:6]
		}

		log.WithField("headers", headers).Debug("Line headers")
		if len(redisMetrics) > 1 {
			parseMetrics(sampleMsg, data, &redisMetrics[1], out)
		} else if headers[1] == "heroku" {
			if headers[2] == "router" {
				parseMetrics(routerMsg, data, &output[1], out)
			} else {
				parseMetrics(sampleMsg, data, &output[1], out)
			}
		} else if headers[1] == "app" {
			if headers[2] == "api" {
				if strings.HasPrefix(output[1], "Release") {
					parseMetrics(releaseMsg, data, &output[1], out)
				} else {
					parseScalingMessage(data, &output[1], out)
				}
			} else {
				dynoType := dynoNumber.ReplaceAllString(headers[2], "")
				tags := append(*data.tags, "source:"+headers[2], "type:"+dynoType)
				data.tags = &tags
				parseMetrics(metricsTag, data, &output[1], out)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"strings"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/atgjack/prob"
	"github.com/rveen/ltspice"
)

// lta - LT data analyzer
//
// TODO support options measdgt, numdgt
// TODO support LTSpice XVII format

type Parameter struct {
	Name   string
	Max    float64
	Min    float64
	Mean   float64
	StdDev float64
	Cpk    float64
	Ppm    float64
	Good   float64
}

var Parameters []Parameter

func main() {

	var summary bool
	var duty int
	var header bool

	flag.IntVar(&duty, "d", 0, "Calculate duty cycle of the specified column")
	flag.BoolVar(&summary, "s", false, "Print summary")
	flag.BoolVar(&header, "v", false, "Print header")
	flag.Parse()

	file := ""
	if len(flag.Args()) > 0 {
		file = flag.Args()[0]
	}

	m, vars, err := ltspice.Raw(file)

	if err != nil {
		log.Println(err)
	}

	if m == nil {
		log.Println("no data matrix found")
	}

	cols := len(m)
	rows := len(m[0])
	n := 1.0

	// Calculate number of runs
	for i := 0; i < rows; i++ {
		// detect LT runs (time == 0)
		if i > 0 && m[0][i] == 0 {
			n++
			i++
		}
	}

	log.Println("runs", n)

	// Correct std.dev for number of samples (c4(n))
	// c4(n) = sqrt( 2 / (n-1) ) * gamma(n/2) / gamma((n-1)/2)
	// See https://en.wikipedia.org/wiki/Unbiased_estimation_of_standard_deviation#Bias_correction
	c4 := 0.0
	if n > 100 {
		c4 = 4 * (n - 1) / (4*n - 3)
	} else {
		c4 = math.Sqrt(2.0/(n-1)) * math.Gamma(n/2) / math.Gamma((n-1)/2)
	}
	log.Println("c4", c4)

	for i := 0; i < cols; i++ {
		p := Parameter{Name: vars[i], Max: math.NaN(), Min: math.NaN()}
		if i > 0 {
			p.Mean = stats.StatsMean(m[i])
			p.StdDev = stats.StatsSampleStandardDeviation(m[i]) / c4 // This includes Bessel correction (which is ok!)
		}
		Parameters = append(Parameters, p)
	}

	// Does the vars list include any _min or _max ?

	for i := 1; i < cols; i++ {

		if strings.HasSuffix(vars[i], "_min)") {
			v := vars[i][0:len(vars[i])-5] + ")"

			j := 1
			for ; j < cols; j++ {
				if vars[j] == v {
					break
				}
			}
			if j > cols {
				continue
			}

			v = v[0:len(v)-1] + "_max)"

			k := 1
			for ; k < cols; k++ {
				if vars[k] == v {
					break
				}
			}
			if k > cols {
				continue
			}
			Parameters[j].Min = Parameters[i].Mean
			Parameters[j].Max = Parameters[k].Mean
			log.Println(Parameters[j].Name, Parameters[j].Min, Parameters[j].Max)
		}
	}

	// For parameters with min,max calculate additional columns
	for i, p := range Parameters {
		if !math.IsNaN(p.Max) {
			log.Println(p.Mean, p.StdDev, p.Max)
			Parameters[i].Cpk = (p.Max - p.Mean) / (3.0 * p.StdDev)
			norm, err := prob.NewNormal(p.Mean, p.StdDev)
			if err != nil {
				log.Println(err.Error())
			} else {
				bad := (1.0 - norm.Cdf(p.Max)) * 2
				Parameters[i].Good = 1.0 - bad
				Parameters[i].Ppm = bad * 1e6
			}
		}
	}

	if duty == 0 {
		if header {
			fmt.Printf("%-20s %30s %30s %30s %20s %20s %20s %10s\n", "parameter", "mean", "sdev(unbiased)", "min", "max", "cpk", "%ok", "ppm")
		}

		for i, p := range Parameters {

			if i == 0 {
				continue
			}

			fmt.Printf("%-20s %30g %30g %30g %20g %20g %20.6f %10.1f\n", "'"+p.Name+"'", p.Mean, p.StdDev, p.Min, p.Max, p.Cpk, p.Good*100.0, p.Ppm)
		}
	} else {

		var dcs []float64

		//fmt.Printf("duty of %s\n", vars[duty])
		min := stats.StatsMin(m[duty])
		max := stats.StatsMax(m[duty])

		mid := (max-min)/2 + min
		//fmt.Printf("min %f max %f, rows %d, thres %f\n", min, max, rows, mid)

		ni := 0
		nf := 0

		for i := 0; i < rows; i++ {

			// detect LT runs (time = 0)
			if i > 0 && m[0][i] == 0 {
				nf = i - 1
				i++

				// Calculate DC
				m, _ := Dutycycle(m[duty][ni:nf], m[0][ni:nf], mid)
				dcs = append(dcs, m)

				ni = i
			}
		}

		ni = nf + 2
		nf = rows - 1

		// Calculate DC
		m, _ := Dutycycle(m[duty][ni:nf], m[0][ni:nf], mid)
		dcs = append(dcs, m)

		if summary {
			mean := stats.StatsMean(dcs)
			max = stats.StatsMax(dcs)
			min = stats.StatsMin(dcs)
			fmt.Printf("mean %f, min %f, max %f\n", mean, min, max)
		}

	}
}

func Dutycycle(a []float64, t []float64, mid float64) (float64, float64) {
	// fmt.Printf("--------------------------\ndc %d len mid %f\n", len(a), mid)

	// Detect transitions

	// initial state = low
	state := false

	if a[0] > mid {
		state = true
	}

	var transitions []int

	for i := 1; i < len(a); i++ {
		switch state {
		case false:
			// detect transition to high
			if a[i] > mid {
				//fmt.Println("low to high @", t[i])
				state = true
				transitions = append(transitions, i)
			}

		case true:
			// detect transition to low
			if a[i] < mid {
				//fmt.Println("high to low @", t[i])
				state = false
				transitions = append(transitions, -i)
			}

		}
	}

	ti := 0.0
	tf := 0.0

	tdiff0 := 0.0

	var dcs []float64
	var dc float64

	for i, tr := range transitions {

		n := tr
		if n < 0 {
			n = -n
		}

		if i == 0 {
			ti = t[n]
			continue
		}

		tf = t[n]

		/*
			if tr < 0 {
				fmt.Printf("HL %d t %f\n", tr, tf-ti)
			} else {
				fmt.Printf("LH %d t %f\n", tr, tf-ti)
			}*/

		if i == 1 {
			continue
		}

		tdiff := tf - ti
		if i&1 == 1 {

			if tr < 0 {
				dc = tdiff / (tdiff + tdiff0)
			} else {
				dc = 1.0 - tdiff/(tdiff+tdiff0)
			}

			dcs = append(dcs, dc)

		}

		tdiff0 = tf - ti
		ti = tf
	}

	mean := stats.StatsMean(dcs)
	std := stats.StatsSampleStandardDeviation(dcs)

	//fmt.Printf("mean %f, std %f\n", mean, std)

	// Clean dcs
	var dcc []float64
	for _, dc = range dcs {
		if dc > mean+std || dc < mean-std {
			continue
		}
		dcc = append(dcc, dc)
	}

	mean = stats.StatsMean(dcc)
	std = stats.StatsSampleStandardDeviation(dcc)

	fmt.Printf("%f, %f\n", mean, std)

	return mean, std
}
<commit_msg>add histogram<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"strings"

	"github.com/GaryBoone/GoStats/stats"
	"github.com/atgjack/prob"
	"github.com/rveen/ltspice"
)

// lta - LT data analyzer
//
// TODO support options measdgt, numdgt
// TODO support LTSpice XVII format

type Parameter struct {
	Name   string
	Max    float64
	Min    float64
	Mean   float64
	StdDev float64
	Cpk    float64
	Ppm    float64
	Good   float64
}

var Parameters []Parameter

var upperLimit = 0.0
var lowerLimit = 0.0

func main() {

	var summary bool
	var duty int
	var header bool
	var hist int

	flag.IntVar(&duty, "d", 0, "Calculate duty cycle of the specified column")
	flag.IntVar(&hist, "hist", 0, "Generate histogram (hist.svg)")
	flag.BoolVar(&summary, "s", false, "Print summary")
	flag.BoolVar(&header, "v", false, "Print header")
	flag.Float64Var(&upperLimit, "max", 0.0, "Establish the upper limit of the parameter under study")
	flag.Float64Var(&lowerLimit, "min", 0.0, "Establish the lower limit of the parameter under study")
	flag.Parse()

	file := ""
	if len(flag.Args()) > 0 {
		file = flag.Args()[0]
	}

	m, vars, err := ltspice.Raw(file)

	if err != nil {
		log.Println(err)
	}

	if m == nil {
		log.Println("no data matrix found")
	}

	cols := len(m)
	rows := len(m[0])
	n := 1.0

	// Calculate number of runs
	for i := 0; i < rows; i++ {
		// detect LT runs (time == 0)
		if i > 0 && m[0][i] == 0 {
			n++
			i++
		}
	}

	log.Println("runs", n)

	// Correct std.dev for number of samples (c4(n))
	// c4(n) = sqrt( 2 / (n-1) ) * gamma(n/2) / gamma((n-1)/2)
	// See https://en.wikipedia.org/wiki/Unbiased_estimation_of_standard_deviation#Bias_correction
	c4 := 0.0
	if n > 100 {
		c4 = 4 * (n - 1) / (4*n - 3)
	} else {
		c4 = math.Sqrt(2.0/(n-1)) * math.Gamma(n/2) / math.Gamma((n-1)/2)
	}
	log.Println("c4", c4)

	for i := 0; i < cols; i++ {
		p := Parameter{Name: vars[i], Max: math.NaN(), Min: math.NaN()}
		if i > 0 {
			p.Mean = stats.StatsMean(m[i])
			p.StdDev = stats.StatsSampleStandardDeviation(m[i]) / c4 // This includes Bessel correction (which is ok!)
		}
		Parameters = append(Parameters, p)
	}

	// Does the vars list include any _min or _max ?

	for i := 1; i < cols; i++ {

		if strings.HasSuffix(vars[i], "_min)") {
			v := vars[i][0:len(vars[i])-5] + ")"

			j := 1
			for ; j < cols; j++ {
				if vars[j] == v {
					break
				}
			}
			if j > cols {
				continue
			}

			v = v[0:len(v)-1] + "_max)"

			k := 1
			for ; k < cols; k++ {
				if vars[k] == v {
					break
				}
			}
			if k > cols {
				continue
			}
			Parameters[j].Min = Parameters[i].Mean
			Parameters[j].Max = Parameters[k].Mean
			log.Println(Parameters[j].Name, Parameters[j].Min, Parameters[j].Max)
		}
	}

	// For parameters with min,max calculate additional columns
	for i, p := range Parameters {
		if !math.IsNaN(p.Max) {
			log.Println(p.Mean, p.StdDev, p.Max)
			Parameters[i].Cpk = (p.Max - p.Mean) / (3.0 * p.StdDev)
			norm, err := prob.NewNormal(p.Mean, p.StdDev)
			if err != nil {
				log.Println(err.Error())
			} else {
				bad := (1.0 - norm.Cdf(p.Max)) * 2
				Parameters[i].Good = 1.0 - bad
				Parameters[i].Ppm = bad * 1e6
			}
		}
	}

	if hist > 0 {
		histogram(m[hist], Parameters[hist])
		return
	}

	if duty == 0 {
		if header {
			fmt.Printf("%-20s %30s %30s %30s %20s %20s %20s %10s\n", "parameter", "mean", "sdev(unbiased)", "min", "max", "cpk", "%ok", "ppm")
		}

		for i, p := range Parameters {

			if i == 0 {
				continue
			}

			fmt.Printf("%-20s %30g %30g %30g %20g %20g %20.6f %10.1f\n", "'"+p.Name+"'", p.Mean, p.StdDev, p.Min, p.Max, p.Cpk, p.Good*100.0, p.Ppm)
		}
	} else {

		var dcs []float64

		//fmt.Printf("duty of %s\n", vars[duty])
		min := stats.StatsMin(m[duty])
		max := stats.StatsMax(m[duty])

		mid := (max-min)/2 + min
		//fmt.Printf("min %f max %f, rows %d, thres %f\n", min, max, rows, mid)

		ni := 0
		nf := 0

		for i := 0; i < rows; i++ {

			// detect LT runs (time = 0)
			if i > 0 && m[0][i] == 0 {
				nf = i - 1
				i++

				// Calculate DC
				m, _ := Dutycycle(m[duty][ni:nf], m[0][ni:nf], mid)
				dcs = append(dcs, m)

				ni = i
			}
		}

		ni = nf + 2
		nf = rows - 1

		// Calculate DC
		m, _ := Dutycycle(m[duty][ni:nf], m[0][ni:nf], mid)
		dcs = append(dcs, m)

		//if summary {
		mean := stats.StatsMean(dcs)
		max = stats.StatsMax(dcs)
		min = stats.StatsMin(dcs)
		sdev := stats.StatsSampleStandardDeviation(dcs) / c4
		fmt.Printf("mean, min, max, sdev, cpk, ok, ppm\n")

		cpk := (upperLimit - mean) / (3.0 * sdev)
		norm, err := prob.NewNormal(mean, sdev)
		if err != nil {
			log.Println(err.Error())
		} else {
			bad := (1.0 - norm.Cdf(upperLimit)) * 2
			good := 1.0 - bad
			ppm := bad * 1e6
			fmt.Printf("%f, %f, %f, %f, %f, %f, %f\n", mean, lowerLimit, upperLimit, sdev, cpk, good*100.0, ppm)
		}

	}
}

func Dutycycle(a []float64, t []float64, mid float64) (float64, float64) {
	// fmt.Printf("--------------------------\ndc %d len mid %f\n", len(a), mid)

	// Detect transitions

	// initial state = low
	state := false

	if a[0] > mid {
		state = true
	}

	var transitions []int

	for i := 1; i < len(a); i++ {
		switch state {
		case false:
			// detect transition to high
			if a[i] > mid {
				//fmt.Println("low to high @", t[i])
				state = true
				transitions = append(transitions, i)
			}

		case true:
			// detect transition to low
			if a[i] < mid {
				//fmt.Println("high to low @", t[i])
				state = false
				transitions = append(transitions, -i)
			}

		}
	}

	ti := 0.0
	tf := 0.0

	tdiff0 := 0.0

	var dcs []float64
	var dc float64

	for i, tr := range transitions {

		n := tr
		if n < 0 {
			n = -n
		}

		if i == 0 {
			ti = t[n]
			continue
		}

		tf = t[n]

		/*
			if tr < 0 {
				fmt.Printf("HL %d t %f\n", tr, tf-ti)
			} else {
				fmt.Printf("LH %d t %f\n", tr, tf-ti)
			}*/

		if i == 1 {
			continue
		}

		tdiff := tf - ti
		if i&1 == 1 {

			if tr < 0 {
				dc = tdiff / (tdiff + tdiff0)
			} else {
				dc = 1.0 - tdiff/(tdiff+tdiff0)
			}

			dcs = append(dcs, dc)

		}

		tdiff0 = tf - ti
		ti = tf
	}

	mean := stats.StatsMean(dcs)
	std := stats.StatsSampleStandardDeviation(dcs)

	//fmt.Printf("mean %f, std %f\n", mean, std)

	// Clean dcs
	var dcc []float64
	for _, dc = range dcs {
		if dc > mean+std || dc < mean-std {
			continue
		}
		dcc = append(dcc, dc)
	}

	mean = stats.StatsMean(dcc)
	std = stats.StatsSampleStandardDeviation(dcc)

	// fmt.Printf("%f, %f\n", mean, std)

	return mean, std
}

type V struct {
	vs []float64
}

func (v V) Len() int {
	return len(v.vs)
}

func (v V) Value(i int) float64 {
	return v.vs[i]
}

func histogram(v []float64, p Parameter) {

	// n is the number of bars
	n := 50.0

	// width of bars
	w := 500 / int(n)

	min := stats.StatsMin(v)
	max := stats.StatsMax(v)

	if !math.IsNaN(p.Max) {
		if p.Max > max {
			max = p.Max
		}
		if p.Min < min {
			min = p.Min
		}
	}

	h := make([]float64, int(n)+1)
	step := (max - min) / n

	// log.Printf("%f bins, min %f, max %f, step %f\n", n, min, max, step)

	for i := 0; i < len(v); i++ {
		e := (v[i] - min) / step
		// log.Println(i, v[i], e)
		h[int(e)]++
	}

	hmax := stats.StatsMax(h)
	for i := 0; i < len(h); i++ {
		h[i] /= hmax

	}

	for i := 0; i < int(n); i++ {
		//fmt.Println(h[i])
	}

	fmt.Println(`<?xml version="1.0"?><svg xmlns="http://www.w3.org/2000/svg" width="520" height="510" viewBox="0,0,520,510"><desc>R SVG Plot!</desc><rect width="100%" height="100%" style="fill:#FFFFFF"/>`)
	y0 := 490
	x1 := 10
	for i := 0; i < int(n); i++ {
		y1 := int(490 - h[i]*480)
		fmt.Printf("<polygon points='%d,%d %d,%d %d,%d, %d,%d' style='stroke-width:1;stroke:#999;fill:#ADD8E6;stroke-opacity:1.000000;fill-opacity:1.000000' />\n", x1, y0, x1, y1, x1+w, y1, x1+w, y0)
		x1 += w
	}
	fmt.Printf("<text x='250' y='505' text-anchor='middle' alignment-baseline='bottom' style='font-size:14; font-family: arial'>min=%f, max=%f</text>", min, max)
	//fmt.Printf("<text x='490' y='500' text-anchor='left' alignment-baseline='bottom' style='font-size:12; font-family: arial'>%f</text>", max)
	fmt.Println("</svg>")
}
<|endoftext|>


<commit_before>// dataloader is an implimentation of facebook's dataloader in go.
// See https://github.com/facebook/dataloader for more information
package dataloader

import (
	"fmt"
	"sync"
	"time"
)

// A `DataLoader` Interface defines a public API for loading data from a particular
// data back-end with unique keys such as the `id` column of a SQL table or
// document name in a MongoDB database, given a batch loading function.
//
// Each `DataLoader` instance should contain a unique memoized cache. Use caution when
// used in long-lived applications or those which serve many users with
// different access permissions and consider creating a new instance per
// web request.
type Interface interface {
	Load(string) Thunk
	LoadMany([]string) ThunkMany
	Clear(string) Interface
	ClearAll() Interface
	Prime(key string, value interface{}) Interface
}

// BatchFunc is a Function, which when given a Slice of keys (string), returns an slice of `results`.
// It's important that the length of the input keys matches the length of the ouput results.
type BatchFunc func([]string) []*Result

// Result is the data structure that is primarily used by the BatchFunc.
// It contains the resolved data, and any errors that may have occured while fetching the data.
type Result struct {
	Data  interface{}
	Error error
}

// ResultMany is used by the loadMany method. It contains a list of resolved data and a list of erros // if any occured.
type ResultMany struct {
	Data  []interface{}
	Error []error
}

// Implements the dataloader.Interface.
type Loader struct {
	// the batch function to be used by this loader
	batchFn BatchFunc
	// the maximum batch size. Set to 0 if you want it to be unbounded.
	cap int
	// the internal cache. This packages contains a basic cache implementation but any custom cache
	// implementation could be used as long as it implements the `Cache` interface.
	cache Cache
	// used to close the input channel early
	forceStartBatch chan bool

	// connt of queued up items
	count int
	// count mutex
	countLock sync.Mutex

	// internal channel that is used to batch items
	input chan *batchRequest
	// input mutex
	inputLock sync.Mutex
}

// Future is a function that will block until the value it contins is resolved.
// After the value it contians is resolved, this function will return the result.
// This function can be called many times, much like a Promise is other languages.
type Thunk func() *Result

// FutureMany is much like the Future func type but it contains a list of results.
type ThunkMany func() *ResultMany

// NewBatchedLoader constructs a new Loader with given options
func NewBatchedLoader(batchFn BatchFunc, cache Cache, cap int) *Loader {
	return &Loader{
		batchFn:         batchFn,
		cache:           cache,
		cap:             cap,
		forceStartBatch: make(chan bool),
		count:           0,
	}
}

// Load load/resolves the given key, returning a channel that will contain the value and error
func (l *Loader) Load(key string) Thunk {
	c := make(chan *Result, 1)
	if v, ok := l.cache.Get(key); ok {
		return v.(func() *Result)
	}

	var value struct {
		value *Result
		lock  sync.Mutex
	}

	thunk := func() *Result {
		if v, ok := <-c; ok {
			value.lock.Lock()
			value.value = v
			value.lock.Unlock()
		}

		return value.value
	}

	l.cache.Set(key, thunk)
	req := &batchRequest{key, c}

	if l.input == nil {
		l.inputLock.Lock()
		l.input = make(chan *batchRequest, l.cap)
		l.inputLock.Unlock()
		go l.batch()
	} else {

	}

	l.input <- req

	l.countLock.Lock()
	l.count = l.count + 1
	l.countLock.Unlock()

	if l.cap > 0 && l.count >= l.cap {
		l.forceStartBatch <- true
	}

	return thunk
}

// LoadMany loads mulitiple keys, returning a channel that will resolve to an array of values and an error
func (l *Loader) LoadMany(keys []string) ThunkMany {
	out := make(chan *ResultMany, 1)
	c := make(chan *result, len(keys))

	for i := range keys {
		go func(i int) {
			getter := l.Load(keys[i])
			c <- &result{getter(), i}
		}(i)
	}

	go func() {
		defer close(out)
		var i int = 1
		outputData := make([]interface{}, len(keys), len(keys))
		outputErrors := make([]error, 0, len(keys))
		for result := range c {
			outputData[result.index] = result.Data
			if result.Error != nil {
				outputErrors = append(outputErrors, resultError{result.Error, result.index})
			}
			if i == len(keys) {
				close(c)
			}
			i++
		}
		out <- &ResultMany{outputData, outputErrors}
	}()

	var value struct {
		value *ResultMany
		lock  sync.Mutex
	}

	thunkMany := func() *ResultMany {
		if v, ok := <-out; ok {
			value.lock.Lock()
			value.value = v
			value.lock.Unlock()
		}

		return value.value
	}

	return thunkMany
}

// Clear clears the value at `key` from the cache, it it exsits. Returs self for method chaining
func (l *Loader) Clear(key string) Interface {
	l.cache.Delete(key)
	return l
}

// ClearAll clears the entire cache. To be used when some event results in unknown invalidations.
// Returs self for method chaining.
func (l *Loader) ClearAll() Interface {
	l.cache.Clear()
	return l
}

// Adds the provided key and value to the cache. If the key already exists, no change is made.
// Returns self for method chaining
func (l *Loader) Prime(key string, value interface{}) Interface {
	if _, ok := l.cache.Get(key); !ok {
		future := func() *Result {
			return &Result{
				Data:  value,
				Error: nil,
			}
		}
		l.cache.Set(key, future)
	}
	return l
}

// type used to on input channel
type batchRequest struct {
	key     string
	channel chan *Result
}

// this help ensure that data return is in same order as data recieved
type result struct {
	*Result
	index int
}

// this help match the error to the key of a specific index
type resultError struct {
	error
	index int
}

// execuite the batch of all items in queue
func (l *Loader) batch() {
	var keys []string
	var reqs []*batchRequest

	go l.sleeper()

	for item := range l.input {
		keys = append(keys, item.key)
		reqs = append(reqs, item)
	}

	items := l.batchFn(keys)

	if len(items) != len(reqs) {
		for _, req := range reqs {
			req.channel <- &Result{
				Error: fmt.Errorf("length of keys must match length of responses"),
				Data:  nil,
			}
			close(req.channel)
		}
	} else {
		for i, req := range reqs {
			req.channel <- items[i]
			close(req.channel)
		}
	}

}

// wait the appropriate amount of time for next batch
func (l *Loader) sleeper() {
	select {
	// used by batch to close early. usually triggered by max batch size
	case <-l.forceStartBatch:
	// this will move this goroutine to the back of the callstack?
	case <-time.After(100 * time.Millisecond):
	}

	l.inputLock.Lock()
	close(l.input)
	l.input = nil
	l.inputLock.Unlock()
}
<commit_msg>set batchtime to 10ms<commit_after>// dataloader is an implimentation of facebook's dataloader in go.
// See https://github.com/facebook/dataloader for more information
package dataloader

import (
	"fmt"
	"sync"
	"time"
)

// A `DataLoader` Interface defines a public API for loading data from a particular
// data back-end with unique keys such as the `id` column of a SQL table or
// document name in a MongoDB database, given a batch loading function.
//
// Each `DataLoader` instance should contain a unique memoized cache. Use caution when
// used in long-lived applications or those which serve many users with
// different access permissions and consider creating a new instance per
// web request.
type Interface interface {
	Load(string) Thunk
	LoadMany([]string) ThunkMany
	Clear(string) Interface
	ClearAll() Interface
	Prime(key string, value interface{}) Interface
}

// BatchFunc is a Function, which when given a Slice of keys (string), returns an slice of `results`.
// It's important that the length of the input keys matches the length of the ouput results.
type BatchFunc func([]string) []*Result

// Result is the data structure that is primarily used by the BatchFunc.
// It contains the resolved data, and any errors that may have occured while fetching the data.
type Result struct {
	Data  interface{}
	Error error
}

// ResultMany is used by the loadMany method. It contains a list of resolved data and a list of erros // if any occured.
type ResultMany struct {
	Data  []interface{}
	Error []error
}

// Implements the dataloader.Interface.
type Loader struct {
	// the batch function to be used by this loader
	batchFn BatchFunc
	// the maximum batch size. Set to 0 if you want it to be unbounded.
	cap int
	// the internal cache. This packages contains a basic cache implementation but any custom cache
	// implementation could be used as long as it implements the `Cache` interface.
	cache Cache
	// used to close the input channel early
	forceStartBatch chan bool

	// connt of queued up items
	count int
	// count mutex
	countLock sync.Mutex

	// internal channel that is used to batch items
	input chan *batchRequest
	// input mutex
	inputLock sync.Mutex
}

// Future is a function that will block until the value it contins is resolved.
// After the value it contians is resolved, this function will return the result.
// This function can be called many times, much like a Promise is other languages.
type Thunk func() *Result

// FutureMany is much like the Future func type but it contains a list of results.
type ThunkMany func() *ResultMany

// NewBatchedLoader constructs a new Loader with given options
func NewBatchedLoader(batchFn BatchFunc, cache Cache, cap int) *Loader {
	return &Loader{
		batchFn:         batchFn,
		cache:           cache,
		cap:             cap,
		forceStartBatch: make(chan bool),
		count:           0,
	}
}

// Load load/resolves the given key, returning a channel that will contain the value and error
func (l *Loader) Load(key string) Thunk {
	c := make(chan *Result, 1)
	if v, ok := l.cache.Get(key); ok {
		return v.(func() *Result)
	}

	var value struct {
		value *Result
		lock  sync.Mutex
	}

	thunk := func() *Result {
		if v, ok := <-c; ok {
			value.lock.Lock()
			value.value = v
			value.lock.Unlock()
		}

		return value.value
	}

	l.cache.Set(key, thunk)
	req := &batchRequest{key, c}

	if l.input == nil {
		l.inputLock.Lock()
		l.input = make(chan *batchRequest, l.cap)
		l.inputLock.Unlock()
		go l.batch()
	} else {

	}

	l.input <- req

	l.countLock.Lock()
	l.count = l.count + 1
	l.countLock.Unlock()

	if l.cap > 0 && l.count >= l.cap {
		l.forceStartBatch <- true
	}

	return thunk
}

// LoadMany loads mulitiple keys, returning a channel that will resolve to an array of values and an error
func (l *Loader) LoadMany(keys []string) ThunkMany {
	out := make(chan *ResultMany, 1)
	c := make(chan *result, len(keys))

	for i := range keys {
		go func(i int) {
			getter := l.Load(keys[i])
			c <- &result{getter(), i}
		}(i)
	}

	go func() {
		defer close(out)
		var i int = 1
		outputData := make([]interface{}, len(keys), len(keys))
		outputErrors := make([]error, 0, len(keys))
		for result := range c {
			outputData[result.index] = result.Data
			if result.Error != nil {
				outputErrors = append(outputErrors, resultError{result.Error, result.index})
			}
			if i == len(keys) {
				close(c)
			}
			i++
		}
		out <- &ResultMany{outputData, outputErrors}
	}()

	var value struct {
		value *ResultMany
		lock  sync.Mutex
	}

	thunkMany := func() *ResultMany {
		if v, ok := <-out; ok {
			value.lock.Lock()
			value.value = v
			value.lock.Unlock()
		}

		return value.value
	}

	return thunkMany
}

// Clear clears the value at `key` from the cache, it it exsits. Returs self for method chaining
func (l *Loader) Clear(key string) Interface {
	l.cache.Delete(key)
	return l
}

// ClearAll clears the entire cache. To be used when some event results in unknown invalidations.
// Returs self for method chaining.
func (l *Loader) ClearAll() Interface {
	l.cache.Clear()
	return l
}

// Adds the provided key and value to the cache. If the key already exists, no change is made.
// Returns self for method chaining
func (l *Loader) Prime(key string, value interface{}) Interface {
	if _, ok := l.cache.Get(key); !ok {
		future := func() *Result {
			return &Result{
				Data:  value,
				Error: nil,
			}
		}
		l.cache.Set(key, future)
	}
	return l
}

// type used to on input channel
type batchRequest struct {
	key     string
	channel chan *Result
}

// this help ensure that data return is in same order as data recieved
type result struct {
	*Result
	index int
}

// this help match the error to the key of a specific index
type resultError struct {
	error
	index int
}

// execuite the batch of all items in queue
func (l *Loader) batch() {
	var keys []string
	var reqs []*batchRequest

	go l.sleeper()

	for item := range l.input {
		keys = append(keys, item.key)
		reqs = append(reqs, item)
	}

	items := l.batchFn(keys)

	if len(items) != len(reqs) {
		for _, req := range reqs {
			req.channel <- &Result{
				Error: fmt.Errorf("length of keys must match length of responses"),
				Data:  nil,
			}
			close(req.channel)
		}
	} else {
		for i, req := range reqs {
			req.channel <- items[i]
			close(req.channel)
		}
	}

}

// wait the appropriate amount of time for next batch
func (l *Loader) sleeper() {
	select {
	// used by batch to close early. usually triggered by max batch size
	case <-l.forceStartBatch:
	// this will move this goroutine to the back of the callstack?
	case <-time.After(10 * time.Millisecond):
	}

	l.inputLock.Lock()
	close(l.input)
	l.input = nil
	l.inputLock.Unlock()
}
<|endoftext|>


<commit_before>package db

import (
	"sync"

	"bitbucket.org/mundipagg/boletoapi/config"
	"bitbucket.org/mundipagg/boletoapi/models"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type mongoDb struct {
	session *mgo.Session
	m       sync.RWMutex
}

//CreateMongo cria uma nova intancia de conexão com o mongodb
func CreateMongo() DB {
	db := new(mongoDb)
	var err error
	db.session, err = mgo.Dial(config.Get().MongoURL)
	if err != nil {
		panic(err)
	}
	db.session.SetMode(mgo.Monotonic, true)
	return db
}

//SaveBoleto salva um boleto no mongoDB
func (e *mongoDb) SaveBoleto(boleto models.BoletoView) error {
	var err error

	e.m.Lock()
	defer e.m.Unlock()
	c := e.session.DB("boletoapi").C("boletos")
	err = c.Insert(boleto)
	return err
}

//GetBoletoById busca um boleto pelo ID que vem na URL
func (e *mongoDb) GetBoletoByID(id string) (models.BoletoView, error) {
	c := e.session.DB("boletoapi").C("boletos")
	result := models.BoletoView{}
	err := c.Find(bson.M{"id": id}).One(&result)
	if err != nil {
		return models.BoletoView{}, err
	}
	return result, nil
}

func (e *mongoDb) Close() {
	e.session.Close()
}
<commit_msg>:construction: muda o modo de escrita do mongo para garantir melhor consistência na leitura e escrita<commit_after>package db

import (
	"sync"

	"bitbucket.org/mundipagg/boletoapi/config"
	"bitbucket.org/mundipagg/boletoapi/models"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type mongoDb struct {
	session *mgo.Session
	m       sync.RWMutex
}

//CreateMongo cria uma nova intancia de conexão com o mongodb
func CreateMongo() DB {
	db := new(mongoDb)
	var err error
	db.session, err = mgo.Dial(config.Get().MongoURL)
	if err != nil {
		panic(err)
	}
	return db
}

//SaveBoleto salva um boleto no mongoDB
func (e *mongoDb) SaveBoleto(boleto models.BoletoView) error {
	var err error

	e.m.Lock()
	defer e.m.Unlock()
	c := e.session.DB("boletoapi").C("boletos")
	err = c.Insert(boleto)
	return err
}

//GetBoletoById busca um boleto pelo ID que vem na URL
func (e *mongoDb) GetBoletoByID(id string) (models.BoletoView, error) {
	c := e.session.DB("boletoapi").C("boletos")
	result := models.BoletoView{}
	err := c.Find(bson.M{"id": id}).One(&result)
	if err != nil {
		return models.BoletoView{}, err
	}
	return result, nil
}

func (e *mongoDb) Close() {
	e.session.Close()
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "os"
  "crypto/tls"
  "encoding/pem"
  "flag"
)

func main() {

  var server = flag.String("server", "", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  var network = flag.String("net", "tcp", "Connect to this kind of network: tcp, udp, or unix")
  var ipv4only = flag.Bool("4", false, "Only connect via IPv4")
  var ipv6only = flag.Bool("6", false, "Only connect via IPv6")
  var verify = flag.Bool("verify", false, "Verify the provided certificate against trusted CAs")
  flag.Parse()

  if (*server == "") {
    panic("Need to specify a server.")
  }

  if (*network != "tcp" && *network != "udp" && *network != "unix") {
    panic("Unknown kind of network type! Try tcp.")
  }

  var network_suffix string = ""
  if (*ipv4only && *ipv6only) {
    fmt.Println("Specifying both -4 & -6 is redundant")
  } else if (*ipv4only) {
    network_suffix = "4"
  } else if (*ipv6only) {
    network_suffix = "6"
  }

  target_host := fmt.Sprintf("%s:%d", *server, *port)
  network_type := fmt.Sprintf("%s%s", *network, network_suffix)

  conn, err := tls.Dial(network_type, target_host, &tls.Config{
    InsecureSkipVerify: !(*verify),
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()

  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    pem_pubkey := &pem.Block{
      Type: "CERTIFICATE",
      Bytes: cert.Raw,
    }
    pem.Encode(os.Stdout, pem_pubkey)
  }
}
<commit_msg>Refactor to not make a variable to pass to pem.Encode<commit_after>package main

import (
  "fmt"
  "os"
  "crypto/tls"
  "encoding/pem"
  "flag"
)

func main() {

  var server = flag.String("server", "", "The TLS host (name or IP) to connect to")
  var port = flag.Int("port", 443, "The port the TLS service is running on")
  var network = flag.String("net", "tcp", "Connect to this kind of network: tcp, udp, or unix")
  var ipv4only = flag.Bool("4", false, "Only connect via IPv4")
  var ipv6only = flag.Bool("6", false, "Only connect via IPv6")
  var verify = flag.Bool("verify", false, "Verify the provided certificate against trusted CAs")
  flag.Parse()

  if (*server == "") {
    panic("Need to specify a server.")
  }

  if (*network != "tcp" && *network != "udp" && *network != "unix") {
    panic("Unknown kind of network type! Try tcp.")
  }

  var network_suffix string = ""
  if (*ipv4only && *ipv6only) {
    fmt.Println("Specifying both -4 & -6 is redundant")
  } else if (*ipv4only) {
    network_suffix = "4"
  } else if (*ipv6only) {
    network_suffix = "6"
  }

  target_host := fmt.Sprintf("%s:%d", *server, *port)
  network_type := fmt.Sprintf("%s%s", *network, network_suffix)

  conn, err := tls.Dial(network_type, target_host, &tls.Config{
    InsecureSkipVerify: !(*verify),
  })
  if err != nil {
    panic("failed to connect: " + err.Error())
  }
  conn.Close()

  tls_state := conn.ConnectionState()
  remote_certs := tls_state.PeerCertificates
  for _, cert := range remote_certs {
    if (cert.BasicConstraintsValid) {
      if (cert.IsCA) {
        continue
      }
    } else {
      continue
    }

    pem.Encode(os.Stdout, &pem.Block{
      Type: "CERTIFICATE",
      Bytes: cert.Raw,
    })
  }
}
<|endoftext|>


<commit_before>package prose

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type TokenTester func(string) bool

type Tokenizer interface {
	Tokenize(string) []*Token
}

// iterTokenizer splits a sentence into words.
type iterTokenizer struct {
	specialRE      *regexp.Regexp
	sanitizer      *strings.Replacer
	contractions   []string
	splitCases     []string
	suffixes       []string
	prefixes       []string
	emoticons      map[string]int
	isUnsplittable TokenTester
}

type TokenizerOptFunc func(*iterTokenizer)

// UsingIsUnsplittableFN gives a function that tests whether a token is splittable or not.
func UsingIsUnsplittable(x TokenTester) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.isUnsplittable = x
	}
}

// Use the provided special regex for unsplittable tokens.
func UsingSpecialRE(x *regexp.Regexp) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.specialRE = x
	}
}

// Use the provided sanitizer.
func UsingSanitizer(x *strings.Replacer) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.sanitizer = x
	}
}

// Use the provided suffixes.
func UsingSuffixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.suffixes = x
	}
}

// Use the provided prefixes.
func UsingPrefixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.prefixes = x
	}
}

// Use the provided map of emoticons.
func UsingEmoticons(x map[string]int) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.emoticons = x
	}
}

// Use the provided contractions.
func UsingContractions(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.contractions = x
	}
}

// Use the provided splitCases.
func UsingSplitCases(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.splitCases = x
	}
}

// Constructor for default iterTokenizer
func NewIterTokenizer(opts ...TokenizerOptFunc) *iterTokenizer {
	tok := new(iterTokenizer)

	// Set default parameters
	tok.contractions = contractions
	tok.emoticons = emoticons
	tok.isUnsplittable = func(_ string) bool { return false }
	tok.prefixes = prefixes
	tok.sanitizer = sanitizer
	tok.specialRE = internalRE
	tok.suffixes = suffixes

	// Apply options if provided
	for _, applyOpt := range opts {
		applyOpt(tok)
	}

	tok.splitCases = append(tok.splitCases, tok.contractions...)

	return tok
}

func addToken(s string, toks []*Token) []*Token {
	if strings.TrimSpace(s) != "" {
		toks = append(toks, &Token{Text: s})
	}
	return toks
}

func (t *iterTokenizer) isSpecial(token string) bool {
	_, found := t.emoticons[token]
	return found || t.specialRE.MatchString(token) || t.isUnsplittable(token)
}

func (t *iterTokenizer) doSplit(token string) []*Token {
	tokens := []*Token{}
	suffs := []*Token{}

	last := 0
	for token != "" && utf8.RuneCountInString(token) != last {
		if t.isSpecial(token) {
			// We've found a special case (e.g., an emoticon) -- so, we add it as a token without
			// any further processing.
			tokens = addToken(token, tokens)
			break
		}
		last = utf8.RuneCountInString(token)
		lower := strings.ToLower(token)
		if hasAnyPrefix(token, t.prefixes) {
			// Remove prefixes -- e.g., $100 -> [$, 100].
			tokens = addToken(string(token[0]), tokens)
			token = token[1:]
		} else if idx := hasAnyIndex(lower, t.splitCases); idx > -1 {
			// Handle "they'll", "I'll", "Don't", "won't", amount($).
			//
			// they'll -> [they, 'll].
			// don't -> [do, n't].
			// amount($) -> [amount, (, $, )].
			tokens = addToken(token[:idx], tokens)
			token = token[idx:]
		} else if hasAnySuffix(token, t.suffixes) {
			// Remove suffixes -- e.g., Well) -> [Well, )].
			suffs = append([]*Token{
				{Text: string(token[len(token)-1])}},
				suffs...)
			token = token[:len(token)-1]
		} else {
			tokens = addToken(token, tokens)
		}
	}

	return append(tokens, suffs...)
}

// tokenize splits a sentence into a slice of words.
func (t *iterTokenizer) Tokenize(text string) []*Token {
	tokens := []*Token{}

	clean, white := t.sanitizer.Replace(text), false
	length := len(clean)

	start, index := 0, 0
	cache := map[string][]*Token{}
	for index <= length {
		uc, size := utf8.DecodeRuneInString(clean[index:])
		if size == 0 {
			break
		} else if index == 0 {
			white = unicode.IsSpace(uc)
		}
		if unicode.IsSpace(uc) != white {
			if start < index {
				span := clean[start:index]
				if toks, found := cache[span]; found {
					tokens = append(tokens, toks...)
				} else {
					toks := t.doSplit(span)
					cache[span] = toks
					tokens = append(tokens, toks...)
				}
			}
			if uc == ' ' {
				start = index + 1
			} else {
				start = index
			}
			white = !white
		}
		index += size
	}

	if start < index {
		tokens = append(tokens, t.doSplit(clean[start:index])...)
	}

	return tokens
}

var internalRE = regexp.MustCompile(`^(?:[A-Za-z]\.){2,}$|^[A-Z][a-z]{1,2}\.$`)
var sanitizer = strings.NewReplacer(
	"\u201c", `"`,
	"\u201d", `"`,
	"\u2018", "'",
	"\u2019", "'",
	"’", "'")
var contractions = []string{"'ll", "'s", "'re", "'m", "n't"}
var suffixes = []string{",", ")", `"`, "]", "!", ";", ".", "?", ":", "'"}
var prefixes = []string{"$", "(", `"`, "["}
var emoticons = map[string]int{
	"(-8":         1,
	"(-;":         1,
	"(-_-)":       1,
	"(._.)":       1,
	"(:":          1,
	"(=":          1,
	"(o:":         1,
	"(¬_¬)":       1,
	"(ಠ_ಠ)":       1,
	"(╯°□°）╯︵┻━┻": 1,
	"-__-":        1,
	"8-)":         1,
	"8-D":         1,
	"8D":          1,
	":(":          1,
	":((":         1,
	":(((":        1,
	":()":         1,
	":)))":        1,
	":-)":         1,
	":-))":        1,
	":-)))":       1,
	":-*":         1,
	":-/":         1,
	":-X":         1,
	":-]":         1,
	":-o":         1,
	":-p":         1,
	":-x":         1,
	":-|":         1,
	":-}":         1,
	":0":          1,
	":3":          1,
	":P":          1,
	":]":          1,
	":`(":         1,
	":`)":         1,
	":`-(":        1,
	":o":          1,
	":o)":         1,
	"=(":          1,
	"=)":          1,
	"=D":          1,
	"=|":          1,
	"@_@":         1,
	"O.o":         1,
	"O_o":         1,
	"V_V":         1,
	"XDD":         1,
	"[-:":         1,
	"^___^":       1,
	"o_0":         1,
	"o_O":         1,
	"o_o":         1,
	"v_v":         1,
	"xD":          1,
	"xDD":         1,
	"¯\\(ツ)/¯":    1,
}
<commit_msg>fix empty slice declaration via literal<commit_after>package prose

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

type TokenTester func(string) bool

type Tokenizer interface {
	Tokenize(string) []*Token
}

// iterTokenizer splits a sentence into words.
type iterTokenizer struct {
	specialRE      *regexp.Regexp
	sanitizer      *strings.Replacer
	contractions   []string
	splitCases     []string
	suffixes       []string
	prefixes       []string
	emoticons      map[string]int
	isUnsplittable TokenTester
}

type TokenizerOptFunc func(*iterTokenizer)

// UsingIsUnsplittableFN gives a function that tests whether a token is splittable or not.
func UsingIsUnsplittable(x TokenTester) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.isUnsplittable = x
	}
}

// Use the provided special regex for unsplittable tokens.
func UsingSpecialRE(x *regexp.Regexp) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.specialRE = x
	}
}

// Use the provided sanitizer.
func UsingSanitizer(x *strings.Replacer) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.sanitizer = x
	}
}

// Use the provided suffixes.
func UsingSuffixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.suffixes = x
	}
}

// Use the provided prefixes.
func UsingPrefixes(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.prefixes = x
	}
}

// Use the provided map of emoticons.
func UsingEmoticons(x map[string]int) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.emoticons = x
	}
}

// Use the provided contractions.
func UsingContractions(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.contractions = x
	}
}

// Use the provided splitCases.
func UsingSplitCases(x []string) TokenizerOptFunc {
	return func(tokenizer *iterTokenizer) {
		tokenizer.splitCases = x
	}
}

// Constructor for default iterTokenizer
func NewIterTokenizer(opts ...TokenizerOptFunc) *iterTokenizer {
	tok := new(iterTokenizer)

	// Set default parameters
	tok.contractions = contractions
	tok.emoticons = emoticons
	tok.isUnsplittable = func(_ string) bool { return false }
	tok.prefixes = prefixes
	tok.sanitizer = sanitizer
	tok.specialRE = internalRE
	tok.suffixes = suffixes

	// Apply options if provided
	for _, applyOpt := range opts {
		applyOpt(tok)
	}

	tok.splitCases = append(tok.splitCases, tok.contractions...)

	return tok
}

func addToken(s string, toks []*Token) []*Token {
	if strings.TrimSpace(s) != "" {
		toks = append(toks, &Token{Text: s})
	}
	return toks
}

func (t *iterTokenizer) isSpecial(token string) bool {
	_, found := t.emoticons[token]
	return found || t.specialRE.MatchString(token) || t.isUnsplittable(token)
}

func (t *iterTokenizer) doSplit(token string) []*Token {
	tokens := []*Token{}
	suffs := []*Token{}

	last := 0
	for token != "" && utf8.RuneCountInString(token) != last {
		if t.isSpecial(token) {
			// We've found a special case (e.g., an emoticon) -- so, we add it as a token without
			// any further processing.
			tokens = addToken(token, tokens)
			break
		}
		last = utf8.RuneCountInString(token)
		lower := strings.ToLower(token)
		if hasAnyPrefix(token, t.prefixes) {
			// Remove prefixes -- e.g., $100 -> [$, 100].
			tokens = addToken(string(token[0]), tokens)
			token = token[1:]
		} else if idx := hasAnyIndex(lower, t.splitCases); idx > -1 {
			// Handle "they'll", "I'll", "Don't", "won't", amount($).
			//
			// they'll -> [they, 'll].
			// don't -> [do, n't].
			// amount($) -> [amount, (, $, )].
			tokens = addToken(token[:idx], tokens)
			token = token[idx:]
		} else if hasAnySuffix(token, t.suffixes) {
			// Remove suffixes -- e.g., Well) -> [Well, )].
			suffs = append([]*Token{
				{Text: string(token[len(token)-1])}},
				suffs...)
			token = token[:len(token)-1]
		} else {
			tokens = addToken(token, tokens)
		}
	}

	return append(tokens, suffs...)
}

// tokenize splits a sentence into a slice of words.
func (t *iterTokenizer) Tokenize(text string) []*Token {
	var tokens []*Token

	clean, white := t.sanitizer.Replace(text), false
	length := len(clean)

	start, index := 0, 0
	cache := map[string][]*Token{}
	for index <= length {
		uc, size := utf8.DecodeRuneInString(clean[index:])
		if size == 0 {
			break
		} else if index == 0 {
			white = unicode.IsSpace(uc)
		}
		if unicode.IsSpace(uc) != white {
			if start < index {
				span := clean[start:index]
				if toks, found := cache[span]; found {
					tokens = append(tokens, toks...)
				} else {
					toks := t.doSplit(span)
					cache[span] = toks
					tokens = append(tokens, toks...)
				}
			}
			if uc == ' ' {
				start = index + 1
			} else {
				start = index
			}
			white = !white
		}
		index += size
	}

	if start < index {
		tokens = append(tokens, t.doSplit(clean[start:index])...)
	}

	return tokens
}

var internalRE = regexp.MustCompile(`^(?:[A-Za-z]\.){2,}$|^[A-Z][a-z]{1,2}\.$`)
var sanitizer = strings.NewReplacer(
	"\u201c", `"`,
	"\u201d", `"`,
	"\u2018", "'",
	"\u2019", "'",
	"’", "'")
var contractions = []string{"'ll", "'s", "'re", "'m", "n't"}
var suffixes = []string{",", ")", `"`, "]", "!", ";", ".", "?", ":", "'"}
var prefixes = []string{"$", "(", `"`, "["}
var emoticons = map[string]int{
	"(-8":         1,
	"(-;":         1,
	"(-_-)":       1,
	"(._.)":       1,
	"(:":          1,
	"(=":          1,
	"(o:":         1,
	"(¬_¬)":       1,
	"(ಠ_ಠ)":       1,
	"(╯°□°）╯︵┻━┻": 1,
	"-__-":        1,
	"8-)":         1,
	"8-D":         1,
	"8D":          1,
	":(":          1,
	":((":         1,
	":(((":        1,
	":()":         1,
	":)))":        1,
	":-)":         1,
	":-))":        1,
	":-)))":       1,
	":-*":         1,
	":-/":         1,
	":-X":         1,
	":-]":         1,
	":-o":         1,
	":-p":         1,
	":-x":         1,
	":-|":         1,
	":-}":         1,
	":0":          1,
	":3":          1,
	":P":          1,
	":]":          1,
	":`(":         1,
	":`)":         1,
	":`-(":        1,
	":o":          1,
	":o)":         1,
	"=(":          1,
	"=)":          1,
	"=D":          1,
	"=|":          1,
	"@_@":         1,
	"O.o":         1,
	"O_o":         1,
	"V_V":         1,
	"XDD":         1,
	"[-:":         1,
	"^___^":       1,
	"o_0":         1,
	"o_O":         1,
	"o_o":         1,
	"v_v":         1,
	"xD":          1,
	"xDD":         1,
	"¯\\(ツ)/¯":    1,
}
<|endoftext|>


<commit_before>package fastjet

import (
	"fmt"
)

// JetDefinition contains a full specification of how to carry out jet clustering.
type JetDefinition struct {
	alg        JetAlgorithm
	r          float64
	extra      float64
	strategy   Strategy
	recombiner Recombiner
	plugin     Plugin
}

func NewJetDefinition(alg JetAlgorithm, r float64, scheme RecombinationScheme, strategy Strategy) JetDefinition {
	return JetDefinition{
		alg:        alg,
		r:          r,
		recombiner: NewRecombiner(scheme),
		strategy:   strategy,
	}
}

func (def JetDefinition) Description() string {
	switch def.alg {
	case PluginAlgorithm:
		return def.plugin.Description()

	case KtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant kt algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case CambridgeAlgorithm:
		return fmt.Sprintf("Longitudinally invariant Cambridge/Aachen algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case AntiKtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant anti-kt algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case GenKtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant generalised kt algorithm with R = %v, p = %v and %s",
			def.R(), def.ExtraParam(), def.Recombiner().Description(),
		)

	case CambridgeForPassiveAlgorithm:
		return fmt.Sprintf("Longitudinally invariant Cambridge/Aache algorithm with R = %v, kt<%v as ghosts",
			def.R(), def.ExtraParam(),
		)

	case EeKtAlgorithm:
		return fmt.Sprintf("e+e- kt (Durham) algorithm with %s", def.Recombiner().Description())

	case EeGenKtAlgorithm:
		return fmt.Sprintf("e+e- generalised kt algorithm with R = %v, p = %s and %s",
			def.R(), def.ExtraParam(), def.Recombiner().Description(),
		)

	case UndefinedJetAlgorithm:
		return "uninitialised JetDefinition"

	default:
		panic(fmt.Errorf("fastjet.Description: invalid jet algorithm (%d)", int(def.alg)))
	}
}

func (def JetDefinition) R() float64 {
	return def.r
}

func (def JetDefinition) ExtraParam() float64 {
	return def.extra
}

func (def JetDefinition) Strategy() Strategy {
	return def.strategy
}

func (def JetDefinition) Recombiner() Recombiner {
	return def.recombiner
}

func (def JetDefinition) RecombinationScheme() RecombinationScheme {
	return def.recombiner.Scheme()
}

func (def JetDefinition) Algorithm() JetAlgorithm {
	return def.alg
}

func (def JetDefinition) Plugin() Plugin {
	return def.plugin
}

// to impl:
//  - ClusterSequence
//  - PseudoJet
//  - Selector + JetMedianBkgEstimator (only if compute-rho)
<commit_msg>definition: fix wrong printf verb<commit_after>package fastjet

import (
	"fmt"
)

// JetDefinition contains a full specification of how to carry out jet clustering.
type JetDefinition struct {
	alg        JetAlgorithm
	r          float64
	extra      float64
	strategy   Strategy
	recombiner Recombiner
	plugin     Plugin
}

func NewJetDefinition(alg JetAlgorithm, r float64, scheme RecombinationScheme, strategy Strategy) JetDefinition {
	return JetDefinition{
		alg:        alg,
		r:          r,
		recombiner: NewRecombiner(scheme),
		strategy:   strategy,
	}
}

func (def JetDefinition) Description() string {
	switch def.alg {
	case PluginAlgorithm:
		return def.plugin.Description()

	case KtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant kt algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case CambridgeAlgorithm:
		return fmt.Sprintf("Longitudinally invariant Cambridge/Aachen algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case AntiKtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant anti-kt algorithm with R = %v and %s",
			def.R(), def.Recombiner().Description(),
		)

	case GenKtAlgorithm:
		return fmt.Sprintf("Longitudinally invariant generalised kt algorithm with R = %v, p = %v and %s",
			def.R(), def.ExtraParam(), def.Recombiner().Description(),
		)

	case CambridgeForPassiveAlgorithm:
		return fmt.Sprintf("Longitudinally invariant Cambridge/Aache algorithm with R = %v, kt<%v as ghosts",
			def.R(), def.ExtraParam(),
		)

	case EeKtAlgorithm:
		return fmt.Sprintf("e+e- kt (Durham) algorithm with %s", def.Recombiner().Description())

	case EeGenKtAlgorithm:
		return fmt.Sprintf("e+e- generalised kt algorithm with R = %v, p = %v and %s",
			def.R(), def.ExtraParam(), def.Recombiner().Description(),
		)

	case UndefinedJetAlgorithm:
		return "uninitialised JetDefinition"

	default:
		panic(fmt.Errorf("fastjet.Description: invalid jet algorithm (%d)", int(def.alg)))
	}
}

func (def JetDefinition) R() float64 {
	return def.r
}

func (def JetDefinition) ExtraParam() float64 {
	return def.extra
}

func (def JetDefinition) Strategy() Strategy {
	return def.strategy
}

func (def JetDefinition) Recombiner() Recombiner {
	return def.recombiner
}

func (def JetDefinition) RecombinationScheme() RecombinationScheme {
	return def.recombiner.Scheme()
}

func (def JetDefinition) Algorithm() JetAlgorithm {
	return def.alg
}

func (def JetDefinition) Plugin() Plugin {
	return def.plugin
}

// to impl:
//  - ClusterSequence
//  - PseudoJet
//  - Selector + JetMedianBkgEstimator (only if compute-rho)
<|endoftext|>


<commit_before>package stripe

import "encoding/json"

// TransferStatus is the list of allowed values for the transfer's status.
// Allowed values are "paid", "pending", "in_transit",  "failed", "canceled".
type TransferStatus string

// TransferType is the list of allowed values for the transfer's type.
// Allowed values are "bank_account", "card", "stripe_account".
type TransferType string

// TransferSourceType is the list of allowed values for the transfer's source_type field.
// Allowed values are "alipay_account", bank_account", "bitcoin_receiver", "card".
type TransferSourceType string

// TransferFailCode is the list of allowed values for the transfer's failure code.
// Allowed values are "insufficient_funds", "account_closed", "no_account",
// "invalid_account_number", "debit_not_authorized", "bank_ownership_changed",
// "account_frozen", "could_not_process", "bank_account_restricted", "invalid_currency".
type TransferFailCode string

// TransferDestinationType consts represent valid transfer destinations.
type TransferDestinationType string

const (
	// TransferDestinationAccount is a constant representing a transfer destination
	// which is a Stripe account.
	TransferDestinationAccount TransferDestinationType = "account"

	// TransferDestinationBankAccount is a constant representing a transfer destination
	// which is a bank account.
	TransferDestinationBankAccount TransferDestinationType = "bank_account"

	// TransferDestinationCard is a constant representing a transfer destination
	// which is a card.
	TransferDestinationCard TransferDestinationType = "card"
)

// TransferDestination describes the destination of a Transfer.
// The Type should indicate which object is fleshed out
// For more details see https://stripe.com/docs/api/go#transfer_object
type TransferDestination struct {
	Type        TransferDestinationType `json:"object"`
	ID          string                  `json:"id"`
	Account     *Account                `json:"-"`
	BankAccount *BankAccount            `json:"-"`
	Card        *Card                   `json:"-"`
}

// TransferParams is the set of parameters that can be used when creating or updating a transfer.
// For more details see https://stripe.com/docs/api#create_transfer and https://stripe.com/docs/api#update_transfer.
type TransferParams struct {
	Params
	Amount                                                 int64
	Fee                                                    uint64
	Currency                                               Currency
	Recipient, Desc, Statement, Bank, Card, SourceTx, Dest string
	SourceType                                             TransferSourceType
}

// TransferListParams is the set of parameters that can be used when listing transfers.
// For more details see https://stripe.com/docs/api#list_transfers.
type TransferListParams struct {
	ListParams
	Created, Date int64
	Recipient     string
	Status        TransferStatus
}

// Transfer is the resource representing a Stripe transfer.
// For more details see https://stripe.com/docs/api#transfers.
type Transfer struct {
	ID             string              `json:"id"`
	Live           bool                `json:"livemode"`
	Amount         int64               `json:"amount"`
	AmountReversed int64               `json:"amount_reversed"`
	Currency       Currency            `json:"currency"`
	Created        int64               `json:"created"`
	Date           int64               `json:"date"`
	Desc           string              `json:"description"`
	Dest           TransferDestination `json:"destination"`
	FailCode       TransferFailCode    `json:"failure_code"`
	FailMsg        string              `json:"failure_message"`
	Status         TransferStatus      `json:"status"`
	Type           TransferType        `json:"type"`
	Tx             *Transaction        `json:"balance_transaction"`
	Meta           map[string]string   `json:"metadata"`
	Bank           *BankAccount        `json:"bank_account"`
	Card           *Card               `json:"card"`
	Recipient      *Recipient          `json:"recipient"`
	Statement      string              `json:"statement_descriptor"`
	Reversals      *ReversalList       `json:"reversals"`
	Reversed       bool                `json:"reversed"`
	SourceTx       *TransactionSource  `json:"source_transaction"`
	SourceType     TransferSourceType  `json:"source_type"`
}

// TransferList is a list of transfers as retrieved from a list endpoint.
type TransferList struct {
	ListMeta
	Values []*Transfer `json:"data"`
}

// UnmarshalJSON handles deserialization of a Transfer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (t *Transfer) UnmarshalJSON(data []byte) error {
	type transfer Transfer
	var tb transfer
	err := json.Unmarshal(data, &tb)
	if err == nil {
		*t = Transfer(tb)
	} else {
		// the id is surrounded by "\" characters, so strip them
		t.ID = string(data[1 : len(data)-1])
	}

	return nil
}

// UnmarshalJSON handles deserialization of a TransferDestination.
// This custom unmarshaling is needed because the specific
// type of destination it refers to is specified in the JSON
func (d *TransferDestination) UnmarshalJSON(data []byte) error {
	type dest TransferDestination
	var dd dest
	err := json.Unmarshal(data, &dd)
	if err == nil {
		*d = TransferDestination(dd)

		switch d.Type {
		case TransferDestinationAccount:
			json.Unmarshal(data, &d.Account)
		case TransferDestinationBankAccount:
			json.Unmarshal(data, &d.BankAccount)
		case TransferDestinationCard:
			json.Unmarshal(data, &d.Card)
		}
	} else {
		// the id is surrounded by "\" characters, so strip them
		d.ID = string(data[1 : len(data)-1])
	}

	return nil
}

// MarshalJSON handles serialization of a TransferDestination.
// This custom marshaling is needed because we can only send a string
// ID as a destination, even though it can be expanded to a full
// object when retrieving
func (d *TransferDestination) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.ID)
}
<commit_msg>add destination_payment to transfers collection<commit_after>package stripe

import "encoding/json"

// TransferStatus is the list of allowed values for the transfer's status.
// Allowed values are "paid", "pending", "in_transit",  "failed", "canceled".
type TransferStatus string

// TransferType is the list of allowed values for the transfer's type.
// Allowed values are "bank_account", "card", "stripe_account".
type TransferType string

// TransferSourceType is the list of allowed values for the transfer's source_type field.
// Allowed values are "alipay_account", bank_account", "bitcoin_receiver", "card".
type TransferSourceType string

// TransferFailCode is the list of allowed values for the transfer's failure code.
// Allowed values are "insufficient_funds", "account_closed", "no_account",
// "invalid_account_number", "debit_not_authorized", "bank_ownership_changed",
// "account_frozen", "could_not_process", "bank_account_restricted", "invalid_currency".
type TransferFailCode string

// TransferDestinationType consts represent valid transfer destinations.
type TransferDestinationType string

const (
	// TransferDestinationAccount is a constant representing a transfer destination
	// which is a Stripe account.
	TransferDestinationAccount TransferDestinationType = "account"

	// TransferDestinationBankAccount is a constant representing a transfer destination
	// which is a bank account.
	TransferDestinationBankAccount TransferDestinationType = "bank_account"

	// TransferDestinationCard is a constant representing a transfer destination
	// which is a card.
	TransferDestinationCard TransferDestinationType = "card"
)

// TransferDestination describes the destination of a Transfer.
// The Type should indicate which object is fleshed out
// For more details see https://stripe.com/docs/api/go#transfer_object
type TransferDestination struct {
	Type        TransferDestinationType `json:"object"`
	ID          string                  `json:"id"`
	Account     *Account                `json:"-"`
	BankAccount *BankAccount            `json:"-"`
	Card        *Card                   `json:"-"`
}

// TransferParams is the set of parameters that can be used when creating or updating a transfer.
// For more details see https://stripe.com/docs/api#create_transfer and https://stripe.com/docs/api#update_transfer.
type TransferParams struct {
	Params
	Amount                                                 int64
	Fee                                                    uint64
	Currency                                               Currency
	Recipient, Desc, Statement, Bank, Card, SourceTx, Dest string
	SourceType                                             TransferSourceType
}

// TransferListParams is the set of parameters that can be used when listing transfers.
// For more details see https://stripe.com/docs/api#list_transfers.
type TransferListParams struct {
	ListParams
	Created, Date int64
	Recipient     string
	Status        TransferStatus
}

// Transfer is the resource representing a Stripe transfer.
// For more details see https://stripe.com/docs/api#transfers.
type Transfer struct {
	ID                 string              `json:"id"`
	Live               bool                `json:"livemode"`
	Amount             int64               `json:"amount"`
	AmountReversed     int64               `json:"amount_reversed"`
	Currency           Currency            `json:"currency"`
	Created            int64               `json:"created"`
	Date               int64               `json:"date"`
	Desc               string              `json:"description"`
	Dest               TransferDestination `json:"destination"`
	FailCode           TransferFailCode    `json:"failure_code"`
	FailMsg            string              `json:"failure_message"`
	Status             TransferStatus      `json:"status"`
	Type               TransferType        `json:"type"`
	Tx                 *Transaction        `json:"balance_transaction"`
	Meta               map[string]string   `json:"metadata"`
	Bank               *BankAccount        `json:"bank_account"`
	Card               *Card               `json:"card"`
	Recipient          *Recipient          `json:"recipient"`
	Statement          string              `json:"statement_descriptor"`
	Reversals          *ReversalList       `json:"reversals"`
	Reversed           bool                `json:"reversed"`
	SourceTx           *TransactionSource  `json:"source_transaction"`
	SourceType         TransferSourceType  `json:"source_type"`
	DestinationPayment string              `json:"destination_payment"`
}

// TransferList is a list of transfers as retrieved from a list endpoint.
type TransferList struct {
	ListMeta
	Values []*Transfer `json:"data"`
}

// UnmarshalJSON handles deserialization of a Transfer.
// This custom unmarshaling is needed because the resulting
// property may be an id or the full struct if it was expanded.
func (t *Transfer) UnmarshalJSON(data []byte) error {
	type transfer Transfer
	var tb transfer
	err := json.Unmarshal(data, &tb)
	if err == nil {
		*t = Transfer(tb)
	} else {
		// the id is surrounded by "\" characters, so strip them
		t.ID = string(data[1 : len(data)-1])
	}

	return nil
}

// UnmarshalJSON handles deserialization of a TransferDestination.
// This custom unmarshaling is needed because the specific
// type of destination it refers to is specified in the JSON
func (d *TransferDestination) UnmarshalJSON(data []byte) error {
	type dest TransferDestination
	var dd dest
	err := json.Unmarshal(data, &dd)
	if err == nil {
		*d = TransferDestination(dd)

		switch d.Type {
		case TransferDestinationAccount:
			json.Unmarshal(data, &d.Account)
		case TransferDestinationBankAccount:
			json.Unmarshal(data, &d.BankAccount)
		case TransferDestinationCard:
			json.Unmarshal(data, &d.Card)
		}
	} else {
		// the id is surrounded by "\" characters, so strip them
		d.ID = string(data[1 : len(data)-1])
	}

	return nil
}

// MarshalJSON handles serialization of a TransferDestination.
// This custom marshaling is needed because we can only send a string
// ID as a destination, even though it can be expanded to a full
// object when retrieving
func (d *TransferDestination) MarshalJSON() ([]byte, error) {
	return json.Marshal(d.ID)
}
<|endoftext|>


<commit_before>//Package api implements the endpoints and utilities necessary to present
//a consistent API to autobd-nodes

package api

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/tywkeene/autobd/logging"
	"github.com/tywkeene/autobd/manifest"
	"github.com/tywkeene/autobd/packing"
	"github.com/tywkeene/autobd/version"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type Node struct {
	Address string
	Version string
	Online  string
	Synced  bool
}

var CurrentNodes map[string]*Node

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

//Check and make sure the client wants or can handle gzip, and replace the writer if it
//can, if not, simply use the normal http.ResponseWriter
func GzipHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

//GetQueryValue() takes a name of a key:value pair to fetchf rom a URL encoded query,
//a http.ResponseWriter 'w', and a http.Request 'r'. In the event that an error is encountered
//the error will be returned to the client via logging facilities that use 'w' and 'r'
func GetQueryValue(name string, w http.ResponseWriter, r *http.Request) string {
	query, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Query parse error"), http.StatusInternalServerError)
		return ""
	}
	value := query.Get(name)
	if len(value) == 0 || value == "" {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Must specify %s", name), http.StatusBadRequest)
		return ""
	}
	return value
}

//ServeManifest() is the http handler for the "/manifest" API endpoint.
//It takes the requested directory passed as a url parameter "dir" i.e "/manifest?dir=/"
//
//It will then generate a manifest by calling api.GetQueryValue(), then writes it to the client as a
//map[string]*manifest.Manifest encoded in json
func ServeManifest(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	dir := GetQueryValue("dir", w, r)
	if dir == "" {
		log.Println("No directory defined")
		logging.LogHttpErr(w, r, fmt.Errorf("Must define directory"), http.StatusInternalServerError)
		return
	}
	dirManifest, err := manifest.GetManifest(dir)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting manifest"), http.StatusInternalServerError)
		return
	}
	serial, _ := json.MarshalIndent(&dirManifest, "  ", "  ")
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Server", "Autobd v"+version.Server())
	io.WriteString(w, string(serial))

	uuid := GetQueryValue("uuid", w, r)
	updateNodeOnline(uuid)
}

//ServeServerVer() is the http handler for the "/version" http API endpoint.
//It writes the json encoded struct version.VersionInfo to the client
func ServeServerVer(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	serialVer, _ := json.MarshalIndent(&version.VersionInfo{version.Server(), version.Commit()}, "  ", "  ")

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Server", "Autobd v"+version.Server())
	io.WriteString(w, string(serialVer))
}

//ServeSync() is the http handler for the "/sync" http API endpoint.
//It takes the requested directory or file name passed as a url parameter "grab" i.e "/sync?grab=file1"
//
//If the requested file is a directory, it will be tarballed and the "Content-Type" http-header will be
//set to "application/x-tar".
//If the file is a normal file, it will be served with http.ServeContent(), with the Content-Type http-header
//set by http.ServeContent()
func ServeSync(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	grab := GetQueryValue("grab", w, r)
	if grab == "" {
		return
	}
	fd, err := os.Open(grab)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting file"), http.StatusInternalServerError)
		return
	}
	defer fd.Close()
	info, err := fd.Stat()
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting file"), http.StatusInternalServerError)
		return
	}
	if info.IsDir() == true {
		w.Header().Set("Content-Type", "application/x-tar")
		if err := packing.PackDir(grab, w); err != nil {
			log.Println(err)
			logging.LogHttpErr(w, r, fmt.Errorf("Error packing directory"), http.StatusInternalServerError)
			return
		}
		return
	}
	http.ServeContent(w, r, grab, info.ModTime(), fd)
	uuid := GetQueryValue("uuid", w, r)
	updateNodeSynced(uuid, true)
	updateNodeOnline(uuid)
}

//Identify() is the http handler for the "/identify" API endpoint
//It takes a node UUID and node version as json encoded strings
//The node is added to the CurrentNodes map, with the RFC850 timestamp
func Identify(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	uuid := GetQueryValue("uuid", w, r)
	version := GetQueryValue("version", w, r)
	log.Printf("New node [UUID: %s Address: %s Version: %s]", uuid, r.RemoteAddr, version)
	if CurrentNodes == nil {
		CurrentNodes = make(map[string]*Node)
		log.Println("Initialized node list")
	}
	CurrentNodes[uuid] = &Node{r.RemoteAddr, version, time.Now().Format(time.RFC850), false}
}

func updateNodeSynced(uuid string, val bool) {
	node := CurrentNodes[uuid]
	node.Synced = val
}

func updateNodeOnline(address string) {
	node := CurrentNodes[address]
	node.Online = time.Now().Format(time.RFC850)
}

//ListNodes() is the http handler for the "/nodes" API endpoint
//It returns the CurrentNodes map encoded in json
func ListNodes(w http.ResponseWriter, r *http.Request) {
	serial, _ := json.MarshalIndent(&CurrentNodes, " ", " ")
	io.WriteString(w, string(serial))
}

func SetupRoutes() {
	http.HandleFunc("/v"+version.Major()+"/manifest", GzipHandler(ServeManifest))
	http.HandleFunc("/v"+version.Major()+"/sync", GzipHandler(ServeSync))
	http.HandleFunc("/v"+version.Major()+"/identify", GzipHandler(Identify))
	http.HandleFunc("/v"+version.Major()+"/nodes", GzipHandler(ListNodes))
	http.HandleFunc("/version", GzipHandler(ServeServerVer))
}
<commit_msg>Added UUID check to /sync and /manifest api endpoints<commit_after>//Package api implements the endpoints and utilities necessary to present
//a consistent API to autobd-nodes

package api

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/tywkeene/autobd/logging"
	"github.com/tywkeene/autobd/manifest"
	"github.com/tywkeene/autobd/packing"
	"github.com/tywkeene/autobd/version"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type Node struct {
	Address string
	Version string
	Online  string
	Synced  bool
}

var CurrentNodes map[string]*Node

type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

//Check and make sure the client wants or can handle gzip, and replace the writer if it
//can, if not, simply use the normal http.ResponseWriter
func GzipHandler(fn http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			fn(w, r)
			return
		}
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzr := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		fn(gzr, r)
	}
}

//GetQueryValue() takes a name of a key:value pair to fetchf rom a URL encoded query,
//a http.ResponseWriter 'w', and a http.Request 'r'. In the event that an error is encountered
//the error will be returned to the client via logging facilities that use 'w' and 'r'
func GetQueryValue(name string, w http.ResponseWriter, r *http.Request) string {
	query, err := url.ParseQuery(r.URL.RawQuery)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Query parse error"), http.StatusInternalServerError)
		return ""
	}
	value := query.Get(name)
	if len(value) == 0 || value == "" {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Must specify %s", name), http.StatusBadRequest)
		return ""
	}
	return value
}

//ServeManifest() is the http handler for the "/manifest" API endpoint.
//It takes the requested directory passed as a url parameter "dir" i.e "/manifest?dir=/"
//
//It will then generate a manifest by calling api.GetQueryValue(), then writes it to the client as a
//map[string]*manifest.Manifest encoded in json
func ServeManifest(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)

	uuid := GetQueryValue("uuid", w, r)
	if validateNode(uuid) == false {
		logging.LogHttpErr(w, r, fmt.Errorf("Invalid node UUID"), http.StatusUnauthorized)
		return
	}
	updateNodeOnline(uuid)

	dir := GetQueryValue("dir", w, r)
	if dir == "" {
		log.Println("No directory defined")
		logging.LogHttpErr(w, r, fmt.Errorf("Must define directory"), http.StatusInternalServerError)
		return
	}
	dirManifest, err := manifest.GetManifest(dir)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting manifest"), http.StatusInternalServerError)
		return
	}
	serial, _ := json.MarshalIndent(&dirManifest, "  ", "  ")
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Server", "Autobd v"+version.Server())
	io.WriteString(w, string(serial))

}

//ServeServerVer() is the http handler for the "/version" http API endpoint.
//It writes the json encoded struct version.VersionInfo to the client
func ServeServerVer(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	serialVer, _ := json.MarshalIndent(&version.VersionInfo{version.Server(), version.Commit()}, "  ", "  ")

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Server", "Autobd v"+version.Server())
	io.WriteString(w, string(serialVer))
}

//ServeSync() is the http handler for the "/sync" http API endpoint.
//It takes the requested directory or file name passed as a url parameter "grab" i.e "/sync?grab=file1"
//
//If the requested file is a directory, it will be tarballed and the "Content-Type" http-header will be
//set to "application/x-tar".
//If the file is a normal file, it will be served with http.ServeContent(), with the Content-Type http-header
//set by http.ServeContent()
func ServeSync(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	uuid := GetQueryValue("uuid", w, r)
	if validateNode(uuid) == false {
		logging.LogHttpErr(w, r, fmt.Errorf("Invalid node UUID"), http.StatusUnauthorized)
		return
	}
	grab := GetQueryValue("grab", w, r)
	if grab == "" {
		return
	}
	fd, err := os.Open(grab)
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting file"), http.StatusInternalServerError)
		return
	}
	defer fd.Close()
	info, err := fd.Stat()
	if err != nil {
		log.Println(err)
		logging.LogHttpErr(w, r, fmt.Errorf("Error getting file"), http.StatusInternalServerError)
		return
	}
	if info.IsDir() == true {
		w.Header().Set("Content-Type", "application/x-tar")
		if err := packing.PackDir(grab, w); err != nil {
			log.Println(err)
			logging.LogHttpErr(w, r, fmt.Errorf("Error packing directory"), http.StatusInternalServerError)
			return
		}
		return
	}
	http.ServeContent(w, r, grab, info.ModTime(), fd)
	updateNodeSynced(uuid, true)
	updateNodeOnline(uuid)
}

//Identify() is the http handler for the "/identify" API endpoint
//It takes a node UUID and node version as json encoded strings
//The node is added to the CurrentNodes map, with the RFC850 timestamp
func Identify(w http.ResponseWriter, r *http.Request) {
	logging.LogHttp(r)
	uuid := GetQueryValue("uuid", w, r)
	version := GetQueryValue("version", w, r)
	log.Printf("New node [UUID: %s Address: %s Version: %s]", uuid, r.RemoteAddr, version)
	if CurrentNodes == nil {
		CurrentNodes = make(map[string]*Node)
		log.Println("Initialized node list")
	}
	CurrentNodes[uuid] = &Node{r.RemoteAddr, version, time.Now().Format(time.RFC850), false}
}

func updateNodeSynced(uuid string, val bool) {
	node := CurrentNodes[uuid]
	node.Synced = val
}

func updateNodeOnline(address string) {
	node := CurrentNodes[address]
	node.Online = time.Now().Format(time.RFC850)
}

func validateNode(uuid string) bool {
	_, ok := CurrentNodes[uuid]
	return ok
}

//ListNodes() is the http handler for the "/nodes" API endpoint
//It returns the CurrentNodes map encoded in json
func ListNodes(w http.ResponseWriter, r *http.Request) {
	serial, _ := json.MarshalIndent(&CurrentNodes, " ", " ")
	io.WriteString(w, string(serial))
}

func SetupRoutes() {
	http.HandleFunc("/v"+version.Major()+"/manifest", GzipHandler(ServeManifest))
	http.HandleFunc("/v"+version.Major()+"/sync", GzipHandler(ServeSync))
	http.HandleFunc("/v"+version.Major()+"/identify", GzipHandler(Identify))
	http.HandleFunc("/v"+version.Major()+"/nodes", GzipHandler(ListNodes))
	http.HandleFunc("/version", GzipHandler(ServeServerVer))
}
<|endoftext|>