<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/grd/stat"
	"log"
	"strings"
	"unicode"
)

func noHyphens(name string) error {
	if strings.Contains(name, "-") {
		return errors.New("Don't put hyphens, that's ugly.")
	}
	return nil
}

func noUnderscore(name string) error {
	if strings.Contains(name, "_") {
		return errors.New("Don't put underscores, that's ugly.")
	}
	return nil
}

func notCapitalized(name string) error {
	for _, r := range []rune(name) {
		if unicode.IsUpper(r) {
			return errors.New("Don't put uppercase characters, it's too enterprisey.")
		}
	}
	return nil
}

func noReferenceToGo(name string) error {
	if strings.Contains(strings.ToLower(name), "go") {
		return errors.New("Don't mention 'go' in your package name. Go is implicit in any package. Go is absolute and infinitesimal. Other languages should rename their packages; for instance 'rails-ruby' and 'python-django' would remove any ambiguity.")
	}
	return nil
}

func noReferenceToGolang(name string) error {
	if strings.Contains(strings.ToLower(name), "golang") {
		return errors.New("The name of Go is Go, not Golang. You don't say Javalang, or Rubylang, or Pythonlang, do you?")
	}
	return nil
}

var errInvalidPackage = "That's not even a valid package name: %s!" +
	" Read the spec: http://golang.org/ref/spec#Package_clause"

func validPackageNames(name string) error {
	if len(name) < 1 {
		return fmt.Errorf(errInvalidPackage, "the name can't be blank")
	}

	for i, r := range []rune(name) {
		if i == 0 {
			if !unicode.IsLetter(r) {
				return fmt.Errorf(errInvalidPackage, "the first character must be a letter")
			}
		}

		switch {
		case unicode.IsLetter(r):
		case unicode.IsDigit(r):
		case r == '-':
		case r == '_':
		case r == '.':
			// ok
		default:
			return fmt.Errorf(errInvalidPackage, "all the characters (but the first) must be either letters or digits")
		}
	}

	return nil
}

func closeToMean(allnames []string, maxDist float64) (f Filter, mean, stdev float64) {
	data := make(stat.IntSlice, len(allnames))
	for i, name := range allnames {
		data[i] = int64(len(name))
	}

	mean = stat.Mean(data)
	stdev = stat.SdMean(data, mean)

	maxMean := int(mean + stdev*maxDist)

	f = func(name string) error {
		diff := float64(len(name)) - mean
		dist := diff / stdev

		log.Printf("len=%d\tdiff=%f\tdist=%f\tmean=%f\tsd=%f", len(name), diff, dist, mean, stdev)

		if dist > maxDist {
			return fmt.Errorf("This package name is %.1f std.dev. longer than normal."+
				" It should be at most %d characters long.", dist, maxMean)
		}
		return nil
	}
	return
}
<commit_msg>Remove debug log entry<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/grd/stat"
	"strings"
	"unicode"
)

func noHyphens(name string) error {
	if strings.Contains(name, "-") {
		return errors.New("Don't put hyphens, that's ugly.")
	}
	return nil
}

func noUnderscore(name string) error {
	if strings.Contains(name, "_") {
		return errors.New("Don't put underscores, that's ugly.")
	}
	return nil
}

func notCapitalized(name string) error {
	for _, r := range []rune(name) {
		if unicode.IsUpper(r) {
			return errors.New("Don't put uppercase characters, it's too enterprisey.")
		}
	}
	return nil
}

func noReferenceToGo(name string) error {
	if strings.Contains(strings.ToLower(name), "go") {
		return errors.New("Don't mention 'go' in your package name. Go is implicit in any package. Go is absolute and infinitesimal. Other languages should rename their packages; for instance 'rails-ruby' and 'python-django' would remove any ambiguity.")
	}
	return nil
}

func noReferenceToGolang(name string) error {
	if strings.Contains(strings.ToLower(name), "golang") {
		return errors.New("The name of Go is Go, not Golang. You don't say Javalang, or Rubylang, or Pythonlang, do you?")
	}
	return nil
}

var errInvalidPackage = "That's not even a valid package name: %s!" +
	" Read the spec: http://golang.org/ref/spec#Package_clause"

func validPackageNames(name string) error {
	if len(name) < 1 {
		return fmt.Errorf(errInvalidPackage, "the name can't be blank")
	}

	for i, r := range []rune(name) {
		if i == 0 {
			if !unicode.IsLetter(r) {
				return fmt.Errorf(errInvalidPackage, "the first character must be a letter")
			}
		}

		switch {
		case unicode.IsLetter(r):
		case unicode.IsDigit(r):
		case r == '-':
		case r == '_':
		case r == '.':
			// ok
		default:
			return fmt.Errorf(errInvalidPackage, "all the characters (but the first) must be either letters or digits")
		}
	}

	return nil
}

func closeToMean(allnames []string, maxDist float64) (f Filter, mean, stdev float64) {
	data := make(stat.IntSlice, len(allnames))
	for i, name := range allnames {
		data[i] = int64(len(name))
	}

	mean = stat.Mean(data)
	stdev = stat.SdMean(data, mean)

	maxMean := int(mean + stdev*maxDist)

	f = func(name string) error {
		diff := float64(len(name)) - mean
		dist := diff / stdev

		if dist > maxDist {
			return fmt.Errorf("This package name is %.1f std.dev. longer than normal."+
				" It should be at most %d characters long.", dist, maxMean)
		}
		return nil
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/go-kit/kit/log/term"
	metrics "github.com/rcrowley/go-metrics"

	"text/tabwriter"

	tmrpc "github.com/tendermint/tendermint/rpc/client"
	"github.com/tendermint/tmlibs/log"
)

var version = "0.3.0"

var logger = log.NewNopLogger()

type statistics struct {
	TxsThroughput    metrics.Histogram `json:"txs_per_sec"`
	BlocksThroughput metrics.Histogram `json:"blocks_per_sec"`
}

func main() {
	var duration, txsRate, connections int
	var verbose bool
	var outputFormat, broadcastTxMethod string

	flag.IntVar(&connections, "c", 1, "Connections to keep open per endpoint")
	flag.IntVar(&duration, "T", 10, "Exit after the specified amount of time in seconds")
	flag.IntVar(&txsRate, "r", 1000, "Txs per second to send in a connection")
	flag.StringVar(&outputFormat, "output-format", "plain", "Output format: plain or json")
	flag.StringVar(&broadcastTxMethod, "broadcast-tx-method", "async", "Broadcast method: async (no guarantees; fastest), sync (ensures tx is checked) or commit (ensures tx is checked and committed; slowest)")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Usage = func() {
		fmt.Println(`Tendermint blockchain benchmarking tool.

Usage:
	tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] [-output-format <plain|json> [-broadcast-tx-method <async|sync|commit>]]

Examples:
	tm-bench localhost:46657`)
		fmt.Println("Flags:")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if verbose {
		if outputFormat == "json" {
			fmt.Fprintln(os.Stderr, "Verbose mode not supported with json output.")
			os.Exit(1)
		}
		// Color errors red
		colorFn := func(keyvals ...interface{}) term.FgBgColor {
			for i := 1; i < len(keyvals); i += 2 {
				if _, ok := keyvals[i].(error); ok {
					return term.FgBgColor{Fg: term.White, Bg: term.Red}
				}
			}
			return term.FgBgColor{}
		}
		logger = log.NewTMLoggerWithColorFn(log.NewSyncWriter(os.Stdout), colorFn)

		fmt.Printf("Running %ds test @ %s\n", duration, flag.Arg(0))
	}

	if broadcastTxMethod != "async" && broadcastTxMethod != "sync" && broadcastTxMethod != "commit" {
		fmt.Fprintln(os.Stderr, "broadcast-tx-method should be either 'sync', 'async' or 'commit'.")
		os.Exit(1)
	}

	endpoints := strings.Split(flag.Arg(0), ",")

	client := tmrpc.NewHTTP(endpoints[0], "/websocket")

	minHeight := latestBlockHeight(client)
	logger.Info("Latest block height", "h", minHeight)

	// record time start
	timeStart := time.Now()
	logger.Info("Time started", "t", timeStart)

	transacters := startTransacters(endpoints, connections, txsRate, "broadcast_tx_"+broadcastTxMethod)

	select {
	case <-time.After(time.Duration(duration) * time.Second):
		for _, t := range transacters {
			t.Stop()
		}
		timeStop := time.Now()
		logger.Info("Time stopped", "t", timeStop)

		stats := calculateStatistics(client, minHeight, timeStart, timeStop)

		printStatistics(stats, outputFormat)

		return
	}
}

func latestBlockHeight(client tmrpc.Client) int64 {
	status, err := client.Status()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	return status.SyncInfo.LatestBlockHeight
}

func calculateStatistics(client tmrpc.Client, minHeight int64, timeStart, timeStop time.Time) *statistics {
	stats := &statistics{
		BlocksThroughput: metrics.NewHistogram(metrics.NewUniformSample(1000)),
		TxsThroughput:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
	}

	// get blocks between minHeight and last height
	info, err := client.BlockchainInfo(minHeight, 0)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	numBlocksPerSec := make(map[int64]int64)
	numTxsPerSec := make(map[int64]int64)
	for _, blockMeta := range info.BlockMetas {
		// check if block was created after timeStart
		if blockMeta.Header.Time.Before(timeStart) {
			continue
		}

		// check if block was created before timeStop
		if blockMeta.Header.Time.After(timeStop) {
			break
		}

		sec := secondsSinceTimeStart(timeStart, blockMeta.Header.Time)

		// increase number of blocks for that second
		if _, ok := numBlocksPerSec[sec]; !ok {
			numBlocksPerSec[sec] = 0
		}
		numBlocksPerSec[sec]++

		// increase number of txs for that second
		if _, ok := numTxsPerSec[sec]; !ok {
			numTxsPerSec[sec] = 0
		}
		numTxsPerSec[sec] += blockMeta.Header.NumTxs
	}

	for _, n := range numBlocksPerSec {
		stats.BlocksThroughput.Update(n)
	}

	for _, n := range numTxsPerSec {
		stats.TxsThroughput.Update(n)
	}

	return stats
}

func secondsSinceTimeStart(timeStart, timePassed time.Time) int64 {
	return int64(timePassed.Sub(timeStart).Seconds())
}

func startTransacters(endpoints []string, connections, txsRate int, broadcastTxMethod string) []*transacter {
	transacters := make([]*transacter, len(endpoints))

	for i, e := range endpoints {
		t := newTransacter(e, connections, txsRate, broadcastTxMethod)
		t.SetLogger(logger)
		if err := t.Start(); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		transacters[i] = t
	}

	return transacters
}

func printStatistics(stats *statistics, outputFormat string) {
	if outputFormat == "json" {
		result, err := json.Marshal(struct {
			TxsThroughput    float64 `json:"txs_per_sec_avg"`
			BlocksThroughput float64 `json:"blocks_per_sec_avg"`
		}{stats.TxsThroughput.Mean(), stats.BlocksThroughput.Mean()})
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		fmt.Println(string(result))
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 0, 5, ' ', 0)
		fmt.Fprintln(w, "Stats\tAvg\tStdDev\tMax\t")
		fmt.Fprintln(w, fmt.Sprintf("Txs/sec\t%.0f\t%.0f\t%d\t",
			stats.TxsThroughput.Mean(),
			stats.TxsThroughput.StdDev(),
			stats.TxsThroughput.Max()))
		fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
			stats.BlocksThroughput.Mean(),
			stats.BlocksThroughput.StdDev(),
			stats.BlocksThroughput.Max()))
		w.Flush()
	}
}
<commit_msg>fix stats calculation<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/go-kit/kit/log/term"
	metrics "github.com/rcrowley/go-metrics"

	"text/tabwriter"

	tmrpc "github.com/tendermint/tendermint/rpc/client"
	"github.com/tendermint/tmlibs/log"
)

var version = "0.3.0"

var logger = log.NewNopLogger()

type statistics struct {
	TxsThroughput    metrics.Histogram `json:"txs_per_sec"`
	BlocksThroughput metrics.Histogram `json:"blocks_per_sec"`
}

func main() {
	var duration, txsRate, connections int
	var verbose bool
	var outputFormat, broadcastTxMethod string

	flag.IntVar(&connections, "c", 1, "Connections to keep open per endpoint")
	flag.IntVar(&duration, "T", 10, "Exit after the specified amount of time in seconds")
	flag.IntVar(&txsRate, "r", 1000, "Txs per second to send in a connection")
	flag.StringVar(&outputFormat, "output-format", "plain", "Output format: plain or json")
	flag.StringVar(&broadcastTxMethod, "broadcast-tx-method", "async", "Broadcast method: async (no guarantees; fastest), sync (ensures tx is checked) or commit (ensures tx is checked and committed; slowest)")
	flag.BoolVar(&verbose, "v", false, "Verbose output")

	flag.Usage = func() {
		fmt.Println(`Tendermint blockchain benchmarking tool.

Usage:
	tm-bench [-c 1] [-T 10] [-r 1000] [endpoints] [-output-format <plain|json> [-broadcast-tx-method <async|sync|commit>]]

Examples:
	tm-bench localhost:46657`)
		fmt.Println("Flags:")
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	if verbose {
		if outputFormat == "json" {
			fmt.Fprintln(os.Stderr, "Verbose mode not supported with json output.")
			os.Exit(1)
		}
		// Color errors red
		colorFn := func(keyvals ...interface{}) term.FgBgColor {
			for i := 1; i < len(keyvals); i += 2 {
				if _, ok := keyvals[i].(error); ok {
					return term.FgBgColor{Fg: term.White, Bg: term.Red}
				}
			}
			return term.FgBgColor{}
		}
		logger = log.NewTMLoggerWithColorFn(log.NewSyncWriter(os.Stdout), colorFn)

		fmt.Printf("Running %ds test @ %s\n", duration, flag.Arg(0))
	}

	if broadcastTxMethod != "async" && broadcastTxMethod != "sync" && broadcastTxMethod != "commit" {
		fmt.Fprintln(os.Stderr, "broadcast-tx-method should be either 'sync', 'async' or 'commit'.")
		os.Exit(1)
	}

	endpoints := strings.Split(flag.Arg(0), ",")

	client := tmrpc.NewHTTP(endpoints[0], "/websocket")

	minHeight := latestBlockHeight(client)
	logger.Info("Latest block height", "h", minHeight)

	// record time start
	timeStart := time.Now()
	logger.Info("Time started", "t", timeStart)

	transacters := startTransacters(endpoints, connections, txsRate, "broadcast_tx_"+broadcastTxMethod)

	select {
	case <-time.After(time.Duration(duration) * time.Second):
		for _, t := range transacters {
			t.Stop()
		}
		timeStop := time.Now()
		logger.Info("Time stopped", "t", timeStop)

		stats := calculateStatistics(client, minHeight, timeStart, timeStop, duration)

		printStatistics(stats, outputFormat)

		return
	}
}

func latestBlockHeight(client tmrpc.Client) int64 {
	status, err := client.Status()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	return status.SyncInfo.LatestBlockHeight
}

func calculateStatistics(client tmrpc.Client, minHeight int64, timeStart, timeStop time.Time, duration int) *statistics {
	stats := &statistics{
		BlocksThroughput: metrics.NewHistogram(metrics.NewUniformSample(1000)),
		TxsThroughput:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
	}

	// get blocks between minHeight and last height
	info, err := client.BlockchainInfo(minHeight, 0)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	numBlocksPerSec := make(map[int64]int64)
	numTxsPerSec := make(map[int64]int64)
	// because during some seconds blocks won't be created...
	for i := int64(0); i < int64(duration); i++ {
		numBlocksPerSec[i] = 0
		numTxsPerSec[i] = 0
	}

	for _, blockMeta := range info.BlockMetas {
		// check if block was created after timeStart
		if blockMeta.Header.Time.Before(timeStart) {
			continue
		}

		// check if block was created before timeStop
		if blockMeta.Header.Time.After(timeStop) {
			break
		}

		sec := secondsSinceTimeStart(timeStart, blockMeta.Header.Time)

		// increase number of blocks for that second
		if _, ok := numBlocksPerSec[sec]; !ok {
			numBlocksPerSec[sec] = 0
		}
		numBlocksPerSec[sec]++

		// increase number of txs for that second
		if _, ok := numTxsPerSec[sec]; !ok {
			numTxsPerSec[sec] = 0
		}
		numTxsPerSec[sec] += blockMeta.Header.NumTxs
	}

	for _, n := range numBlocksPerSec {
		stats.BlocksThroughput.Update(n)
	}

	for _, n := range numTxsPerSec {
		stats.TxsThroughput.Update(n)
	}

	return stats
}

func secondsSinceTimeStart(timeStart, timePassed time.Time) int64 {
	return int64(timePassed.Sub(timeStart).Seconds())
}

func startTransacters(endpoints []string, connections, txsRate int, broadcastTxMethod string) []*transacter {
	transacters := make([]*transacter, len(endpoints))

	for i, e := range endpoints {
		t := newTransacter(e, connections, txsRate, broadcastTxMethod)
		t.SetLogger(logger)
		if err := t.Start(); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		transacters[i] = t
	}

	return transacters
}

func printStatistics(stats *statistics, outputFormat string) {
	if outputFormat == "json" {
		result, err := json.Marshal(struct {
			TxsThroughput    float64 `json:"txs_per_sec_avg"`
			BlocksThroughput float64 `json:"blocks_per_sec_avg"`
		}{stats.TxsThroughput.Mean(), stats.BlocksThroughput.Mean()})
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		fmt.Println(string(result))
	} else {
		w := tabwriter.NewWriter(os.Stdout, 0, 0, 5, ' ', 0)
		fmt.Fprintln(w, "Stats\tAvg\tStdDev\tMax\t")
		fmt.Fprintln(w, fmt.Sprintf("Txs/sec\t%.0f\t%.0f\t%d\t",
			stats.TxsThroughput.Mean(),
			stats.TxsThroughput.StdDev(),
			stats.TxsThroughput.Max()))
		fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
			stats.BlocksThroughput.Mean(),
			stats.BlocksThroughput.StdDev(),
			stats.BlocksThroughput.Max()))
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package bloom

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"testing"
)

func TestBasic(t *testing.T) {
	f := New(1000, 4)
	n1 := []byte("Bess")
	n2 := []byte("Jane")
	n3 := []byte("Emma")
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestBasicUint32(t *testing.T) {
	f := New(1000, 4)
	n1 := make([]byte, 4)
	n2 := make([]byte, 4)
	n3 := make([]byte, 4)
	n4 := make([]byte, 4)
	binary.BigEndian.PutUint32(n1, 100)
	binary.BigEndian.PutUint32(n2, 101)
	binary.BigEndian.PutUint32(n3, 102)
	binary.BigEndian.PutUint32(n4, 103)
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	f.Test(n4)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestDirect20_5(t *testing.T) {
	n := uint(10000)
	k := uint(5)
	load := uint(20)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestDirect15_10(t *testing.T) {
	n := uint(10000)
	k := uint(10)
	load := uint(15)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestEstimated10_0001(t *testing.T) {
	n := uint(10000)
	fp := 0.0001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestEstimated10_001(t *testing.T) {
	n := uint(10000)
	fp := 0.001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestMarshalUnmarshalJSON(t *testing.T) {
	f := New(1000, 4)
	data, err := json.Marshal(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = json.Unmarshal(data, &g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid m value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func BenchmarkDirect(b *testing.B) {
	n := uint(10000)
	max_k := uint(10)
	max_load := uint(20)
	fmt.Printf("m/n")
	for k := uint(2); k <= max_k; k++ {
		fmt.Printf("\tk=%v", k)
	}
	fmt.Println()
	for load := uint(2); load <= max_load; load++ {
		fmt.Print(load)
		for k := uint(2); k <= max_k; k++ {
			f := New(n*load, k)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%f", fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkEstimted(b *testing.B) {
	for n := uint(5000); n <= 50000; n += 5000 {
		fmt.Printf("%v", n)
		for fp := 0.1; fp >= 0.00001; fp /= 10.0 {
			fmt.Printf("\t%f", fp)
			m, k := estimateParameters(n, fp)
			f := NewWithEstimates(n, fp)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%v\t%v\t%f", m, k, fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkSeparateTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.Test(key)
		f.Add(key)
	}
}

func BenchmarkCombinedTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.TestAndAdd(key)
	}
}
<commit_msg>correct typographic error<commit_after>package bloom

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"testing"
)

func TestBasic(t *testing.T) {
	f := New(1000, 4)
	n1 := []byte("Bess")
	n2 := []byte("Jane")
	n3 := []byte("Emma")
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestBasicUint32(t *testing.T) {
	f := New(1000, 4)
	n1 := make([]byte, 4)
	n2 := make([]byte, 4)
	n3 := make([]byte, 4)
	n4 := make([]byte, 4)
	binary.BigEndian.PutUint32(n1, 100)
	binary.BigEndian.PutUint32(n2, 101)
	binary.BigEndian.PutUint32(n3, 102)
	binary.BigEndian.PutUint32(n4, 103)
	f.Add(n1)
	n3a := f.TestAndAdd(n3)
	n1b := f.Test(n1)
	n2b := f.Test(n2)
	n3b := f.Test(n3)
	f.Test(n4)
	if !n1b {
		t.Errorf("%v should be in.", n1)
	}
	if n2b {
		t.Errorf("%v should not be in.", n2)
	}
	if n3a {
		t.Errorf("%v should not be in the first time we look.", n3)
	}
	if !n3b {
		t.Errorf("%v should be in the second time we look.", n3)
	}
}

func TestDirect20_5(t *testing.T) {
	n := uint(10000)
	k := uint(5)
	load := uint(20)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestDirect15_10(t *testing.T) {
	n := uint(10000)
	k := uint(10)
	load := uint(15)
	f := New(n*load, k)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > 0.0001 {
		t.Errorf("False positive rate too high: load=%v, k=%v, %f", load, k, fp_rate)
	}
}

func TestEstimated10_0001(t *testing.T) {
	n := uint(10000)
	fp := 0.0001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestEstimated10_001(t *testing.T) {
	n := uint(10000)
	fp := 0.001
	m, k := estimateParameters(n, fp)
	f := NewWithEstimates(n, fp)
	fp_rate := f.EstimateFalsePositiveRate(n)
	if fp_rate > fp {
		t.Errorf("False positive rate too high: n: %v, fp: %f, n: %v, k: %v result: %f", n, fp, m, k, fp_rate)
	}
}

func TestMarshalUnmarshalJSON(t *testing.T) {
	f := New(1000, 4)
	data, err := json.Marshal(f)
	if err != nil {
		t.Fatal(err.Error())
	}

	var g BloomFilter
	err = json.Unmarshal(data, &g)
	if err != nil {
		t.Fatal(err.Error())
	}
	if g.m != f.m {
		t.Error("invalid m value")
	}
	if g.k != f.k {
		t.Error("invalid m value")
	}
	if g.b == nil {
		t.Fatal("bitset is nil")
	}
	if !g.b.Equal(f.b) {
		t.Error("bitsets are not equal")
	}
}

func BenchmarkDirect(b *testing.B) {
	n := uint(10000)
	max_k := uint(10)
	max_load := uint(20)
	fmt.Printf("m/n")
	for k := uint(2); k <= max_k; k++ {
		fmt.Printf("\tk=%v", k)
	}
	fmt.Println()
	for load := uint(2); load <= max_load; load++ {
		fmt.Print(load)
		for k := uint(2); k <= max_k; k++ {
			f := New(n*load, k)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%f", fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkEstimated(b *testing.B) {
	for n := uint(5000); n <= 50000; n += 5000 {
		fmt.Printf("%v", n)
		for fp := 0.1; fp >= 0.00001; fp /= 10.0 {
			fmt.Printf("\t%f", fp)
			m, k := estimateParameters(n, fp)
			f := NewWithEstimates(n, fp)
			fp_rate := f.EstimateFalsePositiveRate(n)
			fmt.Printf("\t%v\t%v\t%f", m, k, fp_rate)
		}
		fmt.Println()
	}
}

func BenchmarkSeparateTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.Test(key)
		f.Add(key)
	}
}

func BenchmarkCombinedTestAndAdd(b *testing.B) {
	f := NewWithEstimates(uint(b.N), 0.0001)
	key := make([]byte, 100)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		binary.BigEndian.PutUint32(key, uint32(i))
		f.TestAndAdd(key)
	}
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 Arnaud Vazard
//
// See LICENSE file.
package search

import (
	"encoding/json"
	"fmt"
	"github.com/romainletendart/goxxx/core"
	"github.com/thoj/go-ircevent"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
)

//  --- --- --- Constants --- --- ---
const (
	URL_DUCKDUCKGO       string = "https://duckduckgo.com/html/?q=%s"
	URL_WIKIPEDIA        string = "https://%s.wikipedia.org/w/api.php?format=json&action=query&prop=extracts|info&exintro=&explaintext=&inprop=url&titles=%s"
	URL_URBANDICTIONNARY string = "http://api.urbandictionary.com/v0/define?term=%s"

	HELP_DUCKDUCKGO       string = "!d/!dg/!ddg <terms to search> \t=> Search on DuckduckGo"
	HELP_WIKIPEDIA        string = "!w <terms to search> \t\t\t=> Search on Wikipedia EN"
	HELP_WIKIPEDIA_FR     string = "!wf <terms to search> \t\t\t=> Search on Wikipedia FR"
	HELP_URBANDICTIONNARY string = "!u/!ud <terms to search> \t\t=> Search on Urban Dictionnary"
)

// --- --- --- Types  --- --- ---
type wikipedia struct {
	Query struct {
		Pages map[string]struct {
			Extract string `json:"extract"`
			Fullurl string `json:"fullurl"`
			Title   string `json:"title"`
		} `json:"pages"`
	} `json:"query"`
}

type urbanDictionnary struct {
	List []struct {
		Definition string `json:"definition"`
		Example    string `json:"example"`
		Permalink  string `json:"permalink"`
	} `json:"list"`
}

type searchData struct {
	getUrl         func(string, string) []string
	extraParameter string
	// First string is for the first result (URL), second string for the second result (Details/Definition/...)
	text_result    [2]string
	text_no_result string
}

// --- --- --- Global variable --- --- ---
var searchMap = make(map[string]*searchData)

func Init() {
	ddg := &searchData{
		getUrl:         getDuckduckgoSearchResult,
		text_result:    [2]string{"DuckDuckGo: Best result for %q => %s"},
		text_no_result: "DuckDuckGo: No result for %q"}

	searchMap["!d"] = ddg
	searchMap["!dg"] = ddg
	searchMap["!ddg"] = ddg

	searchMap["!w"] = &searchData{
		getUrl:         getWikipediaSearchResult,
		extraParameter: "en",
		text_result:    [2]string{"Wikipedia result for %q => %s"},
		text_no_result: "Wikipedia: No result for %q"}

	wfr := &searchData{
		getUrl:         getWikipediaSearchResult,
		extraParameter: "fr",
		text_result:    [2]string{"Wikipedia result for %q => %s"},
		text_no_result: "Wikipedia: No result for %q"}

	searchMap["!wf"] = wfr
	searchMap["!wfr"] = wfr

	ud := &searchData{
		getUrl:         getUrbanDictionnarySearchResult,
		text_result:    [2]string{"Urban Dictionnary: Best result for %q => %s", "Definition: %s"},
		text_no_result: "Urban Dictionnary: No result for %q"}

	searchMap["!u"] = ud
	searchMap["!ud"] = ud
}

func HandleSearchCmd(event *irc.Event, callback func(*core.ReplyCallbackData)) bool {
	if callback == nil {
		log.Println("Callback nil for the HandleSearchCmd function")
		return false
	}

	fields := strings.Fields(event.Message())
	// fields[0]  => Command
	// fields[1:] => terms to search for
	if len(fields) == 0 {
		return false
	}

	search, present := searchMap[fields[0]]
	if !present {
		return false
	}

	message := strings.Join(fields[1:], " ")
	if message == "" {
		callback(&core.ReplyCallbackData{Message: fmt.Sprintf("Search usage: %s \"terms to search for\"", fields[0])})
		return false
	}

	results := search.getUrl(message, search.extraParameter)
	if results == nil {
		callback(&core.ReplyCallbackData{Message: fmt.Sprintf(search.text_no_result, message)})
		return true
	}
	for index, item := range results {
		switch index {
		// First part of the result is sent to everyone, no nick is sent
		case 0:
			if search.text_result[index] != "" {
				callback(&core.ReplyCallbackData{Message: fmt.Sprintf(search.text_result[index], message, item)})
			} else {
				callback(&core.ReplyCallbackData{Message: item})
			}
		// Second and following parts of the result are sent directly to the user
		case 1:
			if search.text_result[index] != "" {
				callback(&core.ReplyCallbackData{Nick: event.Nick, Message: fmt.Sprintf(search.text_result[index], item)})
			} else {
				callback(&core.ReplyCallbackData{Nick: event.Nick, Message: item})
			}
		default:
			callback(&core.ReplyCallbackData{Nick: event.Nick, Message: item})
		}
	}
	return true
}

// --- --- --- HTTP Functions --- --- ---
func getResponseAsText(url string) []byte {
	response, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()

	text, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return nil
	}
	response.Body.Close()
	return text
}

// --- --- --- Search provider functions --- --- ---

// --- Duckduckgo
func getDuckduckgoResultFromHtml(page []byte) string {
	re := regexp.MustCompile(`<a rel="nofollow" href="(.[^"]*)">`)
	if result := re.FindSubmatch(page); result != nil && len(result) == 2 {
		return string(result[1])
	}
	return ""
}

func getDuckduckgoSearchResult(searchTerms string, extraParameter string) []string {
	HtmlPageFromHttp := getResponseAsText(fmt.Sprintf(URL_DUCKDUCKGO, searchTerms))
	if HtmlPageFromHttp == nil {
		return nil
	}
	result := getDuckduckgoResultFromHtml(HtmlPageFromHttp)
	if result == "" {
		return nil
	}
	return []string{result}
}

// --- Wikipedia
func getWikipediaResultFromJson(jsonDataFromHttp []byte) []string {
	var result wikipedia
	err := json.Unmarshal(jsonDataFromHttp, &result)
	if err != nil {
		log.Println(err)
		return nil
	}

	var returnValues []string
	for key, value := range result.Query.Pages {
		if key != "-1" {
			returnValues = append(returnValues, value.Fullurl)
			returnValues = append(returnValues, strings.Split(value.Extract, ". ")...)
		}
	}
	return returnValues
}

func getWikipediaSearchResult(searchTerms string, extraParameter string) []string {
	searchTerms = strings.Replace(strings.Title(searchTerms), " ", "%20", -1)
	jsonDataFromHttp := getResponseAsText(fmt.Sprintf(URL_WIKIPEDIA, extraParameter, searchTerms))
	if jsonDataFromHttp == nil {
		return nil
	}
	result := getWikipediaResultFromJson(jsonDataFromHttp)
	if result == nil {
		return nil
	}
	return result
}

// --- Urban Dictionnary
func getUrbanDictionnaryResultFromJson(jsonDataFromHttp []byte) []string {
	var result urbanDictionnary
	err := json.Unmarshal(jsonDataFromHttp, &result)
	if err != nil {
		log.Println(err)
		return nil
	}
	if len(result.List) == 0 {
		return nil
	}
	returnValues := []string{result.List[0].Permalink}
	returnValues = append(returnValues, strings.Split(result.List[0].Definition, ". ")...)

	return returnValues
}

func getUrbanDictionnarySearchResult(searchTerms string, extraParameter string) []string {
	searchTerms = strings.Replace(strings.Title(searchTerms), " ", "%20", -1)
	jsonDataFromHttp := getResponseAsText(fmt.Sprintf(URL_URBANDICTIONNARY, searchTerms))
	if jsonDataFromHttp == nil {
		return nil
	}
	result := getUrbanDictionnaryResultFromJson(jsonDataFromHttp)
	if result == nil {
		return nil
	}
	return result
}
<commit_msg>Search: Updated help message for wikipedia FR<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 Arnaud Vazard
//
// See LICENSE file.
package search

import (
	"encoding/json"
	"fmt"
	"github.com/romainletendart/goxxx/core"
	"github.com/thoj/go-ircevent"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
)

//  --- --- --- Constants --- --- ---
const (
	URL_DUCKDUCKGO       string = "https://duckduckgo.com/html/?q=%s"
	URL_WIKIPEDIA        string = "https://%s.wikipedia.org/w/api.php?format=json&action=query&prop=extracts|info&exintro=&explaintext=&inprop=url&titles=%s"
	URL_URBANDICTIONNARY string = "http://api.urbandictionary.com/v0/define?term=%s"

	HELP_DUCKDUCKGO       string = "!d/!dg/!ddg <terms to search> \t=> Search on DuckduckGo"
	HELP_WIKIPEDIA        string = "!w <terms to search> \t\t\t=> Search on Wikipedia EN"
	HELP_WIKIPEDIA_FR     string = "!wf/!wfr <terms to search> \t\t\t=> Search on Wikipedia FR"
	HELP_URBANDICTIONNARY string = "!u/!ud <terms to search> \t\t=> Search on Urban Dictionnary"
)

// --- --- --- Types  --- --- ---
type wikipedia struct {
	Query struct {
		Pages map[string]struct {
			Extract string `json:"extract"`
			Fullurl string `json:"fullurl"`
			Title   string `json:"title"`
		} `json:"pages"`
	} `json:"query"`
}

type urbanDictionnary struct {
	List []struct {
		Definition string `json:"definition"`
		Example    string `json:"example"`
		Permalink  string `json:"permalink"`
	} `json:"list"`
}

type searchData struct {
	getUrl         func(string, string) []string
	extraParameter string
	// First string is for the first result (URL), second string for the second result (Details/Definition/...)
	text_result    [2]string
	text_no_result string
}

// --- --- --- Global variable --- --- ---
var searchMap = make(map[string]*searchData)

func Init() {
	ddg := &searchData{
		getUrl:         getDuckduckgoSearchResult,
		text_result:    [2]string{"DuckDuckGo: Best result for %q => %s"},
		text_no_result: "DuckDuckGo: No result for %q"}

	searchMap["!d"] = ddg
	searchMap["!dg"] = ddg
	searchMap["!ddg"] = ddg

	searchMap["!w"] = &searchData{
		getUrl:         getWikipediaSearchResult,
		extraParameter: "en",
		text_result:    [2]string{"Wikipedia result for %q => %s"},
		text_no_result: "Wikipedia: No result for %q"}

	wfr := &searchData{
		getUrl:         getWikipediaSearchResult,
		extraParameter: "fr",
		text_result:    [2]string{"Wikipedia result for %q => %s"},
		text_no_result: "Wikipedia: No result for %q"}

	searchMap["!wf"] = wfr
	searchMap["!wfr"] = wfr

	ud := &searchData{
		getUrl:         getUrbanDictionnarySearchResult,
		text_result:    [2]string{"Urban Dictionnary: Best result for %q => %s", "Definition: %s"},
		text_no_result: "Urban Dictionnary: No result for %q"}

	searchMap["!u"] = ud
	searchMap["!ud"] = ud
}

func HandleSearchCmd(event *irc.Event, callback func(*core.ReplyCallbackData)) bool {
	if callback == nil {
		log.Println("Callback nil for the HandleSearchCmd function")
		return false
	}

	fields := strings.Fields(event.Message())
	// fields[0]  => Command
	// fields[1:] => terms to search for
	if len(fields) == 0 {
		return false
	}

	search, present := searchMap[fields[0]]
	if !present {
		return false
	}

	message := strings.Join(fields[1:], " ")
	if message == "" {
		callback(&core.ReplyCallbackData{Message: fmt.Sprintf("Search usage: %s \"terms to search for\"", fields[0])})
		return false
	}

	results := search.getUrl(message, search.extraParameter)
	if results == nil {
		callback(&core.ReplyCallbackData{Message: fmt.Sprintf(search.text_no_result, message)})
		return true
	}
	for index, item := range results {
		switch index {
		// First part of the result is sent to everyone, no nick is sent
		case 0:
			if search.text_result[index] != "" {
				callback(&core.ReplyCallbackData{Message: fmt.Sprintf(search.text_result[index], message, item)})
			} else {
				callback(&core.ReplyCallbackData{Message: item})
			}
		// Second and following parts of the result are sent directly to the user
		case 1:
			if search.text_result[index] != "" {
				callback(&core.ReplyCallbackData{Nick: event.Nick, Message: fmt.Sprintf(search.text_result[index], item)})
			} else {
				callback(&core.ReplyCallbackData{Nick: event.Nick, Message: item})
			}
		default:
			callback(&core.ReplyCallbackData{Nick: event.Nick, Message: item})
		}
	}
	return true
}

// --- --- --- HTTP Functions --- --- ---
func getResponseAsText(url string) []byte {
	response, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()

	text, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Println(err)
		return nil
	}
	response.Body.Close()
	return text
}

// --- --- --- Search provider functions --- --- ---

// --- Duckduckgo
func getDuckduckgoResultFromHtml(page []byte) string {
	re := regexp.MustCompile(`<a rel="nofollow" href="(.[^"]*)">`)
	if result := re.FindSubmatch(page); result != nil && len(result) == 2 {
		return string(result[1])
	}
	return ""
}

func getDuckduckgoSearchResult(searchTerms string, extraParameter string) []string {
	HtmlPageFromHttp := getResponseAsText(fmt.Sprintf(URL_DUCKDUCKGO, searchTerms))
	if HtmlPageFromHttp == nil {
		return nil
	}
	result := getDuckduckgoResultFromHtml(HtmlPageFromHttp)
	if result == "" {
		return nil
	}
	return []string{result}
}

// --- Wikipedia
func getWikipediaResultFromJson(jsonDataFromHttp []byte) []string {
	var result wikipedia
	err := json.Unmarshal(jsonDataFromHttp, &result)
	if err != nil {
		log.Println(err)
		return nil
	}

	var returnValues []string
	for key, value := range result.Query.Pages {
		if key != "-1" {
			returnValues = append(returnValues, value.Fullurl)
			returnValues = append(returnValues, strings.Split(value.Extract, ". ")...)
		}
	}
	return returnValues
}

func getWikipediaSearchResult(searchTerms string, extraParameter string) []string {
	searchTerms = strings.Replace(strings.Title(searchTerms), " ", "%20", -1)
	jsonDataFromHttp := getResponseAsText(fmt.Sprintf(URL_WIKIPEDIA, extraParameter, searchTerms))
	if jsonDataFromHttp == nil {
		return nil
	}
	result := getWikipediaResultFromJson(jsonDataFromHttp)
	if result == nil {
		return nil
	}
	return result
}

// --- Urban Dictionnary
func getUrbanDictionnaryResultFromJson(jsonDataFromHttp []byte) []string {
	var result urbanDictionnary
	err := json.Unmarshal(jsonDataFromHttp, &result)
	if err != nil {
		log.Println(err)
		return nil
	}
	if len(result.List) == 0 {
		return nil
	}
	returnValues := []string{result.List[0].Permalink}
	returnValues = append(returnValues, strings.Split(result.List[0].Definition, ". ")...)

	return returnValues
}

func getUrbanDictionnarySearchResult(searchTerms string, extraParameter string) []string {
	searchTerms = strings.Replace(strings.Title(searchTerms), " ", "%20", -1)
	jsonDataFromHttp := getResponseAsText(fmt.Sprintf(URL_URBANDICTIONNARY, searchTerms))
	if jsonDataFromHttp == nil {
		return nil
	}
	result := getUrbanDictionnaryResultFromJson(jsonDataFromHttp)
	if result == nil {
		return nil
	}
	return result
}
<|endoftext|>


<commit_before>package badger

import (
	"bytes"
	"fmt"
	"log"
	"sync"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

type keyRange struct {
	left  []byte
	right []byte
	inf   bool
}

var infRange = keyRange{inf: true}

func (r keyRange) String() string {
	return fmt.Sprintf("[left=%q, right=%q, inf=%v]", r.left, r.right, r.inf)
}

func (r keyRange) equals(dst keyRange) bool {
	return bytes.Compare(r.left, dst.left) == 0 &&
		bytes.Compare(r.right, dst.right) == 0 &&
		r.inf == dst.inf
}

func (r keyRange) overlapsWith(dst keyRange) bool {
	if r.inf || dst.inf {
		return true
	}

	// If my left is greater than dst right, we have no overlap.
	if bytes.Compare(r.left, dst.right) > 0 {
		return false
	}
	// If my right is less than dst left, we have no overlap.
	if bytes.Compare(r.right, dst.left) < 0 {
		return false
	}
	// We have overlap.
	return true
}

func getKeyRange(tables []*table.Table) keyRange {
	y.AssertTrue(len(tables) > 0)
	smallest := tables[0].Smallest()
	biggest := tables[0].Biggest()
	for i := 1; i < len(tables); i++ {
		if bytes.Compare(tables[i].Smallest(), smallest) < 0 {
			smallest = tables[i].Smallest()
		}
		if bytes.Compare(tables[i].Biggest(), biggest) > 0 {
			biggest = tables[i].Biggest()
		}
	}
	return keyRange{left: smallest, right: biggest}
}

type levelCompactStatus struct {
	ranges  []keyRange
	delSize int64
}

func (lcs *levelCompactStatus) debug() string {
	var b bytes.Buffer
	for _, r := range lcs.ranges {
		b.WriteString(r.String())
	}
	return b.String()
}

func (lcs *levelCompactStatus) overlapsWith(dst keyRange) bool {
	for _, r := range lcs.ranges {
		if r.overlapsWith(dst) {
			return true
		}
	}
	return false
}

func (lcs *levelCompactStatus) remove(dst keyRange) bool {
	final := lcs.ranges[:0]
	var found bool
	for _, r := range lcs.ranges {
		if !r.equals(dst) {
			final = append(final, r)
		} else {
			found = true
		}
	}
	lcs.ranges = final
	return found
}

type compactStatus struct {
	sync.RWMutex
	levels []*levelCompactStatus
}

func (cs *compactStatus) toLog(tr trace.Trace) {
	cs.RLock()
	defer cs.RUnlock()

	tr.LazyPrintf("Compaction status:")
	for i, l := range cs.levels {
		if len(l.debug()) == 0 {
			continue
		}
		tr.LazyPrintf("[%d] %s", i, l.debug())
	}
}

func (cs *compactStatus) overlapsWith(level int, this keyRange) bool {
	cs.RLock()
	defer cs.RUnlock()

	thisLevel := cs.levels[level]
	return thisLevel.overlapsWith(this)
}

func (cs *compactStatus) delSize(l int) int64 {
	cs.RLock()
	defer cs.RUnlock()
	return cs.levels[l].delSize
}

// compareAndAdd will check whether we can run this compactDef. That it doesn't overlap with any
// other running compaction. If it can be run, it would store this run in the compactStatus state.
func (cs *compactStatus) compareAndAdd(cd compactDef) bool {
	cs.Lock()
	defer cs.Unlock()

	level := cd.thisLevel.level

	y.AssertTruef(level < len(cs.levels)-1, "Got level %d. Max levels: %d", level, len(cs.levels))
	thisLevel := cs.levels[level]
	nextLevel := cs.levels[level+1]

	if thisLevel.overlapsWith(cd.thisRange) {
		return false
	}
	if nextLevel.overlapsWith(cd.nextRange) {
		return false
	}
	// Check whether this level really needs compaction or not. Otherwise, we'll end up
	// running parallel compactions for the same level.
	if cd.thisLevel.getTotalSize()-thisLevel.delSize < cd.thisLevel.maxTotalSize {
		return false
	}

	thisLevel.ranges = append(thisLevel.ranges, cd.thisRange)
	nextLevel.ranges = append(nextLevel.ranges, cd.nextRange)
	thisLevel.delSize += cd.thisSize

	return true
}

func (cs *compactStatus) delete(cd compactDef) {
	cs.Lock()
	defer cs.Unlock()

	level := cd.thisLevel.level
	y.AssertTruef(level < len(cs.levels)-1, "Got level %d. Max levels: %d", level, len(cs.levels))

	thisLevel := cs.levels[level]
	nextLevel := cs.levels[level+1]

	thisLevel.delSize -= cd.thisSize
	found := thisLevel.remove(cd.thisRange)
	found = nextLevel.remove(cd.nextRange) && found

	if !found {
		this := cd.thisRange
		next := cd.nextRange
		fmt.Printf("Looking for: [%q, %q, %v] in this level.\n", this.left, this.right, this.inf)
		fmt.Printf("This Level:\n%s\n", thisLevel.debug())
		fmt.Println()
		fmt.Printf("Looking for: [%q, %q, %v] in next level.\n", next.left, next.right, next.inf)
		fmt.Printf("Next Level:\n%s\n", nextLevel.debug())
		log.Fatal("keyRange not found")
	}
}
<commit_msg>Fix a deadlock which caused compaction to block.<commit_after>package badger

import (
	"bytes"
	"fmt"
	"log"
	"sync"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

type keyRange struct {
	left  []byte
	right []byte
	inf   bool
}

var infRange = keyRange{inf: true}

func (r keyRange) String() string {
	return fmt.Sprintf("[left=%q, right=%q, inf=%v]", r.left, r.right, r.inf)
}

func (r keyRange) equals(dst keyRange) bool {
	return bytes.Compare(r.left, dst.left) == 0 &&
		bytes.Compare(r.right, dst.right) == 0 &&
		r.inf == dst.inf
}

func (r keyRange) overlapsWith(dst keyRange) bool {
	if r.inf || dst.inf {
		return true
	}

	// If my left is greater than dst right, we have no overlap.
	if bytes.Compare(r.left, dst.right) > 0 {
		return false
	}
	// If my right is less than dst left, we have no overlap.
	if bytes.Compare(r.right, dst.left) < 0 {
		return false
	}
	// We have overlap.
	return true
}

func getKeyRange(tables []*table.Table) keyRange {
	y.AssertTrue(len(tables) > 0)
	smallest := tables[0].Smallest()
	biggest := tables[0].Biggest()
	for i := 1; i < len(tables); i++ {
		if bytes.Compare(tables[i].Smallest(), smallest) < 0 {
			smallest = tables[i].Smallest()
		}
		if bytes.Compare(tables[i].Biggest(), biggest) > 0 {
			biggest = tables[i].Biggest()
		}
	}
	return keyRange{left: smallest, right: biggest}
}

type levelCompactStatus struct {
	ranges  []keyRange
	delSize int64
}

func (lcs *levelCompactStatus) debug() string {
	var b bytes.Buffer
	for _, r := range lcs.ranges {
		b.WriteString(r.String())
	}
	return b.String()
}

func (lcs *levelCompactStatus) overlapsWith(dst keyRange) bool {
	for _, r := range lcs.ranges {
		if r.overlapsWith(dst) {
			return true
		}
	}
	return false
}

func (lcs *levelCompactStatus) remove(dst keyRange) bool {
	final := lcs.ranges[:0]
	var found bool
	for _, r := range lcs.ranges {
		if !r.equals(dst) {
			final = append(final, r)
		} else {
			found = true
		}
	}
	lcs.ranges = final
	return found
}

type compactStatus struct {
	sync.RWMutex
	levels []*levelCompactStatus
}

func (cs *compactStatus) toLog(tr trace.Trace) {
	cs.RLock()
	defer cs.RUnlock()

	tr.LazyPrintf("Compaction status:")
	for i, l := range cs.levels {
		if len(l.debug()) == 0 {
			continue
		}
		tr.LazyPrintf("[%d] %s", i, l.debug())
	}
}

func (cs *compactStatus) overlapsWith(level int, this keyRange) bool {
	cs.RLock()
	defer cs.RUnlock()

	thisLevel := cs.levels[level]
	return thisLevel.overlapsWith(this)
}

func (cs *compactStatus) delSize(l int) int64 {
	cs.RLock()
	defer cs.RUnlock()
	return cs.levels[l].delSize
}

// compareAndAdd will check whether we can run this compactDef. That it doesn't overlap with any
// other running compaction. If it can be run, it would store this run in the compactStatus state.
func (cs *compactStatus) compareAndAdd(cd compactDef) bool {
	cd.thisLevel.AssertRLock()
	cd.nextLevel.AssertRLock()

	cs.Lock()
	defer cs.Unlock()

	level := cd.thisLevel.level

	y.AssertTruef(level < len(cs.levels)-1, "Got level %d. Max levels: %d", level, len(cs.levels))
	thisLevel := cs.levels[level]
	nextLevel := cs.levels[level+1]

	if thisLevel.overlapsWith(cd.thisRange) {
		return false
	}
	if nextLevel.overlapsWith(cd.nextRange) {
		return false
	}
	// Check whether this level really needs compaction or not. Otherwise, we'll end up
	// running parallel compactions for the same level.
	// NOTE: We can directly call thisLevel.totalSize, because we already have acquire a read lock
	// over this and the next level.
	if cd.thisLevel.totalSize-thisLevel.delSize < cd.thisLevel.maxTotalSize {
		return false
	}

	thisLevel.ranges = append(thisLevel.ranges, cd.thisRange)
	nextLevel.ranges = append(nextLevel.ranges, cd.nextRange)
	thisLevel.delSize += cd.thisSize

	return true
}

func (cs *compactStatus) delete(cd compactDef) {
	cs.Lock()
	defer cs.Unlock()

	level := cd.thisLevel.level
	y.AssertTruef(level < len(cs.levels)-1, "Got level %d. Max levels: %d", level, len(cs.levels))

	thisLevel := cs.levels[level]
	nextLevel := cs.levels[level+1]

	thisLevel.delSize -= cd.thisSize
	found := thisLevel.remove(cd.thisRange)
	found = nextLevel.remove(cd.nextRange) && found

	if !found {
		this := cd.thisRange
		next := cd.nextRange
		fmt.Printf("Looking for: [%q, %q, %v] in this level.\n", this.left, this.right, this.inf)
		fmt.Printf("This Level:\n%s\n", thisLevel.debug())
		fmt.Println()
		fmt.Printf("Looking for: [%q, %q, %v] in next level.\n", next.left, next.right, next.inf)
		fmt.Printf("Next Level:\n%s\n", nextLevel.debug())
		log.Fatal("keyRange not found")
	}
}
<|endoftext|>


<commit_before>package securepassctl

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const (
	// DefaultRemote is the default Content-Type header used in HTTP requests
	DefaultRemote = "https://beta.secure-pass.net"
	// ContentType is the default Content-Type header used in HTTP requests
	ContentType = "application/json"
	// UserAgent contains the default User-Agent value used in HTTP requests
	UserAgent = "SecurePass CLI"
)

// DebugLogger collects all debug messages
var DebugLogger = log.New(ioutil.Discard, "", log.LstdFlags)

// SecurePass main object type
type SecurePass struct {
	AppID     string `ini:"APP_ID"`
	AppSecret string `ini:"APP_SECRET"`
	Endpoint  string
}

func (s *SecurePass) setupRequestFieds(req *http.Request) {
	req.Header.Set("Accept", ContentType)
	req.Header.Set("Content-Type", ContentType)
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("X-SecurePass-App-ID", s.AppID)
	req.Header.Set("X-SecurePass-App-Secret", s.AppSecret)
}

func (s *SecurePass) makeRequestURL(path string) (string, error) {
	baseURL, _ := url.Parse(s.Endpoint)
	URL, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(URL).String(), nil
}

// NewRequest initializes and issues an HTTP request to the SecurePass endpoint
func (s *SecurePass) NewRequest(method, path string, data *url.Values) (*http.Request, error) {
	var err error
	var req *http.Request

	URL, err := s.makeRequestURL(path)
	if err != nil {
		return nil, err
	}
	if data != nil {
		req, err = http.NewRequest(method, URL, bytes.NewBufferString(data.Encode()))
	} else {
		req, err = http.NewRequest(method, URL, nil)
	}
	if err != nil {
		return nil, err
	}
	s.setupRequestFieds(req)
	return req, nil
}

// DoRequest issues an HTTP request
func (s *SecurePass) DoRequest(req *http.Request, obj APIResponse, expstatus int) error {
	client := NewClient(nil)
	DebugLogger.Printf("Sending %s request to %s", req.Method, req.URL)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != expstatus {
		return fmt.Errorf("%s", resp.Status)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	if obj.ErrorCode() != 0 {
		return fmt.Errorf("%d: %s", obj.ErrorCode(), obj.ErrorMessage())
	}
	return nil
}

// NewClient initialize http.Client with a certain http.Transport
func NewClient(tr *http.Transport) *http.Client {
	// Skip SSL certificate verification
	if tr == nil {
		tr = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	}
	return &http.Client{Transport: tr}
}

// AppInfo retrieves information on a SecurePass application
func (s *SecurePass) AppInfo(app string) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/info", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppAdd represents /api/v1/apps/add
func (s *SecurePass) AppAdd(app *ApplicationDescriptor) (*AppAddResponse, error) {
	var obj AppAddResponse

	data := url.Values{}
	data.Set("LABEL", app.Label)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/add", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppDel deletes an application from SecurePass
func (s *SecurePass) AppDel(app string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppMod represents /api/v1/apps/modify
func (s *SecurePass) AppMod(appID string, app *ApplicationDescriptor) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("APP_ID", appID)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.Label != "" {
		data.Set("LABEL", app.Label)
	}
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/modify", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppList retrieves the list of applications available in SecurePass
func (s *SecurePass) AppList(realm string) (*AppListResponse, error) {
	var obj AppListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Logs retrieves application logs
func (s *SecurePass) Logs(realm, start, end string) (*LogsResponse, error) {
	var obj LogsResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}
	if start != "" {
		data.Set("START", start)
	}
	if end != "" {
		data.Set("END", end)
	}

	req, err := s.NewRequest("POST", "/api/v1/logs/get", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// GroupMember issues requests to /api/v1/groups/member
func (s *SecurePass) GroupMember(user, group string) (*GroupMemberResponse, error) {
	var obj GroupMemberResponse

	data := url.Values{}
	data.Set("USERNAME", user)
	data.Set("GROUP", group)

	req, err := s.NewRequest("POST", "/api/v1/groups/member", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserInfo issues requests to /api/v1/users/info
func (s *SecurePass) UserInfo(username string) (*UserInfoResponse, error) {
	var obj UserInfoResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/info", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserList issues requests to /api/v1/users/list
func (s *SecurePass) UserList(realm string) (*UserListResponse, error) {
	var obj UserListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAuth issues requests to /api/v1/users/auth
func (s *SecurePass) UserAuth(username, secret string) (*UserAuthResponse, error) {
	var obj UserAuthResponse

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SECRET", secret)

	req, err := s.NewRequest("POST", "/api/v1/users/auth", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAdd issues requests to /api/v1/users/add
func (s *SecurePass) UserAdd(user *UserDescriptor) (*UserAddResponse, error) {
	var obj UserAddResponse

	data := url.Values{}
	data.Set("USERNAME", user.Username)
	data.Set("NAME", user.Name)
	data.Set("SURNAME", user.Surname)
	data.Set("EMAIL", user.Email)
	data.Set("MOBILE", user.Mobile)
	data.Set("NIN", user.Nin)
	data.Set("RFID", user.Rfid)
	data.Set("MANAGER", user.Manager)

	req, err := s.NewRequest("POST", "/api/v1/users/add", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserDel deletes a user from SecurePass
func (s *SecurePass) UserDel(username string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserPasswordChange change user password
func (s *SecurePass) UserPasswordChange(username, password string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if username != "" && password != "" {
		data.Set("USERNAME", username)
		data.Set("PASSWORD", password)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/password/change", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserPasswordDisable disable a user's password
func (s *SecurePass) UserPasswordDisable(username string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/password/disable", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserProvision provisions a user with a token
func (s *SecurePass) UserProvision(username, token string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SWTOKEN", token)

	req, err := s.NewRequest("POST", "/api/v1/users/token/provision", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsDelete deletes an attribute from user's extended attributes
func (s *SecurePass) UserXattrsDelete(username, attribute string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("ATTRIBUTE", attribute)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsList lists user's extended attributes
func (s *SecurePass) UserXattrsList(username string) (*UserXattrsListResponse, error) {
	var obj UserXattrsListResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsSet set user's extended attributes
func (s *SecurePass) UserXattrsSet(username, attribute, value string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("ATTRIBUTE", attribute)
	data.Set("VALUE", value)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/set", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Ping reprenets the /api/v1/ping API call
func (s *SecurePass) Ping() (*PingResponse, error) {
	var obj PingResponse

	req, err := s.NewRequest("GET", "/api/v1/ping", nil)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}
<commit_msg>Improve API call docstring<commit_after>package securepassctl

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const (
	// DefaultRemote is the default Content-Type header used in HTTP requests
	DefaultRemote = "https://beta.secure-pass.net"
	// ContentType is the default Content-Type header used in HTTP requests
	ContentType = "application/json"
	// UserAgent contains the default User-Agent value used in HTTP requests
	UserAgent = "SecurePass CLI"
)

// DebugLogger collects all debug messages
var DebugLogger = log.New(ioutil.Discard, "", log.LstdFlags)

// SecurePass main object type
type SecurePass struct {
	AppID     string `ini:"APP_ID"`
	AppSecret string `ini:"APP_SECRET"`
	Endpoint  string
}

func (s *SecurePass) setupRequestFieds(req *http.Request) {
	req.Header.Set("Accept", ContentType)
	req.Header.Set("Content-Type", ContentType)
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("X-SecurePass-App-ID", s.AppID)
	req.Header.Set("X-SecurePass-App-Secret", s.AppSecret)
}

func (s *SecurePass) makeRequestURL(path string) (string, error) {
	baseURL, _ := url.Parse(s.Endpoint)
	URL, err := url.Parse(path)
	if err != nil {
		return "", err
	}
	return baseURL.ResolveReference(URL).String(), nil
}

// NewRequest initializes and issues an HTTP request to the SecurePass endpoint
func (s *SecurePass) NewRequest(method, path string, data *url.Values) (*http.Request, error) {
	var err error
	var req *http.Request

	URL, err := s.makeRequestURL(path)
	if err != nil {
		return nil, err
	}
	if data != nil {
		req, err = http.NewRequest(method, URL, bytes.NewBufferString(data.Encode()))
	} else {
		req, err = http.NewRequest(method, URL, nil)
	}
	if err != nil {
		return nil, err
	}
	s.setupRequestFieds(req)
	return req, nil
}

// DoRequest issues an HTTP request
func (s *SecurePass) DoRequest(req *http.Request, obj APIResponse, expstatus int) error {
	client := NewClient(nil)
	DebugLogger.Printf("Sending %s request to %s", req.Method, req.URL)
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if resp.StatusCode != expstatus {
		return fmt.Errorf("%s", resp.Status)
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(obj)
	if err != nil {
		return err
	}
	if obj.ErrorCode() != 0 {
		return fmt.Errorf("%d: %s", obj.ErrorCode(), obj.ErrorMessage())
	}
	return nil
}

// NewClient initialize http.Client with a certain http.Transport
func NewClient(tr *http.Transport) *http.Client {
	// Skip SSL certificate verification
	if tr == nil {
		tr = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	}
	return &http.Client{Transport: tr}
}

// AppInfo retrieves information on a SecurePass application
func (s *SecurePass) AppInfo(app string) (*AppInfoResponse, error) {
	var obj AppInfoResponse

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/info", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppAdd represents /api/v1/apps/add
func (s *SecurePass) AppAdd(app *ApplicationDescriptor) (*AppAddResponse, error) {
	var obj AppAddResponse

	data := url.Values{}
	data.Set("LABEL", app.Label)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}
	if app.Realm != "" {
		data.Set("REALM", app.Realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/add", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppDel deletes an application from SecurePass
func (s *SecurePass) AppDel(app string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if app != "" {
		data.Set("APP_ID", app)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppMod represents /api/v1/apps/modify
func (s *SecurePass) AppMod(appID string, app *ApplicationDescriptor) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("APP_ID", appID)
	data.Set("WRITE", fmt.Sprintf("%v", app.Write))
	data.Set("PRIVACY", fmt.Sprintf("%v", app.Privacy))
	if app.Label != "" {
		data.Set("LABEL", app.Label)
	}
	if app.AllowNetworkIPv4 != "" {
		data.Set("ALLOW_NETWORK_IPv4", app.AllowNetworkIPv4)
	}
	if app.AllowNetworkIPv6 != "" {
		data.Set("ALLOW_NETWORK_IPv6", app.AllowNetworkIPv6)
	}
	if app.Group != "" {
		data.Set("GROUP", app.Group)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/modify", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// AppList retrieves the list of applications available in SecurePass
func (s *SecurePass) AppList(realm string) (*AppListResponse, error) {
	var obj AppListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/apps/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Logs retrieves application logs
func (s *SecurePass) Logs(realm, start, end string) (*LogsResponse, error) {
	var obj LogsResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}
	if start != "" {
		data.Set("START", start)
	}
	if end != "" {
		data.Set("END", end)
	}

	req, err := s.NewRequest("POST", "/api/v1/logs/get", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// GroupMember issues requests to /api/v1/groups/member
func (s *SecurePass) GroupMember(user, group string) (*GroupMemberResponse, error) {
	var obj GroupMemberResponse

	data := url.Values{}
	data.Set("USERNAME", user)
	data.Set("GROUP", group)

	req, err := s.NewRequest("POST", "/api/v1/groups/member", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserInfo issues requests to /api/v1/users/info
func (s *SecurePass) UserInfo(username string) (*UserInfoResponse, error) {
	var obj UserInfoResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/info", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserList issues requests to /api/v1/users/list
func (s *SecurePass) UserList(realm string) (*UserListResponse, error) {
	var obj UserListResponse

	data := url.Values{}
	if realm != "" {
		data.Set("REALM", realm)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAuth issues requests to /api/v1/users/auth
func (s *SecurePass) UserAuth(username, secret string) (*UserAuthResponse, error) {
	var obj UserAuthResponse

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SECRET", secret)

	req, err := s.NewRequest("POST", "/api/v1/users/auth", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserAdd issues requests to /api/v1/users/add
func (s *SecurePass) UserAdd(user *UserDescriptor) (*UserAddResponse, error) {
	var obj UserAddResponse

	data := url.Values{}
	data.Set("USERNAME", user.Username)
	data.Set("NAME", user.Name)
	data.Set("SURNAME", user.Surname)
	data.Set("EMAIL", user.Email)
	data.Set("MOBILE", user.Mobile)
	data.Set("NIN", user.Nin)
	data.Set("RFID", user.Rfid)
	data.Set("MANAGER", user.Manager)

	req, err := s.NewRequest("POST", "/api/v1/users/add", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserDel deletes a user from SecurePass
func (s *SecurePass) UserDel(username string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserPasswordChange change user password
func (s *SecurePass) UserPasswordChange(username, password string) (*Response, error) {
	var obj Response

	data := url.Values{}
	if username != "" && password != "" {
		data.Set("USERNAME", username)
		data.Set("PASSWORD", password)
	}

	req, err := s.NewRequest("POST", "/api/v1/users/password/change", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserPasswordDisable disable a user's password
func (s *SecurePass) UserPasswordDisable(username string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/password/disable", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserProvision provisions a user with a token
func (s *SecurePass) UserProvision(username, token string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("SWTOKEN", token)

	req, err := s.NewRequest("POST", "/api/v1/users/token/provision", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsDelete deletes an attribute from user's extended attributes
func (s *SecurePass) UserXattrsDelete(username, attribute string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("ATTRIBUTE", attribute)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/delete", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsList lists user's extended attributes
func (s *SecurePass) UserXattrsList(username string) (*UserXattrsListResponse, error) {
	var obj UserXattrsListResponse

	data := url.Values{}
	data.Set("USERNAME", username)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/list", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// UserXattrsSet set user's extended attributes
func (s *SecurePass) UserXattrsSet(username, attribute, value string) (*Response, error) {
	var obj Response

	data := url.Values{}
	data.Set("USERNAME", username)
	data.Set("ATTRIBUTE", attribute)
	data.Set("VALUE", value)

	req, err := s.NewRequest("POST", "/api/v1/users/xattrs/set", &data)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}

// Ping issues requests to the /api/v1/ping API endpoint
func (s *SecurePass) Ping() (*PingResponse, error) {
	var obj PingResponse

	req, err := s.NewRequest("GET", "/api/v1/ping", nil)
	if err != nil {
		return nil, err
	}
	err = s.DoRequest(req, &obj, 200)
	return &obj, err
}
<|endoftext|>


<commit_before>package security

import (
	"time"

	"github.com/cuigh/auxo/cache"
	"github.com/cuigh/auxo/log"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/auxo/security"
	"github.com/cuigh/auxo/security/certify"
	"github.com/cuigh/auxo/security/certify/ldap"
	"github.com/cuigh/auxo/security/passwd"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/model"
)

const pkgName = "swirl.security"

// Validator is the authenticator function of swirl.
func Validator(setting *model.Setting) func(name, pwd string) (ticket string, err error) {
	ldapRealm := createLDAPRealm(setting)
	return func(name, pwd string) (ticket string, err error) {
		var (
			su security.User
			mu *model.User
		)

		// try find user first
		mu, err = biz.User.GetByName(name)
		if err != nil {
			return
		}

		if mu != nil && mu.Type == model.UserTypeInternal { // internal user
			if !passwd.Validate(pwd, mu.Password, mu.Salt) {
				err = certify.ErrInvalidToken
			}
		} else if ldapRealm != nil { // user not exist or ldap user
			su, err = ldapRealm.Login(certify.NewSimpleToken(name, pwd))
			if err == nil && mu == nil { // create user if not exists
				lu := su.(*ldap.User)
				mu = &model.User{
					Type:      model.UserTypeLDAP,
					LoginName: lu.LoginName(),
					Name:      lu.Name(),
					Email:     lu.Email(),
				}
				err = biz.User.Create(mu, nil)
			}
		}

		// replace user session, one session is allowed per user.
		if err == nil {
			ticket, err = biz.User.UpdateSession(mu.ID)
		}

		// create event
		if err == nil {
			biz.Event.CreateAuthentication(model.EventActionLogin, mu.ID, mu.LoginName, mu.Name)
		}
		return
	}
}

func createLDAPRealm(setting *model.Setting) certify.Realm {
	if setting.LDAP.Enabled {
		opts := []ldap.Option{
			ldap.NameAttr(setting.LDAP.NameAttr),
			ldap.EmailAttr(setting.LDAP.EmailAttr),
			ldap.UserFilter(setting.LDAP.UserFilter),
			ldap.Security(ldap.SecurityPolicy(setting.LDAP.Security)),
		}
		if setting.LDAP.Authentication == 1 {
			opts = append(opts, ldap.Binding(setting.LDAP.BindDN, setting.LDAP.BindPassword))
		}
		return ldap.New(setting.LDAP.Address, setting.LDAP.BaseDN, setting.LDAP.UserDN, opts...)
	}
	return nil
}

// Identifier is used to identity user.
func Identifier(token string) (user web.User) {
	const cacheKey = "auth_user"

	session, err := biz.User.GetSession(token)
	if err != nil {
		log.Get(pkgName).Errorf("Load session failed: %v", err)
		return
	}
	if session == nil || session.Expires.Before(time.Now()) {
		return
	}

	// try find from cache first
	value := cache.Get(cacheKey, session.UserID)
	if !value.IsNil() {
		user = &model.AuthUser{}
		if err = value.Scan(user); err == nil {
			return
		}
		log.Get(pkgName).Warnf("Load auth user from cache failed: %v", err)
	}

	u, err := biz.User.GetByID(session.UserID)
	if err != nil {
		log.Get(pkgName).Errorf("Load user failed: %v", err)
		return
	}
	if u == nil {
		return
	}

	var roles []*model.Role
	if roles, err = getRoles(u); err == nil {
		user = model.NewAuthUser(u, roles)
		cache.Set(user, cacheKey, session.UserID)
	}
	return
}

func getRoles(u *model.User) (roles []*model.Role, err error) {
	if len(u.Roles) > 0 {
		roles = make([]*model.Role, len(u.Roles))
		for i, id := range u.Roles {
			roles[i], err = biz.Role.Get(id)
			if err != nil {
				return nil, err
			}
		}
	}
	return
}
<commit_msg>Fix #8<commit_after>package security

import (
	"time"

	"github.com/cuigh/auxo/cache"
	"github.com/cuigh/auxo/log"
	"github.com/cuigh/auxo/net/web"
	"github.com/cuigh/auxo/security"
	"github.com/cuigh/auxo/security/certify"
	"github.com/cuigh/auxo/security/certify/ldap"
	"github.com/cuigh/auxo/security/passwd"
	"github.com/cuigh/swirl/biz"
	"github.com/cuigh/swirl/model"
)

const pkgName = "swirl.security"

// Validator is the authenticator function of swirl.
func Validator(setting *model.Setting) func(name, pwd string) (ticket string, err error) {
	ldapRealm := createLDAPRealm(setting)
	return func(name, pwd string) (ticket string, err error) {
		var (
			su security.User
			mu *model.User
		)

		// try find user first
		mu, err = biz.User.GetByName(name)
		if err != nil {
			return
		}

		if mu != nil && mu.Type == model.UserTypeInternal { // internal user
			if !passwd.Validate(pwd, mu.Password, mu.Salt) {
				err = certify.ErrInvalidToken
			}
		} else if ldapRealm != nil { // user not exist or ldap user
			su, err = ldapRealm.Login(certify.NewSimpleToken(name, pwd))
			if err == nil && mu == nil { // create user if not exists
				lu := su.(*ldap.User)
				mu = &model.User{
					Type:      model.UserTypeLDAP,
					LoginName: lu.LoginName(),
					Name:      lu.Name(),
					Email:     lu.Email(),
				}
				err = biz.User.Create(mu, nil)
			}
		} else {
			err = certify.ErrInvalidToken
		}

		// replace user session, one session is allowed per user.
		if err == nil {
			ticket, err = biz.User.UpdateSession(mu.ID)
		}

		// create event
		if err == nil {
			biz.Event.CreateAuthentication(model.EventActionLogin, mu.ID, mu.LoginName, mu.Name)
		}
		return
	}
}

func createLDAPRealm(setting *model.Setting) certify.Realm {
	if setting.LDAP.Enabled {
		opts := []ldap.Option{
			ldap.NameAttr(setting.LDAP.NameAttr),
			ldap.EmailAttr(setting.LDAP.EmailAttr),
			ldap.UserFilter(setting.LDAP.UserFilter),
			ldap.Security(ldap.SecurityPolicy(setting.LDAP.Security)),
		}
		if setting.LDAP.Authentication == 1 {
			opts = append(opts, ldap.Binding(setting.LDAP.BindDN, setting.LDAP.BindPassword))
		}
		return ldap.New(setting.LDAP.Address, setting.LDAP.BaseDN, setting.LDAP.UserDN, opts...)
	}
	return nil
}

// Identifier is used to identity user.
func Identifier(token string) (user web.User) {
	const cacheKey = "auth_user"

	session, err := biz.User.GetSession(token)
	if err != nil {
		log.Get(pkgName).Errorf("Load session failed: %v", err)
		return
	}
	if session == nil || session.Expires.Before(time.Now()) {
		return
	}

	// try find from cache first
	value := cache.Get(cacheKey, session.UserID)
	if !value.IsNil() {
		user = &model.AuthUser{}
		if err = value.Scan(user); err == nil {
			return
		}
		log.Get(pkgName).Warnf("Load auth user from cache failed: %v", err)
	}

	u, err := biz.User.GetByID(session.UserID)
	if err != nil {
		log.Get(pkgName).Errorf("Load user failed: %v", err)
		return
	}
	if u == nil {
		return
	}

	var roles []*model.Role
	if roles, err = getRoles(u); err == nil {
		user = model.NewAuthUser(u, roles)
		cache.Set(user, cacheKey, session.UserID)
	}
	return
}

func getRoles(u *model.User) (roles []*model.Role, err error) {
	if len(u.Roles) > 0 {
		roles = make([]*model.Role, len(u.Roles))
		for i, id := range u.Roles {
			roles[i], err = biz.Role.Get(id)
			if err != nil {
				return nil, err
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 256 bytes.
const MaxPayloadSizeBytes = 256

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert interface{} `json:"alert,omitempty"`
	Badge int         `json:"badge,omitempty"`
	Sound string      `json:"sound,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  int32
	Expiry      uint32
	DeviceToken string
	payload     map[string]interface{}
	Priority    uint8
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.payload = make(map[string]interface{})
	pn.Identifier = rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(IdentifierUbound)
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.payload[key]
}

// Set defines the value of a payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.payload[key] = value
}

// PayloadJSON returns the current payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.payload)
}

// PayloadString returns the current payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(payload) > MaxPayloadSizeBytes {
		return nil, errors.New("payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)
	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(payload)))
	binary.Write(frameBuffer, binary.BigEndian, payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)
	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, uint8(pushCommandValue))
	binary.Write(buffer, binary.BigEndian, uint32(frameBuffer.Len()))
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())
	return buffer.Bytes(), nil
}
<commit_msg>Updated max payload size<commit_after>package apns

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"math/rand"
	"strconv"
	"time"
)

// Push commands always start with command value 2.
const pushCommandValue = 2

// Your total notification payload cannot exceed 2KB (starting with iOS 8).
const MaxPayloadSizeBytes = 2048

// Every push notification gets a pseudo-unique identifier;
// this establishes the upper boundary for it. Apple will return
// this identifier if there is an issue sending your notification.
const IdentifierUbound = 9999

// Constants related to the payload fields and their lengths.
const (
	deviceTokenItemid            = 1
	payloadItemid                = 2
	notificationIdentifierItemid = 3
	expirationDateItemid         = 4
	priorityItemid               = 5
	deviceTokenLength            = 32
	notificationIdentifierLength = 4
	expirationDateLength         = 4
	priorityLength               = 1
)

// Payload contains the notification data for your request.
//
// Alert is an interface here because it supports either a string
// or a dictionary, represented within by an AlertDictionary struct.
type Payload struct {
	Alert interface{} `json:"alert,omitempty"`
	Badge int         `json:"badge,omitempty"`
	Sound string      `json:"sound,omitempty"`
}

// NewPayload creates and returns a Payload structure.
func NewPayload() *Payload {
	return new(Payload)
}

// AlertDictionary is a more complex notification payload.
//
// From the APN docs: "Use the ... alert dictionary in general only if you absolutely need to."
// The AlertDictionary is suitable for specific localization needs.
type AlertDictionary struct {
	Body         string   `json:"body,omitempty"`
	ActionLocKey string   `json:"action-loc-key,omitempty"`
	LocKey       string   `json:"loc-key,omitempty"`
	LocArgs      []string `json:"loc-args,omitempty"`
	LaunchImage  string   `json:"launch-image,omitempty"`
}

// NewAlertDictionary creates and returns an AlertDictionary structure.
func NewAlertDictionary() *AlertDictionary {
	return new(AlertDictionary)
}

// PushNotification is the wrapper for the Payload.
// The length fields are computed in ToBytes() and aren't represented here.
type PushNotification struct {
	Identifier  int32
	Expiry      uint32
	DeviceToken string
	payload     map[string]interface{}
	Priority    uint8
}

// NewPushNotification creates and returns a PushNotification structure.
// It also initializes the pseudo-random identifier.
func NewPushNotification() (pn *PushNotification) {
	pn = new(PushNotification)
	pn.payload = make(map[string]interface{})
	pn.Identifier = rand.New(rand.NewSource(time.Now().UnixNano())).Int31n(IdentifierUbound)
	pn.Priority = 10
	return
}

// AddPayload sets the "aps" payload section of the request. It also
// has a hack described within to deal with specific zero values.
func (pn *PushNotification) AddPayload(p *Payload) {
	// This deserves some explanation.
	//
	// Setting an exported field of type int to 0
	// triggers the omitempty behavior if you've set it.
	// Since the badge is optional, we should omit it if
	// it's not set. However, we want to include it if the
	// value is 0, so there's a hack in push_notification.go
	// that exploits the fact that Apple treats -1 for a
	// badge value as though it were 0 (i.e. it clears the
	// badge but doesn't stop the notification from going
	// through successfully.)
	//
	// Still a hack though :)
	if p.Badge == 0 {
		p.Badge = -1
	}
	pn.Set("aps", p)
}

// Get returns the value of a payload key, if it exists.
func (pn *PushNotification) Get(key string) interface{} {
	return pn.payload[key]
}

// Set defines the value of a payload key.
func (pn *PushNotification) Set(key string, value interface{}) {
	pn.payload[key] = value
}

// PayloadJSON returns the current payload in JSON format.
func (pn *PushNotification) PayloadJSON() ([]byte, error) {
	return json.Marshal(pn.payload)
}

// PayloadString returns the current payload in string format.
func (pn *PushNotification) PayloadString() (string, error) {
	j, err := pn.PayloadJSON()
	return string(j), err
}

// ToBytes returns a byte array of the complete PushNotification
// struct. This array is what should be transmitted to the APN Service.
func (pn *PushNotification) ToBytes() ([]byte, error) {
	token, err := hex.DecodeString(pn.DeviceToken)
	if err != nil {
		return nil, err
	}
	payload, err := pn.PayloadJSON()
	if err != nil {
		return nil, err
	}
	if len(payload) > MaxPayloadSizeBytes {
		return nil, errors.New("payload is larger than the " + strconv.Itoa(MaxPayloadSizeBytes) + " byte limit")
	}

	frameBuffer := new(bytes.Buffer)
	binary.Write(frameBuffer, binary.BigEndian, uint8(deviceTokenItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(deviceTokenLength))
	binary.Write(frameBuffer, binary.BigEndian, token)
	binary.Write(frameBuffer, binary.BigEndian, uint8(payloadItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(len(payload)))
	binary.Write(frameBuffer, binary.BigEndian, payload)
	binary.Write(frameBuffer, binary.BigEndian, uint8(notificationIdentifierItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(notificationIdentifierLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Identifier)
	binary.Write(frameBuffer, binary.BigEndian, uint8(expirationDateItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(expirationDateLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Expiry)
	binary.Write(frameBuffer, binary.BigEndian, uint8(priorityItemid))
	binary.Write(frameBuffer, binary.BigEndian, uint16(priorityLength))
	binary.Write(frameBuffer, binary.BigEndian, pn.Priority)

	buffer := bytes.NewBuffer([]byte{})
	binary.Write(buffer, binary.BigEndian, uint8(pushCommandValue))
	binary.Write(buffer, binary.BigEndian, uint32(frameBuffer.Len()))
	binary.Write(buffer, binary.BigEndian, frameBuffer.Bytes())
	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>package gogtm

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/szydell/mstools"
)

func createRoutines(workDir string) {
	// prepare paths
	path, pathO, pathR := generatePaths(workDir)
	// create directory tree for routines
	err := os.MkdirAll(pathR, os.ModePerm)
	mstools.ErrCheck(err) // drop log and fatal close on error
	// create path for objects
	err = os.Mkdir(pathO, os.ModePerm)
	mstools.ErrCheck(err) // drop log and fatal close on error

	// get already configured 'gtmroutines' from environment
	routines := os.Getenv("gtmroutines")
	// concatenate old value with path created for this session
	routines += " " + pathO + "(" + pathR + ")"

	// create file with routines
	generateRoutineFile(pathR)

	// set 'gtmroutines' env variable to access internal gogtm file with routines
	os.Setenv("gtmroutines", routines)

	// prepare path for gtmaccess.ci
	ciPath := filepath.Join(path, "gtmaccess.ci")
	// generate gtmaccess.ci
	generateCiFile(ciPath)
	// set 'GTMCI' env variable to access interface file needed by gt.m api
	os.Setenv("GTMCI", ciPath)

}

func generatePaths(workDir string) (path string, pathO string, pathR string) {
	// add unique directory name for this session (do not mix routines between sessions)
	path = filepath.Join(workDir, "gogtm/"+goSessionID)
	// create directories 'o' for objects, 'r' for routines
	pathR = path + "/r"
	pathO = path + "/o"
	return
}

func cleanRoutines(workDir string) {
	path, _, _ := generatePaths(workDir)

	os.RemoveAll(path)
}

func generateCiFile(path string) {
	data := []byte(`gtminit   : void init^%gtmaccess( O:gtm_char_t* )
gtmdata   : void data^%gtmaccess( I:gtm_char_t*, O:gtm_uint_t*, O:gtm_char_t* )
gtmget    : void get^%gtmaccess( I:gtm_char_t*, I:gtm_string_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmkill   : void kill^%gtmaccess( I:gtm_char_t*, O:gtm_char_t* )
gtmorder  : void order^%gtmaccess( I:gtm_char_t*, I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmquery  : void query^%gtmaccess( I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmset    : void set^%gtmaccess( I:gtm_char_t*, I:gtm_string_t*, O:gtm_char_t*)
gtmxecute : void xecute^%gtmaccess( I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmzkill  : void zkill^%gtmaccess( I:gtm_char_t*, O:gtm_char_t* )
gvstat    : void gvstat^%gtmaccess( O:gtm_char_t*, O:gtm_char_t* )
`)

	err := ioutil.WriteFile(path, data, 0400)
	mstools.ErrCheck(err)
}

func generateRoutineFile(path string) {
	// routines internally used by gogtm. M language.
	data := []byte(`%gtmaccess    ; entry points to access GT.M
    quit
    ;
init(error)
    set $ztrap="new tmp set error=$ecode set tmp=$piece($ecode,"","",2) quit:$quit $extract(tmp,2,$length(tmp)) quit"
    quit:$quit 0 quit
    ;
data(var,value,error)
	set value=$data(@var)
	quit:$quit 0 quit
	;
get(var,opt,value,error)
    set value=$GET(@var,opt)
    quit:$quit 0 quit
    ;
gvstat(stats,error)
    N RET
    S REGION=$V("GVFIRST") S RET=REGION_"->"_$V("GVSTAT",REGION)
    F I=1:1 S REGION=$V("GVNEXT",REGION) Q:REGION=""  S RET=RET_"|"_REGION_"->"_$V("GVSTAT",REGION)
    set stats=RET
    ;
kill(var,error)
    kill @var
    quit:$quit 0 quit
    ;
order(var,dir,value,error)
    set value=$order(@var,dir)
    quit:$quit 0 quit
    ;
query(var,value,error)
    set value=$query(@var)
    quit:$quit 0 quit
    ;
set(var,value,error)
    set @var=value
    quit:$quit 0 quit
    ;
xecute(code,value,error)
    xecute code
    quit:$quit 0 quit
    ;
zkill(var,error)
    zkill @var
    quit:$quit 0 quit
    ;
`)
	path = filepath.Join(path, "_gtmaccess.m")
	err := ioutil.WriteFile(path, data, 0400)
	mstools.ErrCheck(err)
}
<commit_msg>gvstats repaired (m code)<commit_after>package gogtm

import (
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/szydell/mstools"
)

func createRoutines(workDir string) {
	// prepare paths
	path, pathO, pathR := generatePaths(workDir)
	// create directory tree for routines
	err := os.MkdirAll(pathR, os.ModePerm)
	mstools.ErrCheck(err) // drop log and fatal close on error
	// create path for objects
	err = os.Mkdir(pathO, os.ModePerm)
	mstools.ErrCheck(err) // drop log and fatal close on error

	// get already configured 'gtmroutines' from environment
	routines := os.Getenv("gtmroutines")
	// concatenate old value with path created for this session
	routines += " " + pathO + "(" + pathR + ")"

	// create file with routines
	generateRoutineFile(pathR)

	// set 'gtmroutines' env variable to access internal gogtm file with routines
	os.Setenv("gtmroutines", routines)

	// prepare path for gtmaccess.ci
	ciPath := filepath.Join(path, "gtmaccess.ci")
	// generate gtmaccess.ci
	generateCiFile(ciPath)
	// set 'GTMCI' env variable to access interface file needed by gt.m api
	os.Setenv("GTMCI", ciPath)

}

func generatePaths(workDir string) (path string, pathO string, pathR string) {
	// add unique directory name for this session (do not mix routines between sessions)
	path = filepath.Join(workDir, "gogtm/"+goSessionID)
	// create directories 'o' for objects, 'r' for routines
	pathR = path + "/r"
	pathO = path + "/o"
	return
}

func cleanRoutines(workDir string) {
	path, _, _ := generatePaths(workDir)

	os.RemoveAll(path)
}

func generateCiFile(path string) {
	data := []byte(`gtminit   : void init^%gtmaccess( O:gtm_char_t* )
gtmdata   : void data^%gtmaccess( I:gtm_char_t*, O:gtm_uint_t*, O:gtm_char_t* )
gtmget    : void get^%gtmaccess( I:gtm_char_t*, I:gtm_string_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmkill   : void kill^%gtmaccess( I:gtm_char_t*, O:gtm_char_t* )
gtmorder  : void order^%gtmaccess( I:gtm_char_t*, I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmquery  : void query^%gtmaccess( I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmset    : void set^%gtmaccess( I:gtm_char_t*, I:gtm_string_t*, O:gtm_char_t*)
gtmxecute : void xecute^%gtmaccess( I:gtm_char_t*, O:gtm_char_t*, O:gtm_char_t* )
gtmzkill  : void zkill^%gtmaccess( I:gtm_char_t*, O:gtm_char_t* )
gvstat    : void gvstat^%gtmaccess( O:gtm_char_t*, O:gtm_char_t* )
`)

	err := ioutil.WriteFile(path, data, 0400)
	mstools.ErrCheck(err)
}

func generateRoutineFile(path string) {
	// routines internally used by gogtm. M language.
	data := []byte(`%gtmaccess    ; entry points to access GT.M
    quit
    ;
init(error)
    set $ztrap="new tmp set error=$ecode set tmp=$piece($ecode,"","",2) quit:$quit $extract(tmp,2,$length(tmp)) quit"
    quit:$quit 0 quit
    ;
data(var,value,error)
	set value=$data(@var)
	quit:$quit 0 quit
	;
get(var,opt,value,error)
    set value=$GET(@var,opt)
    quit:$quit 0 quit
    ;
gvstat(stats,error)
    N RET
    S REGION=$V("GVFIRST") S RET=REGION_"->"_$V("GVSTAT",REGION)
    F I=1:1 S REGION=$V("GVNEXT",REGION) Q:REGION=""  S RET=RET_"|"_REGION_"->"_$V("GVSTAT",REGION)
	set stats=RET
	quit:$quit 0 quit
    ;
kill(var,error)
    kill @var
    quit:$quit 0 quit
    ;
order(var,dir,value,error)
    set value=$order(@var,dir)
    quit:$quit 0 quit
    ;
query(var,value,error)
    set value=$query(@var)
    quit:$quit 0 quit
    ;
set(var,value,error)
    set @var=value
    quit:$quit 0 quit
    ;
xecute(code,value,error)
    xecute code
    quit:$quit 0 quit
    ;
zkill(var,error)
    zkill @var
    quit:$quit 0 quit
    ;
`)
	path = filepath.Join(path, "_gtmaccess.m")
	err := ioutil.WriteFile(path, data, 0400)
	mstools.ErrCheck(err)
}
<|endoftext|>