<commit_before>/*
	Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package pmm

import (
	"errors"
	"fmt"
	"time"
)

const (
	qanAPIBasePath = "qan-api"
	noMonitoring   = "No monitoring registered for this node identified as"
	apiTimeout     = 10 * time.Second
	NameRegex      = `^[-\w:\.]{2,60}$`
)

var (
	// you can use `-ldflags -X github.com/percona/pmm-client/pmm.Version=`
	// to set build version number
	Version = "1.10.0"

	// you can use `-ldflags -X github.com/percona/pmm-client/pmm.RootDir=`
	// to set root filesystem for pmm-admin
	RootDir = ""

	PMMBaseDir   = RootDir + "/usr/local/percona/pmm-client"
	AgentBaseDir = RootDir + "/usr/local/percona/qan-agent"

	ConfigFile  = fmt.Sprintf("%s/pmm.yml", PMMBaseDir)
	SSLCertFile = fmt.Sprintf("%s/server.crt", PMMBaseDir)
	SSLKeyFile  = fmt.Sprintf("%s/server.key", PMMBaseDir)

	ErrDuplicate  = errors.New("there is already one instance with this name under monitoring.")
	ErrNoService  = errors.New("no service found.")
	ErrOneLinux   = errors.New("there could be only one instance of linux metrics being monitored for this system.")
	errNoInstance = errors.New("no instance found on QAN API.")
)

const nodeExporterArgs = "-collectors.enabled=diskstats,filefd,filesystem,loadavg,meminfo,netdev,netstat,stat,time,uname,vmstat,numastat"

var mysqldExporterArgs = []string{
	"-collect.auto_increment.columns=true",
	"-collect.binlog_size=true",
	"-collect.global_status=true",
	"-collect.global_variables=true",
	"-collect.info_schema.innodb_metrics=true",
	"-collect.info_schema.processlist=true",
	"-collect.info_schema.query_response_time=true",
	"-collect.info_schema.tables=true",
	"-collect.info_schema.tablestats=true",
	"-collect.info_schema.userstats=true",
	"-collect.perf_schema.eventswaits=true",
	"-collect.perf_schema.file_events=true",
	"-collect.perf_schema.indexiowaits=true",
	"-collect.perf_schema.tableiowaits=true",
	"-collect.perf_schema.tablelocks=true",
	"-collect.slave_status=true",
	//"-collect.engine_innodb_status=true",
	//"-collect.engine_tokudb_status=true",
	//"-collect.info_schema.clientstats=true",
	//"-collect.info_schema.innodb_tablespaces=true",
	//"-collect.perf_schema.eventsstatements=true",
}

// mysqld_exporter args to disable optionally.
var mysqldExporterDisableArgs = map[string][]string{
	"tablestats": {
		"-collect.auto_increment.columns=",
		"-collect.info_schema.tables=",
		"-collect.info_schema.tablestats=",
		"-collect.perf_schema.indexiowaits=",
		"-collect.perf_schema.tableiowaits=",
		"-collect.perf_schema.tablelocks=",
	},
	"userstats":   {"-collect.info_schema.userstats="},
	"binlogstats": {"-collect.binlog_size="},
	"processlist": {"-collect.info_schema.processlist="},
}

type Errors []error

func (e Errors) Error() string {
	return join(e, ", ")
}

// join concatenates the elements of a to create a single string. The separator string
// sep is placed between elements in the resulting string.
func join(a []error, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0].Error()
	}
	nilErr := fmt.Sprintf("%v", error(nil))
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		if a[i] == nil {
			n += len(nilErr)
		} else {
			n += len(a[i].Error())
		}
	}

	b := make([]byte, n)
	bp := copy(b, a[0].Error())
	for _, s := range a[1:] {
		bp += copy(b[bp:], sep)
		if s == nil {
			bp += copy(b[bp:], nilErr)
		} else {
			bp += copy(b[bp:], s.Error())
		}
	}
	return string(b)
}
<commit_msg>Update constants.go<commit_after>/*
	Copyright (c) 2016, Percona LLC and/or its affiliates. All rights reserved.

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package pmm

import (
	"errors"
	"fmt"
	"time"
)

const (
	qanAPIBasePath = "qan-api"
	noMonitoring   = "No monitoring registered for this node identified as"
	apiTimeout     = 10 * time.Second
	NameRegex      = `^[-\w:\.]{2,60}$`
)

var (
	// you can use `-ldflags -X github.com/percona/pmm-client/pmm.Version=`
	// to set build version number
	Version = "1.10.0"

	// you can use `-ldflags -X github.com/percona/pmm-client/pmm.RootDir=`
	// to set root filesystem for pmm-admin
	RootDir = ""

	PMMBaseDir   = RootDir + "/usr/local/percona/pmm-client"
	AgentBaseDir = RootDir + "/usr/local/percona/qan-agent"

	ConfigFile  = fmt.Sprintf("%s/pmm.yml", PMMBaseDir)
	SSLCertFile = fmt.Sprintf("%s/server.crt", PMMBaseDir)
	SSLKeyFile  = fmt.Sprintf("%s/server.key", PMMBaseDir)

	ErrDuplicate  = errors.New("there is already one instance with this name under monitoring.")
	ErrNoService  = errors.New("no service found.")
	ErrOneLinux   = errors.New("there could be only one instance of linux metrics being monitored for this system.")
	errNoInstance = errors.New("no instance found on QAN API.")
)

const nodeExporterArgs = "-collectors.enabled=diskstats,filefd,filesystem,loadavg,meminfo,netdev,netstat,stat,time,uname,vmstat,meminfo_numa"

var mysqldExporterArgs = []string{
	"-collect.auto_increment.columns=true",
	"-collect.binlog_size=true",
	"-collect.global_status=true",
	"-collect.global_variables=true",
	"-collect.info_schema.innodb_metrics=true",
	"-collect.info_schema.processlist=true",
	"-collect.info_schema.query_response_time=true",
	"-collect.info_schema.tables=true",
	"-collect.info_schema.tablestats=true",
	"-collect.info_schema.userstats=true",
	"-collect.perf_schema.eventswaits=true",
	"-collect.perf_schema.file_events=true",
	"-collect.perf_schema.indexiowaits=true",
	"-collect.perf_schema.tableiowaits=true",
	"-collect.perf_schema.tablelocks=true",
	"-collect.slave_status=true",
	//"-collect.engine_innodb_status=true",
	//"-collect.engine_tokudb_status=true",
	//"-collect.info_schema.clientstats=true",
	//"-collect.info_schema.innodb_tablespaces=true",
	//"-collect.perf_schema.eventsstatements=true",
}

// mysqld_exporter args to disable optionally.
var mysqldExporterDisableArgs = map[string][]string{
	"tablestats": {
		"-collect.auto_increment.columns=",
		"-collect.info_schema.tables=",
		"-collect.info_schema.tablestats=",
		"-collect.perf_schema.indexiowaits=",
		"-collect.perf_schema.tableiowaits=",
		"-collect.perf_schema.tablelocks=",
	},
	"userstats":   {"-collect.info_schema.userstats="},
	"binlogstats": {"-collect.binlog_size="},
	"processlist": {"-collect.info_schema.processlist="},
}

type Errors []error

func (e Errors) Error() string {
	return join(e, ", ")
}

// join concatenates the elements of a to create a single string. The separator string
// sep is placed between elements in the resulting string.
func join(a []error, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0].Error()
	}
	nilErr := fmt.Sprintf("%v", error(nil))
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		if a[i] == nil {
			n += len(nilErr)
		} else {
			n += len(a[i].Error())
		}
	}

	b := make([]byte, n)
	bp := copy(b, a[0].Error())
	for _, s := range a[1:] {
		bp += copy(b[bp:], sep)
		if s == nil {
			bp += copy(b[bp:], nilErr)
		} else {
			bp += copy(b[bp:], s.Error())
		}
	}
	return string(b)
}
<|endoftext|>


<commit_before>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"strconv"
	"strings"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Println("should have one argument for a directory with test results")
		os.Exit(1)
	}
	currentFile := os.Args[1] + "/index.html"

	i := 0
	s := 0
	global_off := 1.0
	global_on := 1.0
	global_cores := 1.0
	geomean := 0.0

	file, err := os.Open(currentFile)
	if err != nil {
		fmt.Println("Can't open", currentFile)
		os.Exit(1)
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	for i < len(lines) {
		if strings.Contains(lines[i], "onclick=\"toggleVisibility") {
			i++
			name := strings.Split(lines[i][8:], "_off_")
			if len(name[1]) == 2 {
				name[1] = "  " + name[1]
			} else if len(name[1]) == 3 {
				name[1] = " " + name[1]
			}
			j := i + 1
			for strings.Contains(lines[j], "Average") == false {
				j++
			}
			t := strings.Split(lines[j], "	")[2]
			off_MGB, _ := strconv.Atoi(t[:len(t)-5]) // -6 ?
			t = strings.Split(lines[j+1], "	")[0]
			off_cores, _ := strconv.Atoi(t[40:])
			k := j + 1
			for strings.Contains(lines[k], "Average") == false {
				k++
			}
			t = strings.Split(lines[k], "	")[2]
			on_MGB, _ := strconv.Atoi(t[:len(t)-5]) // -6 ?
			t = strings.Split(lines[k+1], "	")[0]
			on_cores, _ := strconv.Atoi(t[40:])

			if (s < 9 && s%3 == 0) || (s >= 9 && (s-9)%4 == 0) {
				fmt.Println("--------", name[0], name[1], "off:", off_MGB, "on:", on_MGB, "off_cores:", off_cores, "on_cores:", on_cores, "--------")
			} else {
				fmt.Println("        ", name[0], name[1], "off:", off_MGB, "on:", on_MGB, "off_cores:", off_cores, "on_cores:", on_cores)
			}
			global_off = global_off * float64(off_MGB)
			global_on = global_on * float64(on_MGB)
			global_cores = global_cores * float64(on_cores)
			geomean++
			i = k + 1
			s = s + 1
		} else {
			i++
		}
	}
	fmt.Println("Overall geomean off:", math.Pow(global_off, 1.0/geomean), "on:", math.Pow(global_on, 1.0/geomean), "cores:", math.Pow(global_cores, 1.0/geomean))
}
<commit_msg>Fix perf_compare<commit_after>// Copyright 2017 Intel Corporation.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"strconv"
	"strings"
)

func main() {
	if len(os.Args) != 2 {
		fmt.Println("should have one argument for a directory with test results")
		os.Exit(1)
	}
	currentFile := os.Args[1] + "/index.html"

	i := 0
	s := 0
	global_off := 1.0
	global_on := 1.0
	global_cores := 1.0
	geomean := 0.0

	file, err := os.Open(currentFile)
	if err != nil {
		fmt.Println("Can't open", currentFile)
		os.Exit(1)
	}
	defer file.Close()

	var lines []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	for i < len(lines) {
		if strings.Contains(lines[i], "onclick=\"toggleVisibility") {
			i++
			name := strings.Split(lines[i][8:], "_off_")
			if len(name[1]) == 2 {
				name[1] = "  " + name[1]
			} else if len(name[1]) == 3 {
				name[1] = " " + name[1]
			}
			j := i + 1
			for strings.Contains(lines[j], "Average") == false {
				j++
			}
			t := strings.Split(lines[j], "	")[1]
			off_MGB, _ := strconv.Atoi(t[:len(t)-5]) // -6 ?
			t = strings.Split(lines[j+1], "	")[0]
			off_cores, _ := strconv.Atoi(t[40:])
			k := j + 1
			for strings.Contains(lines[k], "Average") == false {
				k++
			}
			t = strings.Split(lines[k], "	")[1]
			on_MGB, _ := strconv.Atoi(t[:len(t)-5]) // -6 ?
			t = strings.Split(lines[k+1], "	")[0]
			on_cores, _ := strconv.Atoi(t[40:])

			if (s < 9 && s%3 == 0) || (s >= 9 && (s-9)%4 == 0) {
				fmt.Println("--------", name[0], name[1], "off:", off_MGB, "on:", on_MGB, "off_cores:", off_cores, "on_cores:", on_cores, "--------")
			} else {
				fmt.Println("        ", name[0], name[1], "off:", off_MGB, "on:", on_MGB, "off_cores:", off_cores, "on_cores:", on_cores)
			}
			global_off = global_off * float64(off_MGB)
			global_on = global_on * float64(on_MGB)
			global_cores = global_cores * float64(on_cores)
			geomean++
			i = k + 1
			s = s + 1
		} else {
			i++
		}
	}
	fmt.Println("Overall geomean off:", math.Pow(global_off, 1.0/geomean), "on:", math.Pow(global_on, 1.0/geomean), "cores:", math.Pow(global_cores, 1.0/geomean))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"sync"
	"time"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/badger/badger"
	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/store"
	"github.com/pkg/profile"
)

const mil float64 = 1000000

var (
	which     = flag.String("kv", "both", "Which KV store to use. Options: both, badger, rocksdb")
	numKeys   = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	valueSize = flag.Int("valsz", 128, "Value size in bytes.")
	dir       = flag.String("dir", "/mnt/data", "Base dir for writes.")
)

func fillEntry(e *badger.Entry) {
	k := rand.Int() % int(*numKeys*mil)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *valueSize, k) // 22 bytes.
	if cap(e.Key) < len(key) {
		e.Key = make([]byte, 2*len(key))
	}
	e.Key = e.Key[:len(key)]
	copy(e.Key, key)

	rand.Read(e.Value)
	e.Meta = 0
	e.Offset = 0
}

var ctx = context.Background()
var bdb *badger.KV
var rdb *store.Store

func writeBatch(entries []*badger.Entry) int {
	rb := rdb.NewWriteBatch()
	defer rb.Destroy()

	for _, e := range entries {
		fillEntry(e)
		rb.Put(e.Key, e.Value)
	}
	if bdb != nil {
		y.Check(bdb.Write(ctx, entries))
	}
	if rdb != nil {
		y.Check(rdb.WriteBatch(rb))
	}
	return len(entries)
}

func main() {
	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()
	switch *mode {
	case "cpu":
		defer profile.Start(profile.CPUProfile).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile).Stop()
	case "mutex":
		defer profile.Start(profile.MutexProfile).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile).Stop()
	default:
		// do nothing
	}

	trace.AuthRequest = func(req *http.Request) (any, sensitive bool) {
		return true, true
	}

	nw := *numKeys * mil
	opt := badger.DefaultOptions
	// opt.MapTablesTo = table.Nothing
	opt.Verbose = true
	opt.Dir = *dir + "/badger"
	opt.SyncWrites = false

	var err error

	if *which == "badger" || *which == "both" {
		fmt.Println("Init Badger")
		y.Check(os.RemoveAll(*dir + "/badger"))
		os.MkdirAll(*dir+"/badger", 0777)
		bdb = badger.NewKV(&opt)
	}
	if *which == "rocksdb" || *which == "both" {
		fmt.Println("Init Rocks")
		os.RemoveAll(*dir + "/rocks")
		os.MkdirAll(*dir+"/rocks", 0777)
		rdb, err = store.NewStore(*dir + "/rocks")
		y.Check(err)
	}

	go http.ListenAndServe("0.0.0.0:8080", nil)

	N := 10
	var wg sync.WaitGroup
	for i := 0; i < N; i++ {
		wg.Add(1)
		go func(proc int) {
			entries := make([]*badger.Entry, 100)
			for i := 0; i < len(entries); i++ {
				e := new(badger.Entry)
				e.Key = make([]byte, 22)
				e.Value = make([]byte, *valueSize)
				entries[i] = e
			}

			var written float64
			for written < nw/float64(N) {
				written += float64(writeBatch(entries))
				if int(written)%100000 == 0 {
					fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
				}
			}
			fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
			wg.Done()
		}(i)
	}
	// 	wg.Add(1) // Block
	wg.Wait()
	if bdb != nil {
		fmt.Println("closing badger")
		bdb.Close()
	}
	if rdb != nil {
		fmt.Println("closing rocks")
		rdb.Close()
	}
	time.Sleep(10 * time.Second)
}
<commit_msg>Increase the batch size<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"sync"
	"time"

	"golang.org/x/net/trace"

	"github.com/dgraph-io/badger/badger"
	"github.com/dgraph-io/badger/y"
	"github.com/dgraph-io/dgraph/store"
	"github.com/pkg/profile"
)

const mil float64 = 1000000

var (
	which     = flag.String("kv", "both", "Which KV store to use. Options: both, badger, rocksdb")
	numKeys   = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	valueSize = flag.Int("valsz", 128, "Value size in bytes.")
	dir       = flag.String("dir", "/mnt/data", "Base dir for writes.")
)

func fillEntry(e *badger.Entry) {
	k := rand.Int() % int(*numKeys*mil)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *valueSize, k) // 22 bytes.
	if cap(e.Key) < len(key) {
		e.Key = make([]byte, 2*len(key))
	}
	e.Key = e.Key[:len(key)]
	copy(e.Key, key)

	rand.Read(e.Value)
	e.Meta = 0
	e.Offset = 0
}

var ctx = context.Background()
var bdb *badger.KV
var rdb *store.Store

func writeBatch(entries []*badger.Entry) int {
	rb := rdb.NewWriteBatch()
	defer rb.Destroy()

	for _, e := range entries {
		fillEntry(e)
		rb.Put(e.Key, e.Value)
	}
	if bdb != nil {
		y.Check(bdb.Write(ctx, entries))
	}
	if rdb != nil {
		y.Check(rdb.WriteBatch(rb))
	}
	return len(entries)
}

func main() {
	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]")
	flag.Parse()
	switch *mode {
	case "cpu":
		defer profile.Start(profile.CPUProfile).Stop()
	case "mem":
		defer profile.Start(profile.MemProfile).Stop()
	case "mutex":
		defer profile.Start(profile.MutexProfile).Stop()
	case "block":
		defer profile.Start(profile.BlockProfile).Stop()
	default:
		// do nothing
	}

	trace.AuthRequest = func(req *http.Request) (any, sensitive bool) {
		return true, true
	}

	nw := *numKeys * mil
	opt := badger.DefaultOptions
	// opt.MapTablesTo = table.Nothing
	opt.Verbose = true
	opt.Dir = *dir + "/badger"
	opt.SyncWrites = false

	var err error

	if *which == "badger" || *which == "both" {
		fmt.Println("Init Badger")
		y.Check(os.RemoveAll(*dir + "/badger"))
		os.MkdirAll(*dir+"/badger", 0777)
		bdb = badger.NewKV(&opt)
	}
	if *which == "rocksdb" || *which == "both" {
		fmt.Println("Init Rocks")
		os.RemoveAll(*dir + "/rocks")
		os.MkdirAll(*dir+"/rocks", 0777)
		rdb, err = store.NewStore(*dir + "/rocks")
		y.Check(err)
	}

	go http.ListenAndServe("0.0.0.0:8080", nil)

	N := 12
	var wg sync.WaitGroup
	for i := 0; i < N; i++ {
		wg.Add(1)
		go func(proc int) {
			entries := make([]*badger.Entry, 1000)
			for i := 0; i < len(entries); i++ {
				e := new(badger.Entry)
				e.Key = make([]byte, 22)
				e.Value = make([]byte, *valueSize)
				entries[i] = e
			}

			var written float64
			for written < nw/float64(N) {
				written += float64(writeBatch(entries))
				if int(written)%100000 == 0 {
					fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
				}
			}
			fmt.Printf("[%d] Written %5.2fM key-val pairs\n", proc, written/mil)
			wg.Done()
		}(i)
	}
	// 	wg.Add(1) // Block
	wg.Wait()
	if bdb != nil {
		fmt.Println("closing badger")
		bdb.Close()
	}
	if rdb != nil {
		fmt.Println("closing rocks")
		rdb.Close()
	}
	time.Sleep(10 * time.Second)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"tileserver/ligneous"
	"tileserver/maptiles"

	log "github.com/cihub/seelog"
)

type Config struct {
	Cache  string            `json:"cache"`
	Engine string            `json:"engine"`
	Layers map[string]string `json:"layers"`
	Port   int               `json:"port"`
}

var (
	config Config
	// engine string
	// port string
	// db_cache string
	config_file   string
	print_version bool
	version       string = "0.1.0"
	//logger seelog.LoggerInterface
)

//var logger *log.Logger = log.New(os.Stdout, "[TileServer] ", log.LUTC|log.Ldate|log.Ltime|log.Lshortfile|log.Lmicroseconds)

// Serve a single stylesheet via HTTP. Open view_tileserver.html in your browser
// to see the results.
// The created tiles are cached in an sqlite database (MBTiles 1.2 conform) so
// successive access a tile is much faster.
func TileserverWithCaching(engine string, layer_config map[string]string) {
	bind := fmt.Sprintf("0.0.0.0:%v", config.Port)
	if engine == "postgres" {
		t := maptiles.NewTileServerPostgres(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to postgres database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
	} else {
		t := maptiles.NewTileServerSqlite(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to sqlite3 database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		//log.Error(http.ListenAndServe(bind, t))
	}
}

func init() {
	// TODO: add config file
	// flag.StringVar(&port, "p", "8080", "server port")
	// flag.StringVar(&engine, "e", "sqlite", "database engine [sqlite or postgres]")
	// flag.StringVar(&db_cache, "d", "tilecache.mbtiles", "tile cache database")
	flag.StringVar(&config_file, "c", "", "tile server config")
	flag.BoolVar(&print_version, "v", false, "version")
	flag.Parse()
	// if engine != "sqlite" {
	// 	if engine != "postgres" {
	// 		logger.Fatal("Unsupported database engines")
	// 	}
	// }
	if print_version {
		fmt.Println("TileServer", version)
		os.Exit(1)
	}

	logger, err := ligneous.InitLogger()
	if nil != err {
		fmt.Println("Error starting logging")
		os.Exit(1)
	}
	log.UseLogger(logger)

}

func getConfig() {
	// check if file exists!!!
	if _, err := os.Stat(config_file); err == nil {

		file, err := ioutil.ReadFile(config_file)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(file, &config)
		if err != nil {
			fmt.Println("error:", err)
			os.Exit(1)
		}

		if config.Engine != "sqlite" {
			if config.Engine != "postgres" {
				fmt.Println("Unsupported database engine")
				os.Exit(1)
			}
		}

		log.Debug(config)
	} else {
		fmt.Println("Config file not found")
		os.Exit(1)
	}
}

// Before uncommenting the GenerateOSMTiles call make sure you have
// the neccessary OSM sources. Consult OSM wiki for details.
func main() {
	getConfig()
	TileserverWithCaching(config.Engine, config.Layers)
}

// sudo su mapnik
// psql -d mbtiles -U mapnik -W
<commit_msg>read and write timeouts<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"tileserver/ligneous"
	"tileserver/maptiles"

	log "github.com/cihub/seelog"
)

type Config struct {
	Cache  string            `json:"cache"`
	Engine string            `json:"engine"`
	Layers map[string]string `json:"layers"`
	Port   int               `json:"port"`
}

var (
	config Config
	// engine string
	// port string
	// db_cache string
	config_file   string
	print_version bool
	version       string = "0.1.0"
	//logger seelog.LoggerInterface
)

//var logger *log.Logger = log.New(os.Stdout, "[TileServer] ", log.LUTC|log.Ldate|log.Ltime|log.Lshortfile|log.Lmicroseconds)

// Serve a single stylesheet via HTTP. Open view_tileserver.html in your browser
// to see the results.
// The created tiles are cached in an sqlite database (MBTiles 1.2 conform) so
// successive access a tile is much faster.
func TileserverWithCaching(engine string, layer_config map[string]string) {
	bind := fmt.Sprintf("0.0.0.0:%v", config.Port)
	if engine == "postgres" {
		t := maptiles.NewTileServerPostgres(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to postgres database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		//log.Error(http.ListenAndServe(bind, t))
	} else {
		t := maptiles.NewTileServerSqlite(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to sqlite3 database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		//log.Error(http.ListenAndServe(bind, t))
	}
}

func init() {
	// TODO: add config file
	// flag.StringVar(&port, "p", "8080", "server port")
	// flag.StringVar(&engine, "e", "sqlite", "database engine [sqlite or postgres]")
	// flag.StringVar(&db_cache, "d", "tilecache.mbtiles", "tile cache database")
	flag.StringVar(&config_file, "c", "", "tile server config")
	flag.BoolVar(&print_version, "v", false, "version")
	flag.Parse()
	// if engine != "sqlite" {
	// 	if engine != "postgres" {
	// 		logger.Fatal("Unsupported database engines")
	// 	}
	// }
	if print_version {
		fmt.Println("TileServer", version)
		os.Exit(1)
	}

	logger, err := ligneous.InitLogger()
	if nil != err {
		fmt.Println("Error starting logging")
		os.Exit(1)
	}
	log.UseLogger(logger)

}

func getConfig() {
	// check if file exists!!!
	if _, err := os.Stat(config_file); err == nil {

		file, err := ioutil.ReadFile(config_file)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(file, &config)
		if err != nil {
			fmt.Println("error:", err)
			os.Exit(1)
		}

		if config.Engine != "sqlite" {
			if config.Engine != "postgres" {
				fmt.Println("Unsupported database engine")
				os.Exit(1)
			}
		}

		log.Debug(config)
	} else {
		fmt.Println("Config file not found")
		os.Exit(1)
	}
}

// Before uncommenting the GenerateOSMTiles call make sure you have
// the neccessary OSM sources. Consult OSM wiki for details.
func main() {
	getConfig()
	TileserverWithCaching(config.Engine, config.Layers)
}

// sudo su mapnik
// psql -d mbtiles -U mapnik -W
<|endoftext|>


<commit_before>package timing

import (
	"time"

	"github.com/oakmound/oak/dlog"
)

// A DynamicTicker is a ticker which can
// be sent signals in the form of durations to
// change how often it ticks.
type DynamicTicker struct {
	ticker    *time.Ticker
	C         chan time.Time
	resetCh   chan *time.Ticker
	forceTick chan bool
}

// NewDynamicTicker returns a null-initialized
// dynamic ticker
func NewDynamicTicker() *DynamicTicker {
	dt := &DynamicTicker{
		ticker:    time.NewTicker(1000 * time.Hour),
		C:         make(chan time.Time),
		resetCh:   make(chan *time.Ticker),
		forceTick: make(chan bool),
	}
	go func() {
		for dt.loop() {
		}
	}()
	return dt
}

func (dt *DynamicTicker) loop() bool {
	select {
	case v := <-dt.ticker.C:
		for {
			select {
			case r := <-dt.forceTick:
				if !r {
					dt.close()
					return false
				}
				continue
			case ticker := <-dt.resetCh:
				dt.ticker.Stop()
				dt.ticker = ticker
				return true
			case dt.C <- v:
				return true
			}
		}
	case ticker := <-dt.resetCh:
		dt.ticker.Stop()
		dt.ticker = ticker
	case r := <-dt.forceTick:
		if !r {
			dt.close()
			return false
		}
		for {
			select {
			case r := <-dt.forceTick:
				if !r {
					dt.close()
					return false
				}
				continue
			case ticker := <-dt.resetCh:
				dt.ticker.Stop()
				dt.ticker = ticker
				return true
			case dt.C <- time.Time{}:
				return true
			}
		}
	}
	return true
}

// SetTick changes the rate at which a dynamic ticker
// ticks
func (dt *DynamicTicker) SetTick(d time.Duration) {
	dt.resetCh <- time.NewTicker(d)
}

func (dt *DynamicTicker) close() {
	close(dt.C)
	close(dt.resetCh)
	close(dt.forceTick)
}

// Step will force the dynamic ticker to tick, once.
// If the forced tick is not received, multiple calls
// to step will do nothing.
func (dt *DynamicTicker) Step() {
	select {
	case dt.forceTick <- true:
	default:
	}
}

// ForceStep is the blocking equivalent to Step. After
// this is called, it won't return until the ticker has
// taken the forced step through. A potential use for this
// is in benchmarking how often the work between ticks
// can get done.
func (dt *DynamicTicker) ForceStep() {
	dt.forceTick <- true
}

// Stop closes all internal channels and stops dt's internal ticker
func (dt *DynamicTicker) Stop() {
	defer func() {
		if x := recover(); x != nil {
			dlog.Error("Dynamic Ticker stopped twice")
		}
	}()
	dt.ticker.Stop()
	dt.forceTick <- false
	<-dt.forceTick
}
<commit_msg>DRY in dynamic ticker loop<commit_after>package timing

import (
	"time"

	"github.com/oakmound/oak/dlog"
)

// A DynamicTicker is a ticker which can
// be sent signals in the form of durations to
// change how often it ticks.
type DynamicTicker struct {
	ticker    *time.Ticker
	C         chan time.Time
	resetCh   chan *time.Ticker
	forceTick chan bool
}

// NewDynamicTicker returns a null-initialized
// dynamic ticker
func NewDynamicTicker() *DynamicTicker {
	dt := &DynamicTicker{
		ticker:    time.NewTicker(1000 * time.Hour),
		C:         make(chan time.Time),
		resetCh:   make(chan *time.Ticker),
		forceTick: make(chan bool),
	}
	go func() {
		for dt.loop() {
		}
	}()
	return dt
}

func (dt *DynamicTicker) loop() bool {
	select {
	case v := <-dt.ticker.C:
		return dt.send(v)
	case ticker := <-dt.resetCh:
		dt.ticker.Stop()
		dt.ticker = ticker
	case r := <-dt.forceTick:
		if !r {
			dt.close()
			return false
		}
		return dt.send(time.Time{})
	}
	return true
}

func (dt *DynamicTicker) send(v time.Time) bool {
	for {
		select {
		case r := <-dt.forceTick:
			if !r {
				dt.close()
				return false
			}
			continue
		case ticker := <-dt.resetCh:
			dt.ticker.Stop()
			dt.ticker = ticker
			return true
		case dt.C <- v:
			return true
		}
	}
}

// SetTick changes the rate at which a dynamic ticker
// ticks
func (dt *DynamicTicker) SetTick(d time.Duration) {
	dt.resetCh <- time.NewTicker(d)
}

func (dt *DynamicTicker) close() {
	close(dt.C)
	close(dt.resetCh)
	close(dt.forceTick)
}

// Step will force the dynamic ticker to tick, once.
// If the forced tick is not received, multiple calls
// to step will do nothing.
func (dt *DynamicTicker) Step() {
	select {
	case dt.forceTick <- true:
	default:
	}
}

// ForceStep is the blocking equivalent to Step. After
// this is called, it won't return until the ticker has
// taken the forced step through. A potential use for this
// is in benchmarking how often the work between ticks
// can get done.
func (dt *DynamicTicker) ForceStep() {
	dt.forceTick <- true
}

// Stop closes all internal channels and stops dt's internal ticker
func (dt *DynamicTicker) Stop() {
	defer func() {
		if x := recover(); x != nil {
			dlog.Error("Dynamic Ticker stopped twice")
		}
	}()
	dt.ticker.Stop()
	dt.forceTick <- false
	<-dt.forceTick
}
<|endoftext|>


<commit_before>package url

import (
	"testing"
)

func TestAccessorsNil(t *testing.T) {
	var url *URL
	if url.GetProtocol() != Protocol_Local {
		t.Error("protocol accessor value mismatch for nil message")
	}
	if url.GetUsername() != "" {
		t.Error("username accessor value mismatch for nil message")
	}
	if url.GetHostname() != "" {
		t.Error("hostname accessor value mismatch for nil message")
	}
	if url.GetPort() != 0 {
		t.Error("port accessor value mismatch for nil message")
	}
	if url.GetPath() != "" {
		t.Error("path accessor value mismatch for nil message")
	}
}

func TestAccessors(t *testing.T) {
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port:     23,
		Path:     "/test/path",
	}
	if url.GetProtocol() != url.Protocol {
		t.Error("protocol accessor value mismatch")
	}
	if url.GetUsername() != url.Username {
		t.Error("username accessor value mismatch")
	}
	if url.GetHostname() != url.Hostname {
		t.Error("hostname accessor value mismatch")
	}
	if url.GetPort() != url.Port {
		t.Error("port accessor value mismatch")
	}
	if url.GetPath() != url.Path {
		t.Error("path accessor value mismatch")
	}
}

func TestReset(t *testing.T) {
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port:     23,
		Path:     "/test/path",
	}
	url.Reset()
	if url.Protocol != Protocol_Local {
		t.Error("reset did not reset protocol")
	}
	if url.Username != "" {
		t.Error("reset did not reset username")
	}
	if url.Hostname != "" {
		t.Error("reset did not reset hostname")
	}
	if url.Port != 0 {
		t.Error("reset did not reset port")
	}
	if url.Path != "" {
		t.Error("reset did not reset path")
	}
}

// TestProtocolBuffersMethods tests miscellaneous generated methods that we
// don't actually invoke to (a) make sure they don't panic and (b) stop their
// existence from deflating test coverage.
func TestProtocolBuffersMethods(t *testing.T) {
	// Test Protocol methods.
	protocol := Protocol_Local
	_ = protocol.String()
	_, _ = protocol.EnumDescriptor()

	// Test URL methods.
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port: 12345,
		Path: "/some/path",
	}
	_ = url.String()
	url.ProtoMessage()
	_, _ = url.Descriptor()
	encoded, err := url.Marshal()
	if err != nil {
		t.Error("unable to marshal URL:", err)
	}
	if err = url.Unmarshal(encoded); err != nil {
		t.Error("unable to unmarshal URL:", err)
	}
}
<commit_msg>Fixed another go fmt issue in url_test.go<commit_after>package url

import (
	"testing"
)

func TestAccessorsNil(t *testing.T) {
	var url *URL
	if url.GetProtocol() != Protocol_Local {
		t.Error("protocol accessor value mismatch for nil message")
	}
	if url.GetUsername() != "" {
		t.Error("username accessor value mismatch for nil message")
	}
	if url.GetHostname() != "" {
		t.Error("hostname accessor value mismatch for nil message")
	}
	if url.GetPort() != 0 {
		t.Error("port accessor value mismatch for nil message")
	}
	if url.GetPath() != "" {
		t.Error("path accessor value mismatch for nil message")
	}
}

func TestAccessors(t *testing.T) {
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port:     23,
		Path:     "/test/path",
	}
	if url.GetProtocol() != url.Protocol {
		t.Error("protocol accessor value mismatch")
	}
	if url.GetUsername() != url.Username {
		t.Error("username accessor value mismatch")
	}
	if url.GetHostname() != url.Hostname {
		t.Error("hostname accessor value mismatch")
	}
	if url.GetPort() != url.Port {
		t.Error("port accessor value mismatch")
	}
	if url.GetPath() != url.Path {
		t.Error("path accessor value mismatch")
	}
}

func TestReset(t *testing.T) {
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port:     23,
		Path:     "/test/path",
	}
	url.Reset()
	if url.Protocol != Protocol_Local {
		t.Error("reset did not reset protocol")
	}
	if url.Username != "" {
		t.Error("reset did not reset username")
	}
	if url.Hostname != "" {
		t.Error("reset did not reset hostname")
	}
	if url.Port != 0 {
		t.Error("reset did not reset port")
	}
	if url.Path != "" {
		t.Error("reset did not reset path")
	}
}

// TestProtocolBuffersMethods tests miscellaneous generated methods that we
// don't actually invoke to (a) make sure they don't panic and (b) stop their
// existence from deflating test coverage.
func TestProtocolBuffersMethods(t *testing.T) {
	// Test Protocol methods.
	protocol := Protocol_Local
	_ = protocol.String()
	_, _ = protocol.EnumDescriptor()

	// Test URL methods.
	url := &URL{
		Protocol: Protocol_SSH,
		Username: "user",
		Hostname: "host",
		Port:     12345,
		Path:     "/some/path",
	}
	_ = url.String()
	url.ProtoMessage()
	_, _ = url.Descriptor()
	encoded, err := url.Marshal()
	if err != nil {
		t.Error("unable to marshal URL:", err)
	}
	if err = url.Unmarshal(encoded); err != nil {
		t.Error("unable to unmarshal URL:", err)
	}
}
<|endoftext|>


<commit_before>package db

func (db *Db) CreateSchema() {
}

func createUsersTable() string {
	return `CREATE TABLE users (id VARCHAR(255) NOT NULL, name VARCHAR(255), PRIMARY KEY(id)`;
}
<commit_msg>Add simple migration<commit_after>package db

import "fmt"

func (db *Db) CreateSchema() {
	queries := []string{
		createUsersTable(),
	}

	db.execMany(queries)
}

func (db *Db) execMany(queries []string) {
	for _, q := range queries {
		_, err := db.conn.Exec(q)

		if err != nil {
			fmt.Printf("Executing migration query failed: %v\n", q)
			panic(err)
		}
	}
}

func createUsersTable() string {
	return `CREATE TABLE users (id VARCHAR(255) NOT NULL, name VARCHAR(255), PRIMARY KEY(id)`;
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/vaughan0/go-zmq"
	"log"
	"log/syslog"
)

func main() {

	// Syslog
		syslogLog, err :=syslog.New(syslog.LOG_INFO, "turtledq/server")
		if nil != err {
			log.Fatal("Crap on a stick, could not open up syslog!")
/////////// RETURN
			return
		}

	//DEBUG
	syslogLog.Notice("=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=")
	syslogLog.Notice("TurtleDQ Server BEGIN.")

	// Go!
		go dequeue(syslogLog)

	// Create ZeroMQ context.
		ctx, err := zmq.NewContext()
		if err != nil {
			panic(err)
		}
		defer ctx.Close()

	// Create ZeroMQ socket (from ZeroMQ context).
		sock, err := ctx.Socket(zmq.Router)
		if err != nil {
			panic(err)
		}
		defer sock.Close()

		if err = sock.Bind("tcp://*:5555"); err != nil {
			panic(err)
		}

	// Handle input from ZeroMQ (via channels).
		chans := sock.Channels()
		defer chans.Close()

		for {
			select {

				case msg := <-chans.In():
					go func() {
						resp := handleRequest(syslogLog, msg)
						chans.Out() <- resp
					}()

				case err := <-chans.Errors():
					panic(err)

			} // select
		}

	//DEBUG
	syslogLog.Notice("TurtleDQ Server END.")
}

func handleRequest(syslogLog *syslog.Writer, msg [][]byte) [][]byte {

//@TODO
	syslogLog.Notice( fmt.Sprintf("[handleRequest] msg = [%v]", msg) )

	// Return
//@TODO
		return [][]byte{
			[]byte("apple banana cherry"),
		}
}
<commit_msg>added support for using config file<commit_after>package main

import (
	"fmt"
	"github.com/msbranco/goconfig"
	"github.com/vaughan0/go-zmq"
	"log"
	"log/syslog"
	"os"
	"strings"
)

func main() {

	// Syslog
		syslogLog, err :=syslog.New(syslog.LOG_INFO, "turtledq/server")
		if nil != err {
			log.Fatal("Crap on a stick, could not open up syslog!")
/////////// RETURN
			return
		}

	//DEBUG
	syslogLog.Notice("=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=-<>-=")
	syslogLog.Notice("TurtleDQ Server BEGIN.")

	// Config
		//DEBUG
		syslogLog.Notice("    Settings BEGIN.")

		cmdPath := os.Args[0]
		cmdPathLastSlash := strings.LastIndexFunc(cmdPath, func(c rune) bool {
			return '/' == c
		})

		cmdDirPath := cmdPath[0:cmdPathLastSlash]

		//DEBUG
		syslogLog.Notice( fmt.Sprintf("    command path: [%v]", cmdPath)  )
		syslogLog.Notice( fmt.Sprintf("    command path last slash: [%v]", cmdPathLastSlash) )
		syslogLog.Notice( fmt.Sprintf("    command dir path: [%v]", cmdDirPath) )

		confRelativePath := "turtledq.ini"
		confPath := cmdDirPath + "/" + confRelativePath

		//DEBUG
		syslogLog.Notice( fmt.Sprintf("    settings file relative path: [%v]", confRelativePath) )
		syslogLog.Notice( fmt.Sprintf("    settings file absolute path: [%v]", confPath) )

		//c, err := configfile.ReadConfigFile(confPath);
		c, err := goconfig.ReadConfigFile(confPath);
		if nil != err {
			errMsg := fmt.Sprintf("Error when trying to read config file: err = [%v]", err)
			syslogLog.Err(errMsg)
			log.Fatal(errMsg)
/////////// RETURN
			return
		}

		//DEBUG
		syslogLog.Notice( fmt.Sprintf("config file: [%v]", c) )

		//DEBUG
		syslogLog.Notice("    Settings END.")

	// Go!
		go dequeue(syslogLog)

	// Create ZeroMQ context.
		ctx, err := zmq.NewContext()
		if err != nil {
			panic(err)
		}
		defer ctx.Close()

	// Create ZeroMQ socket (from ZeroMQ context).
		sock, err := ctx.Socket(zmq.Router)
		if err != nil {
			panic(err)
		}
		defer sock.Close()

		if err = sock.Bind("tcp://*:5555"); err != nil {
			panic(err)
		}

	// Handle input from ZeroMQ (via channels).
		chans := sock.Channels()
		defer chans.Close()

		for {
			select {

				case msg := <-chans.In():
					go func() {
						resp := handleRequest(syslogLog, msg)
						chans.Out() <- resp
					}()

				case err := <-chans.Errors():
					panic(err)

			} // select
		}

	//DEBUG
	syslogLog.Notice("TurtleDQ Server END.")
}

func handleRequest(syslogLog *syslog.Writer, msg [][]byte) [][]byte {

//@TODO
	syslogLog.Notice( fmt.Sprintf("[handleRequest] msg = [%v]", msg) )

	// Return
//@TODO
		return [][]byte{
			[]byte("apple banana cherry"),
		}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"koding/db/mongodb/modelhelper"
	"koding/newkite/protocol"
	"koding/tools/slog"
	"net/http"
)

// everyone needs a place for home
func homeHandler(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "Hello world - kontrol!\n")
}

// preparHandler first checks if the incoming POST request is a valid session.
// Every request made to kontrol should be in POST with protocol.Request in
// their body.
func prepareHandler(fn func(w http.ResponseWriter, r *http.Request, msg *protocol.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")

		msg, err := readPostRequest(r.Body)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}

		err = validatePostRequest(msg)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}
		slog.Printf("sessionID '%s' wants '%s'\n", msg.SessionID, msg.RemoteKite)

		session, err := modelhelper.GetSession(msg.SessionID)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}
		slog.Printf("sessionID '%s' is validated as: %s\n", msg.SessionID, session.Username)

		// username is used for matching kites and generating tokens in requestHandler
		msg.Username = session.Username

		fn(w, r, msg)
	}
}

// we assume that the incoming JSON data is in form of protocol.Request. Read
// and return a new protocol.Request from the POST body if succesfull.
func readPostRequest(requestBody io.ReadCloser) (*protocol.Request, error) {
	body, err := ioutil.ReadAll(requestBody)
	if err != nil {
		return nil, err
	}
	defer requestBody.Close()

	req, err := unmarshalRequest(body)
	if err != nil {
		return nil, err
	}

	return req, nil
}

// validate that incoming post request has all necessary (at least the one we
// need) fields.
func validatePostRequest(msg *protocol.Request) error {
	if msg.SessionID == "" {
		return errors.New("sessionID field is empty")
	}

	if msg.RemoteKite == "" {
		return errors.New("remoteKite field is not specified")
	}

	return nil
}

// searchForKites returns a list of kites that matches the variable matchKite
// It also generates a new one-way token that is used between the client and
// kite and appends it to each kite struct
func searchForKites(username, kitename string) ([]protocol.PubResponse, error) {
	kites := make([]protocol.PubResponse, 0)
	token := new(protocol.Token)

	slog.Printf("searching for kite '%s'\n", kitename)

	for _, k := range storage.List() {
		if k.Kitename == kitename {
			token = getToken(username)
			if token == nil {
				token = createToken(username)
			}

			k.Token = token.ID // only token id is important for client
			pubResp := createResponse(protocol.AddKite, k)
			kites = append(kites, pubResp)
		}
	}

	if len(kites) == 0 {
		return nil, fmt.Errorf("'%s' not available\n", kitename)
	}

	return kites, nil
}

// requestHandler sends as response a list of kites that matches kites in form
// of "username/kitename".
func requestHandler(w http.ResponseWriter, r *http.Request, msg *protocol.Request) {
	kites, err := searchForKites(msg.Username, msg.Username+"/"+msg.RemoteKite)
	if err != nil {
		http.Error(w, "{\"err\":\"malformed kite list\"}\n", http.StatusBadRequest)
		return
	}

	kitesJSON, err := json.Marshal(kites)
	if err != nil {
		http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
		return
	}

	w.Write([]byte(kitesJSON))
}
<commit_msg>fix error message<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"koding/db/mongodb/modelhelper"
	"koding/newkite/protocol"
	"koding/tools/slog"
	"net/http"
)

// everyone needs a place for home
func homeHandler(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "Hello world - kontrol!\n")
}

// preparHandler first checks if the incoming POST request is a valid session.
// Every request made to kontrol should be in POST with protocol.Request in
// their body.
func prepareHandler(fn func(w http.ResponseWriter, r *http.Request, msg *protocol.Request)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")

		msg, err := readPostRequest(r.Body)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}

		err = validatePostRequest(msg)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}
		slog.Printf("sessionID '%s' wants '%s'\n", msg.SessionID, msg.RemoteKite)

		session, err := modelhelper.GetSession(msg.SessionID)
		if err != nil {
			http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
			return
		}
		slog.Printf("sessionID '%s' is validated as: %s\n", msg.SessionID, session.Username)

		// username is used for matching kites and generating tokens in requestHandler
		msg.Username = session.Username

		fn(w, r, msg)
	}
}

// we assume that the incoming JSON data is in form of protocol.Request. Read
// and return a new protocol.Request from the POST body if succesfull.
func readPostRequest(requestBody io.ReadCloser) (*protocol.Request, error) {
	body, err := ioutil.ReadAll(requestBody)
	if err != nil {
		return nil, err
	}
	defer requestBody.Close()

	req, err := unmarshalRequest(body)
	if err != nil {
		return nil, err
	}

	return req, nil
}

// validate that incoming post request has all necessary (at least the one we
// need) fields.
func validatePostRequest(msg *protocol.Request) error {
	if msg.SessionID == "" {
		return errors.New("sessionID field is empty")
	}

	if msg.RemoteKite == "" {
		return errors.New("remoteKite field is not specified")
	}

	return nil
}

// searchForKites returns a list of kites that matches the variable matchKite
// It also generates a new one-way token that is used between the client and
// kite and appends it to each kite struct
func searchForKites(username, kitename string) ([]protocol.PubResponse, error) {
	kites := make([]protocol.PubResponse, 0)
	token := new(protocol.Token)

	slog.Printf("searching for kite '%s'\n", kitename)

	for _, k := range storage.List() {
		if k.Kitename == kitename {
			token = getToken(username)
			if token == nil {
				token = createToken(username)
			}

			k.Token = token.ID // only token id is important for client
			pubResp := createResponse(protocol.AddKite, k)
			kites = append(kites, pubResp)
		}
	}

	if len(kites) == 0 {
		return nil, fmt.Errorf("'%s' not available\n", kitename)
	}

	return kites, nil
}

// requestHandler sends as response a list of kites that matches kites in form
// of "username/kitename".
func requestHandler(w http.ResponseWriter, r *http.Request, msg *protocol.Request) {
	kites, err := searchForKites(msg.Username, msg.Username+"/"+msg.RemoteKite)
	if err != nil {
		msg := fmt.Sprintf("{\"err\":\"%s\"}\n", err)
		http.Error(w, msg, http.StatusBadRequest)
		return
	}

	kitesJSON, err := json.Marshal(kites)
	if err != nil {
		http.Error(w, fmt.Sprintf("{\"err\":\"%s\"}\n", err), http.StatusBadRequest)
		return
	}

	w.Write([]byte(kitesJSON))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build solaris

package terminal // import "golang.org/x/crypto/ssh/terminal"

import (
	"golang.org/x/sys/unix"
	"io"
	"syscall"
)

// State contains the state of a terminal.
type State struct {
	termios syscall.Termios
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	// see: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libbc/libc/gen/common/isatty.c
	var termio unix.Termio
	err := unix.IoctlSetTermio(fd, unix.TCGETA, &termio)
	return err == nil
}

// ReadPassword reads a line of input from a terminal without local echo.  This
// is commonly used for inputting passwords and other sensitive data. The slice
// returned does not include the \n.
func ReadPassword(fd int) ([]byte, error) {
	// see also: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libast/common/uwin/getpass.c
	val, err := unix.IoctlGetTermios(fd, unix.TCGETS)
	if err != nil {
		return nil, err
	}
	oldState := *val

	newState := oldState
	newState.Lflag &^= syscall.ECHO
	newState.Lflag |= syscall.ICANON | syscall.ISIG
	newState.Iflag |= syscall.ICRNL
	err = unix.IoctlSetTermios(fd, unix.TCSETS, &newState)
	if err != nil {
		return nil, err
	}

	defer unix.IoctlSetTermios(fd, unix.TCSETS, &oldState)

	var buf [16]byte
	var ret []byte
	for {
		n, err := syscall.Read(fd, buf[:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			if len(ret) == 0 {
				return nil, io.EOF
			}
			break
		}
		if buf[n-1] == '\n' {
			n--
		}
		ret = append(ret, buf[:n]...)
		if n < len(buf) {
			break
		}
	}

	return ret, nil
}
<commit_msg>ssh/terminal: implement missing functions for Solaris/OmniOS<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build solaris

package terminal // import "golang.org/x/crypto/ssh/terminal"

import (
	"golang.org/x/sys/unix"
	"io"
	"syscall"
)

// State contains the state of a terminal.
type State struct {
	state *unix.Termios
}

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	_, err := unix.IoctlGetTermio(fd, unix.TCGETA)
	return err == nil
}

// ReadPassword reads a line of input from a terminal without local echo.  This
// is commonly used for inputting passwords and other sensitive data. The slice
// returned does not include the \n.
func ReadPassword(fd int) ([]byte, error) {
	// see also: http://src.illumos.org/source/xref/illumos-gate/usr/src/lib/libast/common/uwin/getpass.c
	val, err := unix.IoctlGetTermios(fd, unix.TCGETS)
	if err != nil {
		return nil, err
	}
	oldState := *val

	newState := oldState
	newState.Lflag &^= syscall.ECHO
	newState.Lflag |= syscall.ICANON | syscall.ISIG
	newState.Iflag |= syscall.ICRNL
	err = unix.IoctlSetTermios(fd, unix.TCSETS, &newState)
	if err != nil {
		return nil, err
	}

	defer unix.IoctlSetTermios(fd, unix.TCSETS, &oldState)

	var buf [16]byte
	var ret []byte
	for {
		n, err := syscall.Read(fd, buf[:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			if len(ret) == 0 {
				return nil, io.EOF
			}
			break
		}
		if buf[n-1] == '\n' {
			n--
		}
		ret = append(ret, buf[:n]...)
		if n < len(buf) {
			break
		}
	}

	return ret, nil
}

// MakeRaw puts the terminal connected to the given file descriptor into raw
// mode and returns the previous state of the terminal so that it can be
// restored.
// see http://cr.illumos.org/~webrev/andy_js/1060/
func MakeRaw(fd int) (*State, error) {
	oldTermiosPtr, err := unix.IoctlGetTermios(fd, unix.TCGETS)
	if err != nil {
		return nil, err
	}
	oldTermios := *oldTermiosPtr

	newTermios := oldTermios
	newTermios.Iflag &^= syscall.IGNBRK | syscall.BRKINT | syscall.PARMRK | syscall.ISTRIP | syscall.INLCR | syscall.IGNCR | syscall.ICRNL | syscall.IXON
	newTermios.Oflag &^= syscall.OPOST
	newTermios.Lflag &^= syscall.ECHO | syscall.ECHONL | syscall.ICANON | syscall.ISIG | syscall.IEXTEN
	newTermios.Cflag &^= syscall.CSIZE | syscall.PARENB
	newTermios.Cflag |= syscall.CS8
	newTermios.Cc[unix.VMIN] = 1
	newTermios.Cc[unix.VTIME] = 0

	if err := unix.IoctlSetTermios(fd, unix.TCSETS, &newTermios); err != nil {
		return nil, err
	}

	return &State{
		state: oldTermiosPtr,
	}, nil
}

// Restore restores the terminal connected to the given file descriptor to a
// previous state.
func Restore(fd int, oldState *State) error {
	return unix.IoctlSetTermios(fd, unix.TCSETS, oldState.state)
}

// GetState returns the current state of a terminal which may be useful to
// restore the terminal after a signal.
func GetState(fd int) (*State, error) {
	oldTermiosPtr, err := unix.IoctlGetTermios(fd, unix.TCGETS)
	if err != nil {
		return nil, err
	}

	return &State{
		state: oldTermiosPtr,
	}, nil
}

// GetSize returns the dimensions of the given terminal.
func GetSize(fd int) (width, height int, err error) {
	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
	if err != nil {
		return 0, 0, err
	}
	return int(ws.Col), int(ws.Row), nil
}
<|endoftext|>


<commit_before>package utils

import (
	"os"
	"syscall"
	"unsafe"
)

// Isatty returns true if f is a TTY, false otherwise.
func Isatty(f *os.File) bool {
	var t [2]byte
	_, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
		f.Fd(), syscall.TIOCGPGRP,
		uintptr(unsafe.Pointer(&t)))
	return errno == 0
}
<commit_msg>fixes CC-618<commit_after>package utils

import (
	"os"
	"syscall"
	"unsafe"
)

// Isatty returns true if f is a TTY, false otherwise.
func Isatty(f *os.File) bool {
	var t syscall.Termios
	_, _, errno := syscall.Syscall6(syscall.SYS_IOCTL,
		f.Fd(), syscall.TCGETS,
		uintptr(unsafe.Pointer(&t)), 0, 0, 0)
	return errno == 0
}
<|endoftext|>


<commit_before>// Package torrent support for reading and writing torrent files.
package torrent

import (
	"crypto/sha1"
	"errors"
	"io"

	"github.com/zeebo/bencode"
)

// Torrent file dictionary
type Torrent struct {
	Info         *Info              `bencode:"-"`
	RawInfo      bencode.RawMessage `bencode:"info" json:"-"`
	Announce     string             `bencode:"announce"`
	AnnounceList [][]string         `bencode:"announce-list"`
	CreationDate int64              `bencode:"creation date"`
	Comment      string             `bencode:"comment"`
	CreatedBy    string             `bencode:"created by"`
	Encoding     string             `bencode:"encoding"`
}

// Info contains information about torrent.
type Info struct {
	PieceLength uint32 `bencode:"piece length" json:"piece_length"`
	Pieces      []byte `bencode:"pieces" json:"pieces"`
	Private     byte   `bencode:"private" json:"private"`
	Name        string `bencode:"name" json:"name"`
	// Single File Mode
	Length int64  `bencode:"length" json:"length"`
	Md5sum string `bencode:"md5sum" json:"md5sum,omitempty"`
	// Multiple File mode
	Files []FileDict `bencode:"files" json:"files"`
	// Calculated fileds
	Hash        [sha1.Size]byte `bencode:"-" json:"-"`
	TotalLength int64           `bencode:"-" json:"-"`
	NumPieces   uint32          `bencode:"-" json:"-"`
	MultiFile   bool            `bencode:"-" json:"-"`
}

type FileDict struct {
	Length int64    `bencode:"length" json:"length"`
	Path   []string `bencode:"path" json:"path"`
	Md5sum string   `bencode:"md5sum" json:"md5sum,omitempty"`
}

// New returns a torrent from bencoded stream.
func New(r io.Reader) (*Torrent, error) {
	var t Torrent
	err := bencode.NewDecoder(r).Decode(&t)
	if err != nil {
		return nil, err
	}

	if len(t.RawInfo) == 0 {
		return nil, errors.New("no info dict in torrent file")
	}

	t.Info, err = NewInfo(t.RawInfo)
	return &t, err
}

// NewInfo returns info from bencoded bytes in b.
func NewInfo(b []byte) (*Info, error) {
	var i Info
	if err := bencode.DecodeBytes(b, &i); err != nil {
		return nil, err
	}

	hash := sha1.New()
	hash.Write(b)
	copy(i.Hash[:], hash.Sum(nil))

	i.MultiFile = len(i.Files) != 0

	i.NumPieces = uint32(len(i.Pieces)) / sha1.Size

	if !i.MultiFile {
		i.TotalLength = i.Length
	} else {
		for _, f := range i.Files {
			i.TotalLength += f.Length
		}
	}

	return &i, nil
}

func (i *Info) PieceHash(index uint32) []byte {
	if index >= i.NumPieces {
		panic("piece index out of range")
	}
	start := index * sha1.Size
	end := start + sha1.Size
	return i.Pieces[start:end]
}

// GetFiles returns the files in torrent as a slice, even if there is a single file.
func (i *Info) GetFiles() []FileDict {
	if i.MultiFile {
		return i.Files
	}
	return []FileDict{FileDict{i.Length, []string{i.Name}, i.Md5sum}}
}
<commit_msg>fix linter errors<commit_after>// Package torrent support for reading and writing torrent files.
package torrent

import (
	"crypto/sha1"
	"errors"
	"io"

	"github.com/zeebo/bencode"
)

// Torrent file dictionary
type Torrent struct {
	Info         *Info              `bencode:"-"`
	RawInfo      bencode.RawMessage `bencode:"info" json:"-"`
	Announce     string             `bencode:"announce"`
	AnnounceList [][]string         `bencode:"announce-list"`
	CreationDate int64              `bencode:"creation date"`
	Comment      string             `bencode:"comment"`
	CreatedBy    string             `bencode:"created by"`
	Encoding     string             `bencode:"encoding"`
}

// Info contains information about torrent.
type Info struct {
	PieceLength uint32 `bencode:"piece length" json:"piece_length"`
	Pieces      []byte `bencode:"pieces" json:"pieces"`
	Private     byte   `bencode:"private" json:"private"`
	Name        string `bencode:"name" json:"name"`
	// Single File Mode
	Length int64  `bencode:"length" json:"length"`
	Md5sum string `bencode:"md5sum" json:"md5sum,omitempty"`
	// Multiple File mode
	Files []FileDict `bencode:"files" json:"files"`
	// Calculated fileds
	Hash        [sha1.Size]byte `bencode:"-" json:"-"`
	TotalLength int64           `bencode:"-" json:"-"`
	NumPieces   uint32          `bencode:"-" json:"-"`
	MultiFile   bool            `bencode:"-" json:"-"`
}

type FileDict struct {
	Length int64    `bencode:"length" json:"length"`
	Path   []string `bencode:"path" json:"path"`
	Md5sum string   `bencode:"md5sum" json:"md5sum,omitempty"`
}

// New returns a torrent from bencoded stream.
func New(r io.Reader) (*Torrent, error) {
	var t Torrent
	err := bencode.NewDecoder(r).Decode(&t)
	if err != nil {
		return nil, err
	}

	if len(t.RawInfo) == 0 {
		return nil, errors.New("no info dict in torrent file")
	}

	t.Info, err = NewInfo(t.RawInfo)
	return &t, err
}

// NewInfo returns info from bencoded bytes in b.
func NewInfo(b []byte) (*Info, error) {
	var i Info
	if err := bencode.DecodeBytes(b, &i); err != nil {
		return nil, err
	}

	hash := sha1.New()
	hash.Write(b) // nolint: gosec
	copy(i.Hash[:], hash.Sum(nil))

	i.MultiFile = len(i.Files) != 0

	i.NumPieces = uint32(len(i.Pieces)) / sha1.Size

	if !i.MultiFile {
		i.TotalLength = i.Length
	} else {
		for _, f := range i.Files {
			i.TotalLength += f.Length
		}
	}

	return &i, nil
}

func (i *Info) PieceHash(index uint32) []byte {
	if index >= i.NumPieces {
		panic("piece index out of range")
	}
	start := index * sha1.Size
	end := start + sha1.Size
	return i.Pieces[start:end]
}

// GetFiles returns the files in torrent as a slice, even if there is a single file.
func (i *Info) GetFiles() []FileDict {
	if i.MultiFile {
		return i.Files
	}
	return []FileDict{FileDict{i.Length, []string{i.Name}, i.Md5sum}}
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//

package tracker

import (
  "log"
  "net/url"
  "net/http"
  "errors"
  "bytes"
  "time"
)

const (
  DEFAULT_REQ_TYPE        = "POST"
  DEFAULT_PROTOCOL        = "http"
  DEFAULT_SEND_LIMIT      = 500
  DEFAULT_BYTE_LIMIT_GET  = 40000
  DEFAULT_BYTE_LIMIT_POST = 40000
  DEFAULT_DB_NAME         = "events.db"
  POST_WRAPPER_BYTES      = 88 // "schema":"iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-3","data":[]
  POST_STM_BYTES          = 22 // "stm":"1443452851000"
)

type SendResult struct {
  ids    []int
  status int
}

type CallbackResult struct {
  Count  int
  Status int
}

type Emitter struct {
  CollectorUri  string
  CollectorUrl  url.URL
  RequestType   string
  Protocol      string
  SendLimit     int
  ByteLimitGet  int
  ByteLimitPost int
  DbName        string
  Storage       Storage
  SendChannel   chan bool
  Callback      func(successCount []CallbackResult, failureCount []CallbackResult)
  HttpClient    http.Client
}

// InitEmitter creates a new Emitter object which handles
// storing and sending Snowplow Events.
func InitEmitter(options ...func(*Emitter)) *Emitter {
  e := &Emitter{}

  // Set Defaults
  e.RequestType = DEFAULT_REQ_TYPE
  e.Protocol = DEFAULT_PROTOCOL
  e.SendLimit = DEFAULT_SEND_LIMIT
  e.ByteLimitGet = DEFAULT_BYTE_LIMIT_GET
  e.ByteLimitPost = DEFAULT_BYTE_LIMIT_POST
  e.DbName = DEFAULT_DB_NAME

  // Option parameters
  for _, op := range options { op(e) }

  // Check collector URI is not empty
  if e.CollectorUri == "" {
    panic("FATAL: CollectorUri cannot be empty.")
  } else {
    collectorUrl, err := returnCollectorUrl(e.RequestType, e.Protocol, e.CollectorUri)
    if err != nil {
      panic(err.Error())
    } else {
      e.CollectorUrl = *collectorUrl
    }
  }

  // Setup Event Storage
  e.Storage = *InitStorage(e.DbName)

  // Setup HttpClient
  timeout := time.Duration(5 * time.Second)
  e.HttpClient = http.Client{
    Timeout: timeout,
  }

  return e
}

// --- Require

// RequireCollectorUri sets the Emitters collector URI.
func RequireCollectorUri(collectorUri string) func(e *Emitter) {
  return func(e *Emitter) { e.CollectorUri = collectorUri }
}

// --- Option

// OptionRequestType sets the request type to use (GET or POST).
func OptionRequestType(requestType string) func(e *Emitter) {
  return func(e *Emitter) { e.RequestType = requestType }
}

// OptionProtocol sets the protocol type to use (http or https).
func OptionProtocol(protocol string) func(e *Emitter) {
  return func(e *Emitter) { e.Protocol = protocol }
}

// OptionSendLimit sets the send limit for the emitter.
func OptionSendLimit(sendLimit int) func(e *Emitter) {
  return func(e *Emitter) { e.SendLimit = sendLimit }
}

// OptionByteLimitGet sets the byte limit for GET requests.
func OptionByteLimitGet(byteLimitGet int) func(e *Emitter) {
  return func(e *Emitter) { e.ByteLimitGet = byteLimitGet }
}

// OptionByteLimitPost sets the byte limit for POST requests.
func OptionByteLimitPost(byteLimitPost int) func(e *Emitter) {
  return func(e *Emitter) { e.ByteLimitPost = byteLimitPost }
}

// OptionDbName sets the name of the storage database.
func OptionDbName(dbName string) func(e *Emitter) {
  return func(e *Emitter) { e.DbName = dbName }
}

// OptionCallback sets a custom callback for the emitter loop.
func OptionCallback(callback func(successCount []CallbackResult, failureCount []CallbackResult)) func(e *Emitter) {
  return func(e *Emitter) { e.Callback = callback }
}

// --- Event Handlers

// Add will push an event to the database and will then initiate a sending loop.
func (e *Emitter) Add(payload Payload) {
  e.Storage.AddEventRow(payload)
  e.start()
}

// Flush will attempt to start the send loop regardless of an event coming in.
func (e *Emitter) Flush() {
  e.start()
}

// Stop waits for the send channel to have a value and then resets it to nil.
func (e *Emitter) Stop() {
  <-e.SendChannel
  e.SendChannel = nil
}

// start will begin the sending loop.
func (e *Emitter) start() {
  if e.SendChannel == nil || !e.IsSending() {
    e.SendChannel = make(chan bool, 1)
    go func() {
      var done bool
      defer func() {
        e.SendChannel <- done
      }()

      for {
        eventRows := e.Storage.GetEventRowsWithinRange(e.SendLimit)

        // If there are no events in the database exit
        if len(eventRows) == 0 {
          break
        }
        results := e.doSend(eventRows)

        // Process results
        ids := []int{}
        successes := []CallbackResult{}
        failures := []CallbackResult{}

        for _, res := range results {

          count := len(res.ids)
          status := res.status

          if status == 200 {
            ids = append(ids, res.ids...)
            successes = append(successes, CallbackResult{count, status})
          } else {
            failures = append(failures, CallbackResult{count, status})
          }
        }

        if e.Callback != nil {
          e.Callback(successes, failures)
        }

        // If all the events failed to be sent exit
        if len(successes) == 0 && len(failures) > 0 {
          break
        }

        e.Storage.DeleteEventRows(ids)
      }
      done = true
    }()
  }
}

// doSend will send all of the eventsRows it is given.
func (e *Emitter) doSend(eventRows []EventRow) []SendResult {
  futures := []<-chan SendResult{}
  url := e.GetCollectorUrl()

  if e.RequestType == "POST" {
    ids := []int{}
    payloads := []Payload{}
    totalByteSize := 0

    for _, val := range eventRows {
      byteSize := CountBytesInString(val.event.String()) + POST_STM_BYTES
      if byteSize + POST_WRAPPER_BYTES > e.ByteLimitPost {
        // A single payload has exceeded the Byte Limit
        futures = append(futures, e.sendPostRequest(url, []int{val.id}, []Payload{val.event}, true))
      } else if (totalByteSize + byteSize + POST_WRAPPER_BYTES + (len(payloads) - 1)) > e.ByteLimitPost {
        // Byte limit reached
        futures = append(futures, e.sendPostRequest(url, ids, payloads, false))

        // Reset accumulators
        ids = []int{val.id}
        payloads = []Payload{val.event}
        totalByteSize = byteSize
      } else {
        ids = append(ids, val.id)
        payloads = append(payloads, val.event)
        totalByteSize += byteSize
      }
    }
    if len(payloads) > 0 {
      futures = append(futures, e.sendPostRequest(url, ids, payloads, false))
    }
  } else if e.RequestType == "GET" {
    for _, val := range eventRows {
      val.event.Add(SENT_TIMESTAMP, NewString(GetTimestampString()))
      queryString := MapToQueryParams(val.event.Get()).Encode()
      oversize := CountBytesInString(queryString) > e.ByteLimitGet
      futures = append(futures, e.sendGetRequest(url + "?" + queryString, []int{val.id}, oversize))
    }
  }

  // Wait for all Futures to complete
  results := []SendResult{}
  for _, future := range futures {
    results = append(results, <-future)
  }

  return results
}

// SendGetRequest sends a payload to the collector endpoint via GET.
func (e *Emitter) sendGetRequest(url string, ids []int, oversize bool) <-chan SendResult {
  c := make(chan SendResult, 1)
  go func() {
    var result SendResult
    defer func() {
      c <- result
    }()

    status := -1; if oversize { status = 200 }

    req, _ := http.NewRequest("GET", url, nil)
    req.Close = true

    resp, err := e.HttpClient.Do(req)
    if err != nil {
        log.Println(err.Error())
        result = SendResult{ ids: ids, status: status }
        return
    }
    resp.Body.Close()

    status = resp.StatusCode; if oversize { status = 200 }
    result = SendResult{ ids: ids, status: status }
  }()
  return c
}

// SendPostRequest sends an array of Payloads together to the collector endpoint via POST.
func (e *Emitter) sendPostRequest(url string, ids []int, body []Payload, oversize bool) <- chan SendResult {
  c := make(chan SendResult, 1)
  go func() {
    var result SendResult
    defer func() {
      c <- result
    }()

    status := -1; if oversize { status = 200 }

    postEnvelope := map[string]interface{}{
      SCHEMA: SCHEMA_PAYLOAD_DATA,
      DATA: addSentTimeToEvents(body),
    }

    req, _ := http.NewRequest("POST", url, bytes.NewBufferString(MapToJson(postEnvelope)))
    req.Close = true
    req.Header.Set("Content-Type", POST_CONTENT_TYPE)

    resp, err := e.HttpClient.Do(req)
    if err != nil {
        log.Println(err.Error())
        result = SendResult{ ids: ids, status: status }
        return
    }
    resp.Body.Close()

    status = resp.StatusCode; if oversize { status = 200 }
    result = SendResult{ ids: ids, status: status }
  }()
  return c
}

// --- Helpers

// IsSending checks whether the send channel has finished.
func (e Emitter) IsSending() bool {
  return len(e.SendChannel) == 0
}

// returnCollectorUrl builds and returns the full collector URL to be used.
func returnCollectorUrl(requestType string, protocol string, collectorUri string) (*url.URL, error) {
  var rawUrl string
  switch requestType {
    case "POST":
      rawUrl = protocol + "://" + collectorUri + "/" + POST_PROTOCOL_VENDOR + "/" + POST_PROTOCOL_VERSION
    case "GET":
      rawUrl = protocol + "://" + collectorUri + "/" + GET_PROTOCOL_PATH
    default:
      return nil, errors.New("FATAL: RequestType did not match either POST or GET.")
  }
  return url.Parse(rawUrl)
}

// addSentTimeToEvents ranges over an array of events and appends the same timestamp to them all.
func addSentTimeToEvents(events []Payload) []map[string]string {
  eventMaps := []map[string]string{}
  stm := NewString(GetTimestampString())
  for _, p := range events {
    p.Add(SENT_TIMESTAMP, stm)
    eventMaps = append(eventMaps, p.Get())
  }
  return eventMaps
}

// --- Getters & Setters

// GetCollectorUrl returns the stringified collector URL.
func (e Emitter) GetCollectorUrl() string {
  return e.CollectorUrl.String()
}

// SetCollectorUri sets a new Collector URI and updates the Collector URL.
func (e *Emitter) SetCollectorUri(collectorUri string) {
  collectorUrl, err := returnCollectorUrl(e.RequestType, e.Protocol, collectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.CollectorUri = collectorUri
  }
}

// SetRequestType sets a new Request Type and updates the Collector URL.
func (e *Emitter) SetRequestType(requestType string) {
  collectorUrl, err := returnCollectorUrl(requestType, e.Protocol, e.CollectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.RequestType = requestType
  }
}

// SetProtocol sets a new Protocol and updates the Collector URL.
func (e *Emitter) SetProtocol(protocol string) {
  collectorUrl, err := returnCollectorUrl(e.RequestType, protocol, e.CollectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.Protocol = protocol
  }
}
<commit_msg>Update result success to be for any 2xx and 3xx codes (closes #9)<commit_after>//
// Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//

package tracker

import (
  "log"
  "net/url"
  "net/http"
  "errors"
  "bytes"
  "time"
)

const (
  DEFAULT_REQ_TYPE        = "POST"
  DEFAULT_PROTOCOL        = "http"
  DEFAULT_SEND_LIMIT      = 500
  DEFAULT_BYTE_LIMIT_GET  = 40000
  DEFAULT_BYTE_LIMIT_POST = 40000
  DEFAULT_DB_NAME         = "events.db"
  POST_WRAPPER_BYTES      = 88 // "schema":"iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-3","data":[]
  POST_STM_BYTES          = 22 // "stm":"1443452851000"
)

type SendResult struct {
  ids    []int
  status int
}

type CallbackResult struct {
  Count  int
  Status int
}

type Emitter struct {
  CollectorUri  string
  CollectorUrl  url.URL
  RequestType   string
  Protocol      string
  SendLimit     int
  ByteLimitGet  int
  ByteLimitPost int
  DbName        string
  Storage       Storage
  SendChannel   chan bool
  Callback      func(successCount []CallbackResult, failureCount []CallbackResult)
  HttpClient    http.Client
}

// InitEmitter creates a new Emitter object which handles
// storing and sending Snowplow Events.
func InitEmitter(options ...func(*Emitter)) *Emitter {
  e := &Emitter{}

  // Set Defaults
  e.RequestType = DEFAULT_REQ_TYPE
  e.Protocol = DEFAULT_PROTOCOL
  e.SendLimit = DEFAULT_SEND_LIMIT
  e.ByteLimitGet = DEFAULT_BYTE_LIMIT_GET
  e.ByteLimitPost = DEFAULT_BYTE_LIMIT_POST
  e.DbName = DEFAULT_DB_NAME

  // Option parameters
  for _, op := range options { op(e) }

  // Check collector URI is not empty
  if e.CollectorUri == "" {
    panic("FATAL: CollectorUri cannot be empty.")
  } else {
    collectorUrl, err := returnCollectorUrl(e.RequestType, e.Protocol, e.CollectorUri)
    if err != nil {
      panic(err.Error())
    } else {
      e.CollectorUrl = *collectorUrl
    }
  }

  // Setup Event Storage
  e.Storage = *InitStorage(e.DbName)

  // Setup HttpClient
  timeout := time.Duration(5 * time.Second)
  e.HttpClient = http.Client{
    Timeout: timeout,
  }

  return e
}

// --- Require

// RequireCollectorUri sets the Emitters collector URI.
func RequireCollectorUri(collectorUri string) func(e *Emitter) {
  return func(e *Emitter) { e.CollectorUri = collectorUri }
}

// --- Option

// OptionRequestType sets the request type to use (GET or POST).
func OptionRequestType(requestType string) func(e *Emitter) {
  return func(e *Emitter) { e.RequestType = requestType }
}

// OptionProtocol sets the protocol type to use (http or https).
func OptionProtocol(protocol string) func(e *Emitter) {
  return func(e *Emitter) { e.Protocol = protocol }
}

// OptionSendLimit sets the send limit for the emitter.
func OptionSendLimit(sendLimit int) func(e *Emitter) {
  return func(e *Emitter) { e.SendLimit = sendLimit }
}

// OptionByteLimitGet sets the byte limit for GET requests.
func OptionByteLimitGet(byteLimitGet int) func(e *Emitter) {
  return func(e *Emitter) { e.ByteLimitGet = byteLimitGet }
}

// OptionByteLimitPost sets the byte limit for POST requests.
func OptionByteLimitPost(byteLimitPost int) func(e *Emitter) {
  return func(e *Emitter) { e.ByteLimitPost = byteLimitPost }
}

// OptionDbName sets the name of the storage database.
func OptionDbName(dbName string) func(e *Emitter) {
  return func(e *Emitter) { e.DbName = dbName }
}

// OptionCallback sets a custom callback for the emitter loop.
func OptionCallback(callback func(successCount []CallbackResult, failureCount []CallbackResult)) func(e *Emitter) {
  return func(e *Emitter) { e.Callback = callback }
}

// --- Event Handlers

// Add will push an event to the database and will then initiate a sending loop.
func (e *Emitter) Add(payload Payload) {
  e.Storage.AddEventRow(payload)
  e.start()
}

// Flush will attempt to start the send loop regardless of an event coming in.
func (e *Emitter) Flush() {
  e.start()
}

// Stop waits for the send channel to have a value and then resets it to nil.
func (e *Emitter) Stop() {
  <-e.SendChannel
  e.SendChannel = nil
}

// start will begin the sending loop.
func (e *Emitter) start() {
  if e.SendChannel == nil || !e.IsSending() {
    e.SendChannel = make(chan bool, 1)
    go func() {
      var done bool
      defer func() {
        e.SendChannel <- done
      }()

      for {
        eventRows := e.Storage.GetEventRowsWithinRange(e.SendLimit)

        // If there are no events in the database exit
        if len(eventRows) == 0 {
          break
        }
        results := e.doSend(eventRows)

        // Process results
        ids := []int{}
        successes := []CallbackResult{}
        failures := []CallbackResult{}

        for _, res := range results {

          count := len(res.ids)
          status := res.status

          if status >= 200 && status < 400 {
            ids = append(ids, res.ids...)
            successes = append(successes, CallbackResult{count, status})
          } else {
            failures = append(failures, CallbackResult{count, status})
          }
        }

        if e.Callback != nil {
          e.Callback(successes, failures)
        }

        // If all the events failed to be sent exit
        if len(successes) == 0 && len(failures) > 0 {
          break
        }

        e.Storage.DeleteEventRows(ids)
      }
      done = true
    }()
  }
}

// doSend will send all of the eventsRows it is given.
func (e *Emitter) doSend(eventRows []EventRow) []SendResult {
  futures := []<-chan SendResult{}
  url := e.GetCollectorUrl()

  if e.RequestType == "POST" {
    ids := []int{}
    payloads := []Payload{}
    totalByteSize := 0

    for _, val := range eventRows {
      byteSize := CountBytesInString(val.event.String()) + POST_STM_BYTES
      if byteSize + POST_WRAPPER_BYTES > e.ByteLimitPost {
        // A single payload has exceeded the Byte Limit
        futures = append(futures, e.sendPostRequest(url, []int{val.id}, []Payload{val.event}, true))
      } else if (totalByteSize + byteSize + POST_WRAPPER_BYTES + (len(payloads) - 1)) > e.ByteLimitPost {
        // Byte limit reached
        futures = append(futures, e.sendPostRequest(url, ids, payloads, false))

        // Reset accumulators
        ids = []int{val.id}
        payloads = []Payload{val.event}
        totalByteSize = byteSize
      } else {
        ids = append(ids, val.id)
        payloads = append(payloads, val.event)
        totalByteSize += byteSize
      }
    }
    if len(payloads) > 0 {
      futures = append(futures, e.sendPostRequest(url, ids, payloads, false))
    }
  } else if e.RequestType == "GET" {
    for _, val := range eventRows {
      val.event.Add(SENT_TIMESTAMP, NewString(GetTimestampString()))
      queryString := MapToQueryParams(val.event.Get()).Encode()
      oversize := CountBytesInString(queryString) > e.ByteLimitGet
      futures = append(futures, e.sendGetRequest(url + "?" + queryString, []int{val.id}, oversize))
    }
  }

  // Wait for all Futures to complete
  results := []SendResult{}
  for _, future := range futures {
    results = append(results, <-future)
  }

  return results
}

// SendGetRequest sends a payload to the collector endpoint via GET.
func (e *Emitter) sendGetRequest(url string, ids []int, oversize bool) <-chan SendResult {
  c := make(chan SendResult, 1)
  go func() {
    var result SendResult
    defer func() {
      c <- result
    }()

    status := -1; if oversize { status = 200 }

    req, _ := http.NewRequest("GET", url, nil)
    req.Close = true

    resp, err := e.HttpClient.Do(req)
    if err != nil {
        log.Println(err.Error())
        result = SendResult{ ids: ids, status: status }
        return
    }
    resp.Body.Close()

    status = resp.StatusCode; if oversize { status = 200 }
    result = SendResult{ ids: ids, status: status }
  }()
  return c
}

// SendPostRequest sends an array of Payloads together to the collector endpoint via POST.
func (e *Emitter) sendPostRequest(url string, ids []int, body []Payload, oversize bool) <- chan SendResult {
  c := make(chan SendResult, 1)
  go func() {
    var result SendResult
    defer func() {
      c <- result
    }()

    status := -1; if oversize { status = 200 }

    postEnvelope := map[string]interface{}{
      SCHEMA: SCHEMA_PAYLOAD_DATA,
      DATA: addSentTimeToEvents(body),
    }

    req, _ := http.NewRequest("POST", url, bytes.NewBufferString(MapToJson(postEnvelope)))
    req.Close = true
    req.Header.Set("Content-Type", POST_CONTENT_TYPE)

    resp, err := e.HttpClient.Do(req)
    if err != nil {
        log.Println(err.Error())
        result = SendResult{ ids: ids, status: status }
        return
    }
    resp.Body.Close()

    status = resp.StatusCode; if oversize { status = 200 }
    result = SendResult{ ids: ids, status: status }
  }()
  return c
}

// --- Helpers

// IsSending checks whether the send channel has finished.
func (e Emitter) IsSending() bool {
  return len(e.SendChannel) == 0
}

// returnCollectorUrl builds and returns the full collector URL to be used.
func returnCollectorUrl(requestType string, protocol string, collectorUri string) (*url.URL, error) {
  var rawUrl string
  switch requestType {
    case "POST":
      rawUrl = protocol + "://" + collectorUri + "/" + POST_PROTOCOL_VENDOR + "/" + POST_PROTOCOL_VERSION
    case "GET":
      rawUrl = protocol + "://" + collectorUri + "/" + GET_PROTOCOL_PATH
    default:
      return nil, errors.New("FATAL: RequestType did not match either POST or GET.")
  }
  return url.Parse(rawUrl)
}

// addSentTimeToEvents ranges over an array of events and appends the same timestamp to them all.
func addSentTimeToEvents(events []Payload) []map[string]string {
  eventMaps := []map[string]string{}
  stm := NewString(GetTimestampString())
  for _, p := range events {
    p.Add(SENT_TIMESTAMP, stm)
    eventMaps = append(eventMaps, p.Get())
  }
  return eventMaps
}

// --- Getters & Setters

// GetCollectorUrl returns the stringified collector URL.
func (e Emitter) GetCollectorUrl() string {
  return e.CollectorUrl.String()
}

// SetCollectorUri sets a new Collector URI and updates the Collector URL.
func (e *Emitter) SetCollectorUri(collectorUri string) {
  collectorUrl, err := returnCollectorUrl(e.RequestType, e.Protocol, collectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.CollectorUri = collectorUri
  }
}

// SetRequestType sets a new Request Type and updates the Collector URL.
func (e *Emitter) SetRequestType(requestType string) {
  collectorUrl, err := returnCollectorUrl(requestType, e.Protocol, e.CollectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.RequestType = requestType
  }
}

// SetProtocol sets a new Protocol and updates the Collector URL.
func (e *Emitter) SetProtocol(protocol string) {
  collectorUrl, err := returnCollectorUrl(e.RequestType, protocol, e.CollectorUri)
  if err == nil {
    e.CollectorUrl = *collectorUrl
    e.Protocol = protocol
  }
}
<|endoftext|>


<commit_before>// Package wav implements audio data decoding in WAVE format through an audio.StreamSeekCloser.
package wav

import (
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/faiface/pixel/audio"
	"github.com/pkg/errors"
)

// ReadSeekCloser is a union of io.Reader, io.Seeker and io.Closer.
type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

// Decode takes a ReadSeekCloser containing audio data in WAVE format and returns a
// StreamSeekCloser, which streams that audio.
//
// Do not close the supplied ReadSeekCloser, instead, use the Close method of the returned
// StreamSeekCloser when you want to release the resources.
func Decode(rsc ReadSeekCloser) (s audio.StreamSeekCloser, err error) {
	d := decoder{rsc: rsc}
	defer func() { // hacky way to always close rsc if an error occured
		if err != nil {
			d.rsc.Close()
		}
	}()
	herr := binary.Read(rsc, binary.LittleEndian, &d.h)
	if herr != nil {
		return nil, errors.Wrap(herr, "wav")
	}
	if string(d.h.RiffMark[:]) != "RIFF" {
		return nil, errors.New("wav: missing RIFF at the beginning")
	}
	if string(d.h.WaveMark[:]) != "WAVE" {
		return nil, errors.New("wav: unsupported file type")
	}
	if string(d.h.FmtMark[:]) != "fmt " {
		return nil, errors.New("wav: missing format chunk marker")
	}
	if string(d.h.DataMark[:]) != "data" {
		return nil, errors.New("wav: missing data chunk marker")
	}
	if d.h.FormatType != 1 {
		return nil, errors.New("wav: unsupported format type")
	}
	if d.h.NumChans <= 0 {
		return nil, errors.New("wav: invalid number of channels (less than 1)")
	}
	if d.h.BitsPerSample != 8 && d.h.BitsPerSample != 16 {
		return nil, errors.New("wav: unsupported number of bits per sample, 8 or 16 are supported")
	}
	return &d, nil
}

type header struct {
	RiffMark      [4]byte
	FileSize      int32
	WaveMark      [4]byte
	FmtMark       [4]byte
	FormatSize    int32
	FormatType    int16
	NumChans      int16
	SampleRate    int32
	ByteRate      int32
	BytesPerFrame int16
	BitsPerSample int16
	DataMark      [4]byte
	DataSize      int32
}

type decoder struct {
	rsc ReadSeekCloser
	h   header
	pos int32
	err error
}

func (s *decoder) Err() error {
	return s.err
}

func (s *decoder) Duration() time.Duration {
	numBytes := time.Duration(s.h.DataSize)
	perFrame := time.Duration(s.h.BytesPerFrame)
	sampRate := time.Duration(s.h.SampleRate)
	return numBytes / perFrame * time.Second / sampRate
}

func (s *decoder) Position() time.Duration {
	frameIndex := time.Duration(s.pos / int32(s.h.BytesPerFrame))
	frameTime := time.Second / time.Duration(s.h.SampleRate)
	return frameIndex * frameTime
}

func (s *decoder) Seek(d time.Duration) error {
	if d < 0 || s.Duration() < d {
		return fmt.Errorf("wav: seek duration %v out of range [%v, %v]", d, 0, s.Duration())
	}
	frame := int32(d / (time.Second / time.Duration(s.h.SampleRate)))
	pos := frame * int32(s.h.BytesPerFrame)
	_, err := s.rsc.Seek(int64(pos)+44, io.SeekStart) // 44 is the size of the header
	if err != nil {
		return errors.Wrap(err, "wav: seek error")
	}
	s.pos = pos
	return nil
}

func (s *decoder) Stream(samples [][2]float64) (n int, ok bool) {
	if s.err != nil || s.pos >= s.h.DataSize {
		return 0, false
	}
	bytesPerFrame := int(s.h.BytesPerFrame)
	p := make([]byte, len(samples)*bytesPerFrame)
	n, err := s.rsc.Read(p)
	if err != nil {
		s.err = err
	}
	switch {
	case s.h.BitsPerSample == 8 && s.h.NumChans == 1:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			val := float64(p[i])/(1<<8-1)*2 - 1
			samples[j][0] = val
			samples[j][1] = val
		}
	case s.h.BitsPerSample == 8 && s.h.NumChans >= 2:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			samples[j][0] = float64(p[i+0])/(1<<8-1)*2 - 1
			samples[j][1] = float64(p[i+1])/(1<<8-1)*2 - 1
		}
	case s.h.BitsPerSample == 16 && s.h.NumChans == 1:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			val := float64(int16(p[i+0])+int16(p[i+1])*(1<<8)) / (1<<15 - 1)
			samples[j][0] = val
			samples[j][1] = val
		}
	case s.h.BitsPerSample == 16 && s.h.NumChans >= 2:
		for i, j := 0, 0; i <= n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			samples[j][0] = float64(int16(p[i+0])+int16(p[i+1])*(1<<8)) / (1<<15 - 1)
			samples[j][1] = float64(int16(p[i+2])+int16(p[i+3])*(1<<8)) / (1<<15 - 1)
		}
	}
	return n / bytesPerFrame, true
}

func (s *decoder) Close() error {
	err := s.rsc.Close()
	if err != nil {
		return errors.Wrap(err, "wav")
	}
	return nil
}
<commit_msg>audio: wav: fix Stream (move position, forgot to do it previously)<commit_after>// Package wav implements audio data decoding in WAVE format through an audio.StreamSeekCloser.
package wav

import (
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/faiface/pixel/audio"
	"github.com/pkg/errors"
)

// ReadSeekCloser is a union of io.Reader, io.Seeker and io.Closer.
type ReadSeekCloser interface {
	io.Reader
	io.Seeker
	io.Closer
}

// Decode takes a ReadSeekCloser containing audio data in WAVE format and returns a
// StreamSeekCloser, which streams that audio.
//
// Do not close the supplied ReadSeekCloser, instead, use the Close method of the returned
// StreamSeekCloser when you want to release the resources.
func Decode(rsc ReadSeekCloser) (s audio.StreamSeekCloser, err error) {
	d := decoder{rsc: rsc}
	defer func() { // hacky way to always close rsc if an error occured
		if err != nil {
			d.rsc.Close()
		}
	}()
	herr := binary.Read(rsc, binary.LittleEndian, &d.h)
	if herr != nil {
		return nil, errors.Wrap(herr, "wav")
	}
	if string(d.h.RiffMark[:]) != "RIFF" {
		return nil, errors.New("wav: missing RIFF at the beginning")
	}
	if string(d.h.WaveMark[:]) != "WAVE" {
		return nil, errors.New("wav: unsupported file type")
	}
	if string(d.h.FmtMark[:]) != "fmt " {
		return nil, errors.New("wav: missing format chunk marker")
	}
	if string(d.h.DataMark[:]) != "data" {
		return nil, errors.New("wav: missing data chunk marker")
	}
	if d.h.FormatType != 1 {
		return nil, errors.New("wav: unsupported format type")
	}
	if d.h.NumChans <= 0 {
		return nil, errors.New("wav: invalid number of channels (less than 1)")
	}
	if d.h.BitsPerSample != 8 && d.h.BitsPerSample != 16 {
		return nil, errors.New("wav: unsupported number of bits per sample, 8 or 16 are supported")
	}
	return &d, nil
}

type header struct {
	RiffMark      [4]byte
	FileSize      int32
	WaveMark      [4]byte
	FmtMark       [4]byte
	FormatSize    int32
	FormatType    int16
	NumChans      int16
	SampleRate    int32
	ByteRate      int32
	BytesPerFrame int16
	BitsPerSample int16
	DataMark      [4]byte
	DataSize      int32
}

type decoder struct {
	rsc ReadSeekCloser
	h   header
	pos int32
	err error
}

func (s *decoder) Err() error {
	return s.err
}

func (s *decoder) Duration() time.Duration {
	numBytes := time.Duration(s.h.DataSize)
	perFrame := time.Duration(s.h.BytesPerFrame)
	sampRate := time.Duration(s.h.SampleRate)
	return numBytes / perFrame * time.Second / sampRate
}

func (s *decoder) Position() time.Duration {
	frameIndex := time.Duration(s.pos / int32(s.h.BytesPerFrame))
	frameTime := time.Second / time.Duration(s.h.SampleRate)
	return frameIndex * frameTime
}

func (s *decoder) Seek(d time.Duration) error {
	if d < 0 || s.Duration() < d {
		return fmt.Errorf("wav: seek duration %v out of range [%v, %v]", d, 0, s.Duration())
	}
	frame := int32(d / (time.Second / time.Duration(s.h.SampleRate)))
	pos := frame * int32(s.h.BytesPerFrame)
	_, err := s.rsc.Seek(int64(pos)+44, io.SeekStart) // 44 is the size of the header
	if err != nil {
		return errors.Wrap(err, "wav: seek error")
	}
	s.pos = pos
	return nil
}

func (s *decoder) Stream(samples [][2]float64) (n int, ok bool) {
	if s.err != nil || s.pos >= s.h.DataSize {
		return 0, false
	}
	bytesPerFrame := int(s.h.BytesPerFrame)
	p := make([]byte, len(samples)*bytesPerFrame)
	n, err := s.rsc.Read(p)
	if err != nil {
		s.err = err
	}
	switch {
	case s.h.BitsPerSample == 8 && s.h.NumChans == 1:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			val := float64(p[i])/(1<<8-1)*2 - 1
			samples[j][0] = val
			samples[j][1] = val
		}
	case s.h.BitsPerSample == 8 && s.h.NumChans >= 2:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			samples[j][0] = float64(p[i+0])/(1<<8-1)*2 - 1
			samples[j][1] = float64(p[i+1])/(1<<8-1)*2 - 1
		}
	case s.h.BitsPerSample == 16 && s.h.NumChans == 1:
		for i, j := 0, 0; i < n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			val := float64(int16(p[i+0])+int16(p[i+1])*(1<<8)) / (1<<15 - 1)
			samples[j][0] = val
			samples[j][1] = val
		}
	case s.h.BitsPerSample == 16 && s.h.NumChans >= 2:
		for i, j := 0, 0; i <= n-bytesPerFrame; i, j = i+bytesPerFrame, j+1 {
			samples[j][0] = float64(int16(p[i+0])+int16(p[i+1])*(1<<8)) / (1<<15 - 1)
			samples[j][1] = float64(int16(p[i+2])+int16(p[i+3])*(1<<8)) / (1<<15 - 1)
		}
	}
	s.pos += int32(n)
	return n / bytesPerFrame, true
}

func (s *decoder) Close() error {
	err := s.rsc.Close()
	if err != nil {
		return errors.Wrap(err, "wav")
	}
	return nil
}
<|endoftext|>


<commit_before>package transfer

import (
	"encoding/json"
	"io"
	"time"

	log "github.com/Sirupsen/logrus"
)

type Server struct {
	im      io.ReadWriter
	errors  chan error
	done    chan bool
	encoder *json.Encoder
	decoder *json.Decoder
}

func (sv *Server) handleCmd() bool {
	cmd := &Command{}
	if err := sv.decoder.Decode(&cmd); err != nil {
		// TODO: Is there a better way than polling?
		if err == io.EOF {
			time.Sleep(100 * time.Millisecond)
			return true
		}

		log.Warningf("Unable to decode item from json stream: %v", err)
		return true
	}

	handler, ok := handlerMap[cmd.ID]
	if !ok {
		log.Warningf("Unknown command id: %d", cmd.ID)
		return true
	}

	resp, err := handler(sv, cmd)
	if err != nil {
		log.Warningf("Handling %s failed: %v", cmd.ID.String(), err)
		return true
	}

	if resp == nil {
		// We don't want to answer.
		return true
	}

	resp.ID = cmd.ID
	if err := sv.encoder.Encode(&resp); err != nil {
		log.Warningf("Casting response to json failed: %v", err)
		return true
	}

	return cmd.ID != CmdQuit
}

func (sv *Server) loop() {
	for {
		select {
		case _ = <-sv.done:
			sv.errors <- nil
			return
		default:
			if !sv.handleCmd() {
				sv.errors <- nil
				return
			}
		}
	}
}

func NewServer(im io.ReadWriter) *Server {
	sv := &Server{
		im:      im,
		encoder: json.NewEncoder(im),
		decoder: json.NewDecoder(im),
		done:    make(chan bool, 1),
		errors:  make(chan error),
	}

	go sv.loop()
	return sv
}

func (sv *Server) Serve() error {
	err := <-sv.errors
	sv.Quit()
	return err
}

func (sv *Server) Quit() {
	sv.done <- true
}
<commit_msg>transfer: add EOF comment.<commit_after>package transfer

import (
	"encoding/json"
	"io"
	"time"

	log "github.com/Sirupsen/logrus"
)

type Server struct {
	im      io.ReadWriter
	errors  chan error
	done    chan bool
	encoder *json.Encoder
	decoder *json.Decoder
}

func (sv *Server) handleCmd() bool {
	cmd := &Command{}

	// NOTE: We rely on the underlying stream to not return io.EOF
	//       early (i.e. when no data is yet available)
	if err := sv.decoder.Decode(&cmd); err != nil {
		log.Warningf("Unable to decode item from json stream: %v", err)
		return true
	}

	handler, ok := handlerMap[cmd.ID]
	if !ok {
		log.Warningf("Unknown command id: %d", cmd.ID)
		return true
	}

	resp, err := handler(sv, cmd)
	if err != nil {
		log.Warningf("Handling %s failed: %v", cmd.ID.String(), err)
		return true
	}

	if resp == nil {
		// We don't want to answer.
		return true
	}

	resp.ID = cmd.ID
	if err := sv.encoder.Encode(&resp); err != nil {
		log.Warningf("Casting response to json failed: %v", err)
		return true
	}

	return cmd.ID != CmdQuit
}

func (sv *Server) loop() {
	for {
		select {
		case _ = <-sv.done:
			sv.errors <- nil
			return
		default:
			if !sv.handleCmd() {
				sv.errors <- nil
				return
			}
		}
	}
}

func NewServer(im io.ReadWriter) *Server {
	sv := &Server{
		im:      im,
		encoder: json.NewEncoder(im),
		decoder: json.NewDecoder(im),
		done:    make(chan bool, 1),
		errors:  make(chan error),
	}

	go sv.loop()
	return sv
}

func (sv *Server) Serve() error {
	err := <-sv.errors
	sv.Quit()
	return err
}

func (sv *Server) Quit() {
	sv.done <- true
}
<|endoftext|>