<commit_before>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Sync struct {
	Service       *s3.S3
	ACL           string
	Bucket        string
	Prefix        string
	WorkPool      tunny.WorkPool
	Logger        *log.WOFLogger
	Debug         bool
	Success       int64
	Error         int64
	Skipped       int64
	Scheduled     int64
	Completed     int64
	Retried       int64
	TimeToProcess *time.Duration
	Retries       *pool.LIFOPool
	MaxRetries    float64 // max percentage of errors over scheduled
}

func NewSync(region string, acl string, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	logger.Info("creating a new Sync thing-y with %d processes", procs)

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()

	retries := pool.NewLIFOPool()

	cfg := &aws.Config{
		Region: aws.String(region),
		// Credentials: credentials.NewSharedCredentials("", s3cfg.Credentials),
	}

	sess := session.New(cfg)

	/*
	        _, err := sess.Config.Credentials.Get()

		if err != nil {
		   return nil, err
		   }
	*/

	svc := s3.New(sess)

	ttp := new(time.Duration)

	return &Sync{
		Service:       svc,
		Bucket:        bucket,
		Prefix:        prefix,
		WorkPool:      *workpool,
		Debug:         debug,
		Logger:        logger,
		Scheduled:     0,
		Completed:     0,
		Skipped:       0,
		Error:         0,
		Success:       0,
		Retried:       0,
		TimeToProcess: ttp,
		Retries:       retries,
		MaxRetries:    25.0, // maybe allow this to be user-defined ?
	}
}

func WOFSync(bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	return NewSync("us-east-1", "public-read", bucket, prefix, procs, debug, logger)
}

func (sink *Sync) SyncDirectory(root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	callback := func(source string, info os.FileInfo) error {

		if info.IsDir() {
			return nil
		}

		err := sink.SyncFile(source, root, wg)

		if err != nil {
			sink.Logger.Error("failed to sync %s, because '%s'", source, err)
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFiles(files []string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	for _, path := range files {

		go func(path string, root string, wg *sync.WaitGroup) {
			sink.SyncFile(path, root, wg)
		}(path, root, wg)
	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFileList(path string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)

	count := 100000
	ch := make(chan bool, count)

	go func() {
		for i := 0; i < count; i++ {
			ch <- true
		}
	}()

	wg := new(sync.WaitGroup)

	for scanner.Scan() {

		<-ch

		path := scanner.Text()
		sink.Logger.Debug("schedule %s for sync", path)

		wg.Add(1)

		go func(path string, root string, wg *sync.WaitGroup, ch chan bool) {

			defer wg.Done()

			sink.SyncFile(path, root, wg)
			ch <- true

			sink.Logger.Debug("completed sync for %s", path)
		}(path, root, wg, ch)
	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFile(source string, root string, wg *sync.WaitGroup) error {

	atomic.AddInt64(&sink.Scheduled, 1)

	_, err := sink.WorkPool.SendWork(func() {

		wg.Add(1)

		defer wg.Done()

		dest := source

		dest = strings.Replace(dest, root, "", -1)

		if sink.Prefix != "" {
			dest = path.Join(sink.Prefix, dest)
		}

		// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
		// which is a potential waste of time and resource. Or maybe we just
		// don't care? (20150930/thisisaaronland)

		sink.Logger.Debug("Looking for changes to %s (%s)", dest, sink.Prefix)

		change, ch_err := sink.HasChanged(source, dest)

		if ch_err != nil {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Error, 1)
			sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)

			sink.Retries.Push(&pool.PoolString{String: source})
			return
		}

		if sink.Debug == true {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Skipped, 1)
			sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
			return
		}

		if !change {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Skipped, 1)
			sink.Logger.Debug("%s has not changed, skipping", source)
			return
		}

		err := sink.DoSyncFile(source, dest)

		if err != nil {
			sink.Retries.Push(&pool.PoolString{String: source})
			atomic.AddInt64(&sink.Error, 1)
		} else {
			atomic.AddInt64(&sink.Success, 1)
		}

		atomic.AddInt64(&sink.Completed, 1)
	})

	if err != nil {
		atomic.AddInt64(&sink.Error, 1)
		sink.Logger.Error("failed to schedule %s for processing, because %v", source, err)
		return err
	}

	sink.Logger.Debug("schedule %s for processing", source)
	return nil
}

func (sink *Sync) DoSyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

	params := &s3.PutObjectInput{
		Bucket: aws.String(sink.Bucket),
		Key:    aws.String(dest),
		Body:   bytes.NewReader(body),
		ACL:    aws.String(sink.ACL),
	}

	_, err = sink.Service.PutObject(params)

	if err != nil {
		sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		return err
	}

	return nil
}

func (sink *Sync) HasChanged(source string, dest string) (ch bool, err error) {

	sink.Logger.Debug("HEAD %s", dest)

	// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#HeadObjectInput
	// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#HeadObjectOutput

	params := &s3.HeadObjectInput{
		Bucket: aws.String(sink.Bucket),
		Key:    aws.String(source),
	}

	rsp, err := sink.Service.HeadObject(params)

	if err != nil {

		/*
			if e, ok := err.(*aws_s3.Error); ok && e.StatusCode == 404 {
				sink.Logger.Debug("%s is 404 so assuming it has changed (WHOA)", dest)
				return true, nil
			}
		*/

		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := *rsp.ETag
	remote_hash := strings.Replace(etag, "\"", "", -1)

	sink.Logger.Debug("local hash is %s remote hash is %s", local_hash, remote_hash)

	if local_hash == remote_hash {
		return false, nil
	}

	// Okay so we think that things have changed but let's just check
	// modification times to be extra sure (20151112/thisisaaronland)

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()
	mtime_remote := *rsp.LastModified

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	sink.Logger.Debug("local %s %s", mtime_local, source)
	sink.Logger.Debug("remote %s %s", mtime_remote, dest)

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	return true, nil
}

func (sink *Sync) ProcessRetries(root string) bool {

	to_retry := sink.Retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(sink.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > sink.MaxRetries {
			sink.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		sink.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for sink.Retries.Length() > 0 {

			r, ok := sink.Retries.Pop()

			if !ok {
				sink.Logger.Error("failed to pop retries because... computers?")
				break
			}

			source := r.StringValue()

			go func(source string, root string, wg *sync.WaitGroup) {

				atomic.AddInt64(&sink.Scheduled, 1)

				sink.WorkPool.SendWork(func() {

					atomic.AddInt64(&sink.Retried, 1)

					sink.Logger.Info("retry syncing %s", source)

					sink.SyncFile(source, root, wg)

					atomic.AddInt64(&sink.Completed, 1)
				})

			}(source, root, wg)
		}

		wg.Wait()
	}

	return true
}

func (sink *Sync) MonitorStatus() {

	go func() {

		t0 := time.Now()

		for {

			rpt := sink.StatusReport()
			ttp := time.Since(t0)

			sink.Logger.Info("%s Time %v", rpt, ttp)

			time.Sleep(10 * time.Second)

			if sink.Scheduled == sink.Completed {
				break
			}
		}

		sink.Logger.Info(sink.StatusReport())
		sink.Logger.Info("monitoring complete")
	}()
}

func (sink *Sync) StatusReport() string {
	return fmt.Sprintf("Scheduled %d Completed %d Success %d Error %d Skipped %d Retried %d Goroutines %d",
		sink.Scheduled, sink.Completed, sink.Success, sink.Error, sink.Skipped, sink.Retried, runtime.NumGoroutine())
}
<commit_msg>use aws.NewConfig so credentials can be optional<commit_after>package s3

// https://github.com/aws/aws-sdk-go
// https://docs.aws.amazon.com/sdk-for-go/api/service/s3.html

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/jeffail/tunny"
	"github.com/whosonfirst/go-whosonfirst-crawl"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	utils "github.com/whosonfirst/go-whosonfirst-utils"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

type Sync struct {
	Service       *s3.S3
	ACL           string
	Bucket        string
	Prefix        string
	WorkPool      tunny.WorkPool
	Logger        *log.WOFLogger
	Debug         bool
	Success       int64
	Error         int64
	Skipped       int64
	Scheduled     int64
	Completed     int64
	Retried       int64
	TimeToProcess *time.Duration
	Retries       *pool.LIFOPool
	MaxRetries    float64 // max percentage of errors over scheduled
}

func NewSync(creds *credentials.Credentials, region string, acl string, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	logger.Info("creating a new Sync thing-y with %d processes", procs)

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()

	retries := pool.NewLIFOPool()

	cfg := aws.NewConfig()
	cfg.WithRegion(region)

	if creds != nil {
		cfg.WithCredentials(creds)
	}

	sess := session.New(cfg)

	svc := s3.New(sess)

	ttp := new(time.Duration)

	return &Sync{
		Service:       svc,
		Bucket:        bucket,
		Prefix:        prefix,
		WorkPool:      *workpool,
		Debug:         debug,
		Logger:        logger,
		Scheduled:     0,
		Completed:     0,
		Skipped:       0,
		Error:         0,
		Success:       0,
		Retried:       0,
		TimeToProcess: ttp,
		Retries:       retries,
		MaxRetries:    25.0, // maybe allow this to be user-defined ?
	}
}

func WOFSync(bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	return NewSync(nil, "us-east-1", "public-read", bucket, prefix, procs, debug, logger)
}

func (sink *Sync) SyncDirectory(root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	callback := func(source string, info os.FileInfo) error {

		if info.IsDir() {
			return nil
		}

		err := sink.SyncFile(source, root, wg)

		if err != nil {
			sink.Logger.Error("failed to sync %s, because '%s'", source, err)
		}

		return nil
	}

	c := crawl.NewCrawler(root)
	_ = c.Crawl(callback)

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFiles(files []string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	wg := new(sync.WaitGroup)

	for _, path := range files {

		go func(path string, root string, wg *sync.WaitGroup) {
			sink.SyncFile(path, root, wg)
		}(path, root, wg)
	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFileList(path string, root string) error {

	defer sink.WorkPool.Close()

	t0 := time.Now()

	file, err := os.Open(path)

	if err != nil {
		return err
	}

	defer file.Close()

	scanner := bufio.NewScanner(file)

	count := 100000
	ch := make(chan bool, count)

	go func() {
		for i := 0; i < count; i++ {
			ch <- true
		}
	}()

	wg := new(sync.WaitGroup)

	for scanner.Scan() {

		<-ch

		path := scanner.Text()
		sink.Logger.Debug("schedule %s for sync", path)

		wg.Add(1)

		go func(path string, root string, wg *sync.WaitGroup, ch chan bool) {

			defer wg.Done()

			sink.SyncFile(path, root, wg)
			ch <- true

			sink.Logger.Debug("completed sync for %s", path)
		}(path, root, wg, ch)
	}

	wg.Wait()

	sink.ProcessRetries(root)

	ttp := time.Since(t0)
	sink.TimeToProcess = &ttp

	return nil
}

func (sink *Sync) SyncFile(source string, root string, wg *sync.WaitGroup) error {

	atomic.AddInt64(&sink.Scheduled, 1)

	_, err := sink.WorkPool.SendWork(func() {

		wg.Add(1)

		defer wg.Done()

		dest := source

		dest = strings.Replace(dest, root, "", -1)

		if sink.Prefix != "" {
			dest = path.Join(sink.Prefix, dest)
		}

		// Note: both HasChanged and SyncFile will ioutil.ReadFile(source)
		// which is a potential waste of time and resource. Or maybe we just
		// don't care? (20150930/thisisaaronland)

		sink.Logger.Debug("Looking for changes to %s (%s)", dest, sink.Prefix)

		change, ch_err := sink.HasChanged(source, dest)

		if ch_err != nil {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Error, 1)
			sink.Logger.Warning("failed to determine whether %s had changed, because '%s'", source, ch_err)

			sink.Retries.Push(&pool.PoolString{String: source})
			return
		}

		if sink.Debug == true {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Skipped, 1)
			sink.Logger.Debug("has %s changed? the answer is %v but does it really matter since debugging is enabled?", source, change)
			return
		}

		if !change {
			atomic.AddInt64(&sink.Completed, 1)
			atomic.AddInt64(&sink.Skipped, 1)
			sink.Logger.Debug("%s has not changed, skipping", source)
			return
		}

		err := sink.DoSyncFile(source, dest)

		if err != nil {
			sink.Retries.Push(&pool.PoolString{String: source})
			atomic.AddInt64(&sink.Error, 1)
		} else {
			atomic.AddInt64(&sink.Success, 1)
		}

		atomic.AddInt64(&sink.Completed, 1)
	})

	if err != nil {
		atomic.AddInt64(&sink.Error, 1)
		sink.Logger.Error("failed to schedule %s for processing, because %v", source, err)
		return err
	}

	sink.Logger.Debug("schedule %s for processing", source)
	return nil
}

func (sink *Sync) DoSyncFile(source string, dest string) error {

	sink.Logger.Debug("prepare %s for syncing", source)

	body, err := ioutil.ReadFile(source)

	if err != nil {
		sink.Logger.Error("Failed to read %s, because %v", source, err)
		return err
	}

	sink.Logger.Debug("PUT %s as %s", dest, sink.ACL)

	params := &s3.PutObjectInput{
		Bucket: aws.String(sink.Bucket),
		Key:    aws.String(dest),
		Body:   bytes.NewReader(body),
		ACL:    aws.String(sink.ACL),
	}

	_, err = sink.Service.PutObject(params)

	if err != nil {
		sink.Logger.Error("failed to PUT %s, because '%s'", dest, err)
		return err
	}

	return nil
}

func (sink *Sync) HasChanged(source string, dest string) (ch bool, err error) {

	sink.Logger.Debug("HEAD %s", dest)

	// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#HeadObjectInput
	// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#HeadObjectOutput

	params := &s3.HeadObjectInput{
		Bucket: aws.String(sink.Bucket),
		Key:    aws.String(source),
	}

	rsp, err := sink.Service.HeadObject(params)

	if err != nil {

		/*
			if e, ok := err.(*aws_s3.Error); ok && e.StatusCode == 404 {
				sink.Logger.Debug("%s is 404 so assuming it has changed (WHOA)", dest)
				return true, nil
			}
		*/

		sink.Logger.Error("failed to HEAD %s because %s", dest, err)
		return false, err
	}

	local_hash, err := utils.HashFile(source)

	if err != nil {
		sink.Logger.Warning("failed to hash %s, because %v", source, err)
		return false, err
	}

	etag := *rsp.ETag
	remote_hash := strings.Replace(etag, "\"", "", -1)

	sink.Logger.Debug("local hash is %s remote hash is %s", local_hash, remote_hash)

	if local_hash == remote_hash {
		return false, nil
	}

	// Okay so we think that things have changed but let's just check
	// modification times to be extra sure (20151112/thisisaaronland)

	info, err := os.Stat(source)

	if err != nil {
		sink.Logger.Error("failed to stat %s because %s", source, err)
		return false, err
	}

	mtime_local := info.ModTime()
	mtime_remote := *rsp.LastModified

	// Because who remembers this stuff anyway...
	// func (t Time) Before(u Time) bool
	// Before reports whether the time instant t is before u.

	sink.Logger.Debug("local %s %s", mtime_local, source)
	sink.Logger.Debug("remote %s %s", mtime_remote, dest)

	if mtime_local.Before(mtime_remote) {
		sink.Logger.Warning("remote copy of %s has a more recent modification date (local: %s remote: %s)", source, mtime_local, mtime_remote)
		return false, nil
	}

	return true, nil
}

func (sink *Sync) ProcessRetries(root string) bool {

	to_retry := sink.Retries.Length()

	if to_retry > 0 {

		scheduled_f := float64(sink.Scheduled)
		retry_f := float64(to_retry)

		pct := (retry_f / scheduled_f) * 100.0

		if pct > sink.MaxRetries {
			sink.Logger.Warning("E_EXCESSIVE_ERRORS, %f percent of scheduled processes failed thus undermining our faith that they will work now...", pct)
			return false
		}

		sink.Logger.Info("There are %d failed requests that will now be retried", to_retry)

		wg := new(sync.WaitGroup)

		for sink.Retries.Length() > 0 {

			r, ok := sink.Retries.Pop()

			if !ok {
				sink.Logger.Error("failed to pop retries because... computers?")
				break
			}

			source := r.StringValue()

			go func(source string, root string, wg *sync.WaitGroup) {

				atomic.AddInt64(&sink.Scheduled, 1)

				sink.WorkPool.SendWork(func() {

					atomic.AddInt64(&sink.Retried, 1)

					sink.Logger.Info("retry syncing %s", source)

					sink.SyncFile(source, root, wg)

					atomic.AddInt64(&sink.Completed, 1)
				})

			}(source, root, wg)
		}

		wg.Wait()
	}

	return true
}

func (sink *Sync) MonitorStatus() {

	go func() {

		t0 := time.Now()

		for {

			rpt := sink.StatusReport()
			ttp := time.Since(t0)

			sink.Logger.Info("%s Time %v", rpt, ttp)

			time.Sleep(10 * time.Second)

			if sink.Scheduled == sink.Completed {
				break
			}
		}

		sink.Logger.Info(sink.StatusReport())
		sink.Logger.Info("monitoring complete")
	}()
}

func (sink *Sync) StatusReport() string {
	return fmt.Sprintf("Scheduled %d Completed %d Success %d Error %d Skipped %d Retried %d Goroutines %d",
		sink.Scheduled, sink.Completed, sink.Success, sink.Error, sink.Skipped, sink.Retried, runtime.NumGoroutine())
}
<|endoftext|>


<commit_before>// +build tk

package main

import (
	"bytes"
	"encoding/base64"
	"flag"
	"fmt"
	"image"
	"image/draw"
	"image/png"
	"log"
	"os"

	"github.com/nsf/gothic"
)

func main() {
	opt := flag.Bool("s", false, "Use true 16-color solarized palette")
	optVersion := flag.Bool("v", false, "print version number")
	optCenteredCamera := flag.Bool("c", false, "centered camera")
	optMinimalUI := flag.Bool("m", false, "80x24 minimal UI")
	optNoAnim := flag.Bool("n", false, "no animations")
	flag.Parse()
	if *opt {
		SolarizedPalette()
	}
	if *optVersion {
		fmt.Println(Version)
		os.Exit(0)
	}
	if *optCenteredCamera {
		CenteredCamera = true
	}
	if *optNoAnim {
		DisableAnimations = true
	}

	tui := &termui{}
	if *optMinimalUI {
		gameConfig.Small = true
		UIHeight = 24
		UIWidth = 80
	}
	err := tui.Init()
	if err != nil {
		fmt.Fprintf(os.Stderr, "boohu: %v\n", err)
		os.Exit(1)
	}
	defer tui.Close()

	ApplyDefaultKeyBindings()
	tui.PostInit()
	gameConfig.Tiles = true
	LinkColors()
	gameConfig.DarkLOS = true

	g := &game{}
	load, err := g.LoadConfig()
	if load && err != nil {
		g.Print("Error loading config file.")
	} else if load {
		CustomKeys = true
	}

	tui.DrawWelcome()
	load, err = g.Load()
	if !load {
		g.InitLevel()
	} else if err != nil {
		g.InitLevel()
		g.Print("Error loading saved game… starting new game.")
	}
	g.ui = tui
	g.EventLoop()
}

type termui struct {
	ir         *gothic.Interpreter
	cells      []UICell
	backBuffer []UICell
	cursor     position
	stty       string
	cache      map[UICell]string
	width      int
	height     int
	mousepos   position
	menuHover  menu
	itemHover  int
}

func (ui *termui) Init() error {
	ui.cells = make([]UICell, UIWidth*UIHeight)
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	ui.ir = gothic.NewInterpreter(`
set width [expr {16 * 100}]
set height [expr {24 * 26}]
set can [canvas .c -width $width -height $height -background black]
grid $can -row 0 -column 0
focus $can
image create photo gamescreen -width $width -height $height -palette 256/256/256
image create photo bufscreen -width $width -height $height -palette 256/256/256
$can create image 0 0 -anchor nw -image gamescreen
bind $can <Key> {
	GetKey %A %K
}
bind $can <Motion> {
	MouseMotion %x %y
}
bind $can <ButtonPress> {
	MouseDown %x %y %b
}
`)
	ui.ir.RegisterCommand("GetKey", func(c, keysym string) {
		var s string
		if c != "" {
			s = c
		} else {
			s = keysym
		}
		ch <- uiInput{key: s}
	})
	ui.ir.RegisterCommand("MouseDown", func(x, y, b int) {
		ch <- uiInput{mouse: true, mouseX: (x - 1) / ui.width, mouseY: (y - 1) / ui.height, button: b - 1}
	})
	ui.ir.RegisterCommand("MouseMotion", func(x, y int) {
		nx := (x - 1) / ui.width
		ny := (y - 1) / ui.height
		if nx != ui.mousepos.X || ny != ui.mousepos.Y {
			ui.mousepos.X = nx
			ui.mousepos.Y = ny
			ch <- uiInput{mouse: true, mouseX: nx, mouseY: ny, button: -1}
		}
	})
	ui.menuHover = -1
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	ui.InitElements()
	return nil
}

func (ui *termui) InitElements() error {
	ui.width = 16
	ui.height = 24
	ui.cache = make(map[UICell]string)
	return nil
}

var ch chan uiInput
var interrupt chan bool

func init() {
	ch = make(chan uiInput, 100)
	interrupt = make(chan bool)
}

type uiInput struct {
	key       string
	mouse     bool
	mouseX    int
	mouseY    int
	button    int
	interrupt bool
}

func (ui *termui) Close() {
}

func (ui *termui) PostInit() {
	SolarizedPalette()
	ui.HideCursor()
	settingsActions = append(settingsActions, toggleTiles)
}

func (ui *termui) Flush() {
	for i := 0; i < len(ui.cells); i++ {
		if ui.cells[i] == ui.backBuffer[i] {
			continue
		}
		cell := ui.cells[i]
		x, y := ui.GetPos(i)
		ui.Draw(cell, x, y)
		ui.backBuffer[i] = cell
	}
	ui.ir.Eval("gamescreen copy bufscreen") // TODO: optimize this
}

func (ui *termui) ApplyToggleLayout() {
	gameConfig.Small = !gameConfig.Small
	if gameConfig.Small {
		ui.ResetCells()
		ui.Flush()
		UIHeight = 24
		UIWidth = 80
	} else {
		UIHeight = 26
		UIWidth = 100
	}
	ui.cache = make(map[UICell]string)
	ui.cells = make([]UICell, UIWidth*UIHeight)
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
}

func getImage(cell UICell) string {
	var pngImg []byte
	if cell.inMap && gameConfig.Tiles {
		pngImg = TileImgs["map-notile"]
		if im, ok := TileImgs["map-"+string(cell.r)]; ok {
			pngImg = im
		} else if im, ok := TileImgs["map-"+MapNames[cell.r]]; ok {
			pngImg = im
		}
	} else {
		pngImg = TileImgs["map-notile"]
		if im, ok := TileImgs["letter-"+string(cell.r)]; ok {
			pngImg = im
		} else if im, ok := TileImgs["letter-"+LetterNames[cell.r]]; ok {
			pngImg = im
		}
	}
	buf := make([]byte, len(pngImg))
	base64.StdEncoding.Decode(buf, pngImg) // TODO: check error
	br := bytes.NewReader(buf)
	img, err := png.Decode(br)
	if err != nil {
		log.Printf("Could not decode png: %v", err)
		return ""
	}
	rect := img.Bounds()
	rgbaimg := image.NewRGBA(rect)
	draw.Draw(rgbaimg, rect, img, rect.Min, draw.Src)
	bgc := cell.bg.Color()
	fgc := cell.fg.Color()
	for y := 0; y < rect.Max.Y; y++ {
		for x := 0; x < rect.Max.X; x++ {
			c := rgbaimg.At(x, y)
			r, _, _, _ := c.RGBA()
			if r == 0 {
				rgbaimg.Set(x, y, bgc)
			} else {
				rgbaimg.Set(x, y, fgc)
			}
		}
	}
	pngbuf := &bytes.Buffer{}
	png.Encode(pngbuf, rgbaimg)
	return base64.StdEncoding.EncodeToString(pngbuf.Bytes())
}

func (ui *termui) Draw(cell UICell, x, y int) {
	var img string
	if im, ok := ui.cache[cell]; ok {
		img = im
	} else {
		img = getImage(cell)
		ui.cache[cell] = img
	}
	ui.ir.Eval("::bufscreen put %{%s} -format png -to %{%d} %{%d}", img, x*ui.width, ui.height*y)
}
<commit_msg>little optimisation<commit_after>// +build tk

package main

import (
	"bytes"
	"encoding/base64"
	"flag"
	"fmt"
	"image"
	"image/draw"
	"image/png"
	"log"
	"os"

	"github.com/nsf/gothic"
)

func main() {
	opt := flag.Bool("s", false, "Use true 16-color solarized palette")
	optVersion := flag.Bool("v", false, "print version number")
	optCenteredCamera := flag.Bool("c", false, "centered camera")
	optMinimalUI := flag.Bool("m", false, "80x24 minimal UI")
	optNoAnim := flag.Bool("n", false, "no animations")
	flag.Parse()
	if *opt {
		SolarizedPalette()
	}
	if *optVersion {
		fmt.Println(Version)
		os.Exit(0)
	}
	if *optCenteredCamera {
		CenteredCamera = true
	}
	if *optNoAnim {
		DisableAnimations = true
	}

	tui := &termui{}
	if *optMinimalUI {
		gameConfig.Small = true
		UIHeight = 24
		UIWidth = 80
	}
	err := tui.Init()
	if err != nil {
		fmt.Fprintf(os.Stderr, "boohu: %v\n", err)
		os.Exit(1)
	}
	defer tui.Close()

	ApplyDefaultKeyBindings()
	tui.PostInit()
	gameConfig.Tiles = true
	LinkColors()
	gameConfig.DarkLOS = true

	g := &game{}
	load, err := g.LoadConfig()
	if load && err != nil {
		g.Print("Error loading config file.")
	} else if load {
		CustomKeys = true
	}

	tui.DrawWelcome()
	load, err = g.Load()
	if !load {
		g.InitLevel()
	} else if err != nil {
		g.InitLevel()
		g.Print("Error loading saved game… starting new game.")
	}
	g.ui = tui
	g.EventLoop()
}

type termui struct {
	ir         *gothic.Interpreter
	cells      []UICell
	backBuffer []UICell
	cursor     position
	stty       string
	cache      map[UICell]string
	width      int
	height     int
	mousepos   position
	menuHover  menu
	itemHover  int
}

func (ui *termui) Init() error {
	ui.cells = make([]UICell, UIWidth*UIHeight)
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	ui.ir = gothic.NewInterpreter(`
set width [expr {16 * 100}]
set height [expr {24 * 26}]
set can [canvas .c -width $width -height $height -background black]
grid $can -row 0 -column 0
focus $can
image create photo gamescreen -width $width -height $height -palette 256/256/256
image create photo bufscreen -width $width -height $height -palette 256/256/256
$can create image 0 0 -anchor nw -image gamescreen
bind $can <Key> {
	GetKey %A %K
}
bind $can <Motion> {
	MouseMotion %x %y
}
bind $can <ButtonPress> {
	MouseDown %x %y %b
}
`)
	ui.ir.RegisterCommand("GetKey", func(c, keysym string) {
		var s string
		if c != "" {
			s = c
		} else {
			s = keysym
		}
		ch <- uiInput{key: s}
	})
	ui.ir.RegisterCommand("MouseDown", func(x, y, b int) {
		ch <- uiInput{mouse: true, mouseX: (x - 1) / ui.width, mouseY: (y - 1) / ui.height, button: b - 1}
	})
	ui.ir.RegisterCommand("MouseMotion", func(x, y int) {
		nx := (x - 1) / ui.width
		ny := (y - 1) / ui.height
		if nx != ui.mousepos.X || ny != ui.mousepos.Y {
			ui.mousepos.X = nx
			ui.mousepos.Y = ny
			ch <- uiInput{mouse: true, mouseX: nx, mouseY: ny, button: -1}
		}
	})
	ui.menuHover = -1
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
	ui.InitElements()
	return nil
}

func (ui *termui) InitElements() error {
	ui.width = 16
	ui.height = 24
	ui.cache = make(map[UICell]string)
	return nil
}

var ch chan uiInput
var interrupt chan bool

func init() {
	ch = make(chan uiInput, 100)
	interrupt = make(chan bool)
}

type uiInput struct {
	key       string
	mouse     bool
	mouseX    int
	mouseY    int
	button    int
	interrupt bool
}

func (ui *termui) Close() {
}

func (ui *termui) PostInit() {
	SolarizedPalette()
	ui.HideCursor()
	settingsActions = append(settingsActions, toggleTiles)
}

func (ui *termui) Flush() {
	xmin := UIWidth - 1
	xmax := 0
	ymin := UIHeight - 1
	ymax := 0
	for i := 0; i < len(ui.cells); i++ {
		if ui.cells[i] == ui.backBuffer[i] {
			continue
		}
		cell := ui.cells[i]
		x, y := ui.GetPos(i)
		ui.Draw(cell, x, y)
		ui.backBuffer[i] = cell
		if x < xmin {
			xmin = x
		}
		if x > xmax {
			xmax = x
		}
		if y < ymin {
			ymin = y
		}
		if y > ymax {
			ymax = y
		}
	}
	ui.ir.Eval("gamescreen copy bufscreen -from %{0%d} %{1%d} %{2%d} %{3%d} -to %{0%d} %{1%d} %{2%d} %{3%d}",
		xmin*16, ymin*24, (xmax+1)*16, (ymax+1)*24) // TODO: optimize this more
}

func (ui *termui) ApplyToggleLayout() {
	gameConfig.Small = !gameConfig.Small
	if gameConfig.Small {
		ui.ResetCells()
		ui.Flush()
		UIHeight = 24
		UIWidth = 80
	} else {
		UIHeight = 26
		UIWidth = 100
	}
	ui.cache = make(map[UICell]string)
	ui.cells = make([]UICell, UIWidth*UIHeight)
	ui.ResetCells()
	ui.backBuffer = make([]UICell, UIWidth*UIHeight)
}

func getImage(cell UICell) string {
	var pngImg []byte
	if cell.inMap && gameConfig.Tiles {
		pngImg = TileImgs["map-notile"]
		if im, ok := TileImgs["map-"+string(cell.r)]; ok {
			pngImg = im
		} else if im, ok := TileImgs["map-"+MapNames[cell.r]]; ok {
			pngImg = im
		}
	} else {
		pngImg = TileImgs["map-notile"]
		if im, ok := TileImgs["letter-"+string(cell.r)]; ok {
			pngImg = im
		} else if im, ok := TileImgs["letter-"+LetterNames[cell.r]]; ok {
			pngImg = im
		}
	}
	buf := make([]byte, len(pngImg))
	base64.StdEncoding.Decode(buf, pngImg) // TODO: check error
	br := bytes.NewReader(buf)
	img, err := png.Decode(br)
	if err != nil {
		log.Printf("Could not decode png: %v", err)
		return ""
	}
	rect := img.Bounds()
	rgbaimg := image.NewRGBA(rect)
	draw.Draw(rgbaimg, rect, img, rect.Min, draw.Src)
	bgc := cell.bg.Color()
	fgc := cell.fg.Color()
	for y := 0; y < rect.Max.Y; y++ {
		for x := 0; x < rect.Max.X; x++ {
			c := rgbaimg.At(x, y)
			r, _, _, _ := c.RGBA()
			if r == 0 {
				rgbaimg.Set(x, y, bgc)
			} else {
				rgbaimg.Set(x, y, fgc)
			}
		}
	}
	pngbuf := &bytes.Buffer{}
	png.Encode(pngbuf, rgbaimg)
	return base64.StdEncoding.EncodeToString(pngbuf.Bytes())
}

func (ui *termui) Draw(cell UICell, x, y int) {
	var img string
	if im, ok := ui.cache[cell]; ok {
		img = im
	} else {
		img = getImage(cell)
		ui.cache[cell] = img
	}
	ui.ir.Eval("::bufscreen put %{%s} -format png -to %{%d} %{%d}", img, x*ui.width, ui.height*y)
}
<|endoftext|>


<commit_before>package boltplus

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"os"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/golang/snappy"
	"github.com/nytlabs/gojee"
)

// Transaction represents represents a bold db transaction and exposes the query and update routines
// usage:
// ```
// tx, err := db.NewTransaction()
// defer tx.Close()
// tx.Get(...)
// ...
// tx.Put(...)
// tx.Commit()
// ```
type Transaction struct {
	tx         *bolt.Tx
	isFinished bool
}

// Commit commits and closes the transaction
func (tx *Transaction) Commit() error {
	tx.isFinished = true
	return tx.tx.Commit()
}

// Rollback discards all changes and closes the transaction
func (tx *Transaction) Rollback() error {
	tx.isFinished = true
	return tx.tx.Rollback()
}

// Close closes the transaction. If neither Commit nor Rollback were called before,
// it rollbacks the transaction
func (tx *Transaction) Close() error {
	if !tx.isFinished {
		return tx.Rollback()
	}
	return nil
}

// Put inserts a doc into a bucket
func (tx *Transaction) Put(bucketPath, key string, val map[string]interface{}) error {
	bucket, err := tx.getBucketOrCreate(bucketPath)
	if err != nil {
		log.Print("bucket err:", err)
		return err
	}
	bs, err := tx.dataToBytes(val)
	if err != nil {
		return err
	}
	return bucket.Put([]byte(key), bs)
}

// Get retrieves a doc from a bucket
func (tx *Transaction) Get(bucketPath, key string) (map[string]interface{}, error) {
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}
	data := bucket.Get([]byte(key))
	return tx.bytesToData(data)
}

// Delete deletes a doc from a bucket
func (tx *Transaction) Delete(bucketPath, key string) error {
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return err
	}
	return bucket.Delete([]byte(key))
}

// GetAll returns all docs in a bucket
func (tx *Transaction) GetAll(bucketPath string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		e := bucket.ForEach(func(k, v []byte) error {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					return e
				}
				returnChannel <- &Pair{string(k), value}
			}
			return nil
		})
		if e != nil {
			log.Print(e)
		}
	}()

	return returnChannel, nil
}

// GetPrefix returns all docs in a bucket matching a prefix
func (tx *Transaction) GetPrefix(bucketPath, prefix string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)

	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		c := bucket.Cursor()
		for k, v := c.Seek([]byte(prefix)); bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					log.Print(e)
					continue
				}
				returnChannel <- &Pair{string(k), value}
			}
		}
	}()

	return returnChannel, nil
}

// GetRange returns all docs in a bucket matching a prefix
func (tx *Transaction) GetRange(bucketPath, start, end string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)

	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		c := bucket.Cursor()
		for k, v := c.Seek([]byte(start)); k != nil && bytes.Compare(k, []byte(end)) <= 0; k, v = c.Next() {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					log.Print(e)
					continue
				}
				returnChannel <- &Pair{string(k), value}
			}
		}
	}()

	return returnChannel, nil
}

// Find searches a bucket for documents
func (tx *Transaction) Find(bucketPath, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetAll(bucketPath)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// FindPrefix searches a bucket for documents
func (tx *Transaction) FindPrefix(bucketPath, prefix, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetPrefix(bucketPath, prefix)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// FindRange searches a bucket for documents
func (tx *Transaction) FindRange(bucketPath, start, end, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetRange(bucketPath, start, end)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// Backup performs a hot backup of the whole database
func (tx *Transaction) Backup(file string) error {
	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = tx.tx.WriteTo(f)
	return err
}

// Buckets returns a list of all buckets and subbuckets
func (tx *Transaction) Buckets() ([]string, error) {
	var res []string
	err := tx.tx.ForEach(func(k []byte, bucket *bolt.Bucket) error {
		res = append(res, searchSubbuckets(bucket, "")...)
		return nil
	})
	return res, err
}

func searchSubbuckets(bucket *bolt.Bucket, prefix string) []string {
	var res []string
	bucket.ForEach(func(k, v []byte) error {
		if v == nil {
			key := string(k)
			if prefix != "" {
				key = prefix + "." + key
			}
			res = append(res, key)
			res = append(res, searchSubbuckets(bucket.Bucket(k), key)...)
		}
		return nil
	})
	return res
}

func (tx *Transaction) getBucketOrCreate(bucketPath string) (*bolt.Bucket, error) {
	buckets := strings.Split(bucketPath, ".")
	bucket, err := tx.tx.CreateBucketIfNotExists([]byte(buckets[0]))
	if err != nil {
		return nil, err
	}
	if len(bucketPath) > 1 {
		for _, id := range buckets {
			bucket, err = bucket.CreateBucketIfNotExists([]byte(id))
			if err != nil {
				return nil, err
			}
		}
	}
	return bucket, nil
}

func (tx *Transaction) getBucket(bucketPath string) (*bolt.Bucket, error) {
	buckets := strings.Split(bucketPath, ".")
	bucket := tx.tx.Bucket([]byte(buckets[0]))
	if bucket == nil {
		return nil, errors.New("no such bucket")
	}
	if len(bucketPath) > 1 {
		for _, id := range buckets {
			bucket = bucket.Bucket([]byte(id))
			if bucket == nil {
				return nil, errors.New("no such bucket")
			}
		}
	}
	return bucket, nil
}

func (tx *Transaction) dataToBytes(data map[string]interface{}) ([]byte, error) {
	var buff bytes.Buffer
	encoder := json.NewEncoder(snappy.NewWriter(&buff))
	if err := encoder.Encode(data); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

func (tx *Transaction) bytesToData(data []byte) (map[string]interface{}, error) {
	buff := bytes.NewBuffer(data)
	decoder := json.NewDecoder(snappy.NewReader(buff))
	value := make(map[string]interface{})
	err := decoder.Decode(&value)
	if err != nil {
		return nil, err
	}
	return value, nil
}

func (tx *Transaction) findFromStream(stream chan *Pair, filterExpression string) (chan *Pair, error) {
	res := make(chan *Pair, 64)
	tokens, err := jee.Lexer(filterExpression)
	if err != nil {
		return nil, err
	}
	tree, err := jee.Parser(tokens)
	if err != nil {
		return nil, err
	}
	go func() {
		defer close(res)
		for pair := range stream {
			if val, err := jee.Eval(tree, pair.Value); err == nil {
				if match, ok := val.(bool); ok && match {
					res <- pair
				}
			} else {
				log.Print(err)
			}
		}
	}()
	return res, nil
}
<commit_msg>added better errorhandling for missing range and prefix props;<commit_after>package boltplus

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"os"
	"strings"

	"github.com/boltdb/bolt"
	"github.com/golang/snappy"
	"github.com/nytlabs/gojee"
)

// Transaction represents represents a bold db transaction and exposes the query and update routines
// usage:
// ```
// tx, err := db.NewTransaction()
// defer tx.Close()
// tx.Get(...)
// ...
// tx.Put(...)
// tx.Commit()
// ```
type Transaction struct {
	tx         *bolt.Tx
	isFinished bool
}

// Commit commits and closes the transaction
func (tx *Transaction) Commit() error {
	tx.isFinished = true
	return tx.tx.Commit()
}

// Rollback discards all changes and closes the transaction
func (tx *Transaction) Rollback() error {
	tx.isFinished = true
	return tx.tx.Rollback()
}

// Close closes the transaction. If neither Commit nor Rollback were called before,
// it rollbacks the transaction
func (tx *Transaction) Close() error {
	if !tx.isFinished {
		return tx.Rollback()
	}
	return nil
}

// Put inserts a doc into a bucket
func (tx *Transaction) Put(bucketPath, key string, val map[string]interface{}) error {
	bucket, err := tx.getBucketOrCreate(bucketPath)
	if err != nil {
		log.Print("bucket err:", err)
		return err
	}
	bs, err := tx.dataToBytes(val)
	if err != nil {
		return err
	}
	return bucket.Put([]byte(key), bs)
}

// Get retrieves a doc from a bucket
func (tx *Transaction) Get(bucketPath, key string) (map[string]interface{}, error) {
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}
	data := bucket.Get([]byte(key))
	return tx.bytesToData(data)
}

// Delete deletes a doc from a bucket
func (tx *Transaction) Delete(bucketPath, key string) error {
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return err
	}
	return bucket.Delete([]byte(key))
}

// GetAll returns all docs in a bucket
func (tx *Transaction) GetAll(bucketPath string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)
	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		e := bucket.ForEach(func(k, v []byte) error {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					return e
				}
				returnChannel <- &Pair{string(k), value}
			}
			return nil
		})
		if e != nil {
			log.Print(e)
		}
	}()

	return returnChannel, nil
}

// GetPrefix returns all docs in a bucket matching a prefix
func (tx *Transaction) GetPrefix(bucketPath, prefix string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)

	if prefix == "" {
		return nil, errors.New("empty prefix")
	}

	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		c := bucket.Cursor()
		for k, v := c.Seek([]byte(prefix)); bytes.HasPrefix(k, []byte(prefix)); k, v = c.Next() {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					log.Print(e)
					continue
				}
				returnChannel <- &Pair{string(k), value}
			}
		}
	}()

	return returnChannel, nil
}

// GetRange returns all docs in a bucket matching a prefix
func (tx *Transaction) GetRange(bucketPath, start, end string) (chan *Pair, error) {
	returnChannel := make(chan *Pair, 64)

	bucket, err := tx.getBucket(bucketPath)
	if err != nil {
		return nil, err
	}

	if start == "" || end == "" {
		return nil, errors.New("empty start/end")
	}

	go func() {
		defer close(returnChannel)
		defer tx.Close()
		c := bucket.Cursor()
		for k, v := c.Seek([]byte(start)); k != nil && bytes.Compare(k, []byte(end)) <= 0; k, v = c.Next() {
			if v != nil {
				value, e := tx.bytesToData(v)
				if e != nil {
					log.Print(e)
					continue
				}
				returnChannel <- &Pair{string(k), value}
			}
		}
	}()

	return returnChannel, nil
}

// Find searches a bucket for documents
func (tx *Transaction) Find(bucketPath, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetAll(bucketPath)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// FindPrefix searches a bucket for documents
func (tx *Transaction) FindPrefix(bucketPath, prefix, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetPrefix(bucketPath, prefix)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// FindRange searches a bucket for documents
func (tx *Transaction) FindRange(bucketPath, start, end, filterExpression string) (chan *Pair, error) {
	stream, err := tx.GetRange(bucketPath, start, end)
	if err != nil {
		return nil, err
	}
	return tx.findFromStream(stream, filterExpression)
}

// Backup performs a hot backup of the whole database
func (tx *Transaction) Backup(file string) error {
	f, err := os.Create(file)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = tx.tx.WriteTo(f)
	return err
}

// Buckets returns a list of all buckets and subbuckets
func (tx *Transaction) Buckets() ([]string, error) {
	var res []string
	err := tx.tx.ForEach(func(k []byte, bucket *bolt.Bucket) error {
		res = append(res, searchSubbuckets(bucket, "")...)
		return nil
	})
	return res, err
}

func searchSubbuckets(bucket *bolt.Bucket, prefix string) []string {
	var res []string
	bucket.ForEach(func(k, v []byte) error {
		if v == nil {
			key := string(k)
			if prefix != "" {
				key = prefix + "." + key
			}
			res = append(res, key)
			res = append(res, searchSubbuckets(bucket.Bucket(k), key)...)
		}
		return nil
	})
	return res
}

func (tx *Transaction) getBucketOrCreate(bucketPath string) (*bolt.Bucket, error) {
	buckets := strings.Split(bucketPath, ".")
	bucket, err := tx.tx.CreateBucketIfNotExists([]byte(buckets[0]))
	if err != nil {
		return nil, err
	}
	if len(bucketPath) > 1 {
		for _, id := range buckets {
			bucket, err = bucket.CreateBucketIfNotExists([]byte(id))
			if err != nil {
				return nil, err
			}
		}
	}
	return bucket, nil
}

func (tx *Transaction) getBucket(bucketPath string) (*bolt.Bucket, error) {
	buckets := strings.Split(bucketPath, ".")
	bucket := tx.tx.Bucket([]byte(buckets[0]))
	if bucket == nil {
		return nil, errors.New("no such bucket")
	}
	if len(bucketPath) > 1 {
		for _, id := range buckets {
			bucket = bucket.Bucket([]byte(id))
			if bucket == nil {
				return nil, errors.New("no such bucket")
			}
		}
	}
	return bucket, nil
}

func (tx *Transaction) dataToBytes(data map[string]interface{}) ([]byte, error) {
	var buff bytes.Buffer
	encoder := json.NewEncoder(snappy.NewWriter(&buff))
	if err := encoder.Encode(data); err != nil {
		return nil, err
	}
	return buff.Bytes(), nil
}

func (tx *Transaction) bytesToData(data []byte) (map[string]interface{}, error) {
	buff := bytes.NewBuffer(data)
	decoder := json.NewDecoder(snappy.NewReader(buff))
	value := make(map[string]interface{})
	err := decoder.Decode(&value)
	if err != nil {
		return nil, err
	}
	return value, nil
}

func (tx *Transaction) findFromStream(stream chan *Pair, filterExpression string) (chan *Pair, error) {
	res := make(chan *Pair, 64)
	tokens, err := jee.Lexer(filterExpression)
	if err != nil {
		return nil, err
	}
	tree, err := jee.Parser(tokens)
	if err != nil {
		return nil, err
	}
	go func() {
		defer close(res)
		for pair := range stream {
			if val, err := jee.Eval(tree, pair.Value); err == nil {
				if match, ok := val.(bool); ok && match {
					res <- pair
				}
			} else {
				log.Print(err)
			}
		}
	}()
	return res, nil
}
<|endoftext|>


<commit_before>package h2spec

import (
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
)

func DataTestGroup() *TestGroup {
	tg := NewTestGroup("6.1", "DATA")

	tg.AddTestCase(NewTestCase(
		"Sends a DATA frame with 0x0 stream identifier",
		"The endpoint MUST respond with a connection error of type PROTOCOL_ERROR.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			http2Conn.fr.WriteData(0, true, []byte("test"))

			actualCodes := []http2.ErrCode{http2.ErrCodeProtocol}
			return TestConnectionError(ctx, http2Conn, actualCodes)
		},
	))

	tg.AddTestCase(NewTestCase(
		"Sends a DATA frame on the stream that is not opend",
		"The endpoint MUST respond with a stream error of type STREAM_CLOSED.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			hdrs := []hpack.HeaderField{
				pair(":method", "GET"),
				pair(":scheme", "http"),
				pair(":path", "/"),
				pair(":authority", ctx.Authority()),
			}

			var hp http2.HeadersFrameParam
			hp.StreamID = 1
			hp.EndStream = true
			hp.EndHeaders = true
			hp.BlockFragment = http2Conn.EncodeHeader(hdrs)
			http2Conn.fr.WriteHeaders(hp)
			http2Conn.fr.WriteData(1, true, []byte("test"))

			actualCodes := []http2.ErrCode{http2.ErrCodeStreamClosed}
			return TestStreamError(ctx, http2Conn, actualCodes)
		},
	))

	/*
		tg.AddTestCase(NewTestCase(
			"Sends a DATA frame with invalid pad length",
			"The endpoint MUST treat this as a connection error of type PROTOCOL_ERROR."
			func(ctx *Context) (expected []Result, actual Result) {},
		))
	*/

	return tg
}
<commit_msg>Add test case for DATA frame with invalid pad length<commit_after>package h2spec

import (
	"fmt"
	"github.com/bradfitz/http2"
	"github.com/bradfitz/http2/hpack"
)

func DataTestGroup() *TestGroup {
	tg := NewTestGroup("6.1", "DATA")

	tg.AddTestCase(NewTestCase(
		"Sends a DATA frame with 0x0 stream identifier",
		"The endpoint MUST respond with a connection error of type PROTOCOL_ERROR.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			http2Conn.fr.WriteData(0, true, []byte("test"))

			actualCodes := []http2.ErrCode{http2.ErrCodeProtocol}
			return TestConnectionError(ctx, http2Conn, actualCodes)
		},
	))

	tg.AddTestCase(NewTestCase(
		"Sends a DATA frame on the stream that is not opend",
		"The endpoint MUST respond with a stream error of type STREAM_CLOSED.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			hdrs := []hpack.HeaderField{
				pair(":method", "GET"),
				pair(":scheme", "http"),
				pair(":path", "/"),
				pair(":authority", ctx.Authority()),
			}

			var hp http2.HeadersFrameParam
			hp.StreamID = 1
			hp.EndStream = true
			hp.EndHeaders = true
			hp.BlockFragment = http2Conn.EncodeHeader(hdrs)
			http2Conn.fr.WriteHeaders(hp)
			http2Conn.fr.WriteData(1, true, []byte("test"))

			actualCodes := []http2.ErrCode{http2.ErrCodeStreamClosed}
			return TestStreamError(ctx, http2Conn, actualCodes)
		},
	))

	tg.AddTestCase(NewTestCase(
		"Sends a DATA frame with invalid pad length",
		"The endpoint MUST treat this as a connection error of type PROTOCOL_ERROR.",
		func(ctx *Context) (expected []Result, actual Result) {
			http2Conn := CreateHttp2Conn(ctx, true)
			defer http2Conn.conn.Close()

			hdrs := []hpack.HeaderField{
				pair(":method", "GET"),
				pair(":scheme", "http"),
				pair(":path", "/"),
				pair(":authority", ctx.Authority()),
			}

			var hp http2.HeadersFrameParam
			hp.StreamID = 1
			hp.EndStream = true
			hp.EndHeaders = true
			hp.BlockFragment = http2Conn.EncodeHeader(hdrs)
			http2Conn.fr.WriteHeaders(hp)

			fmt.Fprintf(http2Conn.conn, "\x00\x00\x05\x00\x0b\x00\x00\x00\x01")
			fmt.Fprintf(http2Conn.conn, "\x04\x54\x65\x73\x74")

			actualCodes := []http2.ErrCode{http2.ErrCodeStreamClosed}
			return TestStreamError(ctx, http2Conn, actualCodes)
		},
	))

	return tg
}
<|endoftext|>


<commit_before>package fld

import (
	"fmt"
	"net"

	stdetcd "github.com/coreos/go-etcd/etcd"
	"github.com/happypancake/go-integrate/etcd"
	"github.com/op/go-logging"
)

var (
	address   string
	outgoing  = make(chan []byte, 100000)
	conn      net.Conn
	addresses = make(chan string)
	log       = logging.MustGetLogger("fld")
)

type LoggingBackend struct{}

func NewLoggingBackend() *LoggingBackend {
	return &LoggingBackend{}
}

func (*LoggingBackend) Log(level logging.Level, depth int, rec *logging.Record) error {

	switch level {
	case logging.CRITICAL:
		return send("crit", rec.Formatted())
	case logging.ERROR:
		return send("err", rec.Formatted())
	case logging.WARNING:
		return send("warning", rec.Formatted())
	case logging.NOTICE:
		return send("notice", rec.Formatted())
	case logging.INFO:
		return send("info", rec.Formatted())
	case logging.DEBUG:
		return send("debug", rec.Formatted())
	default:
	}
	panic("unhandled log level")
}

func InitWithDynamicConfig(client *stdetcd.Client, hostname string) {
	key := fmt.Sprintf("/%v/logsd/address", hostname)
	go etcd.GetAndWatchStringValue(client, key, addresses, nil)
}

func send(severity string, name string) error {

	length := float64(len(outgoing))
	capacity := float64(cap(outgoing))

	if length < capacity*0.9 {
		payload := severity + "|" + name
		outgoing <- []byte(payload)
	}
	return nil
}

func connect() error {

	c, err := net.Dial("udp", address)
	if err != nil {
		return err
	}

	conn = c
	return nil
}

func hasAddress() bool {
	return address != ""
}

func processOutgoing() {
	for {
		select {
		case outgoing := <-outgoing:
			if !hasAddress() {
				break
			}

			if _, err := conn.Write(outgoing); err != nil {
				connect()
			}
		case address = <-addresses:
			connect()
		}
	}
}
<commit_msg>Fix outgoing work is not processed<commit_after>package fld

import (
	"fmt"
	"net"

	stdetcd "github.com/coreos/go-etcd/etcd"
	"github.com/happypancake/go-integrate/etcd"
	"github.com/op/go-logging"
)

var (
	address   string
	outgoing  = make(chan []byte, 100000)
	conn      net.Conn
	addresses = make(chan string)
	log       = logging.MustGetLogger("fld")
)

type LoggingBackend struct{}

func NewLoggingBackend() *LoggingBackend {
	return &LoggingBackend{}
}

func (*LoggingBackend) Log(level logging.Level, depth int, rec *logging.Record) error {

	switch level {
	case logging.CRITICAL:
		return send("crit", rec.Formatted())
	case logging.ERROR:
		return send("err", rec.Formatted())
	case logging.WARNING:
		return send("warning", rec.Formatted())
	case logging.NOTICE:
		return send("notice", rec.Formatted())
	case logging.INFO:
		return send("info", rec.Formatted())
	case logging.DEBUG:
		return send("debug", rec.Formatted())
	default:
	}
	panic("unhandled log level")
}

func InitWithDynamicConfig(client *stdetcd.Client, hostname string) {
	key := fmt.Sprintf("/%v/logsd/address", hostname)

	log.Info("Configuring fld by watching key: %v", key)
	go etcd.GetAndWatchStringValue(client, key, addresses, nil)
	go processOutgoing()
}

func send(severity string, name string) error {

	length := float64(len(outgoing))
	capacity := float64(cap(outgoing))

	if length < capacity*0.9 {
		payload := severity + "|" + name
		outgoing <- []byte(payload)
	}
	return nil
}

func connect() error {

	c, err := net.Dial("udp", address)
	if err != nil {
		return err
	}

	conn = c
	return nil
}

func hasAddress() bool {
	return address != ""
}

func processOutgoing() {
	for {
		select {
		case outgoing := <-outgoing:
			if !hasAddress() {
				break
			}

			if _, err := conn.Write(outgoing); err != nil {
				connect()
			}
		case address = <-addresses:
			log.Info("logsd address received: %v", address)
			connect()
		}
	}
}
<|endoftext|>


<commit_before>package helix

import (
	"github.com/gocraft/web"
)

func (c *Context) GetHandler(w web.ResponseWriter, req *web.Request) {
	var err error
	ctype := ""
	acceptList, _ := conneg(req.Request)
	if len(acceptList) > 0 && acceptList[0].SubType != "*" {
		ctype, err = acceptList.Negotiate(rdfMimes...)
		if err != nil {
			w.WriteHeader(406)
			w.Write([]byte("HTTP 406 - Accept type not acceptable: " + err.Error()))
			return
		}
		logger.Info().Str("Accept", ctype).Msg("")
	}

	if canSerialize(ctype) {
		c.getRDF(w, req, ctype)
		return
	}
	// w.WriteHeader(400)
}

func (c *Context) getRDF(w web.ResponseWriter, req *web.Request, mime string) {
	URI := absoluteURI(req.Request)
	graph, err := c.getGraph(URI)
	if err != nil {
		w.WriteHeader(404)
		w.Write([]byte(err.Error()))
		return
	}
	// TODO replace with something better for ETag generation
	w.Header().Add("ETag", newETag([]byte(graph.String())))
	graph.Serialize(w, mime)
}
<commit_msg>return 404 by default<commit_after>package helix

import (
	"github.com/gocraft/web"
)

func (c *Context) GetHandler(w web.ResponseWriter, req *web.Request) {
	var err error
	ctype := ""
	acceptList, _ := conneg(req.Request)
	if len(acceptList) > 0 && acceptList[0].SubType != "*" {
		ctype, err = acceptList.Negotiate(rdfMimes...)
		if err != nil {
			w.WriteHeader(406)
			w.Write([]byte("HTTP 406 - Accept type not acceptable: " + err.Error()))
			return
		}
		logger.Info().Str("Accept", ctype).Msg("")
	}

	if canSerialize(ctype) {
		c.getRDF(w, req, ctype)
		return
	}
	w.WriteHeader(404)
}

func (c *Context) getRDF(w web.ResponseWriter, req *web.Request, mime string) {
	URI := absoluteURI(req.Request)
	graph, err := c.getGraph(URI)
	if err != nil {
		w.WriteHeader(404)
		w.Write([]byte(err.Error()))
		return
	}
	// TODO replace with something better for ETag generation
	w.Header().Add("ETag", newETag([]byte(graph.String())))
	graph.Serialize(w, mime)
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
)

type FactoidTxStatus struct {
	TxID string `json:"txid"`
	GeneralTransactionData
}

type EntryStatus struct {
	CommitTxID string `json:"committxid"`
	EntryHash  string `json:"entryhash"`

	CommitData GeneralTransactionData `json:"commitdata"`
	EntryData  GeneralTransactionData `json:"entrydata"`

	ReserveTransactions          []ReserveInfo `json:"reserveinfo,omitempty"`
	ConflictingRevealEntryHashes []string      `json:"conflictingrevealentryhashes,omitempty"`
}

type ReserveInfo struct {
	TxID    string `json:"txid"`
	Timeout int64  `json:"timeout"` //Unix time
}

type GeneralTransactionData struct {
	// TransactionDate in Unix time
	TransactionDate int64 `json:"transactiondate,omitempty"`
	//TransactionDateString ISO8601 time
	TransactionDateString string `json:"transactiondatestring,omitempty"`
	//Unix time
	BlockDate int64 `json:"blockdate,omitempty"`
	//ISO8601 time
	BlockDateString string `json:"blockdatestring,omitempty"`

	Malleated *Malleated `json:"malleated,omitempty"`
	Status    string     `json:"status"`
}

type Malleated struct {
	MalleatedTxIDs []string `json:"malleatedtxids"`
}

func FactoidACK(txID, fullTransaction string) (*FactoidTxStatus, error) {
	params := ackRequest{TxID: txID, FullTransaction: fullTransaction}
	req := NewJSON2Request("factoid-ack", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	eb := new(FactoidTxStatus)
	if err := json.Unmarshal(resp.JSONResult(), eb); err != nil {
		return nil, err
	}

	return eb, nil
}

func EntryACK(txID, fullTransaction string) (*EntryStatus, error) {
	params := ackRequest{TxID: txID, FullTransaction: fullTransaction}
	req := NewJSON2Request("entry-ack", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	eb := new(EntryStatus)
	if err := json.Unmarshal(resp.JSONResult(), eb); err != nil {
		return nil, err
	}

	return eb, nil
}
<commit_msg>added string methods for ack status returns<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/json"
	"fmt"
)

type FactoidTxStatus struct {
	TxID string `json:"txid"`
	GeneralTransactionData
}

func (f *FactoidTxStatus) String() string {
	var s string
	s += fmt.Sprintln("TxID:", f.TxID)
	s += fmt.Sprintln("Status:", f.Status)
	s += fmt.Sprintln("Date:", f.TransactionDateString)

	return s
}

type EntryStatus struct {
	CommitTxID string `json:"committxid"`
	EntryHash  string `json:"entryhash"`

	CommitData GeneralTransactionData `json:"commitdata"`
	EntryData  GeneralTransactionData `json:"entrydata"`

	ReserveTransactions          []ReserveInfo `json:"reserveinfo,omitempty"`
	ConflictingRevealEntryHashes []string      `json:"conflictingrevealentryhashes,omitempty"`
}

func (e *EntryStatus) String() string {
	var s string
	s += fmt.Sprintln("TxID:", e.CommitTxID)
	s += fmt.Sprintln("Status:", e.CommitData.Status)
	s += fmt.Sprintln("Date:", e.CommitData.TransactionDateString)

	return s
}

type ReserveInfo struct {
	TxID    string `json:"txid"`
	Timeout int64  `json:"timeout"` //Unix time
}

type GeneralTransactionData struct {
	// TransactionDate in Unix time
	TransactionDate int64 `json:"transactiondate,omitempty"`
	//TransactionDateString ISO8601 time
	TransactionDateString string `json:"transactiondatestring,omitempty"`
	//Unix time
	BlockDate int64 `json:"blockdate,omitempty"`
	//ISO8601 time
	BlockDateString string `json:"blockdatestring,omitempty"`

	Malleated *Malleated `json:"malleated,omitempty"`
	Status    string     `json:"status"`
}

type Malleated struct {
	MalleatedTxIDs []string `json:"malleatedtxids"`
}

func FactoidACK(txID, fullTransaction string) (*FactoidTxStatus, error) {
	params := ackRequest{TxID: txID, FullTransaction: fullTransaction}
	req := NewJSON2Request("factoid-ack", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	eb := new(FactoidTxStatus)
	if err := json.Unmarshal(resp.JSONResult(), eb); err != nil {
		return nil, err
	}

	return eb, nil
}

func EntryACK(txID, fullTransaction string) (*EntryStatus, error) {
	params := ackRequest{TxID: txID, FullTransaction: fullTransaction}
	req := NewJSON2Request("entry-ack", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.Error != nil {
		return nil, resp.Error
	}

	eb := new(EntryStatus)
	if err := json.Unmarshal(resp.JSONResult(), eb); err != nil {
		return nil, err
	}

	return eb, nil
}
<|endoftext|>


<commit_before>package agi

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// State describes the Asterisk channel state.  There are mapped
// directly to the Asterisk enumerations.
type State int

const (
	// StateDown indicates the channel is down and available
	StateDown State = iota

	// StateReserved indicates the channel is down but reserved
	StateReserved

	// StateOffhook indicates that the channel is offhook
	StateOffhook

	// StateDialing indicates that digits have been dialed
	StateDialing

	// StateRing indicates the channel is ringing
	StateRing

	// StateRinging indicates the channel's remote end is rining (the channel is receiving ringback)
	StateRinging

	// StateUp indicates the channel is up
	StateUp

	// StateBusy indicates the line is busy
	StateBusy

	// StateDialingOffHook indicates digits have been dialed while offhook
	StateDialingOffHook

	// StatePreRing indicates the channel has detected an incoming call and is waiting for ring
	StatePreRing
)

// AGI represents an AGI session
type AGI struct {
	// Variables stored the initial variables
	// transmitted from Asterisk at the start
	// of the AGI session.
	Variables map[string]string

	r    io.Reader
	eagi io.Reader
	w    io.Writer

	conn net.Conn

	mu sync.Mutex
}

// Response represents a response to an AGI
// request.
type Response struct {
	Error        error  // Error received, if any
	Status       int    // HTTP-style status code received
	Result       int    // Result is the numerical return (if parseable)
	ResultString string // Result value as a string
	Value        string // Value is the (optional) string value returned
}

// Err returns the error value from the response
func (r *Response) Err() error {
	return r.Error
}

// Val returns the response value and error
func (r *Response) Val() (string, error) {
	return r.Value, r.Error
}

// Regex for AGI response result code and value
var responseRegex = regexp.MustCompile(`^([\d]{3})\sresult=(\-?[[:alnum:]]*)(\s.*)?$`)

const (
	// StatusOK indicates the AGI command was
	// accepted.
	StatusOK = 200

	// StatusInvalid indicates Asterisk did not
	// understand the command.
	StatusInvalid = 510

	// StatusDeadChannel indicates that the command
	// cannot be performed on a dead (hungup) channel.
	StatusDeadChannel = 511

	// StatusEndUsage indicates...TODO
	StatusEndUsage = 520
)

// HandlerFunc is a function which accepts an AGI instance
type HandlerFunc func(*AGI)

// New creates an AGI session from the given reader and writer.
func New(r io.Reader, w io.Writer) *AGI {
	return NewWithEAGI(r, w, nil)
}

// NewWithEAGI returns a new AGI session to the given `os.Stdin` `io.Reader`,
// EAGI `io.Reader`, and `os.Stdout` `io.Writer`. The initial variables will
// be read in.
func NewWithEAGI(r io.Reader, w io.Writer, eagi io.Reader) *AGI {
	a := AGI{
		Variables: make(map[string]string),
		r:         r,
		w:         w,
		eagi:      eagi,
	}

	s := bufio.NewScanner(a.r)
	for s.Scan() {
		if s.Text() == "" {
			break
		}

		terms := strings.Split(s.Text(), ":")
		if len(terms) == 2 {
			a.Variables[strings.TrimSpace(terms[0])] = strings.TrimSpace(terms[1])
		}
	}

	return &a
}

// NewConn returns a new AGI session bound to the given net.Conn interface
func NewConn(conn net.Conn) *AGI {
	a := New(conn, conn)
	a.conn = conn
	return a
}

// NewStdio returns a new AGI session to stdin and stdout.
func NewStdio() *AGI {
	return New(os.Stdin, os.Stdout)
}

// NewEAGI returns a new AGI session to stdin, the EAGI stream (FD=3), and stdout.
func NewEAGI() *AGI {
	return NewWithEAGI(os.Stdin, os.Stdout, os.NewFile(uintptr(3), "/dev/stdeagi"))
}

// Listen binds an AGI HandlerFunc to the given TCP `host:port` address, creating a FastAGI service.
func Listen(addr string, handler HandlerFunc) error {
	if addr == "" {
		addr = "localhost:4573"
	}

	l, err := net.Listen("tcp", addr)
	if err != nil {
		return errors.Wrap(err, "failed to bind server")
	}
	defer l.Close() // nolint: errcheck

	for {
		conn, err := l.Accept()
		if err != nil {
			return errors.Wrap(err, "failed to accept TCP connection")
		}

		go handler(NewConn(conn))
	}
}

// Close closes any network connection associated with the AGI instance
func (a *AGI) Close() (err error) {
	if a.conn != nil {
		err = a.conn.Close()
		a.conn = nil
	}
	return
}

// EAGI enables access to the EAGI incoming stream (if available).
func (a *AGI) EAGI() io.Reader {
	return a.eagi
}

// Command sends the given command line to stdout
// and returns the response.
// TODO: this does not handle multi-line responses properly
func (a *AGI) Command(cmd ...string) (resp *Response) {
	resp = &Response{}

	a.mu.Lock()
	defer a.mu.Unlock()

	cmdString := strings.Join(cmd, " ") + "\n"
	_, err := a.w.Write([]byte(cmdString))
	if err != nil {
		resp.Error = errors.Wrap(err, "failed to send command")
		return
	}

	s := bufio.NewScanner(a.r)
	for s.Scan() {
		raw := s.Text()
		if raw == "" {
			break
		}

		// Parse and store the result code
		pieces := responseRegex.FindStringSubmatch(raw)
		if pieces == nil {
			resp.Error = fmt.Errorf("failed to parse result: %s", raw)
			return
		}

		// Status code is the first substring
		resp.Status, err = strconv.Atoi(pieces[1])
		if err != nil {
			resp.Error = errors.Wrap(err, "failed to get status code")
			return
		}

		// Result code is the second substring
		resp.ResultString = pieces[2]
		resp.Result, err = strconv.Atoi(pieces[2])
		if err != nil {
			resp.Error = errors.Wrap(err, "failed to parse status code as an integer")
			return
		}

		// Value is the third (and optional) substring
		wrappedVal := strings.TrimSpace(pieces[3])
		resp.Value = strings.TrimSuffix(strings.TrimPrefix(wrappedVal, "("), ")")

		// FIXME: handle multiple line return values
		break // nolint
	}

	// If the Status code is not 200, return an error
	if resp.Status != 200 {
		resp.Error = fmt.Errorf("Non-200 status code")
	}
	return
}

// Answer answers the channel
func (a *AGI) Answer() error {
	return a.Command("ANSWER").Err()
}

// Status returns the channel status
func (a *AGI) Status() (State, error) {
	r, err := a.Command("CHANNEL STATUS").Val()
	if err != nil {
		return StateDown, err
	}
	state, err := strconv.Atoi(r)
	if err != nil {
		return StateDown, fmt.Errorf("Failed to parse state %s", r)
	}
	return State(state), nil
}

// Exec runs a dialplan application
func (a *AGI) Exec(cmd ...string) (string, error) {
	cmd = append([]string{"EXEC"}, cmd...)
	return a.Command(cmd...).Val()
}

// Get gets the value of the given channel variable
func (a *AGI) Get(key string) (string, error) {
	return a.Command("GET VARIABLE", key).Val()
}

// GetData plays a file and receives DTMF, returning the received digits
func (a *AGI) GetData(name string, timeout int, maxdigits int) (digit string, err error) {
	return a.Command("GET DATA", name, strconv.Itoa(timeout), strconv.Itoa(maxdigits)).Val()
}

// Hangup terminates the call
func (a *AGI) Hangup() error {
	return a.Command("HANGUP").Err()
}

// RecordOptions describes the options available when recording
type RecordOptions struct {
	// Format is the format of the audio file to record; defaults to "wav".
	Format string

	// EscapeDigits is the set of digits on receipt of which will terminate the recording. Default is "#".  This may not be blank.
	EscapeDigits string

	// Timeout is the maximum time to allow for the recording.  Defaults to 5 minutes.
	Timeout time.Duration

	// Silence is the maximum amount of silence to allow before ending the recording.  The finest resolution is to the second.   0=disabled, which is the default.
	Silence time.Duration

	// Beep controls whether a beep is played before starting the recording.  Defaults to false.
	Beep bool

	// Offset is the number of samples in the recording to advance before storing to the file.  This is means of clipping the beginning of a recording.  Defaults to 0.
	Offset int
}

// Record records audio to a file
func (a *AGI) Record(name string, opts *RecordOptions) error {
	if opts == nil {
		opts = &RecordOptions{}
	}
	if opts.Format == "" {
		opts.Format = "wav"
	}
	if opts.EscapeDigits == "" {
		opts.EscapeDigits = "#"
	}
	if opts.Timeout == 0 {
		opts.Timeout = 5 * time.Minute
	}

	cmd := strings.Join([]string{
		"RECORD FILE ",
		name,
		opts.Format,
		opts.EscapeDigits,
		toMSec(opts.Timeout),
	}, " ")

	if opts.Offset > 0 {
		cmd += " " + strconv.Itoa(opts.Offset)
	}

	if opts.Beep {
		cmd += " BEEP"
	}

	if opts.Silence > 0 {
		cmd += " s=" + toSec(opts.Silence)
	}

	return a.Command(cmd).Err()
}

// SayAlpha plays a character string, annunciating each character.
func (a *AGI) SayAlpha(label string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY ALPHA", label, escapeDigits).Val()
}

// SayDigits plays a digit string, annunciating each digit.
func (a *AGI) SayDigits(number string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY DIGITS", number, escapeDigits).Val()
}

// SayDate plays a date
func (a *AGI) SayDate(when time.Time, escapeDigits string) (digit string, err error) {
	return a.Command("SAY DATE", toEpoch(when), escapeDigits).Val()
}

// SayDateTime plays a date using the given format.  See `voicemail.conf` for the format syntax; defaults to `ABdY 'digits/at' IMp`.
func (a *AGI) SayDateTime(when time.Time, escapeDigits string, format string) (digit string, err error) {
	// Extract the timezone from the time
	zone, _ := when.Zone()

	// Use the Asterisk default format if we are not given one
	if format == "" {
		format = "ABdY 'digits/at' IMp"
	}

	return a.Command("SAY DATETIME", toEpoch(when), escapeDigits, format, zone).Val()
}

// SayNumber plays the given number.
func (a *AGI) SayNumber(number string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY NUMBER", number, escapeDigits).Val()
}

// SayPhonetic plays the given phrase phonetically
func (a *AGI) SayPhonetic(phrase string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY PHOENTIC", phrase, escapeDigits).Val()
}

// SayTime plays the time part of the given timestamp
func (a *AGI) SayTime(when time.Time, escapeDigits string) (digit string, err error) {
	return a.Command("SAY TIME", toEpoch(when), escapeDigits).Val()
}

// Set sets the given channel variable to
// the provided value.
func (a *AGI) Set(key, val string) error {
	return a.Command("SET VARIABLE", key, val).Err()
}

// StreamFile plays the given file to the channel
func (a *AGI) StreamFile(name string, escapeDigits string, offset int) (digit string, err error) {
	return a.Command("STREAM FILE", name, escapeDigits, strconv.Itoa(offset)).Val()
}

// Verbose logs the given message to the verbose message system
func (a *AGI) Verbose(msg string, level int) error {
	return a.Command("VERBOSE", msg, strconv.Itoa(level)).Err()
}

// WaitForDigit waits for a DTMF digit and returns what is received
func (a *AGI) WaitForDigit(timeout time.Duration) (digit string, err error) {
	return a.Command("WAIT FOR DIGIT", toMSec(timeout)).Val()
}
<commit_msg>Changed how return value of WaitForDigit is built<commit_after>package agi

import (
	"bufio"
	"fmt"
	"io"
	"net"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// State describes the Asterisk channel state.  There are mapped
// directly to the Asterisk enumerations.
type State int

const (
	// StateDown indicates the channel is down and available
	StateDown State = iota

	// StateReserved indicates the channel is down but reserved
	StateReserved

	// StateOffhook indicates that the channel is offhook
	StateOffhook

	// StateDialing indicates that digits have been dialed
	StateDialing

	// StateRing indicates the channel is ringing
	StateRing

	// StateRinging indicates the channel's remote end is rining (the channel is receiving ringback)
	StateRinging

	// StateUp indicates the channel is up
	StateUp

	// StateBusy indicates the line is busy
	StateBusy

	// StateDialingOffHook indicates digits have been dialed while offhook
	StateDialingOffHook

	// StatePreRing indicates the channel has detected an incoming call and is waiting for ring
	StatePreRing
)

// AGI represents an AGI session
type AGI struct {
	// Variables stored the initial variables
	// transmitted from Asterisk at the start
	// of the AGI session.
	Variables map[string]string

	r    io.Reader
	eagi io.Reader
	w    io.Writer

	conn net.Conn

	mu sync.Mutex
}

// Response represents a response to an AGI
// request.
type Response struct {
	Error        error  // Error received, if any
	Status       int    // HTTP-style status code received
	Result       int    // Result is the numerical return (if parseable)
	ResultString string // Result value as a string
	Value        string // Value is the (optional) string value returned
}

// Err returns the error value from the response
func (r *Response) Err() error {
	return r.Error
}

// Val returns the response value and error
func (r *Response) Val() (string, error) {
	return r.Value, r.Error
}

// Regex for AGI response result code and value
var responseRegex = regexp.MustCompile(`^([\d]{3})\sresult=(\-?[[:alnum:]]*)(\s.*)?$`)

const (
	// StatusOK indicates the AGI command was
	// accepted.
	StatusOK = 200

	// StatusInvalid indicates Asterisk did not
	// understand the command.
	StatusInvalid = 510

	// StatusDeadChannel indicates that the command
	// cannot be performed on a dead (hungup) channel.
	StatusDeadChannel = 511

	// StatusEndUsage indicates...TODO
	StatusEndUsage = 520
)

// HandlerFunc is a function which accepts an AGI instance
type HandlerFunc func(*AGI)

// New creates an AGI session from the given reader and writer.
func New(r io.Reader, w io.Writer) *AGI {
	return NewWithEAGI(r, w, nil)
}

// NewWithEAGI returns a new AGI session to the given `os.Stdin` `io.Reader`,
// EAGI `io.Reader`, and `os.Stdout` `io.Writer`. The initial variables will
// be read in.
func NewWithEAGI(r io.Reader, w io.Writer, eagi io.Reader) *AGI {
	a := AGI{
		Variables: make(map[string]string),
		r:         r,
		w:         w,
		eagi:      eagi,
	}

	s := bufio.NewScanner(a.r)
	for s.Scan() {
		if s.Text() == "" {
			break
		}

		terms := strings.Split(s.Text(), ":")
		if len(terms) == 2 {
			a.Variables[strings.TrimSpace(terms[0])] = strings.TrimSpace(terms[1])
		}
	}

	return &a
}

// NewConn returns a new AGI session bound to the given net.Conn interface
func NewConn(conn net.Conn) *AGI {
	a := New(conn, conn)
	a.conn = conn
	return a
}

// NewStdio returns a new AGI session to stdin and stdout.
func NewStdio() *AGI {
	return New(os.Stdin, os.Stdout)
}

// NewEAGI returns a new AGI session to stdin, the EAGI stream (FD=3), and stdout.
func NewEAGI() *AGI {
	return NewWithEAGI(os.Stdin, os.Stdout, os.NewFile(uintptr(3), "/dev/stdeagi"))
}

// Listen binds an AGI HandlerFunc to the given TCP `host:port` address, creating a FastAGI service.
func Listen(addr string, handler HandlerFunc) error {
	if addr == "" {
		addr = "localhost:4573"
	}

	l, err := net.Listen("tcp", addr)
	if err != nil {
		return errors.Wrap(err, "failed to bind server")
	}
	defer l.Close() // nolint: errcheck

	for {
		conn, err := l.Accept()
		if err != nil {
			return errors.Wrap(err, "failed to accept TCP connection")
		}

		go handler(NewConn(conn))
	}
}

// Close closes any network connection associated with the AGI instance
func (a *AGI) Close() (err error) {
	if a.conn != nil {
		err = a.conn.Close()
		a.conn = nil
	}
	return
}

// EAGI enables access to the EAGI incoming stream (if available).
func (a *AGI) EAGI() io.Reader {
	return a.eagi
}

// Command sends the given command line to stdout
// and returns the response.
// TODO: this does not handle multi-line responses properly
func (a *AGI) Command(cmd ...string) (resp *Response) {
	resp = &Response{}

	a.mu.Lock()
	defer a.mu.Unlock()

	cmdString := strings.Join(cmd, " ") + "\n"
	_, err := a.w.Write([]byte(cmdString))
	if err != nil {
		resp.Error = errors.Wrap(err, "failed to send command")
		return
	}

	s := bufio.NewScanner(a.r)
	for s.Scan() {
		raw := s.Text()
		if raw == "" {
			break
		}

		// Parse and store the result code
		pieces := responseRegex.FindStringSubmatch(raw)
		if pieces == nil {
			resp.Error = fmt.Errorf("failed to parse result: %s", raw)
			return
		}

		// Status code is the first substring
		resp.Status, err = strconv.Atoi(pieces[1])
		if err != nil {
			resp.Error = errors.Wrap(err, "failed to get status code")
			return
		}

		// Result code is the second substring
		resp.ResultString = pieces[2]
		resp.Result, err = strconv.Atoi(pieces[2])
		if err != nil {
			resp.Error = errors.Wrap(err, "failed to parse status code as an integer")
			return
		}

		// Value is the third (and optional) substring
		wrappedVal := strings.TrimSpace(pieces[3])
		resp.Value = strings.TrimSuffix(strings.TrimPrefix(wrappedVal, "("), ")")

		// FIXME: handle multiple line return values
		break // nolint
	}

	// If the Status code is not 200, return an error
	if resp.Status != 200 {
		resp.Error = fmt.Errorf("Non-200 status code")
	}
	return
}

// Answer answers the channel
func (a *AGI) Answer() error {
	return a.Command("ANSWER").Err()
}

// Status returns the channel status
func (a *AGI) Status() (State, error) {
	r, err := a.Command("CHANNEL STATUS").Val()
	if err != nil {
		return StateDown, err
	}
	state, err := strconv.Atoi(r)
	if err != nil {
		return StateDown, fmt.Errorf("Failed to parse state %s", r)
	}
	return State(state), nil
}

// Exec runs a dialplan application
func (a *AGI) Exec(cmd ...string) (string, error) {
	cmd = append([]string{"EXEC"}, cmd...)
	return a.Command(cmd...).Val()
}

// Get gets the value of the given channel variable
func (a *AGI) Get(key string) (string, error) {
	return a.Command("GET VARIABLE", key).Val()
}

// GetData plays a file and receives DTMF, returning the received digits
func (a *AGI) GetData(name string, timeout int, maxdigits int) (digit string, err error) {
	return a.Command("GET DATA", name, strconv.Itoa(timeout), strconv.Itoa(maxdigits)).Val()
}

// Hangup terminates the call
func (a *AGI) Hangup() error {
	return a.Command("HANGUP").Err()
}

// RecordOptions describes the options available when recording
type RecordOptions struct {
	// Format is the format of the audio file to record; defaults to "wav".
	Format string

	// EscapeDigits is the set of digits on receipt of which will terminate the recording. Default is "#".  This may not be blank.
	EscapeDigits string

	// Timeout is the maximum time to allow for the recording.  Defaults to 5 minutes.
	Timeout time.Duration

	// Silence is the maximum amount of silence to allow before ending the recording.  The finest resolution is to the second.   0=disabled, which is the default.
	Silence time.Duration

	// Beep controls whether a beep is played before starting the recording.  Defaults to false.
	Beep bool

	// Offset is the number of samples in the recording to advance before storing to the file.  This is means of clipping the beginning of a recording.  Defaults to 0.
	Offset int
}

// Record records audio to a file
func (a *AGI) Record(name string, opts *RecordOptions) error {
	if opts == nil {
		opts = &RecordOptions{}
	}
	if opts.Format == "" {
		opts.Format = "wav"
	}
	if opts.EscapeDigits == "" {
		opts.EscapeDigits = "#"
	}
	if opts.Timeout == 0 {
		opts.Timeout = 5 * time.Minute
	}

	cmd := strings.Join([]string{
		"RECORD FILE ",
		name,
		opts.Format,
		opts.EscapeDigits,
		toMSec(opts.Timeout),
	}, " ")

	if opts.Offset > 0 {
		cmd += " " + strconv.Itoa(opts.Offset)
	}

	if opts.Beep {
		cmd += " BEEP"
	}

	if opts.Silence > 0 {
		cmd += " s=" + toSec(opts.Silence)
	}

	return a.Command(cmd).Err()
}

// SayAlpha plays a character string, annunciating each character.
func (a *AGI) SayAlpha(label string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY ALPHA", label, escapeDigits).Val()
}

// SayDigits plays a digit string, annunciating each digit.
func (a *AGI) SayDigits(number string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY DIGITS", number, escapeDigits).Val()
}

// SayDate plays a date
func (a *AGI) SayDate(when time.Time, escapeDigits string) (digit string, err error) {
	return a.Command("SAY DATE", toEpoch(when), escapeDigits).Val()
}

// SayDateTime plays a date using the given format.  See `voicemail.conf` for the format syntax; defaults to `ABdY 'digits/at' IMp`.
func (a *AGI) SayDateTime(when time.Time, escapeDigits string, format string) (digit string, err error) {
	// Extract the timezone from the time
	zone, _ := when.Zone()

	// Use the Asterisk default format if we are not given one
	if format == "" {
		format = "ABdY 'digits/at' IMp"
	}

	return a.Command("SAY DATETIME", toEpoch(when), escapeDigits, format, zone).Val()
}

// SayNumber plays the given number.
func (a *AGI) SayNumber(number string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY NUMBER", number, escapeDigits).Val()
}

// SayPhonetic plays the given phrase phonetically
func (a *AGI) SayPhonetic(phrase string, escapeDigits string) (digit string, err error) {
	return a.Command("SAY PHOENTIC", phrase, escapeDigits).Val()
}

// SayTime plays the time part of the given timestamp
func (a *AGI) SayTime(when time.Time, escapeDigits string) (digit string, err error) {
	return a.Command("SAY TIME", toEpoch(when), escapeDigits).Val()
}

// Set sets the given channel variable to
// the provided value.
func (a *AGI) Set(key, val string) error {
	return a.Command("SET VARIABLE", key, val).Err()
}

// StreamFile plays the given file to the channel
func (a *AGI) StreamFile(name string, escapeDigits string, offset int) (digit string, err error) {
	return a.Command("STREAM FILE", name, escapeDigits, strconv.Itoa(offset)).Val()
}

// Verbose logs the given message to the verbose message system
func (a *AGI) Verbose(msg string, level int) error {
	return a.Command("VERBOSE", msg, strconv.Itoa(level)).Err()
}

// WaitForDigit waits for a DTMF digit and returns what is received
func (a *AGI) WaitForDigit(timeout time.Duration) (digit string, err error) {
	resp := a.Command("WAIT FOR DIGIT", toMSec(timeout))
	return string(resp.Result), resp.Error
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/go-martini/martini"
    "github.com/martini-contrib/render"
    "github.com/martini-contrib/binding"
    "github.com/Lupino/collect/models"
    "mime/multipart"
    "net/http"
)

type FileForm struct {
    File *multipart.FileHeader `form:"file" binding:"required"`
}

type TagForm struct {
    Tag  string                `form:"tag"  binding:"required"`
}

type DatasetForm struct {
    FileForm
    TagForm
}

func api(mart *martini.ClassicMartini) {
    mart.Post(API + "/datasets/?", binding.Bind(DatasetForm{}), func(form DatasetForm, r render.Render) {
        var err error
        var file *models.File
        var tag *models.Tag
        var dataset *models.Dataset

        if file, err = uploadFile(form.File); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        if tag, err = saveTag(form.Tag); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        if dataset, err = saveDataset(file, tag, 0); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.Dataset{"dataset": dataset})
    })

    mart.Post(API + "/upload/?", binding.Bind(FileForm{}), func(form FileForm, r render.Render) {
        var err error
        var file *models.File

        if file, err = uploadFile(form.File); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.File{"file": file})
    })

    mart.Post(API + "/tags/?", binding.Bind(TagForm{}), func(form TagForm, r render.Render) {
        var err error
        var tag *models.Tag

        if tag, err = saveTag(form.Tag); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.Tag{"file": tag})
    })
}
<commit_msg>Add DataTypeForm<commit_after>package main

import (
    "github.com/go-martini/martini"
    "github.com/martini-contrib/render"
    "github.com/martini-contrib/binding"
    "github.com/Lupino/collect/models"
    "mime/multipart"
    "net/http"
)

type FileForm struct {
    File *multipart.FileHeader `form:"file" binding:"required"`
}

type TagForm struct {
    Tag  string                `form:"tag"  binding:"required"`
}

type DataTypeForm struct {
    DataType uint              `form:"data_type"`
}

func (dt DataTypeForm) Validate(errors binding.Errors, req *http.Request) (binding.Errors) {
    if dt.DataType > 2 {
        errors = append(errors, binding.Error{
            FieldNames: []string{"data_type"},
            Classification: "TypeError",
            Message: "data_type must on [0 1 2] or nil",
        })
    }
    return errors
}

type DatasetForm struct {
    FileForm
    TagForm
    DataTypeForm

}

func api(mart *martini.ClassicMartini) {
    mart.Post(API + "/datasets/?", binding.Bind(DatasetForm{}), func(form DatasetForm, r render.Render) {
        var err error
        var file *models.File
        var tag *models.Tag
        var dataset *models.Dataset

        if file, err = uploadFile(form.File); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        if tag, err = saveTag(form.Tag); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        if dataset, err = saveDataset(file, tag, form.DataType); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.Dataset{"dataset": dataset})
    })

    mart.Post(API + "/upload/?", binding.Bind(FileForm{}), func(form FileForm, r render.Render) {
        var err error
        var file *models.File

        if file, err = uploadFile(form.File); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.File{"file": file})
    })

    mart.Post(API + "/tags/?", binding.Bind(TagForm{}), func(form TagForm, r render.Render) {
        var err error
        var tag *models.Tag

        if tag, err = saveTag(form.Tag); err != nil {
            r.JSON(http.StatusInternalServerError, map[string]interface{}{"err": err.Error()})
        }

        r.JSON(http.StatusOK, map[string]*models.Tag{"file": tag})
    })
}
<|endoftext|>


<commit_before>package b2

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type B2 struct {
	AccountID          string
	ApplicationKey     string
	AuthorizationToken string
	ApiUrl             string
	DownloadUrl        string
	client             *client
}

type client struct {
	Protocol string
	http.Client
}

type authResponse struct {
	AccountID          string `json:"accountId"`
	AuthorizationToken string `json:"authorizationToken"`
	ApiUrl             string `json:"apiUrl"`
	DownloadUrl        string `json:"downloadUrl"`
}

// TODO make error public, add checking for error type
type errorResponse struct {
	Status  int64  `json:"status"`
	Code    string `json:"code"`
	Message string `json:"message"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("Status: %d, Code: %s, Message: %s", e.Status, e.Code, e.Message)
}

func CreateB2(accountId, appKey string) (*B2, error) {
	c := &client{Protocol: "https", Client: http.Client{}}
	return createB2(accountId, appKey, c)
}

func createB2(accountId, appKey string, client *client) (*B2, error) {
	b := &B2{
		AccountID:      accountId,
		ApplicationKey: appKey,
		client:         client,
	}

	req, err := b.CreateRequest("GET", "https://api.backblaze.com/b2api/v1/b2_authorize_account", nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(accountId, appKey)

	resp, err := b.client.Do(req)
	if err != nil {
		return nil, err
	}
	return b.parseCreateB2Response(resp)
}

func (b *B2) parseCreateB2Response(resp *http.Response) (*B2, error) {
	authResp := &authResponse{}
	err := ParseResponse(resp, authResp)
	if err != nil {
		return nil, err
	}

	b.AuthorizationToken = authResp.AuthorizationToken
	b.ApiUrl = authResp.ApiUrl
	b.DownloadUrl = authResp.DownloadUrl

	return b, nil
}

func (b *B2) CreateRequest(method, url string, request interface{}) (*http.Request, error) {
	reqBody, err := json.Marshal(request)
	if err != nil {
		return nil, err
	}
	url, err = b.replaceProtocol(url)
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequest(method, url, bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}
	return req, nil
}

func (b *B2) replaceProtocol(url string) (string, error) {
	protoTrim := strings.Index(url, ":")
	if protoTrim == -1 {
		return url, fmt.Errorf("Invalid url")
	}
	return b.client.Protocol + url[protoTrim:], nil
}

func GetBzInfoHeaders(resp *http.Response) map[string]string {
	out := map[string]string{}
	for k, v := range resp.Header {
		if strings.HasPrefix(k, "X-Bz-Info-") {
			// strip Bz prefix and grab first header
			out[k[10:]] = v[0]
		}
	}
	return out
}

func ParseResponse(resp *http.Response, respBody interface{}) error {
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		return ParseResponseBody(resp, respBody)
	} else {
		return ParseErrorResponse(resp)
	}
}

func ParseResponseBody(resp *http.Response, respBody interface{}) error {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return json.Unmarshal(body, respBody)
}

func ParseErrorResponse(resp *http.Response) error {
	errResp := &errorResponse{}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = json.Unmarshal(body, errResp)
	if err != nil {
		return err
	}
	return errResp
}
<commit_msg>Remove unnecessary error check from B2 CreateRequest<commit_after>package b2

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type B2 struct {
	AccountID          string
	ApplicationKey     string
	AuthorizationToken string
	ApiUrl             string
	DownloadUrl        string
	client             *client
}

type client struct {
	Protocol string
	http.Client
}

type authResponse struct {
	AccountID          string `json:"accountId"`
	AuthorizationToken string `json:"authorizationToken"`
	ApiUrl             string `json:"apiUrl"`
	DownloadUrl        string `json:"downloadUrl"`
}

// TODO make error public, add checking for error type
type errorResponse struct {
	Status  int64  `json:"status"`
	Code    string `json:"code"`
	Message string `json:"message"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("Status: %d, Code: %s, Message: %s", e.Status, e.Code, e.Message)
}

func CreateB2(accountId, appKey string) (*B2, error) {
	c := &client{Protocol: "https", Client: http.Client{}}
	return createB2(accountId, appKey, c)
}

func createB2(accountId, appKey string, client *client) (*B2, error) {
	b := &B2{
		AccountID:      accountId,
		ApplicationKey: appKey,
		client:         client,
	}

	req, err := b.CreateRequest("GET", "https://api.backblaze.com/b2api/v1/b2_authorize_account", nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(accountId, appKey)

	resp, err := b.client.Do(req)
	if err != nil {
		return nil, err
	}
	return b.parseCreateB2Response(resp)
}

func (b *B2) parseCreateB2Response(resp *http.Response) (*B2, error) {
	authResp := &authResponse{}
	err := ParseResponse(resp, authResp)
	if err != nil {
		return nil, err
	}

	b.AuthorizationToken = authResp.AuthorizationToken
	b.ApiUrl = authResp.ApiUrl
	b.DownloadUrl = authResp.DownloadUrl

	return b, nil
}

func (b *B2) CreateRequest(method, url string, request interface{}) (*http.Request, error) {
	reqBody, err := json.Marshal(request)
	if err != nil {
		return nil, err
	}
	url, err = b.replaceProtocol(url)
	if err != nil {
		return nil, err
	}
	return http.NewRequest(method, url, bytes.NewReader(reqBody))
}

func (b *B2) replaceProtocol(url string) (string, error) {
	protoTrim := strings.Index(url, ":")
	if protoTrim == -1 {
		return url, fmt.Errorf("Invalid url")
	}
	return b.client.Protocol + url[protoTrim:], nil
}

func GetBzInfoHeaders(resp *http.Response) map[string]string {
	out := map[string]string{}
	for k, v := range resp.Header {
		if strings.HasPrefix(k, "X-Bz-Info-") {
			// strip Bz prefix and grab first header
			out[k[10:]] = v[0]
		}
	}
	return out
}

func ParseResponse(resp *http.Response, respBody interface{}) error {
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		return ParseResponseBody(resp, respBody)
	} else {
		return ParseErrorResponse(resp)
	}
}

func ParseResponseBody(resp *http.Response, respBody interface{}) error {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	return json.Unmarshal(body, respBody)
}

func ParseErrorResponse(resp *http.Response) error {
	errResp := &errorResponse{}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	err = json.Unmarshal(body, errResp)
	if err != nil {
		return err
	}
	return errResp
}
<|endoftext|>