<commit_before>package frat

import (
	// "testing"
	. "gopkg.in/check.v1"
	"labix.org/v2/mgo/bson"
)

// var key = []byte("asdf1234asdf1234")

// Hook up gocheck into the "go test" runner.
// func Test(t *testing.T) { TestingT(t) }

// type TestSuite struct{}

var _ = Suite(&TestSuite{})

type FooBar struct {
	Id    bson.ObjectId   `bson:"_id"`
	Msg   string        `encrypted:"true",bson="msg"`
	Count int           `encrypted:"true",bson="count"`
}

func (s *TestSuite) TestConnect(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := new(MongoConnection)

	connection.Config = config

	connection.Connect()
	defer connection.Session.Close()

	err := connection.Session.Ping()

	c.Assert(err, Equals, nil)
}

func (s *TestSuite) TestSaveAndFind(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	// This needs to always be a pointer, otherwise the encryption component won't like it.
	message := new(FooBar)
	message.Msg = "Foo"
	message.Count = 5

	err := connection.Save(message)

	c.Assert(err, Equals, nil)

	newMessage := new(FooBar)

	connection.FindById(message.Id, newMessage)

	// Make sure the ids are the same
	c.Assert(newMessage.Id.String(), Equals, message.Id.String())
	c.Assert(newMessage.Msg, Equals, message.Msg)
	c.Assert(newMessage.Count, Equals, message.Count)
}

func (s *TestSuite) TestFindNonExistent(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	newMessage := new(FooBar)

	err := connection.FindById(bson.NewObjectId(), newMessage)

	c.Assert(err.Error(), Equals, "not found")
}

func (s *TestSuite) TestDelete(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	// This needs to always be a pointer, otherwise the encryption component won't like it.
	message := new(FooBar)
	message.Msg = "Foo"
	message.Count = 5

	err := connection.Save(message)

	c.Assert(err, Equals, nil)

	connection.Delete(message)

	newMessage := new(FooBar)
	err = connection.FindById(message.Id, newMessage)
	c.Assert(err.Error(), Equals, "not found")
	// Make sure the ids are the same

}
<commit_msg>Save benchmark<commit_after>package frat

import (
	"testing"
	. "gopkg.in/check.v1"
	"labix.org/v2/mgo/bson"
)

// var key = []byte("asdf1234asdf1234")

// Hook up gocheck into the "go test" runner.
// func Test(t *testing.T) { TestingT(t) }

// type TestSuite struct{}

var _ = Suite(&TestSuite{})

type FooBar struct {
	Id    bson.ObjectId   `bson:"_id"`
	Msg   string        `encrypted:"true",bson="msg"`
	Count int           `encrypted:"true",bson="count"`
}

func (s *TestSuite) TestConnect(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := new(MongoConnection)

	connection.Config = config

	connection.Connect()
	defer connection.Session.Close()

	err := connection.Session.Ping()

	c.Assert(err, Equals, nil)

	connection.Session.DB(config.Database).DropDatabase()
}

func (s *TestSuite) TestSaveAndFind(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	// This needs to always be a pointer, otherwise the encryption component won't like it.
	message := new(FooBar)
	message.Msg = "Foo"
	message.Count = 5

	err := connection.Save(message)

	c.Assert(err, Equals, nil)

	newMessage := new(FooBar)

	connection.FindById(message.Id, newMessage)

	// Make sure the ids are the same
	c.Assert(newMessage.Id.String(), Equals, message.Id.String())
	c.Assert(newMessage.Msg, Equals, message.Msg)
	c.Assert(newMessage.Count, Equals, message.Count)
	connection.Session.DB(config.Database).DropDatabase()
}

func (s *TestSuite) TestFindNonExistent(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	newMessage := new(FooBar)

	err := connection.FindById(bson.NewObjectId(), newMessage)

	c.Assert(err.Error(), Equals, "not found")
	connection.Session.DB(config.Database).DropDatabase()
}

func (s *TestSuite) TestDelete(c *C) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	// This needs to always be a pointer, otherwise the encryption component won't like it.
	message := new(FooBar)
	message.Msg = "Foo"
	message.Count = 5

	err := connection.Save(message)

	c.Assert(err, Equals, nil)

	connection.Delete(message)

	newMessage := new(FooBar)
	err = connection.FindById(message.Id, newMessage)
	c.Assert(err.Error(), Equals, "not found")
	// Make sure the ids are the same
	//
	connection.Session.DB(config.Database).DropDatabase()

}

/////////////////////
/// BENCHMARKS
/////////////////////
func createAndSaveDocument(conn *MongoConnection) {
 	message := &FooBar{
 		Msg:"Foo",
 		Count:5,
 	}

 	err := conn.Save(message)
 	if err != nil {
 		panic(err)
 	}
}

func BenchmarkEncryptedAndSave(b *testing.B) {
	config := &MongoConfig{"localhost","gotest"}

	connection := Connect(config)

	defer connection.Session.Close()

	for i := 0; i < b.N; i++ {
	    createAndSaveDocument(connection)
	}
	connection.Session.DB(config.Database).DropDatabase()
}
<|endoftext|>


<commit_before>// +build experimental

package checkpoint

import (
	"golang.org/x/net/context"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/spf13/cobra"
)

type createOptions struct {
	container    string
	checkpoint   string
	leaveRunning bool
}

func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
	var opts createOptions

	cmd := &cobra.Command{
		Use:   "create CONTAINER CHECKPOINT",
		Short: "Create a checkpoint from a running container",
		Args:  cli.ExactArgs(2),
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.container = args[0]
			opts.checkpoint = args[1]
			return runCreate(dockerCli, opts)
		},
	}

	flags := cmd.Flags()
	flags.BoolVar(&opts.leaveRunning, "leave-running", false, "leave the container running after checkpoing")

	return cmd
}

func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
	client := dockerCli.Client()

	checkpointOpts := types.CheckpointCreateOptions{
		CheckpointID: opts.checkpoint,
		Exit:         !opts.leaveRunning,
	}

	err := client.CheckpointCreate(context.Background(), opts.container, checkpointOpts)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix typo<commit_after>// +build experimental

package checkpoint

import (
	"golang.org/x/net/context"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/cli"
	"github.com/docker/docker/cli/command"
	"github.com/spf13/cobra"
)

type createOptions struct {
	container    string
	checkpoint   string
	leaveRunning bool
}

func newCreateCommand(dockerCli *command.DockerCli) *cobra.Command {
	var opts createOptions

	cmd := &cobra.Command{
		Use:   "create CONTAINER CHECKPOINT",
		Short: "Create a checkpoint from a running container",
		Args:  cli.ExactArgs(2),
		RunE: func(cmd *cobra.Command, args []string) error {
			opts.container = args[0]
			opts.checkpoint = args[1]
			return runCreate(dockerCli, opts)
		},
	}

	flags := cmd.Flags()
	flags.BoolVar(&opts.leaveRunning, "leave-running", false, "leave the container running after checkpoint")

	return cmd
}

func runCreate(dockerCli *command.DockerCli, opts createOptions) error {
	client := dockerCli.Client()

	checkpointOpts := types.CheckpointCreateOptions{
		CheckpointID: opts.checkpoint,
		Exit:         !opts.leaveRunning,
	}

	err := client.CheckpointCreate(context.Background(), opts.container, checkpointOpts)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths     = map[string][]string{}
	extraAdd = flag.String("add", "", "Extra commands or directories to add (full path, comma-separated string)")
	extraCpio = flag.String("cpio", "", "A list of cpio archives to include in the output")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// But does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraAdd != "" {
		copyc := strings.Fields(*extraAdd)
		for _, eachPath := range copyc {
			debug("each path is %s\n", eachPath)
			// Must not include ~ in path
			if strings.Count(eachPath, ":") > 1 {
				log.Fatalf(" Input has more than one :")
			}
			modPath := strings.Replace(eachPath, ":", "/", 1)
			debug("modpath is %s\n", modPath)
			statval, err := os.Stat(modPath)
			if err != nil {
				log.Fatalf("%v", err)
			}
			p := strings.Split(eachPath, ":")
			if len(p) != 2 {
				p = append([]string{"/"}, p...)
			}
			debug("P is %v\n", p)
			debug("Paths currently is %v\n", paths)
			paths[p[0]] = append(paths[p[0]], p[1])
			debug("putps")
			debug("Paths currently is %v\n", paths)
			if !statval.IsDir() {
				tmpSlice := []string{modPath}
				libs, err := uroot.LddList(tmpSlice)
				if err != nil {
					log.Fatalf("%v", err)
				}
				paths["/"] = append(paths["/"], libs...)
				debug("Paths currently is (because command) %v\n", paths)
			}
		}
	}

	if *extraCpio != "" {
		extras := strings.Fields(*extraCpio)
		for _, x := range extras {
			a, err := cpio.Format("newc")
			if err != nil {
				log.Fatalf("Creating archiver: %v", err)
			}
			f, err := os.Open(x)
			if err != nil {
				log.Fatalf("%v: %v", x, err)
			}
			defer f.Close()
			rr := a.Reader(f)
			recs, err := rr.ReadRecords()
			if err != nil {
				log.Fatalf("read records: %v", err)
			}
			cpio.MakeReproducible(recs)
			if err := w.WriteRecords(recs); err != nil {
				log.Fatalf("%v", err)
			}
		}
	}
	//append the ldd'd files to /
	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// We need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<commit_msg>quick change to extraAdd<commit_after>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// bbramfs builds a simple initramfs given an existing built bb; see bb.go
// You have to run bb first, which creates cmds/bb/bbsh. cd to that directory,
// and run bbramfs, and you have a single binary which does all u-root commands.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/cpio"
	_ "github.com/u-root/u-root/pkg/cpio/newc"
	"github.com/u-root/u-root/pkg/ldd"
)

var (
	// Paths contains the paths to put into the initramfs.
	// The index is a root directory, and the value is the place from which to
	// walk. The only required root is the bbsh dir itself, and the starting
	// walk is init -- i.e. we grab only one file. Should you wish to bring in,
	// e.g., /lib/modules/4.04, you would do add the root as / and the
	// starting point for the walk as lib/modules/4.04. That way we only preserve
	// as much of the path as we need, but we can preserve it all.
	paths     = map[string][]string{}
	extraAdd = flag.String("add", "", "Extra commands or directories to add (full path, space-separated string)")
	extraCpio = flag.String("cpio", "", "A list of cpio archives to include in the output")
)

func sanity() {
	goBinGo := filepath.Join(config.Goroot, "bin/go")
	_, err := os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	// But does the one in go/bin/OS_ARCH exist too?
	goBinGo = filepath.Join(config.Goroot, fmt.Sprintf("bin/%s_%s/go", config.Goos, config.Arch))
	_, err = os.Stat(goBinGo)
	if err == nil {
		config.Go = goBinGo
	}
	if config.Go == "" {
		log.Fatalf("Can't find a go binary! Is GOROOT set correctly?")
	}
}

// dirComponents takes a string and returns an array of strings,
// such that we can create the directory records for intermediate
// directories.
func dirComponents(dir string) []string {
	var dirlist []string
	if filepath.Dir(dir) == "." {
		return []string{}
	}
	for d := filepath.Dir(dir); d != "/"; d = filepath.Dir(d) {
		dirlist = append([]string{d}, dirlist...)
	}
	dirlist = append(dirlist, dir)
	return dirlist
}

func ramfs() {
	archiver, err := cpio.Format("newc")
	if err != nil {
		log.Fatalf("Creating newc archiver: %v", err)
	}

	oname := fmt.Sprintf("/tmp/initramfs.%v_%v.cpio", config.Goos, config.Arch)
	f, err := os.Create(oname)
	if err != nil {
		log.Fatalf("%v", err)
	}

	w := archiver.Writer(f)
	cpio.MakeReproducible(devCPIO[:])
	if err := w.WriteRecords(devCPIO[:]); err != nil {
		log.Fatalf("%v", err)
	}

	paths[filepath.Join(config.Gopath, "src/github.com/u-root/u-root/bb/bbsh")] = []string{"init", "ubin"}

	if *extraAdd != "" {
		copyc := strings.Fields(*extraAdd)
		for _, eachPath := range copyc {
			debug("each path is %s\n", eachPath)
			// Must not include ~ in path
			if strings.Count(eachPath, ":") > 1 {
				log.Fatalf(" Input has more than one :")
			}
			modPath := strings.Replace(eachPath, ":", "/", 1)
			debug("modpath is %s\n", modPath)
			statval, err := os.Stat(modPath)
			if err != nil {
				log.Fatalf("%v", err)
			}
			p := strings.Split(eachPath, ":")
			if len(p) != 2 {
				p = append([]string{"/"}, p...)
			}
			debug("P is %v\n", p)
			debug("Paths currently is %v\n", paths)
			paths[p[0]] = append(paths[p[0]], p[1])
			debug("putps")
			debug("Paths currently is %v\n", paths)
			if !statval.IsDir() {
				tmpSlice := []string{modPath}
				libs, err := uroot.LddList(tmpSlice)
				if err != nil {
					log.Fatalf("%v", err)
				}
				paths["/"] = append(paths["/"], libs...)
				debug("Paths currently is (because command) %v\n", paths)
			}
		}
	}

	if *extraCpio != "" {
		extras := strings.Fields(*extraCpio)
		for _, x := range extras {
			a, err := cpio.Format("newc")
			if err != nil {
				log.Fatalf("Creating archiver: %v", err)
			}
			f, err := os.Open(x)
			if err != nil {
				log.Fatalf("%v: %v", x, err)
			}
			defer f.Close()
			rr := a.Reader(f)
			recs, err := rr.ReadRecords()
			if err != nil {
				log.Fatalf("read records: %v", err)
			}
			cpio.MakeReproducible(recs)
			if err := w.WriteRecords(recs); err != nil {
				log.Fatalf("%v", err)
			}
		}
	}
	//append the ldd'd files to /
	// For all the 'roots' in paths, start walking at the name.
	debug("PATHS: %v", paths)
	for r, list := range paths {
		debug("PATHS: root %v", r)
		// We need to make all the path prefix directories.
		for _, n := range list {
			debug("\troot %v, name %v", r, n)
			for _, d := range dirComponents(n) {
				debug("\t\troot %v, name %v, component %v", r, n, d)
				rec, err := cpio.GetRecord(d)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", d, err)
				}
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v", err)
				}
			}
		}
		for _, n := range list {
			if err := filepath.Walk(filepath.Join(r, n), func(name string, fi os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				cn, err := filepath.Rel(r, name)
				if err != nil {
					log.Fatalf("filepath.Rel(%v, %v): %v", r, name, err)
				}
				debug("%v\n", cn)
				rec, err := cpio.GetRecord(name)
				if err != nil {
					log.Fatalf("Getting record of %q failed: %v", cn, err)
				}
				// the name in the cpio is relative to our starting point.
				rec.Name = cn
				recs := []cpio.Record{rec}
				cpio.MakeReproducible(recs)
				if err := w.WriteRecords(recs); err != nil {
					log.Fatalf("%v", err)
				}
				return nil
			}); err != nil {
				log.Fatalf("bbsh walk failed: %v", err)
			}
		}
	}

	if err := w.WriteTrailer(); err != nil {
		log.Fatalf("Error writing trailer record: %v", err)
	}
	fmt.Printf("Output file is in %v\n", oname)
}
<|endoftext|>


<commit_before>// Package data provide simple CRUD operation on couchdb doc
package data

import (
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/permissions"
	"github.com/labstack/echo"
)

func validDoctype(next echo.HandlerFunc) echo.HandlerFunc {
	// TODO extends me to verificate characters allowed in db name.
	return func(c echo.Context) error {
		doctype := c.Param("doctype")
		if doctype == "" {
			return jsonapi.NewError(http.StatusBadRequest, "Invalid doctype '%s'", doctype)
		}
		c.Set("doctype", doctype)
		return next(c)
	}
}

func fixErrorNoDatabaseIsWrongDoctype(err error) error {
	if couchdb.IsNoDatabaseError(err) {
		err.(*couchdb.Error).Reason = "wrong_doctype"
	}
	return err
}

func allDoctypes(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	if err := permissions.AllowWholeType(c, permissions.GET, consts.Doctypes); err != nil {
		return err
	}

	types, err := couchdb.AllDoctypes(instance)
	if err != nil {
		return err
	}
	var doctypes []string
	for _, typ := range types {
		if CheckReadable(typ) == nil {
			doctypes = append(doctypes, typ)
		}
	}
	return c.JSON(http.StatusOK, doctypes)
}

// GetDoc get a doc by its type and id
func getDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	docid := c.Param("docid")

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if docid == "" {
		return dbStatus(c)
	}

	revs := c.QueryParam("revs")
	if revs == "true" {
		return proxy(c, docid)
	}

	var out couchdb.JSONDoc
	err := couchdb.GetDoc(instance, doctype, docid, &out)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	out.Type = doctype

	if err := permissions.Allow(c, permissions.GET, &out); err != nil {
		return err
	}

	return c.JSON(http.StatusOK, out.ToMapWithType())
}

// CreateDoc create doc from the json passed as body
func createDoc(c echo.Context) error {
	doctype := c.Get("doctype").(string)
	instance := middlewares.GetInstance(c)

	doc := couchdb.JSONDoc{Type: doctype}
	if err := c.Bind(&doc.M); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckWritable(doctype); err != nil {
		return err
	}

	if err := permissions.Allow(c, permissions.POST, &doc); err != nil {
		return err
	}

	if err := couchdb.CreateDoc(instance, doc); err != nil {
		return err
	}

	return c.JSON(http.StatusCreated, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func createNamedDoc(c echo.Context, doc couchdb.JSONDoc) error {
	instance := middlewares.GetInstance(c)

	err := permissions.Allow(c, permissions.POST, &doc)
	if err != nil {
		return err
	}

	err = couchdb.CreateNamedDoc(instance, doc)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func updateDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	var doc couchdb.JSONDoc
	if err := c.Bind(&doc); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	doc.Type = c.Param("doctype")

	if err := CheckWritable(doc.Type); err != nil {
		return err
	}

	if (doc.ID() == "") != (doc.Rev() == "") {
		return jsonapi.NewError(http.StatusBadRequest,
			"You must either provide an _id and _rev in document (update) or neither (create with fixed id).")
	}

	if doc.ID() != "" && doc.ID() != c.Param("docid") {
		return jsonapi.NewError(http.StatusBadRequest, "document _id doesnt match url")
	}

	if doc.ID() == "" {
		doc.SetID(c.Param("docid"))
		return createNamedDoc(c, doc)
	}

	errWhole := permissions.AllowWholeType(c, permissions.PUT, doc.DocType())
	if errWhole != nil {

		// we cant apply to whole type, let's fetch old doc and see if it applies there
		var old couchdb.JSONDoc
		errFetch := couchdb.GetDoc(instance, doc.DocType(), doc.ID(), &old)
		if errFetch != nil {
			return errFetch
		}

		// check if permissions set allows manipulating old doc
		errOld := permissions.Allow(c, permissions.PUT, &old)
		if errOld != nil {
			return errOld
		}

		// also check if permissions set allows manipulating new doc
		errNew := permissions.Allow(c, permissions.PUT, &doc)
		if errNew != nil {
			return errNew
		}
	}

	errUpdate := couchdb.UpdateDoc(instance, doc)
	if errUpdate != nil {
		return fixErrorNoDatabaseIsWrongDoctype(errUpdate)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func deleteDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	docid := c.Param("docid")
	revHeader := c.Request().Header.Get("If-Match")
	revQuery := c.QueryParam("rev")
	rev := ""

	if revHeader != "" && revQuery != "" && revQuery != revHeader {
		return jsonapi.NewError(http.StatusBadRequest,
			"If-Match Header and rev query parameters mismatch")
	} else if revHeader != "" {
		rev = revHeader
	} else if revQuery != "" {
		rev = revQuery
	} else {
		return jsonapi.NewError(http.StatusBadRequest, "delete without revision")
	}

	if err := CheckWritable(doctype); err != nil {
		return err
	}

	var doc couchdb.JSONDoc
	err := couchdb.GetDoc(instance, doctype, docid, &doc)
	if err != nil {
		return err
	}
	doc.Type = doctype
	doc.SetRev(rev)

	err = permissions.Allow(c, permissions.DELETE, &doc)
	if err != nil {
		return err
	}

	err = couchdb.DeleteDoc(instance, &doc)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":      true,
		"id":      doc.ID(),
		"rev":     doc.Rev(),
		"type":    doc.DocType(),
		"deleted": true,
	})

}

func defineIndex(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	var definitionRequest map[string]interface{}

	if err := c.Bind(&definitionRequest); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	result, err := couchdb.DefineIndexRaw(instance, doctype, &definitionRequest)
	if couchdb.IsNoDatabaseError(err) {
		if err = couchdb.CreateDB(instance, doctype); err == nil {
			result, err = couchdb.DefineIndexRaw(instance, doctype, &definitionRequest)
		}
	}
	if err != nil {
		return err
	}

	return c.JSON(http.StatusOK, result)
}

const maxMangoLimit = 100

func findDocuments(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	var findRequest map[string]interface{}

	if err := c.Bind(&findRequest); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	limit, hasLimit := findRequest["limit"].(float64)
	if !hasLimit || limit > maxMangoLimit {
		limit = 100
	}

	// add 1 so we know if there is more.
	findRequest["limit"] = limit + 1

	var results []couchdb.JSONDoc
	err := couchdb.FindDocsRaw(instance, doctype, &findRequest, &results)
	if err != nil {
		return err
	}

	out := echo.Map{
		"docs":  results,
		"limit": limit,
		"next":  false,
	}
	if len(results) > int(limit) {
		out["docs"] = results[:len(results)-1]
		out["next"] = true
	}

	return c.JSON(http.StatusOK, out)
}

var allowedChangesParams = map[string]bool{
	"feed":      true,
	"style":     true,
	"since":     true,
	"limit":     true,
	"timeout":   true,
	"heartbeat": true, // Pouchdb sends heartbeet even for non-continuous
	"_nonce":    true, // Pouchdb sends a request hash to avoid agressive caching by some browsers
}

func changesFeed(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	var doctype = c.Get("doctype").(string)

	// Drop a clear error for parameters not supported by stack
	for key := range c.QueryParams() {
		if !allowedChangesParams[key] {
			return jsonapi.NewError(http.StatusBadRequest, "Unsuported query parameter '%s'", key)
		}
	}

	feed, err := couchdb.ValidChangesMode(c.QueryParam("feed"))
	if err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	feedStyle, err := couchdb.ValidChangesStyle(c.QueryParam("style"))
	if err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	limitString := c.QueryParam("limit")
	limit := 0
	if limitString != "" {
		if limit, err = strconv.Atoi(limitString); err != nil {
			return jsonapi.NewError(http.StatusBadRequest, "Invalid limit value '%s'", err.Error())
		}
	}

	if err = permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	results, err := couchdb.GetChanges(instance, &couchdb.ChangesRequest{
		DocType: doctype,
		Feed:    feed,
		Style:   feedStyle,
		Since:   c.QueryParam("since"),
		Limit:   limit,
	})

	if err != nil {
		return err
	}

	return c.JSON(http.StatusOK, results)
}

func allDocs(c echo.Context) error {
	doctype := c.Get("doctype").(string)

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	return proxy(c, "_all_docs")
}

// mostly just to prevent couchdb crash on replications
func dataAPIWelcome(c echo.Context) error {
	return c.JSON(http.StatusOK, echo.Map{
		"message": "welcome to a cozy API",
	})
}

func couchdbStyleErrorHandler(next echo.HandlerFunc) echo.HandlerFunc {
	return func(c echo.Context) error {
		err := next(c)
		if err == nil {
			return nil
		}

		if ce, ok := err.(*couchdb.Error); ok {
			return c.JSON(ce.StatusCode, ce.JSON())
		}

		if he, ok := err.(*echo.HTTPError); ok {
			return c.JSON(he.Code, echo.Map{"error": he.Error()})
		}

		if je, ok := err.(*jsonapi.Error); ok {
			return c.JSON(je.Status, echo.Map{"error": je.Title})
		}

		return c.JSON(http.StatusInternalServerError, echo.Map{
			"error": err.Error(),
		})
	}
}

// Routes sets the routing for the status service
func Routes(router *echo.Group) {
	router.Use(couchdbStyleErrorHandler)

	// API Routes that don't depend on a doctype
	router.GET("/", dataAPIWelcome)
	router.GET("/_all_doctypes", allDoctypes)

	group := router.Group("/:doctype", validDoctype)

	replicationRoutes(group)

	// API Routes under /:doctype
	group.GET("/:docid", getDoc)
	group.PUT("/:docid", updateDoc)
	group.DELETE("/:docid", deleteDoc)
	group.GET("/:docid/relationships/references", listReferencesHandler)
	group.POST("/:docid/relationships/references", addReferencesHandler)
	group.DELETE("/:docid/relationships/references", removeReferencesHandler)
	group.POST("/", createDoc)
	group.GET("/_all_docs", allDocs)
	group.POST("/_all_docs", allDocs)
	group.POST("/_index", defineIndex)
	group.POST("/_find", findDocuments)
	// group.DELETE("/:docid", DeleteDoc)
}
<commit_msg>Add document type for update permission check<commit_after>// Package data provide simple CRUD operation on couchdb doc
package data

import (
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/cozy/cozy-stack/web/permissions"
	"github.com/labstack/echo"
)

func validDoctype(next echo.HandlerFunc) echo.HandlerFunc {
	// TODO extends me to verificate characters allowed in db name.
	return func(c echo.Context) error {
		doctype := c.Param("doctype")
		if doctype == "" {
			return jsonapi.NewError(http.StatusBadRequest, "Invalid doctype '%s'", doctype)
		}
		c.Set("doctype", doctype)
		return next(c)
	}
}

func fixErrorNoDatabaseIsWrongDoctype(err error) error {
	if couchdb.IsNoDatabaseError(err) {
		err.(*couchdb.Error).Reason = "wrong_doctype"
	}
	return err
}

func allDoctypes(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	if err := permissions.AllowWholeType(c, permissions.GET, consts.Doctypes); err != nil {
		return err
	}

	types, err := couchdb.AllDoctypes(instance)
	if err != nil {
		return err
	}
	var doctypes []string
	for _, typ := range types {
		if CheckReadable(typ) == nil {
			doctypes = append(doctypes, typ)
		}
	}
	return c.JSON(http.StatusOK, doctypes)
}

// GetDoc get a doc by its type and id
func getDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	docid := c.Param("docid")

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if docid == "" {
		return dbStatus(c)
	}

	revs := c.QueryParam("revs")
	if revs == "true" {
		return proxy(c, docid)
	}

	var out couchdb.JSONDoc
	err := couchdb.GetDoc(instance, doctype, docid, &out)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	out.Type = doctype

	if err := permissions.Allow(c, permissions.GET, &out); err != nil {
		return err
	}

	return c.JSON(http.StatusOK, out.ToMapWithType())
}

// CreateDoc create doc from the json passed as body
func createDoc(c echo.Context) error {
	doctype := c.Get("doctype").(string)
	instance := middlewares.GetInstance(c)

	doc := couchdb.JSONDoc{Type: doctype}
	if err := c.Bind(&doc.M); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckWritable(doctype); err != nil {
		return err
	}

	if err := permissions.Allow(c, permissions.POST, &doc); err != nil {
		return err
	}

	if err := couchdb.CreateDoc(instance, doc); err != nil {
		return err
	}

	return c.JSON(http.StatusCreated, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func createNamedDoc(c echo.Context, doc couchdb.JSONDoc) error {
	instance := middlewares.GetInstance(c)

	err := permissions.Allow(c, permissions.POST, &doc)
	if err != nil {
		return err
	}

	err = couchdb.CreateNamedDoc(instance, doc)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func updateDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)

	var doc couchdb.JSONDoc
	if err := c.Bind(&doc); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	doc.Type = c.Param("doctype")

	if err := CheckWritable(doc.Type); err != nil {
		return err
	}

	if (doc.ID() == "") != (doc.Rev() == "") {
		return jsonapi.NewError(http.StatusBadRequest,
			"You must either provide an _id and _rev in document (update) or neither (create with fixed id).")
	}

	if doc.ID() != "" && doc.ID() != c.Param("docid") {
		return jsonapi.NewError(http.StatusBadRequest, "document _id doesnt match url")
	}

	if doc.ID() == "" {
		doc.SetID(c.Param("docid"))
		return createNamedDoc(c, doc)
	}

	errWhole := permissions.AllowWholeType(c, permissions.PUT, doc.DocType())
	if errWhole != nil {

		// we cant apply to whole type, let's fetch old doc and see if it applies there
		var old couchdb.JSONDoc
		errFetch := couchdb.GetDoc(instance, doc.DocType(), doc.ID(), &old)
		if errFetch != nil {
			return errFetch
		}
		old.Type = doc.DocType()
		// check if permissions set allows manipulating old doc
		errOld := permissions.Allow(c, permissions.PUT, &old)
		if errOld != nil {
			return errOld
		}

		// also check if permissions set allows manipulating new doc
		errNew := permissions.Allow(c, permissions.PUT, &doc)
		if errNew != nil {
			return errNew
		}
	}

	errUpdate := couchdb.UpdateDoc(instance, doc)
	if errUpdate != nil {
		return fixErrorNoDatabaseIsWrongDoctype(errUpdate)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":   true,
		"id":   doc.ID(),
		"rev":  doc.Rev(),
		"type": doc.DocType(),
		"data": doc.ToMapWithType(),
	})
}

func deleteDoc(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	docid := c.Param("docid")
	revHeader := c.Request().Header.Get("If-Match")
	revQuery := c.QueryParam("rev")
	rev := ""

	if revHeader != "" && revQuery != "" && revQuery != revHeader {
		return jsonapi.NewError(http.StatusBadRequest,
			"If-Match Header and rev query parameters mismatch")
	} else if revHeader != "" {
		rev = revHeader
	} else if revQuery != "" {
		rev = revQuery
	} else {
		return jsonapi.NewError(http.StatusBadRequest, "delete without revision")
	}

	if err := CheckWritable(doctype); err != nil {
		return err
	}

	var doc couchdb.JSONDoc
	err := couchdb.GetDoc(instance, doctype, docid, &doc)
	if err != nil {
		return err
	}
	doc.Type = doctype
	doc.SetRev(rev)

	err = permissions.Allow(c, permissions.DELETE, &doc)
	if err != nil {
		return err
	}

	err = couchdb.DeleteDoc(instance, &doc)
	if err != nil {
		return fixErrorNoDatabaseIsWrongDoctype(err)
	}

	return c.JSON(http.StatusOK, echo.Map{
		"ok":      true,
		"id":      doc.ID(),
		"rev":     doc.Rev(),
		"type":    doc.DocType(),
		"deleted": true,
	})

}

func defineIndex(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	var definitionRequest map[string]interface{}

	if err := c.Bind(&definitionRequest); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	result, err := couchdb.DefineIndexRaw(instance, doctype, &definitionRequest)
	if couchdb.IsNoDatabaseError(err) {
		if err = couchdb.CreateDB(instance, doctype); err == nil {
			result, err = couchdb.DefineIndexRaw(instance, doctype, &definitionRequest)
		}
	}
	if err != nil {
		return err
	}

	return c.JSON(http.StatusOK, result)
}

const maxMangoLimit = 100

func findDocuments(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	doctype := c.Get("doctype").(string)
	var findRequest map[string]interface{}

	if err := c.Bind(&findRequest); err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	limit, hasLimit := findRequest["limit"].(float64)
	if !hasLimit || limit > maxMangoLimit {
		limit = 100
	}

	// add 1 so we know if there is more.
	findRequest["limit"] = limit + 1

	var results []couchdb.JSONDoc
	err := couchdb.FindDocsRaw(instance, doctype, &findRequest, &results)
	if err != nil {
		return err
	}

	out := echo.Map{
		"docs":  results,
		"limit": limit,
		"next":  false,
	}
	if len(results) > int(limit) {
		out["docs"] = results[:len(results)-1]
		out["next"] = true
	}

	return c.JSON(http.StatusOK, out)
}

var allowedChangesParams = map[string]bool{
	"feed":      true,
	"style":     true,
	"since":     true,
	"limit":     true,
	"timeout":   true,
	"heartbeat": true, // Pouchdb sends heartbeet even for non-continuous
	"_nonce":    true, // Pouchdb sends a request hash to avoid agressive caching by some browsers
}

func changesFeed(c echo.Context) error {
	instance := middlewares.GetInstance(c)
	var doctype = c.Get("doctype").(string)

	// Drop a clear error for parameters not supported by stack
	for key := range c.QueryParams() {
		if !allowedChangesParams[key] {
			return jsonapi.NewError(http.StatusBadRequest, "Unsuported query parameter '%s'", key)
		}
	}

	feed, err := couchdb.ValidChangesMode(c.QueryParam("feed"))
	if err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	feedStyle, err := couchdb.ValidChangesStyle(c.QueryParam("style"))
	if err != nil {
		return jsonapi.NewError(http.StatusBadRequest, err)
	}

	limitString := c.QueryParam("limit")
	limit := 0
	if limitString != "" {
		if limit, err = strconv.Atoi(limitString); err != nil {
			return jsonapi.NewError(http.StatusBadRequest, "Invalid limit value '%s'", err.Error())
		}
	}

	if err = permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	results, err := couchdb.GetChanges(instance, &couchdb.ChangesRequest{
		DocType: doctype,
		Feed:    feed,
		Style:   feedStyle,
		Since:   c.QueryParam("since"),
		Limit:   limit,
	})

	if err != nil {
		return err
	}

	return c.JSON(http.StatusOK, results)
}

func allDocs(c echo.Context) error {
	doctype := c.Get("doctype").(string)

	if err := CheckReadable(doctype); err != nil {
		return err
	}

	if err := permissions.AllowWholeType(c, permissions.GET, doctype); err != nil {
		return err
	}

	return proxy(c, "_all_docs")
}

// mostly just to prevent couchdb crash on replications
func dataAPIWelcome(c echo.Context) error {
	return c.JSON(http.StatusOK, echo.Map{
		"message": "welcome to a cozy API",
	})
}

func couchdbStyleErrorHandler(next echo.HandlerFunc) echo.HandlerFunc {
	return func(c echo.Context) error {
		err := next(c)
		if err == nil {
			return nil
		}

		if ce, ok := err.(*couchdb.Error); ok {
			return c.JSON(ce.StatusCode, ce.JSON())
		}

		if he, ok := err.(*echo.HTTPError); ok {
			return c.JSON(he.Code, echo.Map{"error": he.Error()})
		}

		if je, ok := err.(*jsonapi.Error); ok {
			return c.JSON(je.Status, echo.Map{"error": je.Title})
		}

		return c.JSON(http.StatusInternalServerError, echo.Map{
			"error": err.Error(),
		})
	}
}

// Routes sets the routing for the status service
func Routes(router *echo.Group) {
	router.Use(couchdbStyleErrorHandler)

	// API Routes that don't depend on a doctype
	router.GET("/", dataAPIWelcome)
	router.GET("/_all_doctypes", allDoctypes)

	group := router.Group("/:doctype", validDoctype)

	replicationRoutes(group)

	// API Routes under /:doctype
	group.GET("/:docid", getDoc)
	group.PUT("/:docid", updateDoc)
	group.DELETE("/:docid", deleteDoc)
	group.GET("/:docid/relationships/references", listReferencesHandler)
	group.POST("/:docid/relationships/references", addReferencesHandler)
	group.DELETE("/:docid/relationships/references", removeReferencesHandler)
	group.POST("/", createDoc)
	group.GET("/_all_docs", allDocs)
	group.POST("/_all_docs", allDocs)
	group.POST("/_index", defineIndex)
	group.POST("/_find", findDocuments)
	// group.DELETE("/:docid", DeleteDoc)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"testing"
)

func TestMain(m *testing.M) {
	// command to start firestore emulator
	cmd := exec.Command("gcloud", "beta", "emulators", "firestore", "start", fmt.Sprintf("--host-port=localhost:%d", 8181), "--quiet")

	// this makes it killable
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// we need to capture it's output to know when it's started
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	defer stderr.Close()

	// start her up!
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// ensure the process is killed when we're finished, even if an error occurs
	// (thanks to Brian Moran for suggestion)
	var result int
	defer func() {
		syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
		os.Exit(result)
	}()

	// we're going to wait until it's running to start
	var wg sync.WaitGroup
	wg.Add(1)

	// by starting a separate go routine
	go func() {
		// reading it's output
		buf := make([]byte, 256, 256)
		for {
			n, err := stderr.Read(buf[:])
			if err != nil {
				// until it ends
				if err == io.EOF {
					break
				}
				log.Fatalf("reading stderr %v", err)
			}

			if n > 0 {
				d := string(buf[:n])

				// only required if we want to see the emulator output
				log.Printf("%s", d)

				// checking for the message that it's started
				if strings.Contains(d, "Dev App Server is now running") {
					wg.Done()
				}

				// and capturing the FIRESTORE_EMULATOR_HOST value to set
				pos := strings.Index(d, FirestoreEmulatorHost+"=")
				if pos > 0 {
					host := d[pos+len(FirestoreEmulatorHost)+1 : len(d)-1]
					os.Setenv(FirestoreEmulatorHost, host)
				}
			}
		}
	}()

	// wait until the running message has been received
	wg.Wait()

	agentTestSetup()
	cacheTestSetup()
	noisy = false

	// now it's running, we can run our unit tests
	result = m.Run()
}

func TestGetUsername(t *testing.T) {
	cases := []struct {
		in   string
		want string
	}{
		{"accounts.google.com:example@gmail.com", "example@gmail.com"},
		{"", ""},
	}
	for _, c := range cases {
		got := getEmailFromString(c.in)
		if got != c.want {
			t.Errorf("getEmailFromString(%s)  got %s, want %s", c.in, got, c.want)
		}

	}
}

// func TestGetQueries(t *testing.T) {
// 	emptyreq, _ := http.NewRequest("GET", "/", nil)
// 	req, _ := http.NewRequest("GET", "/?g=12345678&email=test@example.com", nil)

// 	postreq, _ := http.NewRequest("POST", "/", strings.NewReader("g=12345678"))
// 	postreq.Header.Set("Content-Type", "application/x-www-form-urlencoded")

// 	var table = []struct {
// 		inReq    *http.Request
// 		inParam  string
// 		outErr   error
// 		outParam string
// 		outOK    bool
// 	}{
// 		{req, "g", nil, "12345678", true},
// 		{emptyreq, "g", fmt.Errorf("query parameter '%s' is missing", "g"), "", false},
// 		{req, "email", nil, "test@example.com", true},
// 		{emptyreq, "email", fmt.Errorf("query parameter '%s' is missing", "email"), "", false},
// 		{req, "name", fmt.Errorf("query parameter '%s' is missing", "name"), "", false},
// 		{postreq, "g", nil, "12345678", true},
// 		{postreq, "name", fmt.Errorf("query parameter '%s' is missing", "name"), "", false},
// 	}

// 	for _, v := range table {
// 		result, err := getQueries(v.inReq, v.inParam)

// 		// Was having a weird condition where comparisonwas always wrong despite
// 		// beig the exact same thing.
// 		errText := fmt.Sprintf("%s", err)
// 		wantText := fmt.Sprintf("%s", v.outErr)
// 		if !(errText == wantText) {
// 			t.Errorf("getQueries()  got '%+v', want '%+v'", err, v.outErr)
// 		}

// 		got, ok := result[v.inParam]
// 		if ok != v.outOK {
// 			t.Errorf("getQueries()  got '%t', want '%t'", ok, v.outOK)
// 		}

// 		if got != v.outParam {
// 			t.Errorf("getQueries()  got '%s', want '%s'", got, v.outParam)
// 		}
// 	}
// }

func TestSimpleHandlers(t *testing.T) {

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/healthz", `{"msg":"ok"}`, http.HandlerFunc(handleHealth)},
		{"/api/player/identify", fmt.Sprintf(`{"name":"","email":"%s@google.com"}`, os.Getenv("USER")), JSONHandler(iapUsernameGetHandle, "none")},
		{"/api/player/isadmin", "false", AdminHandler(isAdminHandle)},
		{"/api/admin/list", "[]", JSONHandler(adminListHandle, "none")},
		{"/api/game/list", "[]", JSONHandler(gameListHandle, "none")},
		{"/api/player/game/list", "[]", JSONHandler(playerGameListHandle, "none")},
		{"/api/cache/clear", `{"msg":"ok"}`, SimpleHandler(clearCacheHandle, "none")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusOK {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusOK)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}

func TestGlobalAuthErrorsGetHandlers(t *testing.T) {

	errmsg := fmt.Sprintf(`{"error":"%s"}`, ErrNotAdmin)

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/api/admin/list", errmsg, JSONHandler(adminListHandle, "global")},
		{"/api/game/list", errmsg, JSONHandler(gameListHandle, "global")},
		{"/api/cache/clear", errmsg, SimpleHandler(clearCacheHandle, "global")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusForbidden {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusForbidden)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}

func TestGlobalAuthSuccessGetHandlers(t *testing.T) {

	player1 := Player{"", fmt.Sprintf("%s@google.com", os.Getenv("USER"))}
	player2 := Player{"", fmt.Sprintf("%s@google.com", "other")}

	game1, err := getNewGame("Test Game 1", player1)
	if err != nil {
		t.Errorf("error in setting up games for testing %v", err)
	}

	game2, err := getNewGame("Test Game 2", player2)
	if err != nil {
		t.Errorf("error in setting up games for testing %v", err)
	}

	if err := a.AddAdmin(player1); err != nil {
		t.Errorf("error in setting up adin for testing %v", err)
	}

	games := Games{}
	games.Add(game1)
	games.Add(game2)

	gamejson, err := games.JSON()
	if err != nil {
		t.Errorf("error in setting up games json for testing %v", err)
	}

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/api/admin/list", fmt.Sprintf(`[{"name":"","email":"%s"}]`, player1.Email), JSONHandler(adminListHandle, "global")},
		{"/api/game/list", gamejson, JSONHandler(gameListHandle, "global")},
		{"/api/cache/clear", `{"msg":"ok"}`, SimpleHandler(clearCacheHandle, "global")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusOK {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusOK)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}
<commit_msg>updated tests to work.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"testing"
)

func TestMain(m *testing.M) {
	// command to start firestore emulator
	cmd := exec.Command("gcloud", "beta", "emulators", "firestore", "start", fmt.Sprintf("--host-port=localhost:%d", 8181), "--quiet")

	// this makes it killable
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// we need to capture it's output to know when it's started
	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	defer stderr.Close()

	// start her up!
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// ensure the process is killed when we're finished, even if an error occurs
	// (thanks to Brian Moran for suggestion)
	var result int
	defer func() {
		syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
		os.Exit(result)
	}()

	// we're going to wait until it's running to start
	var wg sync.WaitGroup
	wg.Add(1)

	// by starting a separate go routine
	go func() {
		// reading it's output
		buf := make([]byte, 256, 256)
		for {
			n, err := stderr.Read(buf[:])
			if err != nil {
				// until it ends
				if err == io.EOF {
					break
				}
				log.Fatalf("reading stderr %v", err)
			}

			if n > 0 {
				d := string(buf[:n])

				// only required if we want to see the emulator output
				log.Printf("%s", d)

				// checking for the message that it's started
				if strings.Contains(d, "Dev App Server is now running") {
					wg.Done()
				}

				// and capturing the FIRESTORE_EMULATOR_HOST value to set
				pos := strings.Index(d, FirestoreEmulatorHost+"=")
				if pos > 0 {
					host := d[pos+len(FirestoreEmulatorHost)+1 : len(d)-1]
					os.Setenv(FirestoreEmulatorHost, host)
				}
			}
		}
	}()

	// wait until the running message has been received
	wg.Wait()

	agentTestSetup()
	cacheTestSetup()
	noisy = false

	// now it's running, we can run our unit tests
	result = m.Run()
}

func TestGetUsername(t *testing.T) {
	cases := []struct {
		in   string
		want string
	}{
		{"accounts.google.com:example@gmail.com", "example@gmail.com"},
		{"", ""},
	}
	for _, c := range cases {
		got := getEmailFromString(c.in)
		if got != c.want {
			t.Errorf("getEmailFromString(%s)  got %s, want %s", c.in, got, c.want)
		}

	}
}

// func TestGetQueries(t *testing.T) {
// 	emptyreq, _ := http.NewRequest("GET", "/", nil)
// 	req, _ := http.NewRequest("GET", "/?g=12345678&email=test@example.com", nil)

// 	postreq, _ := http.NewRequest("POST", "/", strings.NewReader("g=12345678"))
// 	postreq.Header.Set("Content-Type", "application/x-www-form-urlencoded")

// 	var table = []struct {
// 		inReq    *http.Request
// 		inParam  string
// 		outErr   error
// 		outParam string
// 		outOK    bool
// 	}{
// 		{req, "g", nil, "12345678", true},
// 		{emptyreq, "g", fmt.Errorf("query parameter '%s' is missing", "g"), "", false},
// 		{req, "email", nil, "test@example.com", true},
// 		{emptyreq, "email", fmt.Errorf("query parameter '%s' is missing", "email"), "", false},
// 		{req, "name", fmt.Errorf("query parameter '%s' is missing", "name"), "", false},
// 		{postreq, "g", nil, "12345678", true},
// 		{postreq, "name", fmt.Errorf("query parameter '%s' is missing", "name"), "", false},
// 	}

// 	for _, v := range table {
// 		result, err := getQueries(v.inReq, v.inParam)

// 		// Was having a weird condition where comparisonwas always wrong despite
// 		// beig the exact same thing.
// 		errText := fmt.Sprintf("%s", err)
// 		wantText := fmt.Sprintf("%s", v.outErr)
// 		if !(errText == wantText) {
// 			t.Errorf("getQueries()  got '%+v', want '%+v'", err, v.outErr)
// 		}

// 		got, ok := result[v.inParam]
// 		if ok != v.outOK {
// 			t.Errorf("getQueries()  got '%t', want '%t'", ok, v.outOK)
// 		}

// 		if got != v.outParam {
// 			t.Errorf("getQueries()  got '%s', want '%s'", got, v.outParam)
// 		}
// 	}
// }

func TestSimpleHandlers(t *testing.T) {

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/healthz", `{"msg":"ok"}`, http.HandlerFunc(handleHealth)},
		{"/api/player/identify", fmt.Sprintf(`{"name":"","email":"%s@google.com"}`, os.Getenv("USER")), JSONHandler(iapUsernameGetHandle, "none")},
		{"/api/player/isadmin", "false", AdminHandler(isAdminHandle)},
		{"/api/admin/list", "[]", JSONHandler(adminListHandle, "none")},
		{"/api/game/list?l=5&t=1000000", "[]", JSONHandler(gameListHandle, "none")},
		{"/api/player/game/list", "[]", JSONHandler(playerGameListHandle, "none")},
		{"/api/cache/clear", `{"msg":"ok"}`, SimpleHandler(clearCacheHandle, "none")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusOK {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusOK)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}

func TestGlobalAuthErrorsGetHandlers(t *testing.T) {

	errmsg := fmt.Sprintf(`{"error":"%s"}`, ErrNotAdmin)

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/api/admin/list", errmsg, JSONHandler(adminListHandle, "global")},
		{"/api/game/list", errmsg, JSONHandler(gameListHandle, "global")},
		{"/api/cache/clear", errmsg, SimpleHandler(clearCacheHandle, "global")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusForbidden {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusForbidden)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}

func TestGlobalAuthSuccessGetHandlers(t *testing.T) {

	player1 := Player{"", fmt.Sprintf("%s@google.com", os.Getenv("USER"))}
	player2 := Player{"", fmt.Sprintf("%s@google.com", "other")}

	game1, err := getNewGame("Test Game 1", player1)
	if err != nil {
		t.Errorf("error in setting up games for testing %v", err)
	}

	game2, err := getNewGame("Test Game 2", player2)
	if err != nil {
		t.Errorf("error in setting up games for testing %v", err)
	}

	if err := a.AddAdmin(player1); err != nil {
		t.Errorf("error in setting up adin for testing %v", err)
	}

	games := Games{}
	games.Add(game1)
	games.Add(game2)

	var table = []struct {
		in      string
		out     string
		handler http.Handler
	}{
		{"/api/admin/list", fmt.Sprintf(`[{"name":"","email":"%s"}]`, player1.Email), JSONHandler(adminListHandle, "global")},
		{"/api/cache/clear", `{"msg":"ok"}`, SimpleHandler(clearCacheHandle, "global")},
	}

	for _, v := range table {

		// Create a request to pass to our handler. We don't have any query parameters for now, so we'll
		// pass 'nil' as the third parameter.
		req, err := http.NewRequest("GET", v.in, nil)
		if err != nil {
			t.Fatal(err)
		}

		// We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response.
		rr := httptest.NewRecorder()

		// Our handlers satisfy http.Handler, so we can call their ServeHTTP method
		// directly and pass in our Request and ResponseRecorder.
		v.handler.ServeHTTP(rr, req)

		// Check the status code is what we expect.
		if status := rr.Code; status != http.StatusOK {
			t.Errorf("handler returned wrong status code: got %v want %v",
				status, http.StatusOK)
		}

		// Check the response body is what we expect.
		expected := v.out
		if rr.Body.String() != expected {
			t.Errorf("handler returned unexpected body: got %v want %v",
				rr.Body.String(), expected)
		}

	}
}
<|endoftext|>


<commit_before>package transfer

import (
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/disorganizer/brig/id"
	"github.com/disorganizer/brig/repo"
	"github.com/disorganizer/brig/transfer/wire"
	"github.com/disorganizer/brig/util"
	"github.com/disorganizer/brig/util/ipfsutil"
)

type Connector struct {
	layer Layer

	// Open repo. required for answering requests.
	// (might be nil for tests if no handlers are tested)
	rp *repo.Repository

	// Map of open conversations
	open map[id.ID]Conversation

	// Map from hash id to last seen timestamp
	heartbeat map[id.ID]*ipfsutil.Pinger

	// lock for `open`
	mu sync.Mutex
}

// dialer uses ipfs to create a net.Conn to another node.
type dialer struct {
	layer Layer
	node  *ipfsutil.Node
}

func (d *dialer) Dial(peer id.Peer) (net.Conn, error) {
	return d.node.Dial(peer.Hash(), d.layer.ProtocolID())
}

// listenerFilter filters
type listenerFilter struct {
	ls   net.Listener
	rms  repo.RemoteStore
	quit bool
}

func newListenerFilter(ls net.Listener, rms repo.RemoteStore) *listenerFilter {
	return &listenerFilter{
		ls:  ls,
		rms: rms,
	}
}

func (lf *listenerFilter) Accept() (net.Conn, error) {
	for !lf.quit {
		conn, err := lf.ls.Accept()
		if err != nil {
			return nil, err
		}

		streamConn, ok := conn.(*ipfsutil.StreamConn)
		if !ok {
			// TODO
			return nil, fmt.Errorf("Not used with ipfs listener?")
		}

		hash := streamConn.PeerHash()

		// Check if we know of this hash:
		for remote := range lf.rms.Iter() {
			if remote.Hash() == hash {
				return streamConn, nil
			}
		}

		log.Warningf("Denying incoming connection from `%s`", hash)
	}

	return nil, fmt.Errorf("Listener was closed")
}

func (lf *listenerFilter) Close() error {
	lf.quit = true
	return lf.ls.Close()
}

func (lf *listenerFilter) Addr() net.Addr {
	return lf.Addr()
}

// NewConnector returns an unconnected Connector.
func NewConnector(layer Layer, rp *repo.Repository) *Connector {
	// TODO: pass authMgr.
	// authMgr := MockAuthSuccess
	cnc := &Connector{
		rp:        rp,
		layer:     layer,
		open:      make(map[id.ID]Conversation),
		heartbeat: make(map[id.ID]*ipfsutil.Pinger),
	}

	handlerMap := map[wire.RequestType]HandlerFunc{
		wire.RequestType_FETCH:       cnc.handleFetch,
		wire.RequestType_UPDATE_FILE: cnc.handleUpdateFile,
	}

	for typ, handler := range handlerMap {
		layer.RegisterHandler(typ, handler)
	}

	return cnc
}

func partnerIsAllowed(rms repo.RemoteStore, ID id.ID) error {
	_, err := rms.Get(ID)
	if err != nil {
		return err
	}

	return nil
}

func (cn *Connector) Dial(peer id.Peer) (*APIClient, error) {
	if !cn.IsInOnlineMode() {
		return nil, ErrOffline
	}

	if err := partnerIsAllowed(cn.rp.Remotes, peer.ID()); err != nil {
		return nil, err
	}

	// Lookup if a conversation was already established:
	cn.mu.Lock()
	cnv, ok := cn.open[peer.ID()]
	cn.mu.Unlock()

	if ok {
		return newAPIClient(cnv, cn.rp.IPFS)
	}

	cnv, err := cn.layer.Dial(peer)
	if err != nil {
		return nil, err
	}

	// Remember conversation:
	cn.mu.Lock()
	cn.open[peer.ID()] = cnv
	cn.mu.Unlock()

	return newAPIClient(cnv, cn.rp.IPFS)
}

func (cn *Connector) IsOnline(peer id.Peer) bool {
	if !cn.IsInOnlineMode() {
		return false
	}

	cn.mu.Lock()
	defer cn.mu.Unlock()

	pinger, ok := cn.heartbeat[peer.ID()]
	if !ok {
		var err error

		pinger, err = cn.rp.IPFS.Ping(peer.Hash())
		if err != nil {
			return false
		}

		cn.heartbeat[peer.ID()] = pinger
	}

	if time.Since(pinger.LastSeen()) < 5*time.Second {
		return true
	}

	// If creating the pinger worked, remote should be online.
	return true
}

func (cn *Connector) Broadcast(req *wire.Request) error {
	var errs util.Errors

	cn.mu.Lock()
	defer cn.mu.Unlock()

	for _, cnv := range cn.open {
		if err := cnv.SendAsync(req, nil); err != nil {
			errs = append(errs, err)
		}
	}

	return errs
}

func (cn *Connector) Layer() Layer {
	cn.mu.Lock()
	defer cn.mu.Unlock()

	return cn.layer
}

func (cn *Connector) Connect() error {
	ls, err := cn.rp.IPFS.Listen(cn.layer.ProtocolID())
	if err != nil {
		return err
	}

	go func() {
		for remote := range cn.rp.Remotes.Iter() {
			cnv, err := cn.layer.Dial(remote)
			if err != nil {
				log.Warningf("Could not connect to `%s`: %v", remote.ID(), err)
				continue
			}

			cn.mu.Lock()
			cn.open[remote.ID()] = cnv
			cn.mu.Unlock()
		}
	}()

	return cn.layer.Connect(ls, &dialer{cn.layer, cn.rp.IPFS})
}

func (cn *Connector) Disconnect() error {
	var errs util.Errors

	cn.mu.Lock()
	defer cn.mu.Unlock()

	for _, cnv := range cn.open {
		peer := cnv.Peer()

		delete(cn.open, peer.ID())
		delete(cn.heartbeat, peer.ID())

		if err := cnv.Close(); err != nil {
			errs = append(errs, err)
		}
	}

	return cn.layer.Disconnect()
}

func (cn *Connector) Close() error {
	return cn.Disconnect()
}

func (cn *Connector) IsInOnlineMode() bool {
	cn.mu.Lock()
	defer cn.mu.Unlock()

	return cn.layer.IsInOnlineMode()
}
<commit_msg>transfer/connector.go: Actually use ListenFilter<commit_after>package transfer

import (
	"fmt"
	"net"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/disorganizer/brig/id"
	"github.com/disorganizer/brig/repo"
	"github.com/disorganizer/brig/transfer/wire"
	"github.com/disorganizer/brig/util"
	"github.com/disorganizer/brig/util/ipfsutil"
)

type Connector struct {
	layer Layer

	// Open repo. required for answering requests.
	// (might be nil for tests if no handlers are tested)
	rp *repo.Repository

	// Map of open conversations
	open map[id.ID]Conversation

	// Map from hash id to last seen timestamp
	heartbeat map[id.ID]*ipfsutil.Pinger

	// lock for `open`
	mu sync.Mutex
}

// dialer uses ipfs to create a net.Conn to another node.
type dialer struct {
	layer Layer
	node  *ipfsutil.Node
}

func (d *dialer) Dial(peer id.Peer) (net.Conn, error) {
	return d.node.Dial(peer.Hash(), d.layer.ProtocolID())
}

// listenerFilter filters
type listenerFilter struct {
	ls   net.Listener
	rms  repo.RemoteStore
	quit bool
}

func newListenerFilter(ls net.Listener, rms repo.RemoteStore) *listenerFilter {
	return &listenerFilter{
		ls:  ls,
		rms: rms,
	}
}

func (lf *listenerFilter) Accept() (net.Conn, error) {
	for !lf.quit {
		conn, err := lf.ls.Accept()
		if err != nil {
			return nil, err
		}

		streamConn, ok := conn.(*ipfsutil.StreamConn)
		if !ok {
			// TODO
			return nil, fmt.Errorf("Not used with ipfs listener?")
		}

		hash := streamConn.PeerHash()

		// Check if we know of this hash:
		for remote := range lf.rms.Iter() {
			if remote.Hash() == hash {
				return streamConn, nil
			}
		}

		log.Warningf("Denying incoming connection from `%s`", hash)
	}

	return nil, fmt.Errorf("Listener was closed")
}

func (lf *listenerFilter) Close() error {
	lf.quit = true
	return lf.ls.Close()
}

func (lf *listenerFilter) Addr() net.Addr {
	return lf.Addr()
}

// NewConnector returns an unconnected Connector.
func NewConnector(layer Layer, rp *repo.Repository) *Connector {
	// TODO: pass authMgr.
	// authMgr := MockAuthSuccess
	cnc := &Connector{
		rp:        rp,
		layer:     layer,
		open:      make(map[id.ID]Conversation),
		heartbeat: make(map[id.ID]*ipfsutil.Pinger),
	}

	handlerMap := map[wire.RequestType]HandlerFunc{
		wire.RequestType_FETCH:       cnc.handleFetch,
		wire.RequestType_UPDATE_FILE: cnc.handleUpdateFile,
	}

	for typ, handler := range handlerMap {
		layer.RegisterHandler(typ, handler)
	}

	return cnc
}

func partnerIsAllowed(rms repo.RemoteStore, ID id.ID) error {
	_, err := rms.Get(ID)
	if err != nil {
		return err
	}

	return nil
}

func (cn *Connector) Dial(peer id.Peer) (*APIClient, error) {
	if !cn.IsInOnlineMode() {
		return nil, ErrOffline
	}

	if err := partnerIsAllowed(cn.rp.Remotes, peer.ID()); err != nil {
		return nil, err
	}

	// Lookup if a conversation was already established:
	cn.mu.Lock()
	cnv, ok := cn.open[peer.ID()]
	cn.mu.Unlock()

	if ok {
		return newAPIClient(cnv, cn.rp.IPFS)
	}

	cnv, err := cn.layer.Dial(peer)
	if err != nil {
		return nil, err
	}

	// Remember conversation:
	cn.mu.Lock()
	cn.open[peer.ID()] = cnv
	cn.mu.Unlock()

	return newAPIClient(cnv, cn.rp.IPFS)
}

func (cn *Connector) IsOnline(peer id.Peer) bool {
	if !cn.IsInOnlineMode() {
		return false
	}

	cn.mu.Lock()
	defer cn.mu.Unlock()

	pinger, ok := cn.heartbeat[peer.ID()]
	if !ok {
		var err error

		pinger, err = cn.rp.IPFS.Ping(peer.Hash())
		if err != nil {
			return false
		}

		cn.heartbeat[peer.ID()] = pinger
	}

	if time.Since(pinger.LastSeen()) < 5*time.Second {
		return true
	}

	// If creating the pinger worked, remote should be online.
	return true
}

func (cn *Connector) Broadcast(req *wire.Request) error {
	var errs util.Errors

	cn.mu.Lock()
	defer cn.mu.Unlock()

	for _, cnv := range cn.open {
		if err := cnv.SendAsync(req, nil); err != nil {
			errs = append(errs, err)
		}
	}

	return errs
}

func (cn *Connector) Layer() Layer {
	cn.mu.Lock()
	defer cn.mu.Unlock()

	return cn.layer
}

func (cn *Connector) Connect() error {
	ls, err := cn.rp.IPFS.Listen(cn.layer.ProtocolID())
	if err != nil {
		return err
	}

	// Make sure we filter unauthorized incoming connections:
	filter := newListenerFilter(ls, cn.rp.Remotes)

	go func() {
		for remote := range cn.rp.Remotes.Iter() {
			cnv, err := cn.layer.Dial(remote)
			if err != nil {
				log.Warningf("Could not connect to `%s`: %v", remote.ID(), err)
				continue
			}

			cn.mu.Lock()
			cn.open[remote.ID()] = cnv
			cn.mu.Unlock()
		}
	}()

	return cn.layer.Connect(filter, &dialer{cn.layer, cn.rp.IPFS})
}

func (cn *Connector) Disconnect() error {
	var errs util.Errors

	cn.mu.Lock()
	defer cn.mu.Unlock()

	for _, cnv := range cn.open {
		peer := cnv.Peer()

		delete(cn.open, peer.ID())
		delete(cn.heartbeat, peer.ID())

		if err := cnv.Close(); err != nil {
			errs = append(errs, err)
		}
	}

	return cn.layer.Disconnect()
}

func (cn *Connector) Close() error {
	return cn.Disconnect()
}

func (cn *Connector) IsInOnlineMode() bool {
	cn.mu.Lock()
	defer cn.mu.Unlock()

	return cn.layer.IsInOnlineMode()
}
<|endoftext|>


<commit_before>package wav

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"time"

	"github.com/go-audio/audio"
	"github.com/mattetti/audio/riff"
)

// Decoder handles the decoding of wav files.
type Decoder struct {
	r      io.ReadSeeker
	parser *riff.Parser

	NumChans   uint16
	BitDepth   uint16
	SampleRate uint32

	AvgBytesPerSec uint32
	WavAudioFormat uint16

	err             error
	PCMSize         int
	pcmDataAccessed bool
	// pcmChunk is available so we can use the LimitReader
	PCMChunk *riff.Chunk
}

// NewDecoder creates a decoder for the passed wav reader.
// Note that the reader doesn't get rewinded as the container is processed.
func NewDecoder(r io.ReadSeeker) *Decoder {
	return &Decoder{
		r:      r,
		parser: riff.New(r),
	}
}

// SampleBitDepth returns the bit depth encoding of each sample.
func (d *Decoder) SampleBitDepth() int32 {
	if d == nil {
		return 0
	}
	return int32(d.BitDepth)
}

// PCMLen returns the total number of bytes in the PCM data chunk
func (d *Decoder) PCMLen() int64 {
	if d == nil {
		return 0
	}
	return int64(d.PCMSize)
}

// Err returns the first non-EOF error that was encountered by the Decoder.
func (d *Decoder) Err() error {
	if d.err == io.EOF {
		return nil
	}
	return d.err
}

// EOF returns positively if the underlying reader reached the end of file.
func (d *Decoder) EOF() bool {
	if d == nil || d.err == io.EOF {
		return true
	}
	return false
}

// IsValidFile verifies that the file is valid/readable.
func (d *Decoder) IsValidFile() bool {
	d.err = d.readHeaders()
	if d.err != nil {
		return false
	}
	if d.NumChans < 1 {
		return false
	}
	if d.BitDepth < 8 {
		return false
	}
	if d, err := d.Duration(); err != nil || d <= 0 {
		return false
	}

	return true
}

// ReadInfo reads the underlying reader until the comm header is parsed.
// This method is safe to call multiple times.
func (d *Decoder) ReadInfo() {
	d.err = d.readHeaders()
}

// Reset resets the decoder (and rewind the underlying reader)
func (d *Decoder) Reset() {
	d.err = nil
	d.pcmDataAccessed = false
	d.NumChans = 0
	d.BitDepth = 0
	d.SampleRate = 0
	d.AvgBytesPerSec = 0
	d.WavAudioFormat = 0
	d.PCMSize = 0
	d.r.Seek(0, 0)
	d.PCMChunk = nil
	d.parser = riff.New(d.r)
}

// FwdToPCM forwards the underlying reader until the start of the PCM chunk.
// If the PCM chunk was already read, no data will be found (you need to rewind).
func (d *Decoder) FwdToPCM() error {
	if d == nil {
		return fmt.Errorf("PCM data not found")
	}
	d.err = d.readHeaders()
	if d.err != nil {
		return nil
	}

	var chunk *riff.Chunk
	for d.err == nil {
		chunk, d.err = d.NextChunk()
		if d.err != nil {
			return d.err
		}
		if chunk.ID == riff.DataFormatID {
			d.PCMSize = chunk.Size
			d.PCMChunk = chunk
			break
		}
		chunk.Drain()
	}
	if chunk == nil {
		return fmt.Errorf("PCM data not found")
	}
	d.pcmDataAccessed = true

	return nil
}

// WasPCMAccessed returns positively if the PCM data was previously accessed.
func (d *Decoder) WasPCMAccessed() bool {
	if d == nil {
		return false
	}
	return d.pcmDataAccessed
}

// FullPCMBuffer is an inneficient way to access all the PCM data contained in the
// audio container. The entire PCM data is held in memory.
// Consider using Buffer() instead.
func (d *Decoder) FullPCMBuffer() (*audio.IntBuffer, error) {
	if !d.WasPCMAccessed() {
		err := d.FwdToPCM()
		if err != nil {
			return nil, d.err
		}
	}
	if d.PCMChunk == nil {
		return nil, errors.New("PCM chunk not found")
	}
	format := &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}

	buf := &audio.IntBuffer{Data: make([]int, 4096), Format: format, SourceBitDepth: int(d.BitDepth)}
	bytesPerSample := (d.BitDepth-1)/8 + 1
	sampleBufData := make([]byte, bytesPerSample)
	decodeF, err := sampleDecodeFunc(int(d.BitDepth))
	if err != nil {
		return nil, fmt.Errorf("could not get sample decode func %v", err)
	}

	i := 0
	for err == nil {
		buf.Data[i], err = decodeF(d.PCMChunk, sampleBufData)
		if err != nil {
			break
		}
		i++
		// grow the underlying slice if needed
		if i == len(buf.Data) {
			buf.Data = append(buf.Data, make([]int, 4096)...)
		}
	}
	buf.Data = buf.Data[:i]

	if err == io.EOF {
		err = nil
	}

	return buf, err
}

// PCMBuffer populates the passed PCM buffer
func (d *Decoder) PCMBuffer(buf *audio.IntBuffer) (n int, err error) {
	if buf == nil {
		return 0, nil
	}

	if !d.pcmDataAccessed {
		err := d.FwdToPCM()
		if err != nil {
			return 0, d.err
		}
	}

	format := &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}

	buf.SourceBitDepth = int(d.BitDepth)
	decodeF, err := sampleDecodeFunc(int(d.BitDepth))
	if err != nil {
		return 0, fmt.Errorf("could not get sample decode func %v", err)
	}

	// populate a file buffer to avoid multiple very small reads
	// we need to cap the buffer size to not be bigger than the pcm chunk.
	size := len(buf.Data) * (int(d.BitDepth) / 8)
	tmpBuf := make([]byte, size)
	var m int
	m, err = d.PCMChunk.R.Read(tmpBuf)
	if err != nil {
		if err == io.EOF {
			return m, nil
		}
		return m, err
	}
	if m == 0 {
		return m, nil
	}
	bufR := bytes.NewReader(tmpBuf[:m])
	sampleBuf := make([]byte, 4, 4)

	// Note that we populate the buffer even if the
	// size of the buffer doesn't fit an even number of frames.
	for n = 0; n < len(buf.Data); n++ {
		buf.Data[n], err = decodeF(bufR, sampleBuf)
		if err != nil {
			break
		}
	}
	buf.Format = format
	if err == io.EOF {
		err = nil
	}

	return n, err
}

// Format returns the audio format of the decoded content.
func (d *Decoder) Format() *audio.Format {
	if d == nil {
		return nil
	}
	return &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}
}

// NextChunk returns the next available chunk
func (d *Decoder) NextChunk() (*riff.Chunk, error) {
	if d.err = d.readHeaders(); d.err != nil {
		d.err = fmt.Errorf("failed to read header - %v", d.err)
		return nil, d.err
	}

	var (
		id   [4]byte
		size uint32
	)

	id, size, d.err = d.parser.IDnSize()
	if d.err != nil {
		d.err = fmt.Errorf("error reading chunk header - %v", d.err)
		return nil, d.err
	}

	c := &riff.Chunk{
		ID:   id,
		Size: int(size),
		R:    io.LimitReader(d.r, int64(size)),
	}
	return c, d.err
}

// Duration returns the time duration for the current audio container
func (d *Decoder) Duration() (time.Duration, error) {
	if d == nil || d.parser == nil {
		return 0, errors.New("can't calculate the duration of a nil pointer")
	}
	return d.parser.Duration()
}

// String implements the Stringer interface.
func (d *Decoder) String() string {
	return d.parser.String()
}

// readHeaders is safe to call multiple times
func (d *Decoder) readHeaders() error {
	if d == nil || d.NumChans > 0 {
		return nil
	}

	id, size, err := d.parser.IDnSize()
	if err != nil {
		return err
	}
	d.parser.ID = id
	if d.parser.ID != riff.RiffID {
		return fmt.Errorf("%s - %s", d.parser.ID, riff.ErrFmtNotSupported)
	}
	d.parser.Size = size
	if err := binary.Read(d.r, binary.BigEndian, &d.parser.Format); err != nil {
		return err
	}

	var chunk *riff.Chunk
	var rewindBytes int64

	for err == nil {
		chunk, err = d.parser.NextChunk()
		if err != nil {
			break
		}

		if chunk.ID == riff.FmtID {
			chunk.DecodeWavHeader(d.parser)
			d.NumChans = d.parser.NumChannels
			d.BitDepth = d.parser.BitsPerSample
			d.SampleRate = d.parser.SampleRate
			d.WavAudioFormat = d.parser.WavAudioFormat
			d.AvgBytesPerSec = d.parser.AvgBytesPerSec

			if rewindBytes > 0 {
				d.r.Seek(-(rewindBytes + int64(chunk.Size) + 8), 1)
			}
			break
		} else {
			// unexpected chunk order, might be a bext chunk
			rewindBytes += int64(chunk.Size) + 8
			// drain the chunk
			io.CopyN(ioutil.Discard, d.r, int64(chunk.Size))
		}

	}

	return d.err
}

// sampleDecodeFunc returns a function that can be used to convert
// a byte range into an int value based on the amount of bits used per sample.
// Note that 8bit samples are unsigned, all other values are signed.
func sampleDecodeFunc(bitsPerSample int) (func(io.Reader, []byte) (int, error), error) {
	// NOTE: WAV PCM data is stored using little-endian
	switch bitsPerSample {
	case 8:
		// 8bit values are unsigned
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:1])
			return int(buf[0]), err
		}, nil
	case 16:
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:2])
			return int(int16(binary.LittleEndian.Uint16(buf[:2]))), err
		}, nil
	case 24:
		// -34,359,738,367 (0x7FFFFF) to 34,359,738,368	(0x800000)
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:3])
			if err != nil {
				return 0, err
			}
			return int(audio.Int24LETo32(buf[:3])), nil
		}, nil
	case 32:
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:4])
			return int(int32(binary.LittleEndian.Uint32(buf[:4]))), err
		}, nil
	default:
		return nil, fmt.Errorf("unhandled byte depth:%d", bitsPerSample)
	}
}

// sampleDecodeFloat64Func returns a function that can be used to convert
// a byte range into a float64 value based on the amount of bits used per sample.
func sampleFloat64DecodeFunc(bitsPerSample int) (func([]byte) float64, error) {
	bytesPerSample := bitsPerSample / 8
	switch bytesPerSample {
	case 1:
		// 8bit values are unsigned
		return func(s []byte) float64 {
			return float64(uint8(s[0]))
		}, nil
	case 2:
		return func(s []byte) float64 {
			return float64(int(s[0]) + int(s[1])<<8)
		}, nil
	case 3:
		return func(s []byte) float64 {
			var output int32
			output |= int32(s[2]) << 0
			output |= int32(s[1]) << 8
			output |= int32(s[0]) << 16
			return float64(output)
		}, nil
	case 4:
		// TODO: fix the float64 conversion (current int implementation)
		return func(s []byte) float64 {
			return float64(int(s[0]) + int(s[1])<<8 + int(s[2])<<16 + int(s[3])<<24)
		}, nil
	default:
		return nil, fmt.Errorf("unhandled byte depth:%d", bitsPerSample)
	}
}
<commit_msg>fix reading 24b padded samples (IOU a test)<commit_after>package wav

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"time"

	"github.com/go-audio/audio"
	"github.com/mattetti/audio/riff"
)

// Decoder handles the decoding of wav files.
type Decoder struct {
	r      io.ReadSeeker
	parser *riff.Parser

	NumChans   uint16
	BitDepth   uint16
	SampleRate uint32

	AvgBytesPerSec uint32
	WavAudioFormat uint16

	err             error
	PCMSize         int
	pcmDataAccessed bool
	// pcmChunk is available so we can use the LimitReader
	PCMChunk *riff.Chunk
}

// NewDecoder creates a decoder for the passed wav reader.
// Note that the reader doesn't get rewinded as the container is processed.
func NewDecoder(r io.ReadSeeker) *Decoder {
	return &Decoder{
		r:      r,
		parser: riff.New(r),
	}
}

// SampleBitDepth returns the bit depth encoding of each sample.
func (d *Decoder) SampleBitDepth() int32 {
	if d == nil {
		return 0
	}
	return int32(d.BitDepth)
}

// PCMLen returns the total number of bytes in the PCM data chunk
func (d *Decoder) PCMLen() int64 {
	if d == nil {
		return 0
	}
	return int64(d.PCMSize)
}

// Err returns the first non-EOF error that was encountered by the Decoder.
func (d *Decoder) Err() error {
	if d.err == io.EOF {
		return nil
	}
	return d.err
}

// EOF returns positively if the underlying reader reached the end of file.
func (d *Decoder) EOF() bool {
	if d == nil || d.err == io.EOF {
		return true
	}
	return false
}

// IsValidFile verifies that the file is valid/readable.
func (d *Decoder) IsValidFile() bool {
	d.err = d.readHeaders()
	if d.err != nil {
		return false
	}
	if d.NumChans < 1 {
		return false
	}
	if d.BitDepth < 8 {
		return false
	}
	if d, err := d.Duration(); err != nil || d <= 0 {
		return false
	}

	return true
}

// ReadInfo reads the underlying reader until the comm header is parsed.
// This method is safe to call multiple times.
func (d *Decoder) ReadInfo() {
	d.err = d.readHeaders()
}

// Reset resets the decoder (and rewind the underlying reader)
func (d *Decoder) Reset() {
	d.err = nil
	d.pcmDataAccessed = false
	d.NumChans = 0
	d.BitDepth = 0
	d.SampleRate = 0
	d.AvgBytesPerSec = 0
	d.WavAudioFormat = 0
	d.PCMSize = 0
	d.r.Seek(0, 0)
	d.PCMChunk = nil
	d.parser = riff.New(d.r)
}

// FwdToPCM forwards the underlying reader until the start of the PCM chunk.
// If the PCM chunk was already read, no data will be found (you need to rewind).
func (d *Decoder) FwdToPCM() error {
	if d == nil {
		return fmt.Errorf("PCM data not found")
	}
	d.err = d.readHeaders()
	if d.err != nil {
		return nil
	}

	var chunk *riff.Chunk
	for d.err == nil {
		chunk, d.err = d.NextChunk()
		if d.err != nil {
			return d.err
		}
		if chunk.ID == riff.DataFormatID {
			d.PCMSize = chunk.Size
			d.PCMChunk = chunk
			break
		}
		chunk.Drain()
	}
	if chunk == nil {
		return fmt.Errorf("PCM data not found")
	}
	d.pcmDataAccessed = true

	return nil
}

// WasPCMAccessed returns positively if the PCM data was previously accessed.
func (d *Decoder) WasPCMAccessed() bool {
	if d == nil {
		return false
	}
	return d.pcmDataAccessed
}

// FullPCMBuffer is an inneficient way to access all the PCM data contained in the
// audio container. The entire PCM data is held in memory.
// Consider using Buffer() instead.
func (d *Decoder) FullPCMBuffer() (*audio.IntBuffer, error) {
	if !d.WasPCMAccessed() {
		err := d.FwdToPCM()
		if err != nil {
			return nil, d.err
		}
	}
	if d.PCMChunk == nil {
		return nil, errors.New("PCM chunk not found")
	}
	format := &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}

	buf := &audio.IntBuffer{Data: make([]int, 4096), Format: format, SourceBitDepth: int(d.BitDepth)}
	bytesPerSample := (d.BitDepth-1)/8 + 1
	sampleBufData := make([]byte, bytesPerSample)
	decodeF, err := sampleDecodeFunc(int(d.BitDepth))
	if err != nil {
		return nil, fmt.Errorf("could not get sample decode func %v", err)
	}

	i := 0
	for err == nil {
		buf.Data[i], err = decodeF(d.PCMChunk, sampleBufData)
		if err != nil {
			break
		}
		i++
		// grow the underlying slice if needed
		if i == len(buf.Data) {
			buf.Data = append(buf.Data, make([]int, 4096)...)
		}
	}
	buf.Data = buf.Data[:i]

	if err == io.EOF {
		err = nil
	}

	return buf, err
}

// PCMBuffer populates the passed PCM buffer
func (d *Decoder) PCMBuffer(buf *audio.IntBuffer) (n int, err error) {
	if buf == nil {
		return 0, nil
	}

	if !d.pcmDataAccessed {
		err := d.FwdToPCM()
		if err != nil {
			return 0, d.err
		}
	}

	format := &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}

	buf.SourceBitDepth = int(d.BitDepth)
	decodeF, err := sampleDecodeFunc(int(d.BitDepth))
	if err != nil {
		return 0, fmt.Errorf("could not get sample decode func %v", err)
	}

	bPerSample := bytesPerSample(int(d.BitDepth))
	// populate a file buffer to avoid multiple very small reads
	// we need to cap the buffer size to not be bigger than the pcm chunk.
	size := len(buf.Data) * bPerSample
	tmpBuf := make([]byte, size)
	var m int
	m, err = d.PCMChunk.R.Read(tmpBuf)
	if err != nil {
		if err == io.EOF {
			return m, nil
		}
		return m, err
	}
	if m == 0 {
		return m, nil
	}
	bufR := bytes.NewReader(tmpBuf[:m])
	sampleBuf := make([]byte, bPerSample, bPerSample)
	var misaligned bool
	if m%bPerSample > 0 {
		misaligned = true
	}

	// Note that we populate the buffer even if the
	// size of the buffer doesn't fit an even number of frames.
	for n = 0; n < len(buf.Data); n++ {
		buf.Data[n], err = decodeF(bufR, sampleBuf)
		if err != nil {
			// the last sample isn't a full sample but just padding.
			if misaligned {
				n--
			}
			break
		}
	}
	buf.Format = format
	if err == io.EOF {
		err = nil
	}

	return n, err
}

// Format returns the audio format of the decoded content.
func (d *Decoder) Format() *audio.Format {
	if d == nil {
		return nil
	}
	return &audio.Format{
		NumChannels: int(d.NumChans),
		SampleRate:  int(d.SampleRate),
	}
}

// NextChunk returns the next available chunk
func (d *Decoder) NextChunk() (*riff.Chunk, error) {
	if d.err = d.readHeaders(); d.err != nil {
		d.err = fmt.Errorf("failed to read header - %v", d.err)
		return nil, d.err
	}

	var (
		id   [4]byte
		size uint32
	)

	id, size, d.err = d.parser.IDnSize()
	if d.err != nil {
		d.err = fmt.Errorf("error reading chunk header - %v", d.err)
		return nil, d.err
	}

	c := &riff.Chunk{
		ID:   id,
		Size: int(size),
		R:    io.LimitReader(d.r, int64(size)),
	}
	return c, d.err
}

// Duration returns the time duration for the current audio container
func (d *Decoder) Duration() (time.Duration, error) {
	if d == nil || d.parser == nil {
		return 0, errors.New("can't calculate the duration of a nil pointer")
	}
	return d.parser.Duration()
}

// String implements the Stringer interface.
func (d *Decoder) String() string {
	return d.parser.String()
}

// readHeaders is safe to call multiple times
func (d *Decoder) readHeaders() error {
	if d == nil || d.NumChans > 0 {
		return nil
	}

	id, size, err := d.parser.IDnSize()
	if err != nil {
		return err
	}
	d.parser.ID = id
	if d.parser.ID != riff.RiffID {
		return fmt.Errorf("%s - %s", d.parser.ID, riff.ErrFmtNotSupported)
	}
	d.parser.Size = size
	if err := binary.Read(d.r, binary.BigEndian, &d.parser.Format); err != nil {
		return err
	}

	var chunk *riff.Chunk
	var rewindBytes int64

	for err == nil {
		chunk, err = d.parser.NextChunk()
		if err != nil {
			break
		}

		if chunk.ID == riff.FmtID {
			chunk.DecodeWavHeader(d.parser)
			d.NumChans = d.parser.NumChannels
			d.BitDepth = d.parser.BitsPerSample
			d.SampleRate = d.parser.SampleRate
			d.WavAudioFormat = d.parser.WavAudioFormat
			d.AvgBytesPerSec = d.parser.AvgBytesPerSec

			if rewindBytes > 0 {
				d.r.Seek(-(rewindBytes + int64(chunk.Size) + 8), 1)
			}
			break
		} else {
			// unexpected chunk order, might be a bext chunk
			rewindBytes += int64(chunk.Size) + 8
			// drain the chunk
			io.CopyN(ioutil.Discard, d.r, int64(chunk.Size))
		}

	}

	return d.err
}

func bytesPerSample(bitDepth int) int {
	return bitDepth / 8
}

// sampleDecodeFunc returns a function that can be used to convert
// a byte range into an int value based on the amount of bits used per sample.
// Note that 8bit samples are unsigned, all other values are signed.
func sampleDecodeFunc(bitsPerSample int) (func(io.Reader, []byte) (int, error), error) {
	// NOTE: WAV PCM data is stored using little-endian
	switch bitsPerSample {
	case 8:
		// 8bit values are unsigned
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:1])
			return int(buf[0]), err
		}, nil
	case 16:
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:2])
			return int(int16(binary.LittleEndian.Uint16(buf[:2]))), err
		}, nil
	case 24:
		// -34,359,738,367 (0x7FFFFF) to 34,359,738,368	(0x800000)
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:3])
			if err != nil {
				return 0, err
			}
			return int(audio.Int24LETo32(buf[:3])), nil
		}, nil
	case 32:
		return func(r io.Reader, buf []byte) (int, error) {
			_, err := r.Read(buf[:4])
			return int(int32(binary.LittleEndian.Uint32(buf[:4]))), err
		}, nil
	default:
		return nil, fmt.Errorf("unhandled byte depth:%d", bitsPerSample)
	}
}

// sampleDecodeFloat64Func returns a function that can be used to convert
// a byte range into a float64 value based on the amount of bits used per sample.
func sampleFloat64DecodeFunc(bitsPerSample int) (func([]byte) float64, error) {
	bytesPerSample := bitsPerSample / 8
	switch bytesPerSample {
	case 1:
		// 8bit values are unsigned
		return func(s []byte) float64 {
			return float64(uint8(s[0]))
		}, nil
	case 2:
		return func(s []byte) float64 {
			return float64(int(s[0]) + int(s[1])<<8)
		}, nil
	case 3:
		return func(s []byte) float64 {
			var output int32
			output |= int32(s[2]) << 0
			output |= int32(s[1]) << 8
			output |= int32(s[0]) << 16
			return float64(output)
		}, nil
	case 4:
		// TODO: fix the float64 conversion (current int implementation)
		return func(s []byte) float64 {
			return float64(int(s[0]) + int(s[1])<<8 + int(s[2])<<16 + int(s[3])<<24)
		}, nil
	default:
		return nil, fmt.Errorf("unhandled byte depth:%d", bitsPerSample)
	}
}
<|endoftext|>


<commit_before>package flightdb2

import (
	"fmt"
	"sort"
	"time"

	"github.com/skypies/util/date"
)

// FlightForBigQuery is a represenation of a Flight that is slightly denormalized, with
// a track summary instead of a track. It is designed for import into BigQuery, for analysis.
// It has a *lot* in common with CondensedFlight ... perhaps they should be combined ?
type FlightForBigQuery struct {
	FdbId          string // ID back into the flight database

	ModeS          string
	Registration   string
	Equip          string // e.g. B744, A320 etc

	Start,End      time.Time // start and end of track data points
	DatePST        string // Bleargh. This is somewhat approximate.
	TrackSources []string
	Tags         []string

	Waypoint     []WaypointForBigQuery  // Not 'Waypoints', so that the SQL reads more naturally
	Procedure    []FlownProcedure

	// These fields only defined if we have schedule data for the flight
	FlightNumber   string // IATA scheduled flight number
	FlightKey      string // A {flightnumber+date} value; can be used to join against complaints
	Airline        string // IATA airline code, if known
	Callsign       string
	Orig,Dest      string // airport codes
}

type WaypointForBigQuery struct {
	Name string
	Time time.Time
}

func (fbq FlightForBigQuery)String() string {
	proc := ""
	if len(fbq.Procedure) > 0 { proc = fmt.Sprintf("%v", fbq.Procedure[0]) }
	str := fmt.Sprintf("%s %s {%s} %v %v",
		fbq.FlightNumber,
		date.InPdt(fbq.End).Format("2006/01/02"),
		proc,
		fbq.Waypoint,
		fbq.Tags)
	return str
}

func (f *Flight)ForBigQuery() *FlightForBigQuery {
	s,e := f.Times()

	// We need to pick a 'date' for this flight; but we don't have schedule data.
	// Pick the midpoint of the time range we knew about this flight.
	mid := s.Add(e.Sub(s) / 2)

	fbq := FlightForBigQuery{
		FdbId: f.IdSpec().String(),
		ModeS: f.IcaoId,
		Registration: f.Registration,
		Equip: f.EquipmentType,

		Start: s,
		End: e,
		DatePST: date.InPdt(mid).Format("2006/01/02"),
		TrackSources: f.ListTracks(),
		Tags: f.TagList(),

		Waypoint: []WaypointForBigQuery{},
		Procedure: f.DetermineFlownProcedures(),

		FlightNumber: f.IataFlight(),
		FlightKey: fmt.Sprintf("%s-%s", f.IataFlight(), date.InPdt(mid).Format("20060102")),
		Airline: f.Schedule.IATA,
		Callsign: f.Callsign,
		Orig: f.Schedule.Origin,
		Dest: f.Schedule.Destination,
	}

	wptl := []WaypointAndTime{}
	for k,v := range f.Waypoints { wptl = append(wptl, WaypointAndTime{k,v}) }
	sort.Sort(WaypointAndTimeList(wptl))
	for _,wpt := range wptl {
		fbq.Waypoint = append(fbq.Waypoint, WaypointForBigQuery{wpt.WP, wpt.Time})
	}

	return &fbq
}
<commit_msg>Move to BigQuery's date syntax, for easier SQL writing.<commit_after>package flightdb2

import (
	"fmt"
	"sort"
	"time"

	"github.com/skypies/util/date"
)

// FlightForBigQuery is a represenation of a Flight that is slightly denormalized, with
// a track summary instead of a track. It is designed for import into BigQuery, for analysis.
// It has a *lot* in common with CondensedFlight ... perhaps they should be combined ?
type FlightForBigQuery struct {
	FdbId          string // ID back into the flight database

	ModeS          string
	Registration   string
	Equip          string // e.g. B744, A320 etc

	Start,End      time.Time // start and end of track data points
	DatePST        string // Bleargh. This is somewhat approximate.
	TrackSources []string
	Tags         []string

	Waypoint     []WaypointForBigQuery  // Not 'Waypoints', so that the SQL reads more naturally
	Procedure    []FlownProcedure

	// These fields only defined if we have schedule data for the flight
	FlightNumber   string // IATA scheduled flight number
	FlightKey      string // A {flightnumber+date} value; can be used to join against complaints
	Airline        string // IATA airline code, if known
	Callsign       string
	Orig,Dest      string // airport codes
}

type WaypointForBigQuery struct {
	Name string
	Time time.Time
}

func (fbq FlightForBigQuery)String() string {
	proc := ""
	if len(fbq.Procedure) > 0 { proc = fmt.Sprintf("%v", fbq.Procedure[0]) }
	str := fmt.Sprintf("%s %s {%s} %v %v",
		fbq.FlightNumber,
		date.InPdt(fbq.End).Format("2006/01/02"),
		proc,
		fbq.Waypoint,
		fbq.Tags)
	return str
}

func (f *Flight)ForBigQuery() *FlightForBigQuery {
	s,e := f.Times()

	// We need to pick a 'date' for this flight; but we don't have schedule data.
	// Pick the midpoint of the time range we knew about this flight.
	mid := s.Add(e.Sub(s) / 2)

	fbq := FlightForBigQuery{
		FdbId: f.IdSpec().String(),
		ModeS: f.IcaoId,
		Registration: f.Registration,
		Equip: f.EquipmentType,

		Start: s,
		End: e,
		DatePST: date.InPdt(mid).Format("2006-01-02"), // Use the same format as BQ's DATE() function
		TrackSources: f.ListTracks(),
		Tags: f.TagList(),

		Waypoint: []WaypointForBigQuery{},
		Procedure: f.DetermineFlownProcedures(),

		FlightNumber: f.IataFlight(),
		FlightKey: fmt.Sprintf("%s-%s", f.IataFlight(), date.InPdt(mid).Format("20060102")),
		Airline: f.Schedule.IATA,
		Callsign: f.Callsign,
		Orig: f.Schedule.Origin,
		Dest: f.Schedule.Destination,
	}

	wptl := []WaypointAndTime{}
	for k,v := range f.Waypoints { wptl = append(wptl, WaypointAndTime{k,v}) }
	sort.Sort(WaypointAndTimeList(wptl))
	for _,wpt := range wptl {
		fbq.Waypoint = append(fbq.Waypoint, WaypointForBigQuery{wpt.WP, wpt.Time})
	}

	return &fbq
}
<|endoftext|>


<commit_before>// Bijectiv - A library for URL shortener encoding/decoding in Go
// Copyright 2015 Leonardo Eloy
// https://github.com/leonardoeloy/bijectiv
// Algorithm based on http://stackoverflow.com/questions/742013/how-to-code-a-url-shortener

package bijectiv

import (
	"bytes"
	"strings"
)

const (
	default_alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
)

type Bijectiv struct {
	Alphabet string
	Base int
}

func NewAlphabet(alphabet string) *Bijectiv {
	return &Bijectiv{Alphabet: alphabet, Base: len(alphabet)}
}

func New() *Bijectiv {
	return NewAlphabet(default_alphabet)
}

func (b *Bijectiv) Encode(number int) string {
	if number == 0 {
		return b.Alphabet[:0]
	}
	var buffer bytes.Buffer

	for number > 0 {
		base := number % b.Base
		buffer.WriteString(b.Alphabet[base:base+1])

		number /= b.Base
	}

	return reverse(buffer.String())
}

func (b *Bijectiv) Decode(value string) int {
	var number int

	for i := 0; i < len(value); i++ {
		number = number * b.Base + strings.Index(b.Alphabet, value[i:i+1])
	}

	return number
}

// Taken from https://github.com/golang/example/blob/master/stringutil/reverse.go
// Copyright 2014 Google Inc.
func reverse(s string) string {
	r := []rune(s)
	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
		r[i], r[j] = r[j], r[i]
	}
	return string(r)
}

/*
public static String encode(int num)
{
StringBuilder sb = new StringBuilder();

while ( num > 0 )
{
sb.append( ALPHABET.charAt( num % BASE ) );
num /= BASE;
}

return sb.reverse().toString();
}

public static int decode(String str)
{
int num = 0;

for ( int i = 0, len = str.length(); i < len; i++ )
{
num = num * BASE + ALPHABET.indexOf( str.charAt(i) );
}

return num;
}
*/<commit_msg>fixed comments<commit_after>// Bijectiv - A library for URL shortener encoding/decoding in Go
// Copyright 2015 Leonardo Eloy
// https://github.com/leonardoeloy/bijectiv
// Algorithm based on http://stackoverflow.com/questions/742013/how-to-code-a-url-shortener

package bijectiv

import (
	"bytes"
	"strings"
)

const (
	default_alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
)

type Bijectiv struct {
	Alphabet string
	Base int
}

func NewAlphabet(alphabet string) *Bijectiv {
	return &Bijectiv{Alphabet: alphabet, Base: len(alphabet)}
}

func New() *Bijectiv {
	return NewAlphabet(default_alphabet)
}

func (b *Bijectiv) Encode(number int) string {
	if number == 0 {
		return b.Alphabet[:0]
	}
	var buffer bytes.Buffer

	for number > 0 {
		base := number % b.Base
		buffer.WriteString(b.Alphabet[base:base+1])

		number /= b.Base
	}

	return reverse(buffer.String())
}

func (b *Bijectiv) Decode(value string) int {
	var number int

	for i := 0; i < len(value); i++ {
		number = number * b.Base + strings.Index(b.Alphabet, value[i:i+1])
	}

	return number
}

// Taken from https://github.com/golang/example/blob/master/stringutil/reverse.go
// Copyright 2014 Google Inc.
func reverse(s string) string {
	r := []rune(s)
	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
		r[i], r[j] = r[j], r[i]
	}
	return string(r)
}
<|endoftext|>