<commit_before>package gorums_test

import (
	"bytes"
	"fmt"
	"log"
	"testing"

	"github.com/relab/gorums"
)

func TestNewManager(t *testing.T) {
	_, err := gorums.NewManager()
	if err == nil {
		t.Errorf("NewManager(): expected error: '%s'", "could not create manager: no nodes provided")
	}

	nodes := []string{"127.0.0.1:9080", "127.0.0.1:9081", "127.0.0.1:9082"}
	mgr, err := gorums.NewManager(gorums.WithNodeList(nodes), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodes) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodes))
	}

	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err = gorums.NewManager(gorums.WithNodeMap(nodeMap), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodeMap) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodeMap))
	}
}

func TestManagerLogging(t *testing.T) {
	var (
		buf    bytes.Buffer
		logger = log.New(&buf, "logger: ", log.Lshortfile)
	)
	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err := gorums.NewManager(
		gorums.WithNodeMap(nodeMap),
		gorums.WithNoConnect(),
		gorums.WithLogger(logger),
	)
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodeMap) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodeMap))
	}
	fmt.Println(buf.String())
}

func TestManagerAddNode(t *testing.T) {
	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err := gorums.NewManager(gorums.WithNodeMap(nodeMap), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	tests := []struct {
		addr string
		id   uint32
		err  string
	}{
		{"127.0.1.1:1234", 1, "node ID 1 already exists (127.0.1.1:1234)"},
		{"127.0.1.1:1234", 5, ""},
		{"127.0.1.1:1234", 6, ""}, // TODO(meling) does it make sense to allow same addr:port for different IDs?
		{"127.0.1.1:1234", 2, "node ID 2 already exists (127.0.1.1:1234)"},
	}
	for _, test := range tests {
		err = mgr.AddNode(test.addr, test.id)
		if err != nil && err.Error() != test.err {
			t.Errorf("mgr.AddNode(%s, %d) = %s, expected %s", test.addr, test.id, err.Error(), test.err)
		}

	}
}
<commit_msg>Replaced fmt.Println with t.Log<commit_after>package gorums_test

import (
	"bytes"
	"log"
	"testing"

	"github.com/relab/gorums"
)

func TestNewManager(t *testing.T) {
	_, err := gorums.NewManager()
	if err == nil {
		t.Errorf("NewManager(): expected error: '%s'", "could not create manager: no nodes provided")
	}

	nodes := []string{"127.0.0.1:9080", "127.0.0.1:9081", "127.0.0.1:9082"}
	mgr, err := gorums.NewManager(gorums.WithNodeList(nodes), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodes) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodes))
	}

	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err = gorums.NewManager(gorums.WithNodeMap(nodeMap), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodeMap) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodeMap))
	}
}

func TestManagerLogging(t *testing.T) {
	var (
		buf    bytes.Buffer
		logger = log.New(&buf, "logger: ", log.Lshortfile)
	)
	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err := gorums.NewManager(
		gorums.WithNodeMap(nodeMap),
		gorums.WithNoConnect(),
		gorums.WithLogger(logger),
	)
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	if mgr.Size() != len(nodeMap) {
		t.Errorf("mgr.Size() = %d, expected %d", mgr.Size(), len(nodeMap))
	}
	t.Log(buf.String())
}

func TestManagerAddNode(t *testing.T) {
	nodeMap := map[string]uint32{"127.0.0.1:9080": 1, "127.0.0.1:9081": 2, "127.0.0.1:9082": 3, "127.0.0.1:9083": 4}
	mgr, err := gorums.NewManager(gorums.WithNodeMap(nodeMap), gorums.WithNoConnect())
	if err != nil {
		t.Fatalf("NewManager(): unexpected error: %s", err)
	}
	tests := []struct {
		addr string
		id   uint32
		err  string
	}{
		{"127.0.1.1:1234", 1, "node ID 1 already exists (127.0.1.1:1234)"},
		{"127.0.1.1:1234", 5, ""},
		{"127.0.1.1:1234", 6, ""}, // TODO(meling) does it make sense to allow same addr:port for different IDs?
		{"127.0.1.1:1234", 2, "node ID 2 already exists (127.0.1.1:1234)"},
	}
	for _, test := range tests {
		err = mgr.AddNode(test.addr, test.id)
		if err != nil && err.Error() != test.err {
			t.Errorf("mgr.AddNode(%s, %d) = %s, expected %s", test.addr, test.id, err.Error(), test.err)
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"testing"

	"github.com/mtojek/localserver"
	"github.com/stretchr/testify/assert"
)

func TestReadConfiguration(t *testing.T) {
	assert := assert.New(t)

	// given
	scheme := "http"
	hostPort := "127.0.0.1:10600"
	server := localserver.NewLocalServer(hostPort, scheme)
	server.Start()

	setCommandLineArgs("resources/input-data/fuzz_01.txt", scheme+"://"+hostPort)
	sut := newURLFuzzer()

	// when
	configuration := sut.readConfiguration()
	server.Stop()

	// then
	assert.NotNil(configuration, "Simple configuration should be read from command line.")
}

func setCommandLineArgs(customArguments ...string) {
	os.Args = os.Args[:len(os.Args)-1] // remove test.v flag
	for _, customArgument := range customArguments {
		os.Args = append(os.Args, customArgument)
	}
}
<commit_msg>Fix: skip testlogfile flag<commit_after>package main

import (
	"os"
	"testing"

	"github.com/mtojek/localserver"
	"github.com/stretchr/testify/assert"
)

func TestReadConfiguration(t *testing.T) {
	assert := assert.New(t)

	// given
	scheme := "http"
	hostPort := "127.0.0.1:10600"
	server := localserver.NewLocalServer(hostPort, scheme)
	server.Start()

	setCommandLineArgs("resources/input-data/fuzz_01.txt", scheme+"://"+hostPort)
	sut := newURLFuzzer()

	// when
	configuration := sut.readConfiguration()
	server.Stop()

	// then
	assert.NotNil(configuration, "Simple configuration should be read from command line.")
}

func setCommandLineArgs(customArguments ...string) {
	os.Args = os.Args[:len(os.Args)-2] // remove test.v and test.testlogfile flag
	for _, customArgument := range customArguments {
		os.Args = append(os.Args, customArgument)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"path"
	"strings"
)

const programName = "s3upload"

// variables set by command line flags
var bucketName string
var baseDir string
var showHelp bool
var verbose bool
var recursive bool
var includeUnknownMimeTypes bool
var ignore string

// contains information about every object in the bucket
// maps the object key name to its etag
var s3Objects = make(map[string]string)
var ignoreNames = make(map[string]string)

func main() {
	flag.StringVar(&bucketName, "bucket", "", "S3 Bucket Name (required)")
	flag.StringVar(&baseDir, "dir", "", "Local directory (required)")
	flag.BoolVar(&verbose, "verbose", false, "Print extra log messages")
	flag.BoolVar(&showHelp, "help", false, "Show this help")
	flag.BoolVar(&recursive, "recursive", false, "recurse into sub-directories")
	flag.BoolVar(&includeUnknownMimeTypes, "include-unknown-mime-types", false, "upload files with unknown mime types")
	flag.StringVar(&ignore, "ignore", "", "Comma-separated list of files/directories to ignore")

	flag.Parse()
	if showHelp {
		fmt.Fprintf(os.Stderr, "usage: %s [ options ]\noptions:\n", programName)
		flag.PrintDefaults()
		return
	}

	if bucketName == "" {
		log.Fatalf("Must specify bucket: use '%s -help' for usage", programName)
	}

	if baseDir == "" {
		log.Fatalf("Must specify directory: use '%s -help' for usage", programName)
	}

	for _, name := range strings.Split(ignore, ",") {
		ignoreNames[name] = name
	}

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatal(err)
	}

	s3Config := s3.New(auth, aws.APSoutheast2)
	bucket := &s3.Bucket{S3: s3Config, Name: bucketName}

	if verbose {
		log.Println("Listing objects in bucket")
	}

	marker := ""
	for {
		listResp, err := bucket.List("", "/", marker, 1000)
		if err != nil {
			log.Fatal(err)
		}

		for _, key := range listResp.Contents {
			s3Objects[key.Key] = key.ETag
			marker = key.Key
		}

		if !listResp.IsTruncated {
			break
		}
		if verbose {
			log.Printf("%d objects loaded", len(s3Objects))
		}
	}

	processDir(baseDir, "", bucket)
}

func processDir(dirName string, s3KeyPrefix string, bucket *s3.Bucket) {
	if verbose {
		log.Printf("Processing directory %s", dirName)
	}

	fileInfos, err := ioutil.ReadDir(dirName)
	if err != nil {
		log.Fatal(err)
	}

	for _, fileInfo := range fileInfos {
		filePath := path.Join(dirName, fileInfo.Name())

		// Ignore symlinks for now.
		// TODO: add option to follow symlinks
		if (fileInfo.Mode() & os.ModeSymlink) != 0 {
			continue
		}

		if fileInfo.IsDir() {
			if shouldRecurseInto(fileInfo.Name()) {
				subDirName := path.Join(dirName, fileInfo.Name())
				processDir(subDirName, s3KeyPrefix+fileInfo.Name()+"/", bucket)
			}
			continue
		}
		if ignoreNames[fileInfo.Name()] != "" {
			continue
		}
		s3Key := s3KeyPrefix + fileInfo.Name()

		putRequired := false
		var data []byte

		s3ETag := s3Objects[s3Key]
		if s3ETag == "" {
			if verbose {
				log.Printf("Not found in S3 bucket: %s", s3Key)
			}
			putRequired = true
		}

		data, err := ioutil.ReadFile(filePath)
		if err != nil {
			log.Fatal(err)
		}

		// if the object exists, then we check the MD5 of the file to determine whether
		// the file needs to be uploaded
		if !putRequired {
			digest := md5.Sum(data)
			// note the need to convert digest to a slice because it is a byte array ([16]byte)
			fileETag := "\"" + hex.EncodeToString(digest[:]) + "\""

			if fileETag != s3ETag {
				if verbose {
					log.Printf("Need to upload %s: expected ETag = %s, actual = %s", filePath, fileETag, s3ETag)
				}
				putRequired = true
			}
		}

		if putRequired {
			// TODO: this should be configurable, but for now if the mime-type cannot
			// be determined, do not upload
			contentType := mime.TypeByExtension(path.Ext(fileInfo.Name()))
			if contentType == "" && includeUnknownMimeTypes {
				contentType = "application/octet-stream"
			}

			if contentType != "" {
				err = bucket.Put(s3Key, data, contentType, s3.Private, s3.Options{})
				if err != nil {
					log.Fatal(err)
				}
				log.Printf("Uploaded %s\n", s3Key)
			}

		} else {
			if verbose {
				log.Printf("Identical file, no upload required: %s", filePath)
			}
		}

	}
}

func shouldRecurseInto(dirName string) bool {
	if !recursive {
		return false
	}

	if strings.HasPrefix(dirName, ".") || strings.HasPrefix(dirName, "_") {
		return false
	}

	if ignoreNames[dirName] != "" {
		return false
	}

	if dirName == "lost+found" {
		return false
	}

	return true
}
<commit_msg>fixbug: removed prefix from list bucket request<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"flag"
	"fmt"
	"github.com/goamz/goamz/aws"
	"github.com/goamz/goamz/s3"
	"io/ioutil"
	"log"
	"mime"
	"os"
	"path"
	"strings"
)

const programName = "s3upload"

// variables set by command line flags
var bucketName string
var baseDir string
var showHelp bool
var verbose bool
var recursive bool
var includeUnknownMimeTypes bool
var ignore string

// contains information about every object in the bucket
// maps the object key name to its etag
var s3Objects = make(map[string]string)
var ignoreNames = make(map[string]string)

func main() {
	flag.StringVar(&bucketName, "bucket", "", "S3 Bucket Name (required)")
	flag.StringVar(&baseDir, "dir", "", "Local directory (required)")
	flag.BoolVar(&verbose, "verbose", false, "Print extra log messages")
	flag.BoolVar(&showHelp, "help", false, "Show this help")
	flag.BoolVar(&recursive, "recursive", false, "recurse into sub-directories")
	flag.BoolVar(&includeUnknownMimeTypes, "include-unknown-mime-types", false, "upload files with unknown mime types")
	flag.StringVar(&ignore, "ignore", "", "Comma-separated list of files/directories to ignore")

	flag.Parse()
	if showHelp {
		fmt.Fprintf(os.Stderr, "usage: %s [ options ]\noptions:\n", programName)
		flag.PrintDefaults()
		return
	}

	if bucketName == "" {
		log.Fatalf("Must specify bucket: use '%s -help' for usage", programName)
	}

	if baseDir == "" {
		log.Fatalf("Must specify directory: use '%s -help' for usage", programName)
	}

	for _, name := range strings.Split(ignore, ",") {
		ignoreNames[name] = name
	}

	auth, err := aws.EnvAuth()
	if err != nil {
		log.Fatal(err)
	}

	s3Config := s3.New(auth, aws.APSoutheast2)
	bucket := &s3.Bucket{S3: s3Config, Name: bucketName}

	if verbose {
		log.Println("Listing objects in bucket")
	}

	marker := ""
	for {
		listResp, err := bucket.List("", "", marker, 1000)
		if err != nil {
			log.Fatal(err)
		}

		for _, key := range listResp.Contents {
			s3Objects[key.Key] = key.ETag
			marker = key.Key
		}

		if verbose {
			log.Printf("%d objects loaded", len(s3Objects))
		}

		if !listResp.IsTruncated {
			break
		}
	}

	processDir(baseDir, "", bucket)
}

func processDir(dirName string, s3KeyPrefix string, bucket *s3.Bucket) {
	if verbose {
		log.Printf("Processing directory %s", dirName)
	}

	fileInfos, err := ioutil.ReadDir(dirName)
	if err != nil {
		log.Fatal(err)
	}

	for _, fileInfo := range fileInfos {
		filePath := path.Join(dirName, fileInfo.Name())

		// Ignore symlinks for now.
		// TODO: add option to follow symlinks
		if (fileInfo.Mode() & os.ModeSymlink) != 0 {
			continue
		}

		if fileInfo.IsDir() {
			if shouldRecurseInto(fileInfo.Name()) {
				subDirName := path.Join(dirName, fileInfo.Name())
				processDir(subDirName, s3KeyPrefix+fileInfo.Name()+"/", bucket)
			}
			continue
		}
		if ignoreNames[fileInfo.Name()] != "" {
			continue
		}
		s3Key := s3KeyPrefix + fileInfo.Name()

		putRequired := false
		var data []byte

		s3ETag := s3Objects[s3Key]
		if s3ETag == "" {
			if verbose {
				log.Printf("Not found in S3 bucket: %s", s3Key)
			}
			putRequired = true
		}

		data, err := ioutil.ReadFile(filePath)
		if err != nil {
			log.Fatal(err)
		}

		// if the object exists, then we check the MD5 of the file to determine whether
		// the file needs to be uploaded
		if !putRequired {
			digest := md5.Sum(data)
			// note the need to convert digest to a slice because it is a byte array ([16]byte)
			fileETag := "\"" + hex.EncodeToString(digest[:]) + "\""

			if fileETag != s3ETag {
				if verbose {
					log.Printf("Need to upload %s: expected ETag = %s, actual = %s", filePath, fileETag, s3ETag)
				}
				putRequired = true
			}
		}

		if putRequired {
			// TODO: this should be configurable, but for now if the mime-type cannot
			// be determined, do not upload
			contentType := mime.TypeByExtension(path.Ext(fileInfo.Name()))
			if contentType == "" && includeUnknownMimeTypes {
				contentType = "application/octet-stream"
			}

			if contentType != "" {
				err = bucket.Put(s3Key, data, contentType, s3.Private, s3.Options{})
				if err != nil {
					log.Fatal(err)
				}
				log.Printf("Uploaded %s\n", s3Key)
			}

		} else {
			if verbose {
				log.Printf("Identical file, no upload required: %s", filePath)
			}
		}

	}
}

func shouldRecurseInto(dirName string) bool {
	if !recursive {
		return false
	}

	if strings.HasPrefix(dirName, ".") || strings.HasPrefix(dirName, "_") {
		return false
	}

	if ignoreNames[dirName] != "" {
		return false
	}

	if dirName == "lost+found" {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package sqls

import (
	"bytes"
	"fmt"
	"text/template"

	"gopkg.in/goyy/goyy.v0/comm/xtype"
	"gopkg.in/goyy/goyy.v0/data/dialect"
	"gopkg.in/goyy/goyy.v0/util/errors"
	"gopkg.in/goyy/goyy.v0/util/strings"
	"gopkg.in/goyy/goyy.v0/util/templates"
)

// select ... from ... -> select count(*) from ...
func ParseCountSql(sql string) string {
	stack := &xtype.Stack{}
	ss := strings.Split(sql, " ")
	p := 0
	for _, v := range ss {
		if strings.Contains(strings.ToLower(v), "select") {
			p++
			stack.Push(p)
			continue
		}
		if strings.Contains(strings.ToLower(v), "from") {
			if stack.Len() == 1 {
				break
			}
			stack.Pop()
			continue
		}
	}
	pfrom := strings.IndexOrdinal(strings.ToLower(sql), "from", p)
	return "select count(*) " + sql[pfrom:]
}

// ParseNamedSql takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by a database.
func ParseNamedSql(dia dialect.Interface, sql string, args map[string]interface{}) (sqlout string, argsout []interface{}, err error) {
	if dia == nil || strings.IsBlank(sql) || args == nil {
		err = errors.NewNotBlank("dia/sql/args")
		return
	}
	if !strings.Contains(sql, "#{") {
		sqlout = sql
		argsout = make([]interface{}, 0)
		return
	}
	sqls := strings.Betweens(sql, "#{", "}")
	if sqls != nil && len(sqls) > 0 {
		i := 0
		for _, v := range sqls {
			if strings.IsNotBlank(v) {
				if dia.Type() == dialect.ORACLE {
					sql = strings.Replace(sql, "#{"+v+"}", fmt.Sprintf(":%d", i), -1)
					i++
				} else {
					sql = strings.Replace(sql, "#{"+v+"}", "?", -1)
				}
				if _, ok := args[v]; ok {
					argsout = append(argsout, args[v])
				} else {
					err = errors.NewNotBlank("map[" + v + "]")
					sqlout = sql
					return
				}
			}
		}
		sqlout = sql
	}
	return
}

// ParseTemplateSql takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by a database.
func ParseTemplateSql(sql string, args map[string]interface{}) (out string, err error) {
	t, err := template.New("sqls-tmpl").Funcs(templates.Text.FuncMap).Parse(sql)
	if err != nil {
		logger.Error(err.Error())
		return
	}
	var v bytes.Buffer
	err = t.Execute(&v, args)
	if err != nil {
		logger.Error(err.Error())
		return
	}
	out = v.String()
	return
}
<commit_msg>ParseCountSql : Solve the problem of analyzing from<commit_after>// Copyright 2014 The goyy Authors.  All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package sqls

import (
	"bytes"
	"fmt"
	"text/template"

	"gopkg.in/goyy/goyy.v0/comm/xtype"
	"gopkg.in/goyy/goyy.v0/data/dialect"
	"gopkg.in/goyy/goyy.v0/util/errors"
	"gopkg.in/goyy/goyy.v0/util/strings"
	"gopkg.in/goyy/goyy.v0/util/templates"
)

// select ... from ... -> select count(*) from ...
func ParseCountSql(sql string) string {
	stack := &xtype.Stack{}
	ss := strings.Split(sql, " ")
	p := 0
	for _, v := range ss {
		if strings.Contains(strings.ToLower(v), "select ") {
			p++
			stack.Push(p)
			continue
		}
		if strings.Contains(strings.ToLower(v), " from ") {
			if stack.Len() == 1 {
				break
			}
			stack.Pop()
			continue
		}
	}
	pfrom := strings.IndexOrdinal(strings.ToLower(sql), " from ", p)
	return "select count(*) " + sql[pfrom:]
}

// ParseNamedSql takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by a database.
func ParseNamedSql(dia dialect.Interface, sql string, args map[string]interface{}) (sqlout string, argsout []interface{}, err error) {
	if dia == nil || strings.IsBlank(sql) || args == nil {
		err = errors.NewNotBlank("dia/sql/args")
		return
	}
	if !strings.Contains(sql, "#{") {
		sqlout = sql
		argsout = make([]interface{}, 0)
		return
	}
	sqls := strings.Betweens(sql, "#{", "}")
	if sqls != nil && len(sqls) > 0 {
		i := 0
		for _, v := range sqls {
			if strings.IsNotBlank(v) {
				if dia.Type() == dialect.ORACLE {
					sql = strings.Replace(sql, "#{"+v+"}", fmt.Sprintf(":%d", i), -1)
					i++
				} else {
					sql = strings.Replace(sql, "#{"+v+"}", "?", -1)
				}
				if _, ok := args[v]; ok {
					argsout = append(argsout, args[v])
				} else {
					err = errors.NewNotBlank("map[" + v + "]")
					sqlout = sql
					return
				}
			}
		}
		sqlout = sql
	}
	return
}

// ParseTemplateSql takes a query using named parameters and an argument and
// returns a new query with a list of args that can be executed by a database.
func ParseTemplateSql(sql string, args map[string]interface{}) (out string, err error) {
	t, err := template.New("sqls-tmpl").Funcs(templates.Text.FuncMap).Parse(sql)
	if err != nil {
		logger.Error(err.Error())
		return
	}
	var v bytes.Buffer
	err = t.Execute(&v, args)
	if err != nil {
		logger.Error(err.Error())
		return
	}
	out = v.String()
	return
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/types"
	"os"
	"strings"

	"golang.org/x/tools/go/loader"

	"github.com/gojuno/generator"
)

type (
	options struct {
		InputFile     string
		OutputFile    string
		InterfaceName string
		StructName    string
		Package       string
	}

	visitor struct {
		gen             *generator.Generator
		methods         map[string]*types.Signature
		sourceInterface string
	}
)

func main() {
	opts := processFlags()

	gen := generator.New()
	gen.SetPackageName(opts.Package)
	gen.SetVar("structName", opts.StructName)
	gen.SetVar("interfaceName", opts.InterfaceName)
	gen.SetHeader(fmt.Sprintf(`
		This is automatically generated code. Please DO NOT review/modify/comment.
		Original interface can be found in %s
	`, opts.InputFile))

	packagePath, err := generator.PackageOf(opts.InputFile)
	if err != nil {
		die(err)
	}

	cfg := loader.Config{}
	cfg.Import(packagePath)

	prog, err := cfg.Load()
	if err != nil {
		die(fmt.Errorf("failed to load API package %q: %v", packagePath, err))
	}

	pkg := prog.Package(packagePath)
	gen.Info = &pkg.Info

	v := &visitor{
		gen:             gen,
		methods:         map[string]*types.Signature{},
		sourceInterface: opts.InterfaceName,
	}

	for _, file := range pkg.Files {
		ast.Walk(v, file)
	}

	if len(v.methods) == 0 {
		die(fmt.Errorf("interface %s was not found in %s or it's an empty interface", opts.InterfaceName, packagePath))
	}

	if err := gen.ProcessTemplate("interface", template, v.methods); err != nil {
		die(err)
	}

	if err := gen.WriteToFilename(opts.OutputFile); err != nil {
		die(err)
	}
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if ts, ok := node.(*ast.TypeSpec); ok {
		switch t := v.gen.Info.Types[ts.Type].Type.(type) {
		case *types.Interface:
			if ts.Name.Name != v.sourceInterface {
				return v
			}

			v.processInterface(t)
		}
	}

	return v
}

func (v *visitor) processInterface(t *types.Interface) {
	for i := 0; i < t.NumMethods(); i++ {
		v.methods[t.Method(i).Name()] = t.Method(i).Type().(*types.Signature)
	}
}

const template = `
	type {{$structName}} struct {
		t *testing.T
		m *sync.RWMutex

		{{ range $methodName, $method := . }} {{$methodName}}Func func{{ signature $method }}
		{{ end }}
		{{ range $methodName, $method := . }} {{$methodName}}Counter int
		{{ end }}
	}

	func New{{$structName}}(t *testing.T) *{{$structName}} {
		return &{{$structName}}{t: t, m: &sync.Mutex{} }
	}

	{{ range $methodName, $method := . }}
		func (m *{{$structName}}) {{$methodName}}{{signature $method}} {
			m.m.Lock()
			m.{{$methodName}}Counter += 1
			m.m.Unlock()

			if m.{{$methodName}}Func == nil {
				m.t.Fatalf("Unexpected call to {{$structName}}.{{$methodName}}")
			}

			{{if gt (len (results $method)) 0 }}
				return
			{{ end }} m.{{$methodName}}Func({{(params $method).Names}})
		}
	{{ end }}

	func (m *{{$structName}}) ValidateCallCounters() {
		m.t.Log("ValidateCallCounters is deprecated please use CheckMocksCalled")

		{{ range $methodName, $method := . }}
			if m.{{$methodName}}Func != nil && m.{{$methodName}}Counter == 0 {
				m.t.Error("Expected call to {{$structName}}.{{$methodName}}")
			}
		{{ end }}
	}

	//AllMocksCalled returns true if all mocked methods were called before the call to AllMocksCalled,
	//it can be used with assert/require, i.e. assert.True(mock.AllMocksCalled())
	func (m *{{$structName}}) AllMocksCalled() bool {
		m.t.Log("ValidateCallCounters is deprecated please use CheckMocksCalled")
		m.m.RLock()
		defer m.m.RUnlock()

		{{ range $methodName, $method := . }}
			if m.{{$methodName}}Func != nil && m.{{$methodName}}Counter == 0 {
				return false
			}
		{{ end }}

		return true
	}

	`

func processFlags() *options {
	var (
		input  = flag.String("f", "", "input file or the name of the package containing interface declaration")
		name   = flag.String("i", "", "interface name")
		output = flag.String("o", "", "destination file for interface implementation")
		pkg    = flag.String("p", "", "destination package name")
		sname  = flag.String("t", "", "target struct name, default: <interface name>Mock")
	)

	flag.Parse()

	if *pkg == "" || *input == "" || *output == "" || *name == "" || !strings.HasSuffix(*output, ".go") {
		flag.Usage()
		os.Exit(1)
	}

	if *sname == "" {
		*sname = *name + "Mock"
	}

	return &options{
		InputFile:     *input,
		OutputFile:    *output,
		InterfaceName: *name,
		Package:       *pkg,
		StructName:    *sname,
	}
}

func die(err error) {
	fmt.Fprintf(os.Stderr, "%v\n", err)
	os.Exit(1)
}
<commit_msg>fixed template<commit_after>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/types"
	"os"
	"strings"

	"golang.org/x/tools/go/loader"

	"github.com/gojuno/generator"
)

type (
	options struct {
		InputFile     string
		OutputFile    string
		InterfaceName string
		StructName    string
		Package       string
	}

	visitor struct {
		gen             *generator.Generator
		methods         map[string]*types.Signature
		sourceInterface string
	}
)

func main() {
	opts := processFlags()

	gen := generator.New()
	gen.SetPackageName(opts.Package)
	gen.SetVar("structName", opts.StructName)
	gen.SetVar("interfaceName", opts.InterfaceName)
	gen.SetHeader(fmt.Sprintf(`
		This is automatically generated code. Please DO NOT review/modify/comment.
		Original interface can be found in %s
	`, opts.InputFile))

	packagePath, err := generator.PackageOf(opts.InputFile)
	if err != nil {
		die(err)
	}

	cfg := loader.Config{}
	cfg.Import(packagePath)

	prog, err := cfg.Load()
	if err != nil {
		die(fmt.Errorf("failed to load API package %q: %v", packagePath, err))
	}

	pkg := prog.Package(packagePath)
	gen.Info = &pkg.Info

	v := &visitor{
		gen:             gen,
		methods:         map[string]*types.Signature{},
		sourceInterface: opts.InterfaceName,
	}

	for _, file := range pkg.Files {
		ast.Walk(v, file)
	}

	if len(v.methods) == 0 {
		die(fmt.Errorf("interface %s was not found in %s or it's an empty interface", opts.InterfaceName, packagePath))
	}

	if err := gen.ProcessTemplate("interface", template, v.methods); err != nil {
		die(err)
	}

	if err := gen.WriteToFilename(opts.OutputFile); err != nil {
		die(err)
	}
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	if ts, ok := node.(*ast.TypeSpec); ok {
		switch t := v.gen.Info.Types[ts.Type].Type.(type) {
		case *types.Interface:
			if ts.Name.Name != v.sourceInterface {
				return v
			}

			v.processInterface(t)
		}
	}

	return v
}

func (v *visitor) processInterface(t *types.Interface) {
	for i := 0; i < t.NumMethods(); i++ {
		v.methods[t.Method(i).Name()] = t.Method(i).Type().(*types.Signature)
	}
}

const template = `
	type {{$structName}} struct {
		t *testing.T
		m *sync.RWMutex

		{{ range $methodName, $method := . }} {{$methodName}}Func func{{ signature $method }}
		{{ end }}
		{{ range $methodName, $method := . }} {{$methodName}}Counter int
		{{ end }}
	}

	func New{{$structName}}(t *testing.T) *{{$structName}} {
		return &{{$structName}}{t: t, m: &sync.RWMutex{} }
	}

	{{ range $methodName, $method := . }}
		func (m *{{$structName}}) {{$methodName}}{{signature $method}} {
			m.m.Lock()
			m.{{$methodName}}Counter += 1
			m.m.Unlock()

			if m.{{$methodName}}Func == nil {
				m.t.Fatalf("Unexpected call to {{$structName}}.{{$methodName}}")
			}

			{{if gt (len (results $method)) 0 }}
			return {{ end }} m.{{$methodName}}Func({{(params $method).Names}})
		}
	{{ end }}

	func (m *{{$structName}}) ValidateCallCounters() {
		m.t.Log("ValidateCallCounters is deprecated please use CheckMocksCalled")

		{{ range $methodName, $method := . }}
			if m.{{$methodName}}Func != nil && m.{{$methodName}}Counter == 0 {
				m.t.Error("Expected call to {{$structName}}.{{$methodName}}")
			}
		{{ end }}
	}

	//AllMocksCalled returns true if all mocked methods were called before the call to AllMocksCalled,
	//it can be used with assert/require, i.e. assert.True(mock.AllMocksCalled())
	func (m *{{$structName}}) AllMocksCalled() bool {
		m.t.Log("ValidateCallCounters is deprecated please use CheckMocksCalled")
		m.m.RLock()
		defer m.m.RUnlock()

		{{ range $methodName, $method := . }}
			if m.{{$methodName}}Func != nil && m.{{$methodName}}Counter == 0 {
				return false
			}
		{{ end }}

		return true
	}

	`

func processFlags() *options {
	var (
		input  = flag.String("f", "", "input file or the name of the package containing interface declaration")
		name   = flag.String("i", "", "interface name")
		output = flag.String("o", "", "destination file for interface implementation")
		pkg    = flag.String("p", "", "destination package name")
		sname  = flag.String("t", "", "target struct name, default: <interface name>Mock")
	)

	flag.Parse()

	if *pkg == "" || *input == "" || *output == "" || *name == "" || !strings.HasSuffix(*output, ".go") {
		flag.Usage()
		os.Exit(1)
	}

	if *sname == "" {
		*sname = *name + "Mock"
	}

	return &options{
		InputFile:     *input,
		OutputFile:    *output,
		InterfaceName: *name,
		Package:       *pkg,
		StructName:    *sname,
	}
}

func die(err error) {
	fmt.Fprintf(os.Stderr, "%v\n", err)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package acl

import (
    "github.com/revel/revel/cache"
    "github.com/revel/revel"
    "time"
)

const (
    ACL_ENTRY_ID = "ACL_ENTRY"
    defaultExpiration = 8 * time.Hour
)

// Default is always DENY
// This is not interface{} or anything similar to prevent deep cascades

type ACLEntry struct {
    // Parseable reference to the object ACL entry belongs to, eg. "wiki:example1"
    ObjReference string
    // All the defined ACLs for object
    ACLs []ACL
    // Should be use inheritation with ACLs?
    Inheritation bool
    // Parent for calculating inheritation, eg. "page:level2page" or ""
    Parent string
}

type ACL struct {
    // The permission, eg. "read", "write", "admin"
    Permission string
    // Who has the permission, eg. "u:michael", "g:administrators"
    Principal string
}

func BuildPermissionACLs(permission string, principals []string) []ACL {
    a := []ACL{}
    for _, principal := range principals {
        i := ACL{}
        i.Permission = permission
        i.Principal = principal
        a = append(a, i)
    }
    return a
}

func SetEntry(a ACLEntry) {
     go cache.Set(ACL_ENTRY_ID + a.ObjReference, a, defaultExpiration)
}

func GetEntry(reference string) ACLEntry {
    a := ACLEntry{}
    if err := cache.Get(ACL_ENTRY_ID + reference, &a); err != nil {
        revel.ERROR.Println("Unable to get ACL entry %s", reference)
    }
    return a
}

// TODO: inheritation!
func GetPermissions(principals []string, acl ACLEntry) map[string]bool {
    permissions := make(map[string]bool)

    for _, entry := range acl.ACLs {
        for _, principal := range principals {
            if entry.Principal == principal {
                permissions[entry.Permission] = true
            }
        }
    }

    return permissions
}
/*

c.Args["user_details"]
() {

}
*/<commit_msg>Beginning of filtering<commit_after>package acl

import (
    "github.com/revel/revel/cache"
    "github.com/revel/revel"
    "time"
    "reflect"
    "github.com/mikkolehtisalo/revel/ldapuserdetails"
)

const (
    ACL_ENTRY_ID = "ACL_ENTRY"
    defaultExpiration = 8 * time.Hour
)

// Default is always DENY
// This is not interface{} or anything similar to prevent deep cascades

type ACLEntry struct {
    // Parseable reference to the object ACL entry belongs to, eg. "wiki:example1"
    ObjReference string
    // All the defined ACLs for object
    ACLs []ACL
    // Should be use inheritation with ACLs?
    Inheritation bool
    // Parent for calculating inheritation, eg. "page:level2page" or ""
    Parent string
}

type ACL struct {
    // The permission, eg. "read", "write", "admin"
    Permission string
    // Who has the permission, eg. "u:michael", "g:administrators"
    Principal string
}

func BuildPermissionACLs(permission string, principals []string) []ACL {
    a := []ACL{}
    for _, principal := range principals {
        i := ACL{}
        i.Permission = permission
        i.Principal = principal
        a = append(a, i)
    }
    return a
}

func SetEntry(a ACLEntry) {
     go cache.Set(ACL_ENTRY_ID + a.ObjReference, a, defaultExpiration)
}

func GetEntry(reference string) ACLEntry {
    a := ACLEntry{}
    if err := cache.Get(ACL_ENTRY_ID + reference, &a); err != nil {
        revel.ERROR.Println("Unable to get ACL entry %s", reference)
    }
    return a
}

// TODO: inheritation!
func GetPermissions(principals []string, acl ACLEntry) map[string]bool {
    permissions := make(map[string]bool)

    for _, entry := range acl.ACLs {
        for _, principal := range principals {
            if entry.Principal == principal {
                permissions[entry.Permission] = true
            }
        }
    }

    return permissions
}

type Filterable interface {
    BuildACLReference() string
    GetACLEntry(reference string) ACLEntry
}

func takeSliceArg(arg interface{}) (out []interface{}, ok bool) {
    slice, success := takeArg(arg, reflect.Slice)
    if !success {
        ok = false
          return
    }
    c := slice.Len()
    out = make([]interface{}, c)
    for i := 0; i < c; i++ {
        out[i] = slice.Index(i).Interface()
    }
    return out, true
}

func takeArg(arg interface{}, kind reflect.Kind) (val reflect.Value, ok bool) {
    val = reflect.ValueOf(arg)
    if val.Kind() == kind {
        ok = true
    }
    return
}

// Takes any interface{} and attempt to convert it to []Filterable
func get_filterable (items interface{}) []Filterable {
    slice := reflect.ValueOf(items)
    if slice.Kind() != reflect.Slice {
        // Panic?
    }

    co := slice.Len()
    filterableslice := make([]Filterable, co)
    for i := 0; i < co; i++ {
           filterableslice[i] = slice.Index(i).Interface().(Filterable)
    }

    return filterableslice
}

func Filter(c map[string]interface {}, permission string, i interface{}) {
    // Get the items
    items := get_filterable(i)
    revel.INFO.Printf("Items: %+v", items)

    // Get roles for the user
    dets := c["user_details"].(ldapuserdetails.User_details)
    roles := dets.Roles
    revel.INFO.Printf("Roles: %+v", roles)

    // Get the ACL for item
    for _, item := range items {
        ref := item.BuildACLReference()
        revel.INFO.Printf("Reference: %+v", ref)
        acl := item.GetACLEntry(ref)
        revel.INFO.Printf("ACL entry: %+v", acl)
    }
}

/*

c.Args["user_details"]
() {

}
*/<|endoftext|>


<commit_before>package models

import (
	"errors"
	"path"
	"strings"

	"github.com/readium/r2-streamer-go/parser/epub"
)

// Publication Main structure for a publication
type Publication struct {
	Context      []string `json:"@context,omitempty"`
	Metadata     Metadata `json:"metadata"`
	Links        []Link   `json:"links"`
	ReadingOrder []Link   `json:"readingOrder,omitempty"`
	Resources    []Link   `json:"resources,omitempty"` //Replaces the manifest but less redundant
	TOC          []Link   `json:"toc,omitempty"`
	PageList     []Link   `json:"page-list,omitempty"`
	Landmarks    []Link   `json:"landmarks,omitempty"`
	LOI          []Link   `json:"loi,omitempty"` //List of illustrations
	LOA          []Link   `json:"loa,omitempty"` //List of audio files
	LOV          []Link   `json:"lov,omitempty"` //List of videos
	LOT          []Link   `json:"lot,omitempty"` //List of tables

	OtherLinks       []Link                  `json:"-"` //Extension point for links that shouldn't show up in the manifest
	OtherCollections []PublicationCollection `json:"-"` //Extension point for collections that shouldn't show up in the manifest
	Internal         []Internal              `json:"-"`
	LCP              epub.LCP                `json:"-"`
}

// Internal TODO
type Internal struct {
	Name  string
	Value interface{}
}

// Link object used in collections and links
type Link struct {
	Href          string             `json:"href"`
	TypeLink      string             `json:"type,omitempty"`
	Rel           []string           `json:"rel,omitempty"`
	Height        int                `json:"height,omitempty"`
	Width         int                `json:"width,omitempty"`
	Title         string             `json:"title,omitempty"`
	Properties    *Properties        `json:"properties,omitempty"`
	Duration      string             `json:"duration,omitempty"`
	Templated     bool               `json:"templated,omitempty"`
	Children      []Link             `json:"children,omitempty"`
	Bitrate       int                `json:"bitrate,omitempty"`
	MediaOverlays []MediaOverlayNode `json:"-"`
}

// PublicationCollection is used as an extension points for other collections in a Publication
type PublicationCollection struct {
	Role     string
	Metadata []Meta
	Links    []Link
	Children []PublicationCollection
}

// LCPHandler struct to generate json to return to the navigator for the lcp information
type LCPHandler struct {
	Identifier string `json:"identifier,omitempty"`
	Profile    string `json:"profile,omitempty"`
	Key        struct {
		Ready bool   `json:"ready,omitempty"`
		Check string `json:"check,omitempty"`
	} `json:"key,omitempty"`
	Hint struct {
		Text string `json:"text,omitempty"`
		URL  string `json:"url,omitempty"`
	} `json:"hint,omitempty"`
	Support struct {
		Mail string `json:"mail,omitempty"`
		URL  string `json:"url,omitempty"`
		Tel  string `json:"tel,omitempty"`
	} `json:"support"`
}

// LCPHandlerPost struct to unmarshal hash send for decrypting lcp
type LCPHandlerPost struct {
	Key struct {
		Hash string `json:"hash"`
	} `json:"key"`
}

// GetCover return the link for the cover
func (publication *Publication) GetCover() (Link, error) {
	return publication.searchLinkByRel("cover")
}

// GetNavDoc return the link for the navigation document
func (publication *Publication) GetNavDoc() (Link, error) {
	return publication.searchLinkByRel("contents")
}

func (publication *Publication) searchLinkByRel(rel string) (Link, error) {
	for _, resource := range publication.Resources {
		for _, resRel := range resource.Rel {
			if resRel == rel {
				return resource, nil
			}
		}
	}

	for _, item := range publication.ReadingOrder {
		for _, spineRel := range item.Rel {
			if spineRel == rel {
				return item, nil
			}
		}
	}

	for _, link := range publication.Links {
		for _, linkRel := range link.Rel {
			if linkRel == rel {
				return link, nil
			}
		}
	}

	return Link{}, errors.New("Can't find " + rel + " in publication")
}

// AddLink Add link in publication link self or search
func (publication *Publication) AddLink(typeLink string, rel []string, url string, templated bool) {
	link := Link{
		Href:     url,
		TypeLink: typeLink,
	}
	if len(rel) > 0 {
		link.Rel = rel
	}

	if templated == true {
		link.Templated = true
	}

	publication.Links = append(publication.Links, link)
}

// FindAllMediaOverlay return all media overlay structure from struct
func (publication *Publication) FindAllMediaOverlay() []MediaOverlayNode {
	var overlay []MediaOverlayNode

	for _, l := range publication.ReadingOrder {
		if len(l.MediaOverlays) > 0 {
			for _, ov := range l.MediaOverlays {
				overlay = append(overlay, ov)
			}
		}
	}

	return overlay
}

// FindMediaOverlayByHref search in media overlay structure for url that match
func (publication *Publication) FindMediaOverlayByHref(href string) []MediaOverlayNode {
	var overlay []MediaOverlayNode

	for _, l := range publication.ReadingOrder {
		if strings.Contains(l.Href, href) {
			if len(l.MediaOverlays) > 0 {
				for _, ov := range l.MediaOverlays {
					overlay = append(overlay, ov)
				}
			}
		}
	}

	return overlay
}

// AddLCPPassphrase function to add internal metadata for decrypting LCP resources
func (publication *Publication) AddLCPPassphrase(passphrase string) {
	publication.Internal = append(publication.Internal, Internal{Name: "lcp_passphrase", Value: passphrase})
}

// AddLCPHash function to add internal metadata for decrypting LCP resources
func (publication *Publication) AddLCPHash(token []byte) {
	publication.AddToInternal("lcp_hash_passphrase", token)
}

func (publication *Publication) findFromInternal(key string) Internal {
	for _, data := range publication.Internal {
		if data.Name == key {
			return data
		}
	}
	return Internal{}
}

// GetStringFromInternal get data store in internal struct in string
func (publication *Publication) GetStringFromInternal(key string) string {

	data := publication.findFromInternal(key)
	if data.Name != "" {
		return data.Value.(string)
	}
	return ""
}

// GetBytesFromInternal get data store in internal structure in byte
func (publication *Publication) GetBytesFromInternal(key string) []byte {

	data := publication.findFromInternal(key)
	if data.Name != "" {
		return data.Value.([]byte)
	}
	return []byte("")
}

// AddToInternal push data to internal struct in publication
func (publication *Publication) AddToInternal(key string, value interface{}) {
	publication.Internal = append(publication.Internal, Internal{Name: key, Value: value})
}

// GetLCPJSON return the raw lcp license json from META-INF/license.lcpl
// if the data is present else return emtpy string
func (publication *Publication) GetLCPJSON() []byte {
	data := publication.GetBytesFromInternal("lcpl")

	return data
}

// GetLCPHandlerInfo return the lcp handler struct for marshalling
func (publication *Publication) GetLCPHandlerInfo() (LCPHandler, error) {
	var info LCPHandler

	if publication.LCP.ID != "" {
		info.Identifier = publication.LCP.ID
		info.Hint.Text = publication.LCP.Encryption.UserKey.TextHint
		info.Key.Check = publication.LCP.Encryption.UserKey.KeyCheck
		info.Key.Ready = false
		info.Profile = publication.LCP.Encryption.Profile
		for _, l := range publication.LCP.Links {
			if l.Rel == "hint" {
				info.Hint.URL = l.Href
			}
		}

		return info, nil
	}

	return info, errors.New("no LCP information")
}

// GetPreFetchResources select resources that match media type we want to
// prefetch with the manifest
func (publication *Publication) GetPreFetchResources() []Link {
	var resources []Link

	mediaTypes := []string{"text/css", "application/vnd.ms-opentype", "text/javascript"}

	for _, l := range publication.Resources {
		for _, m := range mediaTypes {
			if l.TypeLink == m {
				resources = append(resources, l)
			}
		}
	}

	return resources
}

// AddRel add rel information to Link, will check if the
func (link *Link) AddRel(rel string) {
	relAlreadyPresent := false

	for _, r := range link.Rel {
		if r == rel {
			relAlreadyPresent = true
		}
	}

	if relAlreadyPresent == false {
		link.Rel = append(link.Rel, rel)
	}
}

// AddHrefAbsolute modify Href field with a calculated path based on a
// referend file
func (link *Link) AddHrefAbsolute(href string, baseFile string) {
	link.Href = path.Join(path.Dir(baseFile), href)
}

//TransformLinkToFullURL concatenate a base url to all links
func (publication *Publication) TransformLinkToFullURL(baseURL string) {

	for i := range publication.ReadingOrder {
		if !(strings.Contains(publication.ReadingOrder[i].Href, "http://") || strings.Contains(publication.ReadingOrder[i].Href, "https://")) {
			publication.ReadingOrder[i].Href = baseURL + publication.ReadingOrder[i].Href
		}
	}

	for i := range publication.Resources {
		if !(strings.Contains(publication.Resources[i].Href, "http://") || strings.Contains(publication.Resources[i].Href, "https://")) {
			publication.Resources[i].Href = baseURL + publication.Resources[i].Href
		}
	}

	for i := range publication.TOC {
		if !(strings.Contains(publication.TOC[i].Href, "http://") || strings.Contains(publication.TOC[i].Href, "https://")) {
			publication.TOC[i].Href = baseURL + publication.TOC[i].Href
		}
	}

	for i := range publication.Landmarks {
		if !(strings.Contains(publication.Landmarks[i].Href, "http://") || strings.Contains(publication.Landmarks[i].Href, "https://")) {
			publication.Landmarks[i].Href = baseURL + publication.Landmarks[i].Href
		}
	}
}
<commit_msg>Update JSON mapping name for pageList (#52)<commit_after>package models

import (
	"errors"
	"path"
	"strings"

	"github.com/readium/r2-streamer-go/parser/epub"
)

// Publication Main structure for a publication
type Publication struct {
	Context      []string `json:"@context,omitempty"`
	Metadata     Metadata `json:"metadata"`
	Links        []Link   `json:"links"`
	ReadingOrder []Link   `json:"readingOrder,omitempty"`
	Resources    []Link   `json:"resources,omitempty"` //Replaces the manifest but less redundant
	TOC          []Link   `json:"toc,omitempty"`
	PageList     []Link   `json:"pageList,omitempty"`
	Landmarks    []Link   `json:"landmarks,omitempty"`
	LOI          []Link   `json:"loi,omitempty"` //List of illustrations
	LOA          []Link   `json:"loa,omitempty"` //List of audio files
	LOV          []Link   `json:"lov,omitempty"` //List of videos
	LOT          []Link   `json:"lot,omitempty"` //List of tables

	OtherLinks       []Link                  `json:"-"` //Extension point for links that shouldn't show up in the manifest
	OtherCollections []PublicationCollection `json:"-"` //Extension point for collections that shouldn't show up in the manifest
	Internal         []Internal              `json:"-"`
	LCP              epub.LCP                `json:"-"`
}

// Internal TODO
type Internal struct {
	Name  string
	Value interface{}
}

// Link object used in collections and links
type Link struct {
	Href          string             `json:"href"`
	TypeLink      string             `json:"type,omitempty"`
	Rel           []string           `json:"rel,omitempty"`
	Height        int                `json:"height,omitempty"`
	Width         int                `json:"width,omitempty"`
	Title         string             `json:"title,omitempty"`
	Properties    *Properties        `json:"properties,omitempty"`
	Duration      string             `json:"duration,omitempty"`
	Templated     bool               `json:"templated,omitempty"`
	Children      []Link             `json:"children,omitempty"`
	Bitrate       int                `json:"bitrate,omitempty"`
	MediaOverlays []MediaOverlayNode `json:"-"`
}

// PublicationCollection is used as an extension points for other collections in a Publication
type PublicationCollection struct {
	Role     string
	Metadata []Meta
	Links    []Link
	Children []PublicationCollection
}

// LCPHandler struct to generate json to return to the navigator for the lcp information
type LCPHandler struct {
	Identifier string `json:"identifier,omitempty"`
	Profile    string `json:"profile,omitempty"`
	Key        struct {
		Ready bool   `json:"ready,omitempty"`
		Check string `json:"check,omitempty"`
	} `json:"key,omitempty"`
	Hint struct {
		Text string `json:"text,omitempty"`
		URL  string `json:"url,omitempty"`
	} `json:"hint,omitempty"`
	Support struct {
		Mail string `json:"mail,omitempty"`
		URL  string `json:"url,omitempty"`
		Tel  string `json:"tel,omitempty"`
	} `json:"support"`
}

// LCPHandlerPost struct to unmarshal hash send for decrypting lcp
type LCPHandlerPost struct {
	Key struct {
		Hash string `json:"hash"`
	} `json:"key"`
}

// GetCover return the link for the cover
func (publication *Publication) GetCover() (Link, error) {
	return publication.searchLinkByRel("cover")
}

// GetNavDoc return the link for the navigation document
func (publication *Publication) GetNavDoc() (Link, error) {
	return publication.searchLinkByRel("contents")
}

func (publication *Publication) searchLinkByRel(rel string) (Link, error) {
	for _, resource := range publication.Resources {
		for _, resRel := range resource.Rel {
			if resRel == rel {
				return resource, nil
			}
		}
	}

	for _, item := range publication.ReadingOrder {
		for _, spineRel := range item.Rel {
			if spineRel == rel {
				return item, nil
			}
		}
	}

	for _, link := range publication.Links {
		for _, linkRel := range link.Rel {
			if linkRel == rel {
				return link, nil
			}
		}
	}

	return Link{}, errors.New("Can't find " + rel + " in publication")
}

// AddLink Add link in publication link self or search
func (publication *Publication) AddLink(typeLink string, rel []string, url string, templated bool) {
	link := Link{
		Href:     url,
		TypeLink: typeLink,
	}
	if len(rel) > 0 {
		link.Rel = rel
	}

	if templated == true {
		link.Templated = true
	}

	publication.Links = append(publication.Links, link)
}

// FindAllMediaOverlay return all media overlay structure from struct
func (publication *Publication) FindAllMediaOverlay() []MediaOverlayNode {
	var overlay []MediaOverlayNode

	for _, l := range publication.ReadingOrder {
		if len(l.MediaOverlays) > 0 {
			for _, ov := range l.MediaOverlays {
				overlay = append(overlay, ov)
			}
		}
	}

	return overlay
}

// FindMediaOverlayByHref search in media overlay structure for url that match
func (publication *Publication) FindMediaOverlayByHref(href string) []MediaOverlayNode {
	var overlay []MediaOverlayNode

	for _, l := range publication.ReadingOrder {
		if strings.Contains(l.Href, href) {
			if len(l.MediaOverlays) > 0 {
				for _, ov := range l.MediaOverlays {
					overlay = append(overlay, ov)
				}
			}
		}
	}

	return overlay
}

// AddLCPPassphrase function to add internal metadata for decrypting LCP resources
func (publication *Publication) AddLCPPassphrase(passphrase string) {
	publication.Internal = append(publication.Internal, Internal{Name: "lcp_passphrase", Value: passphrase})
}

// AddLCPHash function to add internal metadata for decrypting LCP resources
func (publication *Publication) AddLCPHash(token []byte) {
	publication.AddToInternal("lcp_hash_passphrase", token)
}

func (publication *Publication) findFromInternal(key string) Internal {
	for _, data := range publication.Internal {
		if data.Name == key {
			return data
		}
	}
	return Internal{}
}

// GetStringFromInternal get data store in internal struct in string
func (publication *Publication) GetStringFromInternal(key string) string {

	data := publication.findFromInternal(key)
	if data.Name != "" {
		return data.Value.(string)
	}
	return ""
}

// GetBytesFromInternal get data store in internal structure in byte
func (publication *Publication) GetBytesFromInternal(key string) []byte {

	data := publication.findFromInternal(key)
	if data.Name != "" {
		return data.Value.([]byte)
	}
	return []byte("")
}

// AddToInternal push data to internal struct in publication
func (publication *Publication) AddToInternal(key string, value interface{}) {
	publication.Internal = append(publication.Internal, Internal{Name: key, Value: value})
}

// GetLCPJSON return the raw lcp license json from META-INF/license.lcpl
// if the data is present else return emtpy string
func (publication *Publication) GetLCPJSON() []byte {
	data := publication.GetBytesFromInternal("lcpl")

	return data
}

// GetLCPHandlerInfo return the lcp handler struct for marshalling
func (publication *Publication) GetLCPHandlerInfo() (LCPHandler, error) {
	var info LCPHandler

	if publication.LCP.ID != "" {
		info.Identifier = publication.LCP.ID
		info.Hint.Text = publication.LCP.Encryption.UserKey.TextHint
		info.Key.Check = publication.LCP.Encryption.UserKey.KeyCheck
		info.Key.Ready = false
		info.Profile = publication.LCP.Encryption.Profile
		for _, l := range publication.LCP.Links {
			if l.Rel == "hint" {
				info.Hint.URL = l.Href
			}
		}

		return info, nil
	}

	return info, errors.New("no LCP information")
}

// GetPreFetchResources select resources that match media type we want to
// prefetch with the manifest
func (publication *Publication) GetPreFetchResources() []Link {
	var resources []Link

	mediaTypes := []string{"text/css", "application/vnd.ms-opentype", "text/javascript"}

	for _, l := range publication.Resources {
		for _, m := range mediaTypes {
			if l.TypeLink == m {
				resources = append(resources, l)
			}
		}
	}

	return resources
}

// AddRel add rel information to Link, will check if the
func (link *Link) AddRel(rel string) {
	relAlreadyPresent := false

	for _, r := range link.Rel {
		if r == rel {
			relAlreadyPresent = true
		}
	}

	if relAlreadyPresent == false {
		link.Rel = append(link.Rel, rel)
	}
}

// AddHrefAbsolute modify Href field with a calculated path based on a
// referend file
func (link *Link) AddHrefAbsolute(href string, baseFile string) {
	link.Href = path.Join(path.Dir(baseFile), href)
}

//TransformLinkToFullURL concatenate a base url to all links
func (publication *Publication) TransformLinkToFullURL(baseURL string) {

	for i := range publication.ReadingOrder {
		if !(strings.Contains(publication.ReadingOrder[i].Href, "http://") || strings.Contains(publication.ReadingOrder[i].Href, "https://")) {
			publication.ReadingOrder[i].Href = baseURL + publication.ReadingOrder[i].Href
		}
	}

	for i := range publication.Resources {
		if !(strings.Contains(publication.Resources[i].Href, "http://") || strings.Contains(publication.Resources[i].Href, "https://")) {
			publication.Resources[i].Href = baseURL + publication.Resources[i].Href
		}
	}

	for i := range publication.TOC {
		if !(strings.Contains(publication.TOC[i].Href, "http://") || strings.Contains(publication.TOC[i].Href, "https://")) {
			publication.TOC[i].Href = baseURL + publication.TOC[i].Href
		}
	}

	for i := range publication.Landmarks {
		if !(strings.Contains(publication.Landmarks[i].Href, "http://") || strings.Contains(publication.Landmarks[i].Href, "https://")) {
			publication.Landmarks[i].Href = baseURL + publication.Landmarks[i].Href
		}
	}
}
<|endoftext|>


<commit_before>package store

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/justphil/denatify-service/models"
)

type Store interface {
	CreateUser(u *models.User) (string, error)
	GetUsers() ([]*models.User, error)
	GetUserById(id string) (*models.User, error)
	UpdateUser(u *models.User) error
	DeleteUserById(id string) error
	Close()
}

const (
	DB        = "nat-busters"
	USERS_COL = "users"
)

type MongoStore struct {
	sess *mgo.Session
}

func NewMongoStore(url string) *MongoStore {
	sess, err := mgo.Dial(url)

	if err != nil {
		panic(err)
	}

	return &MongoStore{sess}
}

func (mongo *MongoStore) query(db string, col string, op func(sess *mgo.Collection)) {
	sess := mongo.sess.Copy()
	defer sess.Close()
	op(sess.DB(db).C(col))
}

func (mongo *MongoStore) CreateUser(u *models.User) (string, error) {
	var userId string
	var err error
	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		u.Id = bson.NewObjectId()
		err = c.Insert(u)
		if err == nil {
			userId = u.Id.Hex()
		}
	})

	return userId, err
}

func (mongo *MongoStore) GetUsers() ([]*models.User, error) {
	var users []*models.User
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.Find(nil).All(&users)
	})

	return users, err
}

func (mongo *MongoStore) GetUserById(id string) (*models.User, error) {
	user := &models.User{}
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.FindId(bson.ObjectIdHex(id)).One(user)
	})

	return user, err
}

func (mongo *MongoStore) UpdateUser(u *models.User) error {
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.UpdateId(u.Id, u)
	})

	return err
}

func (mongo *MongoStore) DeleteUserById(id string) error {
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.RemoveId(bson.ObjectIdHex(id))
	})

	return err
}

func (mongo *MongoStore) Close() {
	mongo.sess.Close()
}
<commit_msg>add todo comments<commit_after>package store

import (
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/justphil/denatify-service/models"
)

type Store interface {
	CreateUser(u *models.User) (string, error)
	GetUsers() ([]*models.User, error)
	GetUserById(id string) (*models.User, error)
	UpdateUser(u *models.User) error
	DeleteUserById(id string) error
	Close()
}

const (
	DB        = "nat-busters"
	USERS_COL = "users"
)

type MongoStore struct {
	sess *mgo.Session
}

func NewMongoStore(url string) *MongoStore {
	sess, err := mgo.Dial(url)

	// TODO: add indexes

	if err != nil {
		panic(err)
	}

	return &MongoStore{sess}
}

func (mongo *MongoStore) query(db string, col string, op func(sess *mgo.Collection)) {
	sess := mongo.sess.Copy()
	defer sess.Close()
	op(sess.DB(db).C(col))
}

func (mongo *MongoStore) CreateUser(u *models.User) (string, error) {
	var userId string
	var err error
	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		u.Id = bson.NewObjectId()
		err = c.Insert(u)
		if err == nil {
			userId = u.Id.Hex()
		}
	})

	return userId, err
}

func (mongo *MongoStore) GetUsers() ([]*models.User, error) {
	var users []*models.User
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.Find(nil).All(&users)
	})

	return users, err
}

func (mongo *MongoStore) GetUserById(id string) (*models.User, error) {
	user := &models.User{}
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.FindId(bson.ObjectIdHex(id)).One(user)
	})

	return user, err
}

func (mongo *MongoStore) UpdateUser(u *models.User) error {
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.UpdateId(u.Id, u)
	})

	return err
}

func (mongo *MongoStore) DeleteUserById(id string) error {
	var err error

	mongo.query(DB, USERS_COL, func(c *mgo.Collection) {
		err = c.RemoveId(bson.ObjectIdHex(id))
	})

	return err
}

func (mongo *MongoStore) Close() {
	mongo.sess.Close()
}
<|endoftext|>


<commit_before>// bulk_load_mongo loads a Mongo daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	flatbuffers "github.com/google/flatbuffers/go"
	"github.com/pkg/profile"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/influxdata/influxdb-comparisons/mongo_serialization"
)

// Program option vars:
var (
	daemonUrl    string
	workers      int
	batchSize    int
	limit        int64
	doLoad       bool
	writeTimeout time.Duration
)

// Global vars
var (
	batchChan    chan *Batch
	inputDone    chan struct{}
	workersGroup sync.WaitGroup
)

// Magic database constants
const (
	dbName              = "benchmark_db"
	pointCollectionName = "point_data"
)

// bufPool holds []byte instances to reduce heap churn.
var bufPool = &sync.Pool{
	New: func() interface{} {
		return make([]byte, 0, 1024)
	},
}

// Batch holds byte slices that will become mongo_serialization.Item instances.
type Batch [][]byte

func (b *Batch) ClearReferences() {
	*b = (*b)[:0]
}

// batchPool holds *Batch instances to reduce heap churn.
var batchPool = &sync.Pool{
	New: func() interface{} {
		return &Batch{}
	},
}

// Parse args:
func init() {
	flag.StringVar(&daemonUrl, "url", "localhost:27017", "Mongo URL.")

	flag.IntVar(&batchSize, "batch-size", 100, "Batch size (input items).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&limit, "limit", -1, "Number of items to insert (default unlimited).")
	flag.DurationVar(&writeTimeout, "write-timeout", 10*time.Second, "Write timeout.")

	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")

	flag.Parse()

	for i := 0; i < workers*batchSize; i++ {
		bufPool.Put(bufPool.New())
	}
}

func main() {
	//_ = profile.Start
	p := profile.Start(profile.MemProfile)
	defer p.Stop()
	if doLoad {
		mustCreateCollections(daemonUrl)
	}

	var session *mgo.Session

	if doLoad {
		var err error
		session, err = mgo.Dial(daemonUrl)
		if err != nil {
			log.Fatal(err)
		}
		session.SetMode(mgo.Eventual, false)
		defer session.Close()
	}

	batchChan = make(chan *Batch, workers*10)
	inputDone = make(chan struct{})

	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processBatches(session)
	}

	start := time.Now()
	itemsRead := scan(session, batchSize)

	<-inputDone
	close(batchChan)
	workersGroup.Wait()
	end := time.Now()
	took := end.Sub(start)
	rate := float64(itemsRead) / float64(took.Seconds())

	fmt.Printf("loaded %d values in %fsec with %d workers (mean rate %f values/sec)\n", itemsRead, took.Seconds(), workers, rate)
}

// scan reads length-delimited flatbuffers items from stdin.
func scan(session *mgo.Session, itemsPerBatch int) int64 {
	//var batch *gocql.Batch
	if doLoad {
		//batch = session.NewBatch(gocql.LoggedBatch)
	}

	var n int
	var itemsRead int64
	r := bufio.NewReaderSize(os.Stdin, 32<<20)

	start := time.Now()
	batch := batchPool.Get().(*Batch)
	lenBuf := make([]byte, 8)

	for {
		if itemsRead == limit {
			break
		}
		// get the serialized item length (this is the framing format)
		_, err := r.Read(lenBuf)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err.Error())
		}

		// ensure correct len of receiving buffer
		l := int(binary.LittleEndian.Uint64(lenBuf))
		itemBuf := bufPool.Get().([]byte)
		if cap(itemBuf) < l {
			itemBuf = make([]byte, l)
		}
		itemBuf = itemBuf[:l]

		// read the bytes and init the flatbuffer object
		totRead := 0
		for totRead < l {
			m, err := r.Read(itemBuf[totRead:])
			// (EOF is also fatal)
			if err != nil {
				log.Fatal(err.Error())
			}
			totRead += m
		}
		if totRead != len(itemBuf) {
			panic(fmt.Sprintf("reader/writer logic error, %d != %d", n, len(itemBuf)))
		}

		*batch = append(*batch, itemBuf)

		itemsRead++
		n++

		if n >= batchSize {
			batchChan <- batch
			n = 0
			batch = batchPool.Get().(*Batch)
		}

		_ = start
		//if itemsRead > 0 && itemsRead%100000 == 0 {
		//	_ = start
		//	//took := (time.Now().UnixNano() - start.UnixNano())
		//	//if took >= 1e9 {
		//	//	tookUs := float64(took) / 1e3
		//	//	tookSec := float64(took) / 1e9
		//	//	fmt.Fprintf(os.Stderr, "itemsRead: %d, rate: %.0f/sec, lag: %.2fus/op\n",
		//	//		itemsRead, float64(itemsRead)/tookSec, tookUs/float64(itemsRead))
		//	//}
		//}
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead
}

// processBatches reads byte buffers from batchChan, interprets them and writes
// them to the target server. Note that mgo forcibly incurs serialization
// overhead (it always encodes to BSON).
func processBatches(session *mgo.Session) {
	db := session.DB(dbName)

	type Tag struct {
		Key string `bson:"key"`
		Val string `bson:"val"`
	}

	type Point struct {
		// Use `string` here even though they are really `[]byte`.
		// This is so the mongo data is human-readable.
		MeasurementName string      `bson:"measurement"`
		FieldName       string      `bson:"field"`
		Timestamp       int64       `bson:"timestamp_ns"`
		Tags            []Tag       `bson:"tags"`
		Value           interface{} `bson:"value"`

		// a private union-like section
		longValue int64
		doubleValue float64
	}
	pPool := &sync.Pool{New: func() interface{} { return &Point{} }}
	pvs := []interface{}{}

	item := &mongo_serialization.Item{}
	destTag := &mongo_serialization.Tag{}
	collection := db.C(pointCollectionName)
	for batch := range batchChan {
		bulk := collection.Bulk()

		if cap(pvs) < len(*batch) {
			pvs = make([]interface{}, len(*batch))
		}
		pvs = pvs[:len(*batch)]

		for i, itemBuf := range *batch {
			// this ui could be improved on the library side:
			n := flatbuffers.GetUOffsetT(itemBuf)
			item.Init(itemBuf, n)
			x := pPool.Get().(*Point)

			x.MeasurementName = unsafeBytesToString(item.MeasurementNameBytes())
			x.FieldName = unsafeBytesToString(item.FieldNameBytes())
			x.Timestamp = item.TimestampNanos()

			tagLength := item.TagsLength()
			if cap(x.Tags) < tagLength {
				x.Tags = make([]Tag, 0, tagLength)
			}
			x.Tags = x.Tags[:tagLength]
			for i := 0; i < tagLength; i++ {
				*destTag = mongo_serialization.Tag{} // clear
				item.Tags(destTag, i)
				x.Tags[i].Key = unsafeBytesToString(destTag.KeyBytes())
				x.Tags[i].Val = unsafeBytesToString(destTag.ValBytes())
			}

			// this complexity is the result of trying to minimize
			// allocs while using an interface{} type for
			// (*Point).Value.
			switch item.ValueType() {
			case mongo_serialization.ValueTypeLong:
				x.longValue = item.LongValue()
				x.Value = &x.longValue
			case mongo_serialization.ValueTypeDouble:
				x.doubleValue = item.DoubleValue()
				x.Value = &x.doubleValue
			default:
				panic("logic error")
			}
			pvs[i] = x

		}
		bulk.Insert(pvs...)

		if doLoad {
			_, err := bulk.Run()
			if err != nil {
				log.Fatalf("Bulk err: %s\n", err.Error())
			}

		}

		// cleanup pvs
		for _, x := range pvs {
			p := x.(*Point)
			p.Timestamp = 0
			p.Value = nil
			p.longValue = 0
			p.doubleValue = 0
			p.Tags = p.Tags[:0]
			pPool.Put(p)
		}

		// cleanup item data
		for _, itemBuf := range *batch {
			bufPool.Put(itemBuf)
		}
		batch.ClearReferences()
		batchPool.Put(batch)
	}
	workersGroup.Done()
}

func mustCreateCollections(daemonUrl string) {
	session, err := mgo.Dial(daemonUrl)
	if err != nil {
		log.Fatal(err)
	}
	defer session.Close()

	// collection C: point data
	// from (*mgo.Collection).Create
	cmd := make(bson.D, 0, 4)
	cmd = append(cmd, bson.DocElem{"create", pointCollectionName})

	// wiredtiger settings
	cmd = append(cmd, bson.DocElem{
		"storageEngine", map[string]interface{}{
			"wiredTiger": map[string]interface{}{
				"configString": "block_compressor=snappy",
			},
		},
	})

	err = session.DB("benchmark_db").Run(cmd, nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Create mongo index at the beginning of load.<commit_after>// bulk_load_mongo loads a Mongo daemon with data from stdin.
//
// The caller is responsible for assuring that the database is empty before
// bulk load.
package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
	"time"

	flatbuffers "github.com/google/flatbuffers/go"
	"github.com/pkg/profile"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/influxdata/influxdb-comparisons/mongo_serialization"
)

// Program option vars:
var (
	daemonUrl    string
	workers      int
	batchSize    int
	limit        int64
	doLoad       bool
	writeTimeout time.Duration
)

// Global vars
var (
	batchChan    chan *Batch
	inputDone    chan struct{}
	workersGroup sync.WaitGroup
)

// Magic database constants
const (
	dbName              = "benchmark_db"
	pointCollectionName = "point_data"
)

// bufPool holds []byte instances to reduce heap churn.
var bufPool = &sync.Pool{
	New: func() interface{} {
		return make([]byte, 0, 1024)
	},
}

// Batch holds byte slices that will become mongo_serialization.Item instances.
type Batch [][]byte

func (b *Batch) ClearReferences() {
	*b = (*b)[:0]
}

// batchPool holds *Batch instances to reduce heap churn.
var batchPool = &sync.Pool{
	New: func() interface{} {
		return &Batch{}
	},
}

// Parse args:
func init() {
	flag.StringVar(&daemonUrl, "url", "localhost:27017", "Mongo URL.")

	flag.IntVar(&batchSize, "batch-size", 100, "Batch size (input items).")
	flag.IntVar(&workers, "workers", 1, "Number of parallel requests to make.")
	flag.Int64Var(&limit, "limit", -1, "Number of items to insert (default unlimited).")
	flag.DurationVar(&writeTimeout, "write-timeout", 10*time.Second, "Write timeout.")

	flag.BoolVar(&doLoad, "do-load", true, "Whether to write data. Set this flag to false to check input read speed.")

	flag.Parse()

	for i := 0; i < workers*batchSize; i++ {
		bufPool.Put(bufPool.New())
	}
}

func main() {
	//_ = profile.Start
	p := profile.Start(profile.MemProfile)
	defer p.Stop()
	if doLoad {
		mustCreateCollections(daemonUrl)
	}

	var session *mgo.Session

	if doLoad {
		var err error
		session, err = mgo.Dial(daemonUrl)
		if err != nil {
			log.Fatal(err)
		}

		session.SetMode(mgo.Eventual, false)

		defer session.Close()
	}

	batchChan = make(chan *Batch, workers*10)
	inputDone = make(chan struct{})

	for i := 0; i < workers; i++ {
		workersGroup.Add(1)
		go processBatches(session)
	}

	start := time.Now()
	itemsRead := scan(session, batchSize)

	<-inputDone
	close(batchChan)
	workersGroup.Wait()
	end := time.Now()
	took := end.Sub(start)
	rate := float64(itemsRead) / float64(took.Seconds())

	fmt.Printf("loaded %d values in %fsec with %d workers (mean rate %f values/sec)\n", itemsRead, took.Seconds(), workers, rate)
}

// scan reads length-delimited flatbuffers items from stdin.
func scan(session *mgo.Session, itemsPerBatch int) int64 {
	//var batch *gocql.Batch
	if doLoad {
		//batch = session.NewBatch(gocql.LoggedBatch)
	}

	var n int
	var itemsRead int64
	r := bufio.NewReaderSize(os.Stdin, 32<<20)

	start := time.Now()
	batch := batchPool.Get().(*Batch)
	lenBuf := make([]byte, 8)

	for {
		if itemsRead == limit {
			break
		}
		// get the serialized item length (this is the framing format)
		_, err := r.Read(lenBuf)
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err.Error())
		}

		// ensure correct len of receiving buffer
		l := int(binary.LittleEndian.Uint64(lenBuf))
		itemBuf := bufPool.Get().([]byte)
		if cap(itemBuf) < l {
			itemBuf = make([]byte, l)
		}
		itemBuf = itemBuf[:l]

		// read the bytes and init the flatbuffer object
		totRead := 0
		for totRead < l {
			m, err := r.Read(itemBuf[totRead:])
			// (EOF is also fatal)
			if err != nil {
				log.Fatal(err.Error())
			}
			totRead += m
		}
		if totRead != len(itemBuf) {
			panic(fmt.Sprintf("reader/writer logic error, %d != %d", n, len(itemBuf)))
		}

		*batch = append(*batch, itemBuf)

		itemsRead++
		n++

		if n >= batchSize {
			batchChan <- batch
			n = 0
			batch = batchPool.Get().(*Batch)
		}

		_ = start
		//if itemsRead > 0 && itemsRead%100000 == 0 {
		//	_ = start
		//	//took := (time.Now().UnixNano() - start.UnixNano())
		//	//if took >= 1e9 {
		//	//	tookUs := float64(took) / 1e3
		//	//	tookSec := float64(took) / 1e9
		//	//	fmt.Fprintf(os.Stderr, "itemsRead: %d, rate: %.0f/sec, lag: %.2fus/op\n",
		//	//		itemsRead, float64(itemsRead)/tookSec, tookUs/float64(itemsRead))
		//	//}
		//}
	}

	// Closing inputDone signals to the application that we've read everything and can now shut down.
	close(inputDone)

	return itemsRead
}

// processBatches reads byte buffers from batchChan, interprets them and writes
// them to the target server. Note that mgo forcibly incurs serialization
// overhead (it always encodes to BSON).
func processBatches(session *mgo.Session) {
	db := session.DB(dbName)

	type Tag struct {
		Key string `bson:"key"`
		Val string `bson:"val"`
	}

	type Point struct {
		// Use `string` here even though they are really `[]byte`.
		// This is so the mongo data is human-readable.
		MeasurementName string      `bson:"measurement"`
		FieldName       string      `bson:"field"`
		Timestamp       int64       `bson:"timestamp_ns"`
		Tags            []Tag       `bson:"tags"`
		Value           interface{} `bson:"value"`

		// a private union-like section
		longValue int64
		doubleValue float64
	}
	pPool := &sync.Pool{New: func() interface{} { return &Point{} }}
	pvs := []interface{}{}

	item := &mongo_serialization.Item{}
	destTag := &mongo_serialization.Tag{}
	collection := db.C(pointCollectionName)
	for batch := range batchChan {
		bulk := collection.Bulk()

		if cap(pvs) < len(*batch) {
			pvs = make([]interface{}, len(*batch))
		}
		pvs = pvs[:len(*batch)]

		for i, itemBuf := range *batch {
			// this ui could be improved on the library side:
			n := flatbuffers.GetUOffsetT(itemBuf)
			item.Init(itemBuf, n)
			x := pPool.Get().(*Point)

			x.MeasurementName = unsafeBytesToString(item.MeasurementNameBytes())
			x.FieldName = unsafeBytesToString(item.FieldNameBytes())
			x.Timestamp = item.TimestampNanos()

			tagLength := item.TagsLength()
			if cap(x.Tags) < tagLength {
				x.Tags = make([]Tag, 0, tagLength)
			}
			x.Tags = x.Tags[:tagLength]
			for i := 0; i < tagLength; i++ {
				*destTag = mongo_serialization.Tag{} // clear
				item.Tags(destTag, i)
				x.Tags[i].Key = unsafeBytesToString(destTag.KeyBytes())
				x.Tags[i].Val = unsafeBytesToString(destTag.ValBytes())
			}

			// this complexity is the result of trying to minimize
			// allocs while using an interface{} type for
			// (*Point).Value.
			switch item.ValueType() {
			case mongo_serialization.ValueTypeLong:
				x.longValue = item.LongValue()
				x.Value = &x.longValue
			case mongo_serialization.ValueTypeDouble:
				x.doubleValue = item.DoubleValue()
				x.Value = &x.doubleValue
			default:
				panic("logic error")
			}
			pvs[i] = x

		}
		bulk.Insert(pvs...)

		if doLoad {
			_, err := bulk.Run()
			if err != nil {
				log.Fatalf("Bulk err: %s\n", err.Error())
			}

		}

		// cleanup pvs
		for _, x := range pvs {
			p := x.(*Point)
			p.Timestamp = 0
			p.Value = nil
			p.longValue = 0
			p.doubleValue = 0
			p.Tags = p.Tags[:0]
			pPool.Put(p)
		}

		// cleanup item data
		for _, itemBuf := range *batch {
			bufPool.Put(itemBuf)
		}
		batch.ClearReferences()
		batchPool.Put(batch)
	}
	workersGroup.Done()
}

func mustCreateCollections(daemonUrl string) {
	session, err := mgo.Dial(daemonUrl)
	if err != nil {
		log.Fatal(err)
	}
	defer session.Close()

	// collection C: point data
	// from (*mgo.Collection).Create
	cmd := make(bson.D, 0, 4)
	cmd = append(cmd, bson.DocElem{"create", pointCollectionName})

	// wiredtiger settings
	cmd = append(cmd, bson.DocElem{
		"storageEngine", map[string]interface{}{
			"wiredTiger": map[string]interface{}{
				"configString": "block_compressor=snappy",
			},
		},
	})

	err = session.DB("benchmark_db").Run(cmd, nil)
	if err != nil {
		log.Fatal(err)
	}

	collection := session.DB("benchmark_db").C("point_data")
	index := mgo.Index{
		Key: []string{"measurement", "tags", "field", "timestamp_ns"},
		Unique: false, // Unique does not work on the entire array of tags!
		DropDups: true,
		Background: false,
		Sparse: false,
	}
	err = collection.EnsureIndex(index)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The list package implements a doubly linked list.
package list

// Element is an element in the linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// The front of the list has prev = nil, and the back has next = nil.
	next, prev *Element

	// Thie list to which this element belongs.
	list *List

	// The contents of this list element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element { return e.next }

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element { return e.prev }

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	front, back *Element
	len         int
}

// Init initializes or clears a List.
func (l *List) Init() *List {
	l.front = nil
	l.back = nil
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List).Init() }

// Front returns the first element in the list.
func (l *List) Front() *Element { return l.front }

// Back returns the last element in the list.
func (l *List) Back() *Element { return l.back }

// Remove removes the element from the list.
func (l *List) Remove(e *Element) {
	l.remove(e)
	e.list = nil // do what remove does not
}

// remove the element from the list, but do not clear the Element's list field.
// This is so that other List methods may use remove when relocating Elements
// without needing to restore the list field.
func (l *List) remove(e *Element) {
	if e.list != l {
		return
	}
	if e.prev == nil {
		l.front = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		l.back = e.prev
	} else {
		e.next.prev = e.prev
	}

	e.prev = nil
	e.next = nil
	l.len--
}

func (l *List) insertBefore(e *Element, mark *Element) {
	if mark.prev == nil {
		// new front of the list
		l.front = e
	} else {
		mark.prev.next = e
	}
	e.prev = mark.prev
	mark.prev = e
	e.next = mark
	l.len++
}

func (l *List) insertAfter(e *Element, mark *Element) {
	if mark.next == nil {
		// new back of the list
		l.back = e
	} else {
		mark.next.prev = e
	}
	e.next = mark.next
	mark.next = e
	e.prev = mark
	l.len++
}

func (l *List) insertFront(e *Element) {
	if l.front == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertBefore(e, l.front)
}

func (l *List) insertBack(e *Element) {
	if l.back == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertAfter(e, l.back)
}

// PushFront inserts the value at the front of the list and returns a new Element containing the value.
func (l *List) PushFront(value interface{}) *Element {
	if l == nil {
		l.Init()
	}
	e := &Element{nil, nil, l, value}
	l.insertFront(e)
	return e
}

// PushBack inserts the value at the back of the list and returns a new Element containing the value.
func (l *List) PushBack(value interface{}) *Element {
	if l == nil {
		l.Init()
	}
	e := &Element{nil, nil, l, value}
	l.insertBack(e)
	return e
}

// InsertBefore inserts the value immediately before mark and returns a new Element containing the value.
func (l *List) InsertBefore(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertBefore(e, mark)
	return e
}

// InsertAfter inserts the value immediately after mark and returns a new Element containing the value.
func (l *List) InsertAfter(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertAfter(e, mark)
	return e
}

// MoveToFront moves the element to the front of the list.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.front == e {
		return
	}
	l.remove(e)
	l.insertFront(e)
}

// MoveToBack moves the element to the back of the list.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.back == e {
		return
	}
	l.remove(e)
	l.insertBack(e)
}

// Len returns the number of elements in the list.
func (l *List) Len() int { return l.len }

// PushBackList inserts each element of ol at the back of the list.
func (l *List) PushBackList(ol *List) {
	last := ol.Back()
	for e := ol.Front(); e != nil; e = e.Next() {
		l.PushBack(e.Value)
		if e == last {
			break
		}
	}
}

// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved.
func (l *List) PushFrontList(ol *List) {
	first := ol.Front()
	for e := ol.Back(); e != nil; e = e.Prev() {
		l.PushFront(e.Value)
		if e == first {
			break
		}
	}
}
<commit_msg>container/list: elide redundant tests and fix comment typo<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The list package implements a doubly linked list.
package list

// Element is an element in the linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// The front of the list has prev = nil, and the back has next = nil.
	next, prev *Element

	// The list to which this element belongs.
	list *List

	// The contents of this list element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element { return e.next }

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element { return e.prev }

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	front, back *Element
	len         int
}

// Init initializes or clears a List.
func (l *List) Init() *List {
	l.front = nil
	l.back = nil
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List) }

// Front returns the first element in the list.
func (l *List) Front() *Element { return l.front }

// Back returns the last element in the list.
func (l *List) Back() *Element { return l.back }

// Remove removes the element from the list.
func (l *List) Remove(e *Element) {
	l.remove(e)
	e.list = nil // do what remove does not
}

// remove the element from the list, but do not clear the Element's list field.
// This is so that other List methods may use remove when relocating Elements
// without needing to restore the list field.
func (l *List) remove(e *Element) {
	if e.list != l {
		return
	}
	if e.prev == nil {
		l.front = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		l.back = e.prev
	} else {
		e.next.prev = e.prev
	}

	e.prev = nil
	e.next = nil
	l.len--
}

func (l *List) insertBefore(e *Element, mark *Element) {
	if mark.prev == nil {
		// new front of the list
		l.front = e
	} else {
		mark.prev.next = e
	}
	e.prev = mark.prev
	mark.prev = e
	e.next = mark
	l.len++
}

func (l *List) insertAfter(e *Element, mark *Element) {
	if mark.next == nil {
		// new back of the list
		l.back = e
	} else {
		mark.next.prev = e
	}
	e.next = mark.next
	mark.next = e
	e.prev = mark
	l.len++
}

func (l *List) insertFront(e *Element) {
	if l.front == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertBefore(e, l.front)
}

func (l *List) insertBack(e *Element) {
	if l.back == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertAfter(e, l.back)
}

// PushFront inserts the value at the front of the list and returns a new Element containing the value.
func (l *List) PushFront(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertFront(e)
	return e
}

// PushBack inserts the value at the back of the list and returns a new Element containing the value.
func (l *List) PushBack(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertBack(e)
	return e
}

// InsertBefore inserts the value immediately before mark and returns a new Element containing the value.
func (l *List) InsertBefore(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertBefore(e, mark)
	return e
}

// InsertAfter inserts the value immediately after mark and returns a new Element containing the value.
func (l *List) InsertAfter(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertAfter(e, mark)
	return e
}

// MoveToFront moves the element to the front of the list.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.front == e {
		return
	}
	l.remove(e)
	l.insertFront(e)
}

// MoveToBack moves the element to the back of the list.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.back == e {
		return
	}
	l.remove(e)
	l.insertBack(e)
}

// Len returns the number of elements in the list.
func (l *List) Len() int { return l.len }

// PushBackList inserts each element of ol at the back of the list.
func (l *List) PushBackList(ol *List) {
	last := ol.Back()
	for e := ol.Front(); e != nil; e = e.Next() {
		l.PushBack(e.Value)
		if e == last {
			break
		}
	}
}

// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved.
func (l *List) PushFrontList(ol *List) {
	first := ol.Front()
	for e := ol.Back(); e != nil; e = e.Prev() {
		l.PushFront(e.Value)
		if e == first {
			break
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The block package implements standard block cipher modes
// that can be wrapped around low-level block cipher implementations.
// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html
// and NIST Special Publication 800-38A.
package block

// A Cipher represents an implementation of block cipher
// using a given key.  It provides the capability to encrypt
// or decrypt individual blocks.  The mode implementations
// extend that capability to streams of blocks.
type Cipher interface {
	// BlockSize returns the cipher's block size.
	BlockSize() int

	// Encrypt encrypts the first block in src into dst.
	// Src and dst may point at the same memory.
	Encrypt(dst, src []byte)

	// Decrypt decrypts the first block in src into dst.
	// Src and dst may point at the same memory.
	Decrypt(dst, src []byte)
}

// Utility routines

func shift1(dst, src []byte) byte {
	var b byte
	for i := len(src) - 1; i >= 0; i-- {
		bb := src[i] >> 7
		dst[i] = src[i]<<1 | b
		b = bb
	}
	return b
}

func same(p, q []byte) bool {
	if len(p) != len(q) {
		return false
	}
	for i := 0; i < len(p); i++ {
		if p[i] != q[i] {
			return false
		}
	}
	return true
}

func dup(p []byte) []byte {
	q := make([]byte, len(p))
	copy(q, p)
	return q
}
<commit_msg>crypto/block: mark as deprecated.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The block package is deprecated, use cipher instead.
// The block package implements standard block cipher modes
// that can be wrapped around low-level block cipher implementations.
// See http://csrc.nist.gov/groups/ST/toolkit/BCM/current_modes.html
// and NIST Special Publication 800-38A.
package block

// A Cipher represents an implementation of block cipher
// using a given key.  It provides the capability to encrypt
// or decrypt individual blocks.  The mode implementations
// extend that capability to streams of blocks.
type Cipher interface {
	// BlockSize returns the cipher's block size.
	BlockSize() int

	// Encrypt encrypts the first block in src into dst.
	// Src and dst may point at the same memory.
	Encrypt(dst, src []byte)

	// Decrypt decrypts the first block in src into dst.
	// Src and dst may point at the same memory.
	Decrypt(dst, src []byte)
}

// Utility routines

func shift1(dst, src []byte) byte {
	var b byte
	for i := len(src) - 1; i >= 0; i-- {
		bb := src[i] >> 7
		dst[i] = src[i]<<1 | b
		b = bb
	}
	return b
}

func same(p, q []byte) bool {
	if len(p) != len(q) {
		return false
	}
	for i := 0; i < len(p); i++ {
		if p[i] != q[i] {
			return false
		}
	}
	return true
}

func dup(p []byte) []byte {
	q := make([]byte, len(p))
	copy(q, p)
	return q
}
<|endoftext|>