<commit_before>// Local filesystem interface
package local

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "local",
		NewFs: NewFs,
	})
}

// FsLocal represents a local filesystem rooted at root
type FsLocal struct {
	root        string        // The root directory
	precisionOk sync.Once     // Whether we need to read the precision
	precision   time.Duration // precision of local filesystem
}

// FsObjectLocal represents a local filesystem object
type FsObjectLocal struct {
	local  fs.Fs       // The Fs this object is part of
	remote string      // The remote path
	path   string      // The local path
	info   os.FileInfo // Interface for file info (always present)
	md5sum string      // the md5sum of the object or "" if not calculated
}

// ------------------------------------------------------------

// NewFs contstructs an FsLocal from the path
func NewFs(name, root string) (fs.Fs, error) {
	root = path.Clean(root)
	f := &FsLocal{root: root}
	// Check to see if this points to a file
	fi, err := os.Lstat(f.root)
	if err == nil && fi.Mode().IsRegular() {
		// It is a file, so use the parent as the root
		remote := path.Base(root)
		f.root = path.Dir(root)
		obj := f.NewFsObject(remote)
		// return a Fs Limited to this object
		return fs.NewLimited(f, obj), nil
	}
	return f, nil
}

// String converts this FsLocal to a string
func (f *FsLocal) String() string {
	return fmt.Sprintf("Local file system at %s", f.root)
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
	path := filepath.Join(f.root, remote)
	o := &FsObjectLocal{local: f, remote: remote, path: path}
	if info != nil {
		o.info = info
	} else {
		err := o.lstat()
		if err != nil {
			fs.Debug(o, "Failed to stat %s: %s", path, err)
			return nil
		}
	}
	return o
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object {
	return f.NewFsObjectWithInfo(remote, nil)
}

// List the path returning a channel of FsObjects
//
// Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	go func() {
		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				fs.Stats.Error()
				log.Printf("Failed to open directory: %s: %s", path, err)
			} else {
				remote, err := filepath.Rel(f.root, path)
				if err != nil {
					fs.Stats.Error()
					log.Printf("Failed to get relative path %s: %s", path, err)
					return nil
				}
				if remote == "." {
					return nil
					// remote = ""
				}
				if fs := f.NewFsObjectWithInfo(remote, fi); fs != nil {
					if fs.Storable() {
						out <- fs
					}
				}
			}
			return nil
		})
		if err != nil {
			fs.Stats.Error()
			log.Printf("Failed to open directory: %s: %s", f.root, err)
		}
		close(out)
	}()
	return out
}

// Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	go func() {
		defer close(out)
		items, err := ioutil.ReadDir(f.root)
		if err != nil {
			fs.Stats.Error()
			log.Printf("Couldn't find read directory: %s", err)
		} else {
			for _, item := range items {
				if item.IsDir() {
					dir := &fs.Dir{
						Name:  item.Name(),
						When:  item.ModTime(),
						Bytes: 0,
						Count: 0,
					}
					// Go down the tree to count the files and directories
					dirpath := path.Join(f.root, item.Name())
					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
						if err != nil {
							fs.Stats.Error()
							log.Printf("Failed to open directory: %s: %s", path, err)
						} else {
							dir.Count += 1
							dir.Bytes += fi.Size()
						}
						return nil
					})
					if err != nil {
						fs.Stats.Error()
						log.Printf("Failed to open directory: %s: %s", dirpath, err)
					}
					out <- dir
				}
			}
		}
		// err := f.findRoot(false)
	}()
	return out
}

// Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	dstPath := filepath.Join(f.root, remote)
	// Temporary FsObject under construction - info filled in by Update()
	o := &FsObjectLocal{local: f, remote: remote, path: dstPath}
	err := o.Update(in, modTime, size)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error {
	return os.MkdirAll(f.root, 0770)
}

// Rmdir removes the directory
//
// If it isn't empty it will return an error
func (f *FsLocal) Rmdir() error {
	return os.Remove(f.root)
}

// Return the precision
func (f *FsLocal) Precision() (precision time.Duration) {
	f.precisionOk.Do(func() {
		f.precision = f.readPrecision()
	})
	return f.precision
}

// Read the precision
func (f *FsLocal) readPrecision() (precision time.Duration) {
	// Default precision of 1s
	precision = time.Second

	// Create temporary file and test it
	fd, err := ioutil.TempFile("", "rclone")
	if err != nil {
		// If failed return 1s
		// fmt.Println("Failed to create temp file", err)
		return time.Second
	}
	path := fd.Name()
	// fmt.Println("Created temp file", path)
	fd.Close()

	// Delete it on return
	defer func() {
		// fmt.Println("Remove temp file")
		os.Remove(path)
	}()

	// Find the minimum duration we can detect
	for duration := time.Duration(1); duration < time.Second; duration *= 10 {
		// Current time with delta
		t := time.Unix(time.Now().Unix(), int64(duration))
		err := os.Chtimes(path, t, t)
		if err != nil {
			// fmt.Println("Failed to Chtimes", err)
			break
		}

		// Read the actual time back
		fi, err := os.Stat(path)
		if err != nil {
			// fmt.Println("Failed to Stat", err)
			break
		}

		// If it matches - have found the precision
		// fmt.Println("compare", fi.ModTime(), t)
		if fi.ModTime() == t {
			// fmt.Println("Precision detected as", duration)
			return duration
		}
	}
	return
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs {
	return o.local
}

// Return a string version
func (o *FsObjectLocal) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectLocal) Remote() string {
	return o.remote
}

// Md5sum calculates the Md5sum of a file returning a lowercase hex string
func (o *FsObjectLocal) Md5sum() (string, error) {
	if o.md5sum != "" {
		return o.md5sum, nil
	}
	in, err := os.Open(o.path)
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to open: %s", err)
		return "", err
	}
	hash := md5.New()
	_, err = io.Copy(hash, in)
	closeErr := in.Close()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read: %s", err)
		return "", err
	}
	if closeErr != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to close: %s", closeErr)
		return "", closeErr
	}
	o.md5sum = hex.EncodeToString(hash.Sum(nil))
	return o.md5sum, nil
}

// Size returns the size of an object in bytes
func (o *FsObjectLocal) Size() int64 {
	return o.info.Size()
}

// ModTime returns the modification time of the object
func (o *FsObjectLocal) ModTime() time.Time {
	return o.info.ModTime()
}

// Sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
	err := os.Chtimes(o.path, modTime, modTime)
	if err != nil {
		fs.Debug(o, "Failed to set mtime on file: %s", err)
	}
}

// Is this object storable
func (o *FsObjectLocal) Storable() bool {
	mode := o.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debug(o, "Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		fs.Debug(o, "FIXME Skipping directory")
		return false
	}
	return true
}

// localOpenFile wraps an io.ReadCloser and updates the md5sum of the
// object that is read
type localOpenFile struct {
	o    *FsObjectLocal // object that is open
	in   io.ReadCloser  // handle we are wrapping
	hash hash.Hash      // currently accumulating MD5
}

// Read bytes from the object - see io.Reader
func (file *localOpenFile) Read(p []byte) (n int, err error) {
	n, err = file.in.Read(p)
	if n > 0 {
		// Hash routines never return an error
		_, _ = file.hash.Write(p[:n])
	}
	return
}

// Close the object and update the md5sum
func (file *localOpenFile) Close() (err error) {
	err = file.in.Close()
	if err == nil {
		file.o.md5sum = hex.EncodeToString(file.hash.Sum(nil))
	} else {
		file.o.md5sum = ""
	}
	return err
}

// Open an object for read
func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
	in, err = os.Open(o.path)
	if err != nil {
		return
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   in,
		hash: md5.New(),
	}
	return
}

// Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
	dir := path.Dir(o.path)
	err := os.MkdirAll(dir, 0770)
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the md5sum of the object we are reading as we go along
	hash := md5.New()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	outErr := out.Close()
	if err != nil {
		return err
	}
	if outErr != nil {
		return outErr
	}

	// All successful so update the md5sum
	o.md5sum = hex.EncodeToString(hash.Sum(nil))

	// Set the mtime
	o.SetModTime(modTime)

	// ReRead info now that we have finished
	return o.lstat()
}

// Stat a FsObject into info
func (o *FsObjectLocal) lstat() error {
	info, err := os.Lstat(o.path)
	o.info = info
	return err
}

// Remove an object
func (o *FsObjectLocal) Remove() error {
	return os.Remove(o.path)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.Object = &FsObjectLocal{}
<commit_msg>local: Implement Purger interface<commit_after>// Local filesystem interface
package local

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
)

// Register with Fs
func init() {
	fs.Register(&fs.FsInfo{
		Name:  "local",
		NewFs: NewFs,
	})
}

// FsLocal represents a local filesystem rooted at root
type FsLocal struct {
	root        string        // The root directory
	precisionOk sync.Once     // Whether we need to read the precision
	precision   time.Duration // precision of local filesystem
}

// FsObjectLocal represents a local filesystem object
type FsObjectLocal struct {
	local  fs.Fs       // The Fs this object is part of
	remote string      // The remote path
	path   string      // The local path
	info   os.FileInfo // Interface for file info (always present)
	md5sum string      // the md5sum of the object or "" if not calculated
}

// ------------------------------------------------------------

// NewFs contstructs an FsLocal from the path
func NewFs(name, root string) (fs.Fs, error) {
	root = path.Clean(root)
	f := &FsLocal{root: root}
	// Check to see if this points to a file
	fi, err := os.Lstat(f.root)
	if err == nil && fi.Mode().IsRegular() {
		// It is a file, so use the parent as the root
		remote := path.Base(root)
		f.root = path.Dir(root)
		obj := f.NewFsObject(remote)
		// return a Fs Limited to this object
		return fs.NewLimited(f, obj), nil
	}
	return f, nil
}

// String converts this FsLocal to a string
func (f *FsLocal) String() string {
	return fmt.Sprintf("Local file system at %s", f.root)
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObjectWithInfo(remote string, info os.FileInfo) fs.Object {
	path := filepath.Join(f.root, remote)
	o := &FsObjectLocal{local: f, remote: remote, path: path}
	if info != nil {
		o.info = info
	} else {
		err := o.lstat()
		if err != nil {
			fs.Debug(o, "Failed to stat %s: %s", path, err)
			return nil
		}
	}
	return o
}

// Return an FsObject from a path
//
// May return nil if an error occurred
func (f *FsLocal) NewFsObject(remote string) fs.Object {
	return f.NewFsObjectWithInfo(remote, nil)
}

// List the path returning a channel of FsObjects
//
// Ignores everything which isn't Storable, eg links etc
func (f *FsLocal) List() fs.ObjectsChan {
	out := make(fs.ObjectsChan, fs.Config.Checkers)
	go func() {
		err := filepath.Walk(f.root, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				fs.Stats.Error()
				log.Printf("Failed to open directory: %s: %s", path, err)
			} else {
				remote, err := filepath.Rel(f.root, path)
				if err != nil {
					fs.Stats.Error()
					log.Printf("Failed to get relative path %s: %s", path, err)
					return nil
				}
				if remote == "." {
					return nil
					// remote = ""
				}
				if fs := f.NewFsObjectWithInfo(remote, fi); fs != nil {
					if fs.Storable() {
						out <- fs
					}
				}
			}
			return nil
		})
		if err != nil {
			fs.Stats.Error()
			log.Printf("Failed to open directory: %s: %s", f.root, err)
		}
		close(out)
	}()
	return out
}

// Walk the path returning a channel of FsObjects
func (f *FsLocal) ListDir() fs.DirChan {
	out := make(fs.DirChan, fs.Config.Checkers)
	go func() {
		defer close(out)
		items, err := ioutil.ReadDir(f.root)
		if err != nil {
			fs.Stats.Error()
			log.Printf("Couldn't find read directory: %s", err)
		} else {
			for _, item := range items {
				if item.IsDir() {
					dir := &fs.Dir{
						Name:  item.Name(),
						When:  item.ModTime(),
						Bytes: 0,
						Count: 0,
					}
					// Go down the tree to count the files and directories
					dirpath := path.Join(f.root, item.Name())
					err := filepath.Walk(dirpath, func(path string, fi os.FileInfo, err error) error {
						if err != nil {
							fs.Stats.Error()
							log.Printf("Failed to open directory: %s: %s", path, err)
						} else {
							dir.Count += 1
							dir.Bytes += fi.Size()
						}
						return nil
					})
					if err != nil {
						fs.Stats.Error()
						log.Printf("Failed to open directory: %s: %s", dirpath, err)
					}
					out <- dir
				}
			}
		}
		// err := f.findRoot(false)
	}()
	return out
}

// Puts the FsObject to the local filesystem
func (f *FsLocal) Put(in io.Reader, remote string, modTime time.Time, size int64) (fs.Object, error) {
	dstPath := filepath.Join(f.root, remote)
	// Temporary FsObject under construction - info filled in by Update()
	o := &FsObjectLocal{local: f, remote: remote, path: dstPath}
	err := o.Update(in, modTime, size)
	if err != nil {
		return nil, err
	}
	return o, nil
}

// Mkdir creates the directory if it doesn't exist
func (f *FsLocal) Mkdir() error {
	return os.MkdirAll(f.root, 0770)
}

// Rmdir removes the directory
//
// If it isn't empty it will return an error
func (f *FsLocal) Rmdir() error {
	return os.Remove(f.root)
}

// Return the precision
func (f *FsLocal) Precision() (precision time.Duration) {
	f.precisionOk.Do(func() {
		f.precision = f.readPrecision()
	})
	return f.precision
}

// Read the precision
func (f *FsLocal) readPrecision() (precision time.Duration) {
	// Default precision of 1s
	precision = time.Second

	// Create temporary file and test it
	fd, err := ioutil.TempFile("", "rclone")
	if err != nil {
		// If failed return 1s
		// fmt.Println("Failed to create temp file", err)
		return time.Second
	}
	path := fd.Name()
	// fmt.Println("Created temp file", path)
	fd.Close()

	// Delete it on return
	defer func() {
		// fmt.Println("Remove temp file")
		os.Remove(path)
	}()

	// Find the minimum duration we can detect
	for duration := time.Duration(1); duration < time.Second; duration *= 10 {
		// Current time with delta
		t := time.Unix(time.Now().Unix(), int64(duration))
		err := os.Chtimes(path, t, t)
		if err != nil {
			// fmt.Println("Failed to Chtimes", err)
			break
		}

		// Read the actual time back
		fi, err := os.Stat(path)
		if err != nil {
			// fmt.Println("Failed to Stat", err)
			break
		}

		// If it matches - have found the precision
		// fmt.Println("compare", fi.ModTime(), t)
		if fi.ModTime() == t {
			// fmt.Println("Precision detected as", duration)
			return duration
		}
	}
	return
}

// Purge deletes all the files and directories
//
// Optional interface: Only implement this if you have a way of
// deleting all the files quicker than just running Remove() on the
// result of List()
func (f *FsLocal) Purge() error {
	return os.RemoveAll(f.root)
}

// ------------------------------------------------------------

// Return the parent Fs
func (o *FsObjectLocal) Fs() fs.Fs {
	return o.local
}

// Return a string version
func (o *FsObjectLocal) String() string {
	if o == nil {
		return "<nil>"
	}
	return o.remote
}

// Return the remote path
func (o *FsObjectLocal) Remote() string {
	return o.remote
}

// Md5sum calculates the Md5sum of a file returning a lowercase hex string
func (o *FsObjectLocal) Md5sum() (string, error) {
	if o.md5sum != "" {
		return o.md5sum, nil
	}
	in, err := os.Open(o.path)
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to open: %s", err)
		return "", err
	}
	hash := md5.New()
	_, err = io.Copy(hash, in)
	closeErr := in.Close()
	if err != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to read: %s", err)
		return "", err
	}
	if closeErr != nil {
		fs.Stats.Error()
		fs.Log(o, "Failed to close: %s", closeErr)
		return "", closeErr
	}
	o.md5sum = hex.EncodeToString(hash.Sum(nil))
	return o.md5sum, nil
}

// Size returns the size of an object in bytes
func (o *FsObjectLocal) Size() int64 {
	return o.info.Size()
}

// ModTime returns the modification time of the object
func (o *FsObjectLocal) ModTime() time.Time {
	return o.info.ModTime()
}

// Sets the modification time of the local fs object
func (o *FsObjectLocal) SetModTime(modTime time.Time) {
	err := os.Chtimes(o.path, modTime, modTime)
	if err != nil {
		fs.Debug(o, "Failed to set mtime on file: %s", err)
	}
}

// Is this object storable
func (o *FsObjectLocal) Storable() bool {
	mode := o.info.Mode()
	if mode&(os.ModeSymlink|os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
		fs.Debug(o, "Can't transfer non file/directory")
		return false
	} else if mode&os.ModeDir != 0 {
		fs.Debug(o, "FIXME Skipping directory")
		return false
	}
	return true
}

// localOpenFile wraps an io.ReadCloser and updates the md5sum of the
// object that is read
type localOpenFile struct {
	o    *FsObjectLocal // object that is open
	in   io.ReadCloser  // handle we are wrapping
	hash hash.Hash      // currently accumulating MD5
}

// Read bytes from the object - see io.Reader
func (file *localOpenFile) Read(p []byte) (n int, err error) {
	n, err = file.in.Read(p)
	if n > 0 {
		// Hash routines never return an error
		_, _ = file.hash.Write(p[:n])
	}
	return
}

// Close the object and update the md5sum
func (file *localOpenFile) Close() (err error) {
	err = file.in.Close()
	if err == nil {
		file.o.md5sum = hex.EncodeToString(file.hash.Sum(nil))
	} else {
		file.o.md5sum = ""
	}
	return err
}

// Open an object for read
func (o *FsObjectLocal) Open() (in io.ReadCloser, err error) {
	in, err = os.Open(o.path)
	if err != nil {
		return
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   in,
		hash: md5.New(),
	}
	return
}

// Update the object from in with modTime and size
func (o *FsObjectLocal) Update(in io.Reader, modTime time.Time, size int64) error {
	dir := path.Dir(o.path)
	err := os.MkdirAll(dir, 0770)
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the md5sum of the object we are reading as we go along
	hash := md5.New()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	outErr := out.Close()
	if err != nil {
		return err
	}
	if outErr != nil {
		return outErr
	}

	// All successful so update the md5sum
	o.md5sum = hex.EncodeToString(hash.Sum(nil))

	// Set the mtime
	o.SetModTime(modTime)

	// ReRead info now that we have finished
	return o.lstat()
}

// Stat a FsObject into info
func (o *FsObjectLocal) lstat() error {
	info, err := os.Lstat(o.path)
	o.info = info
	return err
}

// Remove an object
func (o *FsObjectLocal) Remove() error {
	return os.Remove(o.path)
}

// Check the interfaces are satisfied
var _ fs.Fs = &FsLocal{}
var _ fs.Purger = &FsLocal{}
var _ fs.Object = &FsObjectLocal{}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sort"

	_ "github.com/go-sql-driver/mysql"
	"github.com/valyala/fasthttp"
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

// Databases
const (
	connectionString   = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?interpolateParams=true"
	worldRowCount      = 10000
	maxConnectionCount = 256
)

var (
	worldSelectStmt   *sql.Stmt
	worldUpdateStmt   *sql.Stmt
	fortuneSelectStmt *sql.Stmt
)

const helloWorldString = "Hello, World!"

var (
	// Templates
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	// Database
	db *sql.DB

	helloWorldBytes = []byte(helloWorldString)
)

var prefork = flag.Bool("prefork", false, "use prefork")
var child = flag.Bool("child", false, "is child proc")

func main() {
	var listener net.Listener
	flag.Parse()
	if !*prefork {
		runtime.GOMAXPROCS(runtime.NumCPU())
	} else {
		listener = doPrefork()
	}

	var err error
	db, err = sql.Open("mysql", connectionString)
	if err != nil {
		log.Fatalf("Error opening database: %s", err)
	}
	if err = db.Ping(); err != nil {
		log.Fatalf("Cannot connect to db: %s", err)
	}
	db.SetMaxIdleConns(maxConnectionCount)
	db.SetMaxOpenConns(maxConnectionCount * 2)

	worldSelectStmt = mustPrepare(db, "SELECT id, randomNumber FROM World WHERE id = ?")
	worldUpdateStmt = mustPrepare(db, "UPDATE World SET randomNumber = ? WHERE id = ?")
	fortuneSelectStmt = mustPrepare(db, "SELECT id, message FROM Fortune")

	s := &fasthttp.Server{
		Handler: func(ctx *fasthttp.RequestCtx) {
			path := ctx.Path()
			switch {
			case fasthttp.EqualBytesStr(path, "/plaintext"):
				plaintextHandler(ctx)
			case fasthttp.EqualBytesStr(path, "/json"):
				jsonHandler(ctx)
			case fasthttp.EqualBytesStr(path, "/db"):
				dbHandler(ctx)
			case fasthttp.EqualBytesStr(path, "/queries"):
				queriesHandler(ctx)
			case fasthttp.EqualBytesStr(path, "/fortune"):
				fortuneHandler(ctx)
			case fasthttp.EqualBytesStr(path, "/update"):
				updateHandler(ctx)
			default:
				ctx.Error("unexpected path", fasthttp.StatusBadRequest)
			}
		},
		Name: "fasthttp",
	}
	if !*prefork {
		s.ListenAndServe(":8080")
	} else {
		s.Serve(listener)
	}
}

func mustPrepare(db *sql.DB, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatalf("Error when preparing statement %q: %s", query, err)
	}
	return stmt
}

func doPrefork() (listener net.Listener) {
	var err error
	var fl *os.File
	var tcplistener *net.TCPListener
	if !*child {
		var addr *net.TCPAddr
		addr, err = net.ResolveTCPAddr("tcp", ":8080")
		if err != nil {
			log.Fatal(err)
		}
		tcplistener, err = net.ListenTCP("tcp", addr)
		if err != nil {
			log.Fatal(err)
		}
		fl, err = tcplistener.File()
		if err != nil {
			log.Fatal(err)
		}
		children := make([]*exec.Cmd, runtime.NumCPU())
		for i := range children {
			children[i] = exec.Command(os.Args[0], "-prefork", "-child")
			children[i].Stdout = os.Stdout
			children[i].Stderr = os.Stderr
			children[i].ExtraFiles = []*os.File{fl}
			err = children[i].Start()
			if err != nil {
				log.Fatal(err)
			}
		}
		for _, ch := range children {
			var err error = ch.Wait()
			if err != nil {
				log.Print(err)
			}
		}
		os.Exit(0)
	} else {
		fl = os.NewFile(3, "")
		listener, err = net.FileListener(fl)
		if err != nil {
			log.Fatal(err)
		}
		runtime.GOMAXPROCS(1)
	}
	return listener
}

func jsonMarshal(ctx *fasthttp.RequestCtx, v interface{}) {
	ctx.SetContentType("application/json")
	if err := json.NewEncoder(ctx).Encode(v); err != nil {
		log.Fatalf("error in json.Encoder.Encode: %s", err)
	}
}

// Test 1: JSON serialization
func jsonHandler(ctx *fasthttp.RequestCtx) {
	jsonMarshal(ctx, &Message{helloWorldString})
}

// Test 2: Single database query
func dbHandler(ctx *fasthttp.RequestCtx) {
	var world World
	err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world.Id, &world.RandomNumber)
	if err != nil {
		log.Fatalf("Error scanning world row: %s", err)
	}

	jsonMarshal(ctx, &world)
}

// Test 3: Multiple database queries
func queriesHandler(ctx *fasthttp.RequestCtx) {
	n := ctx.QueryArgs().GetUintOrZero("queries")
	if n < 1 {
		n = 1
	} else if n > 500 {
		n = 500
	}

	world := make([]World, n)
	for i := 0; i < n; i++ {
		err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world[i].Id, &world[i].RandomNumber)
		if err != nil {
			log.Fatalf("Error scanning world row: %s", err)
		}
	}

	jsonMarshal(ctx, world)
}

// Test 4: Fortunes
func fortuneHandler(ctx *fasthttp.RequestCtx) {
	rows, err := fortuneSelectStmt.Query()
	if err != nil {
		log.Fatalf("Error preparing statement: %v", err)
	}

	fortunes := make(Fortunes, 0, 16)
	for rows.Next() { //Fetch rows
		fortune := Fortune{}
		if err := rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			log.Fatalf("Error scanning fortune row: %s", err)
		}
		fortunes = append(fortunes, &fortune)
	}
	rows.Close()
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})

	ctx.SetContentType("text/html")
	if err := tmpl.Execute(ctx, fortunes); err != nil {
		log.Fatalf("Error executing fortune: %s", err)
	}
}

// Test 5: Database updates
func updateHandler(ctx *fasthttp.RequestCtx) {
	n := ctx.QueryArgs().GetUintOrZero("queries")
	if n < 1 {
		n = 1
	} else if n > 500 {
		n = 500
	}
	world := make([]World, n)
	for i := 0; i < n; i++ {
		if err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world[i].Id, &world[i].RandomNumber); err != nil {
			log.Fatalf("Error scanning world row: %s", err)
		}
		world[i].RandomNumber = uint16(rand.Intn(worldRowCount) + 1)
		if _, err := worldUpdateStmt.Exec(world[i].RandomNumber, world[i].Id); err != nil {
			log.Fatalf("Error updating world row: %s", err)
		}
	}

	jsonMarshal(ctx, world)
}

// Test 6: Plaintext
func plaintextHandler(ctx *fasthttp.RequestCtx) {
	ctx.Success("text/plain", helloWorldBytes)
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<commit_msg>fasthttp: use SO_REUSEPORT if -prefork flag is passed. This should improve performance on Peak hardware<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"html/template"
	"log"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sort"

	_ "github.com/go-sql-driver/mysql"
	"github.com/valyala/fasthttp"
	"github.com/valyala/fasthttp/reuseport"
)

type Message struct {
	Message string `json:"message"`
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

// Databases
const (
	connectionString   = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?interpolateParams=true"
	worldRowCount      = 10000
	maxConnectionCount = 256
)

var (
	worldSelectStmt   *sql.Stmt
	worldUpdateStmt   *sql.Stmt
	fortuneSelectStmt *sql.Stmt
)

const helloWorldString = "Hello, World!"

var (
	// Templates
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	// Database
	db *sql.DB

	helloWorldBytes = []byte(helloWorldString)
)

var (
	listenAddr = flag.String("listenAddr", ":8080", "Address to listen to")
	prefork    = flag.Bool("prefork", false, "use prefork")
	child      = flag.Bool("child", false, "is child proc")
)

func main() {
	flag.Parse()

	var listener net.Listener
	if !*prefork {
		runtime.GOMAXPROCS(runtime.NumCPU())
	} else {
		listener = doPrefork()
	}

	var err error
	db, err = sql.Open("mysql", connectionString)
	if err != nil {
		log.Fatalf("Error opening database: %s", err)
	}
	if err = db.Ping(); err != nil {
		log.Fatalf("Cannot connect to db: %s", err)
	}
	db.SetMaxIdleConns(maxConnectionCount)
	db.SetMaxOpenConns(maxConnectionCount * 2)

	worldSelectStmt = mustPrepare(db, "SELECT id, randomNumber FROM World WHERE id = ?")
	worldUpdateStmt = mustPrepare(db, "UPDATE World SET randomNumber = ? WHERE id = ?")
	fortuneSelectStmt = mustPrepare(db, "SELECT id, message FROM Fortune")

	s := &fasthttp.Server{
		Handler: mainHandler,
		Name:    "fasthttp",
	}
	if !*prefork {
		s.ListenAndServe(*listenAddr)
	} else {
		s.Serve(listener)
	}
}

func mustPrepare(db *sql.DB, query string) *sql.Stmt {
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatalf("Error when preparing statement %q: %s", query, err)
	}
	return stmt
}

func doPrefork() net.Listener {
	if !*child {
		children := make([]*exec.Cmd, runtime.NumCPU())
		for i := range children {
			children[i] = exec.Command(os.Args[0], "-prefork", "-child")
			children[i].Stdout = os.Stdout
			children[i].Stderr = os.Stderr
			if err := children[i].Start(); err != nil {
				log.Fatal(err)
			}
		}
		for _, ch := range children {
			if err := ch.Wait(); err != nil {
				log.Print(err)
			}
		}
		os.Exit(0)
		return nil
	}

	runtime.GOMAXPROCS(1)
	listener, err := reuseport.Listen("tcp4", *listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	return listener
}

func jsonMarshal(ctx *fasthttp.RequestCtx, v interface{}) {
	ctx.SetContentType("application/json")
	if err := json.NewEncoder(ctx).Encode(v); err != nil {
		log.Fatalf("error in json.Encoder.Encode: %s", err)
	}
}

func mainHandler(ctx *fasthttp.RequestCtx) {
	path := ctx.Path()
	switch {
	case fasthttp.EqualBytesStr(path, "/plaintext"):
		plaintextHandler(ctx)
	case fasthttp.EqualBytesStr(path, "/json"):
		jsonHandler(ctx)
	case fasthttp.EqualBytesStr(path, "/db"):
		dbHandler(ctx)
	case fasthttp.EqualBytesStr(path, "/queries"):
		queriesHandler(ctx)
	case fasthttp.EqualBytesStr(path, "/fortune"):
		fortuneHandler(ctx)
	case fasthttp.EqualBytesStr(path, "/update"):
		updateHandler(ctx)
	default:
		ctx.Error("unexpected path", fasthttp.StatusBadRequest)
	}
}

// Test 1: JSON serialization
func jsonHandler(ctx *fasthttp.RequestCtx) {
	jsonMarshal(ctx, &Message{helloWorldString})
}

// Test 2: Single database query
func dbHandler(ctx *fasthttp.RequestCtx) {
	var world World
	err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world.Id, &world.RandomNumber)
	if err != nil {
		log.Fatalf("Error scanning world row: %s", err)
	}

	jsonMarshal(ctx, &world)
}

// Test 3: Multiple database queries
func queriesHandler(ctx *fasthttp.RequestCtx) {
	n := ctx.QueryArgs().GetUintOrZero("queries")
	if n < 1 {
		n = 1
	} else if n > 500 {
		n = 500
	}

	world := make([]World, n)
	for i := 0; i < n; i++ {
		err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world[i].Id, &world[i].RandomNumber)
		if err != nil {
			log.Fatalf("Error scanning world row: %s", err)
		}
	}

	jsonMarshal(ctx, world)
}

// Test 4: Fortunes
func fortuneHandler(ctx *fasthttp.RequestCtx) {
	rows, err := fortuneSelectStmt.Query()
	if err != nil {
		log.Fatalf("Error preparing statement: %v", err)
	}

	fortunes := make(Fortunes, 0, 16)
	for rows.Next() { //Fetch rows
		fortune := Fortune{}
		if err := rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			log.Fatalf("Error scanning fortune row: %s", err)
		}
		fortunes = append(fortunes, &fortune)
	}
	rows.Close()
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})

	ctx.SetContentType("text/html")
	if err := tmpl.Execute(ctx, fortunes); err != nil {
		log.Fatalf("Error executing fortune: %s", err)
	}
}

// Test 5: Database updates
func updateHandler(ctx *fasthttp.RequestCtx) {
	n := ctx.QueryArgs().GetUintOrZero("queries")
	if n < 1 {
		n = 1
	} else if n > 500 {
		n = 500
	}
	world := make([]World, n)
	for i := 0; i < n; i++ {
		if err := worldSelectStmt.QueryRow(rand.Intn(worldRowCount)+1).Scan(&world[i].Id, &world[i].RandomNumber); err != nil {
			log.Fatalf("Error scanning world row: %s", err)
		}
		world[i].RandomNumber = uint16(rand.Intn(worldRowCount) + 1)
		if _, err := worldUpdateStmt.Exec(world[i].RandomNumber, world[i].Id); err != nil {
			log.Fatalf("Error updating world row: %s", err)
		}
	}

	jsonMarshal(ctx, world)
}

// Test 6: Plaintext
func plaintextHandler(ctx *fasthttp.RequestCtx) {
	ctx.Success("text/plain", helloWorldBytes)
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"bufio"
	"os"
	"errors"
	"path/filepath"
	"github.com/webdevops/go-stubfilegenerator"
)

type FileStubs struct {
	Positional struct {
		SourceFile string `description:"Source file"`
	} `positional-args:"true"`
	RootPath    string  `long:"path"                  description:"Prefix path for stub files"  default:"./"`
	SourceStdin bool    `long:"stdin"                 description:"Use stdin as file list source"`
	Force       bool    `short:"f"  long:"force"      description:"Overwrite existing files"`
}

func (conf *FileStubs) Execute(args []string) error {
	var fileSource *os.File

	if conf.Positional.SourceFile == "" && !conf.SourceStdin {
		return errors.New("[ERROR] No source defined, either specify file as argument or use --stdin")
	}

	defer NewSigIntHandler(func() {});

	stubgen := stubfilegenerator.NewStubGenerator()

	if conf.Force {
		// use --force
		stubgen.Overwrite = true
	}

	if conf.SourceStdin {
		// use --stdin
		fileSource = os.Stdin
	} else if conf.Positional.SourceFile != "" {
		// use --source
		f, err := os.Open(conf.Positional.SourceFile)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		fileSource = f
	}

	scanner := bufio.NewScanner(fileSource)
	for scanner.Scan() {
		relPath := scanner.Text()
		path := scanner.Text()

		if conf.RootPath != "" {
			path = filepath.Join(conf.RootPath, path)
		}

		fmt.Println(fmt.Sprintf(" >> %s", path))
		stubgen.TemplateVariables["PATH"] = relPath
		stubgen.Generate(path)
	}

	return nil
}
<commit_msg>file:stubs: Don’t process empty file paths<commit_after>package command

import (
	"fmt"
	"bufio"
	"os"
	"errors"
	"path/filepath"
	"github.com/webdevops/go-stubfilegenerator"
)

type FileStubs struct {
	Positional struct {
		SourceFile string `description:"Source file"`
	} `positional-args:"true"`
	RootPath    string  `long:"path"                  description:"Prefix path for stub files"  default:"./"`
	SourceStdin bool    `long:"stdin"                 description:"Use stdin as file list source"`
	Force       bool    `short:"f"  long:"force"      description:"Overwrite existing files"`
}

func (conf *FileStubs) Execute(args []string) error {
	var fileSource *os.File

	if conf.Positional.SourceFile == "" && !conf.SourceStdin {
		return errors.New("[ERROR] No source defined, either specify file as argument or use --stdin")
	}

	defer NewSigIntHandler(func() {});

	stubgen := stubfilegenerator.NewStubGenerator()

	if conf.Force {
		// use --force
		stubgen.Overwrite = true
	}

	if conf.SourceStdin {
		// use --stdin
		fileSource = os.Stdin
	} else if conf.Positional.SourceFile != "" {
		// use --source
		f, err := os.Open(conf.Positional.SourceFile)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		fileSource = f
	}

	scanner := bufio.NewScanner(fileSource)
	for scanner.Scan() {
		relPath := scanner.Text()
		path := relPath

		if path == "" {
			continue
		}

		if conf.RootPath != "" {
			path = filepath.Join(conf.RootPath, path)
		}

		fmt.Println(fmt.Sprintf(" >> %s", path))
		stubgen.TemplateVariables["PATH"] = relPath
		stubgen.Generate(path)
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/github/git-lfs/lfs"
	"github.com/spf13/cobra"
)

var (
	Debugging    = false
	ErrorBuffer  = &bytes.Buffer{}
	ErrorWriter  = io.MultiWriter(os.Stderr, ErrorBuffer)
	OutputWriter = io.MultiWriter(os.Stdout, ErrorBuffer)
	RootCmd      = &cobra.Command{
		Use:   "git-lfs",
		Short: "Git LFS provides large file storage to Git.",
		Run: func(cmd *cobra.Command, args []string) {
			versionCommand(cmd, args)
			cmd.Usage()
		},
	}
)

// Error prints a formatted message to Stderr.  It also gets printed to the
// panic log if one is created for this command.
func Error(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(ErrorWriter, line)
}

// Print prints a formatted message to Stdout.  It also gets printed to the
// panic log if one is created for this command.
func Print(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(OutputWriter, line)
}

// Exit prints a formatted message and exits.
func Exit(format string, args ...interface{}) {
	Error(format, args...)
	os.Exit(2)
}

// Debug prints a formatted message if debugging is enabled.  The formatted
// message also shows up in the panic log, if created.
func Debug(format string, args ...interface{}) {
	if !Debugging {
		return
	}
	log.Printf(format, args...)
}

// LoggedError prints a formatted message to Stderr and writes a stack trace for
// the error to a log file without exiting.
func LoggedError(err error, format string, args ...interface{}) {
	Error(format, args...)
	file := handlePanic(err)

	if len(file) > 0 {
		fmt.Fprintf(os.Stderr, "\nErrors logged to %s\nUse `git lfs logs last` to view the log.\n", file)
	}
}

// Panic prints a formatted message, and writes a stack trace for the error to
// a log file before exiting.
func Panic(err error, format string, args ...interface{}) {
	LoggedError(err, format, args...)
	os.Exit(2)
}

func Run() {
	RootCmd.Execute()
}

func PipeMediaCommand(name string, args ...string) error {
	return PipeCommand("bin/"+name, args...)
}

func PipeCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func requireStdin(msg string) {
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		Error("Cannot read from STDIN. %s", msg)
		os.Exit(1)
	}
}

func handlePanic(err error) string {
	if err == nil {
		return ""
	}

	return logPanic(err, false)
}

func logEnv(w io.Writer) {
	for _, env := range lfs.Environ() {
		fmt.Fprintln(w, env)
	}
}

func logPanic(loggedError error, recursive bool) string {
	var fmtWriter io.Writer = os.Stderr

	if err := os.MkdirAll(lfs.LocalLogDir, 0755); err != nil {
		fmt.Fprintf(fmtWriter, "Unable to log panic to %s: %s\n\n", lfs.LocalLogDir, err.Error())
		return ""
	}

	now := time.Now()
	name := now.Format("20060102T150405.999999999")
	full := filepath.Join(lfs.LocalLogDir, name+".log")

	if file, err := os.Create(full); err != nil {
		if !recursive {
			filename := full
			full = ""
			defer func() {
				fmt.Fprintf(fmtWriter, "Unable to log panic to %s\n\n", filename)
				logPanic(err, true)
			}()
		}
	} else {
		fmtWriter = file
		defer file.Close()
	}

	fmt.Fprintf(fmtWriter, "> %s", filepath.Base(os.Args[0]))
	if len(os.Args) > 0 {
		fmt.Fprintf(fmtWriter, " %s", strings.Join(os.Args[1:], " "))
	}
	fmt.Fprintln(fmtWriter)

	logEnv(fmtWriter)
	fmt.Fprintln(fmtWriter)

	fmtWriter.Write(ErrorBuffer.Bytes())
	fmt.Fprintln(fmtWriter)

	fmt.Fprintln(fmtWriter, loggedError.Error())

	if wErr, ok := loggedError.(ErrorWithStack); ok {
		fmt.Fprintln(fmtWriter, wErr.InnerError())
		for key, value := range wErr.Context() {
			fmt.Fprintf(fmtWriter, "%s=%s\n", key, value)
		}
		fmtWriter.Write(wErr.Stack())
	} else {
		fmtWriter.Write(lfs.Stack())
	}

	return full
}

type ErrorWithStack interface {
	Context() map[string]string
	InnerError() string
	Stack() []byte
}

func init() {
	log.SetOutput(ErrorWriter)
}
<commit_msg>アアー アアアア アーアア<commit_after>package commands

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/github/git-lfs/lfs"
	"github.com/spf13/cobra"
)

var (
	Debugging    = false
	ErrorBuffer  = &bytes.Buffer{}
	ErrorWriter  = io.MultiWriter(os.Stderr, ErrorBuffer)
	OutputWriter = io.MultiWriter(os.Stdout, ErrorBuffer)
	RootCmd      = &cobra.Command{
		Use:   "git-lfs",
		Short: "Git LFS provides large file storage to Git.",
		Run: func(cmd *cobra.Command, args []string) {
			versionCommand(cmd, args)
			cmd.Usage()
		},
	}
)

// Error prints a formatted message to Stderr.  It also gets printed to the
// panic log if one is created for this command.
func Error(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(ErrorWriter, line)
}

// Print prints a formatted message to Stdout.  It also gets printed to the
// panic log if one is created for this command.
func Print(format string, args ...interface{}) {
	line := fmt.Sprintf(format, args...)
	fmt.Fprintln(OutputWriter, line)
}

// Exit prints a formatted message and exits.
func Exit(format string, args ...interface{}) {
	Error(format, args...)
	os.Exit(2)
}

// Debug prints a formatted message if debugging is enabled.  The formatted
// message also shows up in the panic log, if created.
func Debug(format string, args ...interface{}) {
	if !Debugging {
		return
	}
	log.Printf(format, args...)
}

// LoggedError prints a formatted message to Stderr and writes a stack trace for
// the error to a log file without exiting.
func LoggedError(err error, format string, args ...interface{}) {
	Error(format, args...)
	file := handlePanic(err)

	if len(file) > 0 {
		fmt.Fprintf(os.Stderr, "\nErrors logged to %s\nUse `git lfs logs last` to view the log.\n", file)
	}
}

// Panic prints a formatted message, and writes a stack trace for the error to
// a log file before exiting.
func Panic(err error, format string, args ...interface{}) {
	LoggedError(err, format, args...)
	os.Exit(2)
}

func Run() {
	RootCmd.Execute()
}

func PipeMediaCommand(name string, args ...string) error {
	return PipeCommand("bin/"+name, args...)
}

func PipeCommand(name string, args ...string) error {
	cmd := exec.Command(name, args...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func requireStdin(msg string) {
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) != 0 {
		Error("Cannot read from STDIN. %s", msg)
		os.Exit(1)
	}
}

func handlePanic(err error) string {
	if err == nil {
		return ""
	}

	return logPanic(err, false)
}

func logEnv(w io.Writer) {
	for _, env := range lfs.Environ() {
		fmt.Fprintln(w, env)
	}
}

func logPanic(loggedError error, recursive bool) string {
	var fmtWriter io.Writer = os.Stderr

	if !recursive {
		if err := os.MkdirAll(lfs.LocalLogDir, 0755); err != nil {
			fmt.Fprintf(fmtWriter, "Unable to log panic to %s: %s\n\n", lfs.LocalLogDir, err.Error())
			return ""
		}

		now := time.Now()
		name := now.Format("20060102T150405.999999999")
		full := filepath.Join(lfs.LocalLogDir, name+".log")

		if file, err := os.Create(full); err != nil {
			filename := full
			full = ""
			defer func() {
				fmt.Fprintf(fmtWriter, "Unable to log panic to %s\n\n", filename)
				logPanic(err, true)
			}()
		} else {
			fmtWriter = file
			defer file.Close()
		}
	}

	fmt.Fprintf(fmtWriter, "> %s", filepath.Base(os.Args[0]))
	if len(os.Args) > 0 {
		fmt.Fprintf(fmtWriter, " %s", strings.Join(os.Args[1:], " "))
	}
	fmt.Fprintln(fmtWriter)

	logEnv(fmtWriter)
	fmt.Fprintln(fmtWriter)

	fmtWriter.Write(ErrorBuffer.Bytes())
	fmt.Fprintln(fmtWriter)

	fmt.Fprintln(fmtWriter, loggedError.Error())

	if wErr, ok := loggedError.(ErrorWithStack); ok {
		fmt.Fprintln(fmtWriter, wErr.InnerError())
		for key, value := range wErr.Context() {
			fmt.Fprintf(fmtWriter, "%s=%s\n", key, value)
		}
		fmtWriter.Write(wErr.Stack())
	} else {
		fmtWriter.Write(lfs.Stack())
	}

	return full
}

type ErrorWithStack interface {
	Context() map[string]string
	InnerError() string
	Stack() []byte
}

func init() {
	log.SetOutput(ErrorWriter)
}
<|endoftext|>


<commit_before>// Package commands is where all cli logic is, including starting hoarder as a server.
package commands

import (
	"crypto/tls"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jcelliott/lumber"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/nanopack/hoarder/api"
	"github.com/nanopack/hoarder/backends"
	"github.com/nanopack/hoarder/collector"
)

var (
	body     io.ReadWriter        // what to read/write requests body
	verbose  bool                 // whether to display request info
	insecure bool          = true // whether to ignore cert or not

	key  string // blob key
	data string // blob raw data (or '-' for stdin)
	file string // blob file location

	config   string // config file location
	showVers bool   // whether to print version info or not

	// to be populated by linker
	version string
	commit  string

	// HoarderCmd ...
	HoarderCmd = &cobra.Command{
		Use:           "hoarder",
		Short:         "hoarder - data storage",
		Long:          ``,
		SilenceErrors: true,
		SilenceUsage:  true,

		// parse the config if one is provided, or use the defaults
		PersistentPreRunE: readConfig,

		// print version or help, or continue, depending on flag settings
		PreRunE: preFlight,

		// either run hoarder as a server, or run it as a CLI depending on what flags
		// are provided
		RunE: startHoarder,
	}
)

func readConfig(ccmd *cobra.Command, args []string) error {
	// if --config is passed, attempt to parse the config file
	if config != "" {
		filename := filepath.Base(config)
		viper.SetConfigName(filename[:len(filename)-len(filepath.Ext(filename))])
		viper.AddConfigPath(filepath.Dir(config))

		err := viper.ReadInConfig()
		if err != nil {
			return fmt.Errorf("Failed to read config file - %s", err)
		}
	}

	return nil
}

func preFlight(ccmd *cobra.Command, args []string) error {
	// if --version is passed print the version info
	if showVers {
		fmt.Printf("hoarder %s (%s)\n", version, commit)
		return fmt.Errorf("")
	}

	// if --server is not passed, print help
	if !viper.GetBool("server") {
		ccmd.HelpFunc()(ccmd, args)
		return fmt.Errorf("") // no error, just exit
	}

	return nil
}

func startHoarder(ccmd *cobra.Command, args []string) error {
	// convert the log level
	logLvl := lumber.LvlInt(viper.GetString("log-level"))

	// configure the logger
	lumber.Prefix("[hoader]")
	lumber.Level(logLvl)

	// enable/start garbage collection if age config was changed
	if ccmd.Flag("clean-after").Changed {
		lumber.Debug("Starting garbage collector (data older than %ds)...\n", ccmd.Flag("clean-after").Value)

		// start garbage collector
		go collector.Start()
	}

	// set, and initialize, the backend driver
	if err := backends.Initialize(); err != nil {
		return fmt.Errorf("Failed to initialize backend - %s", err)
	}

	// start the API
	if err := api.Start(); err != nil {
		return fmt.Errorf("Failed to start API - %s", err)
	}

	return nil
}

func init() {
	// set config defaults
	backend := "file:///var/db/hoarder"
	cleanAfter := uint64(time.Now().Unix())
	listenAddr := "https://127.0.0.1:7410"
	logLevel := "INFO"
	token := ""
	server := false

	// cli flags
	HoarderCmd.PersistentFlags().StringP("backend", "b", backend, "Hoarder backend")
	HoarderCmd.PersistentFlags().Uint64P("clean-after", "g", cleanAfter, "Age, in seconds, after which data is deemed garbage")
	HoarderCmd.PersistentFlags().StringP("listen-addr", "H", listenAddr, "Hoarder listen uri (scheme defaults to https)")
	HoarderCmd.PersistentFlags().String("log-level", logLevel, "Output level of logs (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)")
	HoarderCmd.PersistentFlags().StringP("token", "t", token, "Auth token used when connecting to a secure Hoarder")
	HoarderCmd.Flags().BoolP("server", "s", server, "Run hoarder as a server")

	// bind config to cli flags
	viper.BindPFlag("backend", HoarderCmd.PersistentFlags().Lookup("backend"))
	viper.BindPFlag("clean-after", HoarderCmd.PersistentFlags().Lookup("clean-after"))
	viper.BindPFlag("listen-addr", HoarderCmd.PersistentFlags().Lookup("listen-addr"))
	viper.BindPFlag("log-level", HoarderCmd.PersistentFlags().Lookup("log-level"))
	viper.BindPFlag("token", HoarderCmd.PersistentFlags().Lookup("token"))
	viper.BindPFlag("server", HoarderCmd.Flags().Lookup("server"))

	// cli-only flags
	HoarderCmd.Flags().StringVarP(&config, "config", "c", "", "Path to config file (with extension)")
	HoarderCmd.Flags().BoolVarP(&showVers, "version", "v", false, "Display the current version of this CLI")

	// commands
	HoarderCmd.AddCommand(addCmd)
	HoarderCmd.AddCommand(listCmd)
	HoarderCmd.AddCommand(removeCmd)
	HoarderCmd.AddCommand(showCmd)
	HoarderCmd.AddCommand(updateCmd)

	// hidden/aliased commands
	HoarderCmd.AddCommand(createCmd)
	HoarderCmd.AddCommand(deleteCmd)
	HoarderCmd.AddCommand(destroyCmd)
	HoarderCmd.AddCommand(fetchCmd)
	HoarderCmd.AddCommand(getCmd)
}

func rest(_method, _key string, _body io.ReadWriter) io.Reader {
	if verbose {
		fmt.Fprintf(os.Stderr, "%s: %s/blobs%s", _method, viper.GetString("listen-addr"), _key)
	}

	req, err := http.NewRequest(_method, fmt.Sprintf("%s/blobs%s", viper.GetString("listen-addr"), _key), _body)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to create request")
	}

	// skip cert check or not (only applies to https)
	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: insecure}

	// set auth token
	req.Header.Add("X-AUTH-TOKEN", viper.GetString("token"))

	// make request to hoarder server
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		// print error and exit
		fmt.Fprintf(os.Stderr, "Failed to make request - %s\n", err)
		os.Exit(1)
	}

	if verbose {
		fmt.Fprintf(os.Stderr, "%5d\n", res.StatusCode)
	}

	return res.Body
}
<commit_msg>Fix typo in logger prefix<commit_after>// Package commands is where all cli logic is, including starting hoarder as a server.
package commands

import (
	"crypto/tls"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/jcelliott/lumber"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/nanopack/hoarder/api"
	"github.com/nanopack/hoarder/backends"
	"github.com/nanopack/hoarder/collector"
)

var (
	body     io.ReadWriter        // what to read/write requests body
	verbose  bool                 // whether to display request info
	insecure bool          = true // whether to ignore cert or not

	key  string // blob key
	data string // blob raw data (or '-' for stdin)
	file string // blob file location

	config   string // config file location
	showVers bool   // whether to print version info or not

	// to be populated by linker
	version string
	commit  string

	// HoarderCmd ...
	HoarderCmd = &cobra.Command{
		Use:           "hoarder",
		Short:         "hoarder - data storage",
		Long:          ``,
		SilenceErrors: true,
		SilenceUsage:  true,

		// parse the config if one is provided, or use the defaults
		PersistentPreRunE: readConfig,

		// print version or help, or continue, depending on flag settings
		PreRunE: preFlight,

		// either run hoarder as a server, or run it as a CLI depending on what flags
		// are provided
		RunE: startHoarder,
	}
)

func readConfig(ccmd *cobra.Command, args []string) error {
	// if --config is passed, attempt to parse the config file
	if config != "" {
		filename := filepath.Base(config)
		viper.SetConfigName(filename[:len(filename)-len(filepath.Ext(filename))])
		viper.AddConfigPath(filepath.Dir(config))

		err := viper.ReadInConfig()
		if err != nil {
			return fmt.Errorf("Failed to read config file - %s", err)
		}
	}

	return nil
}

func preFlight(ccmd *cobra.Command, args []string) error {
	// if --version is passed print the version info
	if showVers {
		fmt.Printf("hoarder %s (%s)\n", version, commit)
		return fmt.Errorf("")
	}

	// if --server is not passed, print help
	if !viper.GetBool("server") {
		ccmd.HelpFunc()(ccmd, args)
		return fmt.Errorf("") // no error, just exit
	}

	return nil
}

func startHoarder(ccmd *cobra.Command, args []string) error {
	// convert the log level
	logLvl := lumber.LvlInt(viper.GetString("log-level"))

	// configure the logger
	lumber.Prefix("[hoarder]")
	lumber.Level(logLvl)

	// enable/start garbage collection if age config was changed
	if ccmd.Flag("clean-after").Changed {
		lumber.Debug("Starting garbage collector (data older than %ds)...\n", ccmd.Flag("clean-after").Value)

		// start garbage collector
		go collector.Start()
	}

	// set, and initialize, the backend driver
	if err := backends.Initialize(); err != nil {
		return fmt.Errorf("Failed to initialize backend - %s", err)
	}

	// start the API
	if err := api.Start(); err != nil {
		return fmt.Errorf("Failed to start API - %s", err)
	}

	return nil
}

func init() {
	// set config defaults
	backend := "file:///var/db/hoarder"
	cleanAfter := uint64(time.Now().Unix())
	listenAddr := "https://127.0.0.1:7410"
	logLevel := "INFO"
	token := ""
	server := false

	// cli flags
	HoarderCmd.PersistentFlags().StringP("backend", "b", backend, "Hoarder backend")
	HoarderCmd.PersistentFlags().Uint64P("clean-after", "g", cleanAfter, "Age, in seconds, after which data is deemed garbage")
	HoarderCmd.PersistentFlags().StringP("listen-addr", "H", listenAddr, "Hoarder listen uri (scheme defaults to https)")
	HoarderCmd.PersistentFlags().String("log-level", logLevel, "Output level of logs (TRACE, DEBUG, INFO, WARN, ERROR, FATAL)")
	HoarderCmd.PersistentFlags().StringP("token", "t", token, "Auth token used when connecting to a secure Hoarder")
	HoarderCmd.Flags().BoolP("server", "s", server, "Run hoarder as a server")

	// bind config to cli flags
	viper.BindPFlag("backend", HoarderCmd.PersistentFlags().Lookup("backend"))
	viper.BindPFlag("clean-after", HoarderCmd.PersistentFlags().Lookup("clean-after"))
	viper.BindPFlag("listen-addr", HoarderCmd.PersistentFlags().Lookup("listen-addr"))
	viper.BindPFlag("log-level", HoarderCmd.PersistentFlags().Lookup("log-level"))
	viper.BindPFlag("token", HoarderCmd.PersistentFlags().Lookup("token"))
	viper.BindPFlag("server", HoarderCmd.Flags().Lookup("server"))

	// cli-only flags
	HoarderCmd.Flags().StringVarP(&config, "config", "c", "", "Path to config file (with extension)")
	HoarderCmd.Flags().BoolVarP(&showVers, "version", "v", false, "Display the current version of this CLI")

	// commands
	HoarderCmd.AddCommand(addCmd)
	HoarderCmd.AddCommand(listCmd)
	HoarderCmd.AddCommand(removeCmd)
	HoarderCmd.AddCommand(showCmd)
	HoarderCmd.AddCommand(updateCmd)

	// hidden/aliased commands
	HoarderCmd.AddCommand(createCmd)
	HoarderCmd.AddCommand(deleteCmd)
	HoarderCmd.AddCommand(destroyCmd)
	HoarderCmd.AddCommand(fetchCmd)
	HoarderCmd.AddCommand(getCmd)
}

func rest(_method, _key string, _body io.ReadWriter) io.Reader {
	if verbose {
		fmt.Fprintf(os.Stderr, "%s: %s/blobs%s", _method, viper.GetString("listen-addr"), _key)
	}

	req, err := http.NewRequest(_method, fmt.Sprintf("%s/blobs%s", viper.GetString("listen-addr"), _key), _body)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Failed to create request")
	}

	// skip cert check or not (only applies to https)
	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: insecure}

	// set auth token
	req.Header.Add("X-AUTH-TOKEN", viper.GetString("token"))

	// make request to hoarder server
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		// print error and exit
		fmt.Fprintf(os.Stderr, "Failed to make request - %s\n", err)
		os.Exit(1)
	}

	if verbose {
		fmt.Fprintf(os.Stderr, "%5d\n", res.StatusCode)
	}

	return res.Body
}
<|endoftext|>


<commit_before><commit_msg>Trying to fix the build<commit_after><|endoftext|>


<commit_before>package logstreamer

import (
	"bytes"
	"io"
	"log"
	"os"
	"strings"
)

type Logstreamer struct {
	Logger *log.Logger
	buf    *bytes.Buffer
	// If prefix == stdout, colors green
	// If prefix == stderr, colors red
	// Else, prefix is taken as-is, and prepended to anything
	// you throw at Write()
	prefix string
	// if true, saves output in memory
	record  bool
	persist string

	// Adds color to stdout & stderr if terminal supports it
	colorOkay  string
	colorFail  string
	colorReset string
}

func NewLogstreamerForWriter(prefix string, writer io.Writer) *Logstreamer {
	logger := log.New(writer, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamerForStdout(prefix string) *Logstreamer {
	// logger := log.New(os.Stdout, prefix, log.Ldate|log.Ltime)
	logger := log.New(os.Stdout, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamerForStderr(prefix string) *Logstreamer {
	logger := log.New(os.Stderr, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer {
	streamer := &Logstreamer{
		Logger:     logger,
		buf:        bytes.NewBuffer([]byte("")),
		prefix:     prefix,
		record:     record,
		persist:    "",
		colorOkay:  "",
		colorFail:  "",
		colorReset: "",
	}

	if strings.HasPrefix(os.Getenv("TERM"), "xterm") {
		streamer.colorOkay = "\x1b[32m"
		streamer.colorFail = "\x1b[31m"
		streamer.colorReset = "\x1b[0m"
	}

	return streamer
}

func (l *Logstreamer) Write(p []byte) (n int, err error) {
	if n, err = l.buf.Write(p); err != nil {
		return
	}

	err = l.OutputLines()
	return
}

func (l *Logstreamer) Close() error {
	if err := l.Flush(); err != nil {
		return err
	}
	l.buf = bytes.NewBuffer([]byte(""))
	return nil
}

func (l *Logstreamer) Flush() error {
	var p []byte
	if _, err := l.buf.Read(p); err != nil {
		return err
	}

	l.out(string(p))
	return nil
}

func (l *Logstreamer) OutputLines() error {
	for {
		line, err := l.buf.ReadString('\n')
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		l.out(line)
	}

	return nil
}

func (l *Logstreamer) FlushRecord() string {
	buffer := l.persist
	l.persist = ""
	return buffer
}

func (l *Logstreamer) out(str string) {
	if l.record == true {
		l.persist = l.persist + str
	}

	if l.prefix == "stdout" {
		str = l.colorOkay + l.prefix + l.colorReset + " " + str
	} else if l.prefix == "stderr" {
		str = l.colorFail + l.prefix + l.colorReset + " " + str
	} else {
		str = l.prefix + str
	}

	l.Logger.Print(str)
}
<commit_msg>switched the order of l.out and error checking, because as it is stated in the Go docs (https://golang.org/pkg/bytes/#Buffer.ReadString):     ReadString reads until the first occurrence of delim in the input, returning a string containing the data up to and including the delimiter. If ReadString encounters an error before finding a delimiter, it returns the data read before the error and the error itself (often io.EOF). ReadString returns err != nil if and only if the returned data does not end in delim.<commit_after>package logstreamer

import (
	"bytes"
	"io"
	"log"
	"os"
	"strings"
)

type Logstreamer struct {
	Logger *log.Logger
	buf    *bytes.Buffer
	// If prefix == stdout, colors green
	// If prefix == stderr, colors red
	// Else, prefix is taken as-is, and prepended to anything
	// you throw at Write()
	prefix string
	// if true, saves output in memory
	record  bool
	persist string

	// Adds color to stdout & stderr if terminal supports it
	colorOkay  string
	colorFail  string
	colorReset string
}

func NewLogstreamerForWriter(prefix string, writer io.Writer) *Logstreamer {
	logger := log.New(writer, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamerForStdout(prefix string) *Logstreamer {
	// logger := log.New(os.Stdout, prefix, log.Ldate|log.Ltime)
	logger := log.New(os.Stdout, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamerForStderr(prefix string) *Logstreamer {
	logger := log.New(os.Stderr, prefix, 0)
	return NewLogstreamer(logger, "", false)
}

func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer {
	streamer := &Logstreamer{
		Logger:     logger,
		buf:        bytes.NewBuffer([]byte("")),
		prefix:     prefix,
		record:     record,
		persist:    "",
		colorOkay:  "",
		colorFail:  "",
		colorReset: "",
	}

	if strings.HasPrefix(os.Getenv("TERM"), "xterm") {
		streamer.colorOkay = "\x1b[32m"
		streamer.colorFail = "\x1b[31m"
		streamer.colorReset = "\x1b[0m"
	}

	return streamer
}

func (l *Logstreamer) Write(p []byte) (n int, err error) {
	if n, err = l.buf.Write(p); err != nil {
		return
	}

	err = l.OutputLines()
	return
}

func (l *Logstreamer) Close() error {
	if err := l.Flush(); err != nil {
		return err
	}
	l.buf = bytes.NewBuffer([]byte(""))
	return nil
}

func (l *Logstreamer) Flush() error {
	var p []byte
	if _, err := l.buf.Read(p); err != nil {
		return err
	}

	l.out(string(p))
	return nil
}

func (l *Logstreamer) OutputLines() error {
	for {
		line, err := l.buf.ReadString('\n')

		if len(line) > 0 {
			l.out(line)
		}

		if err == io.EOF {
			break
		}

		if err != nil {
			return err
		}
	}

	return nil
}

func (l *Logstreamer) FlushRecord() string {
	buffer := l.persist
	l.persist = ""
	return buffer
}

func (l *Logstreamer) out(str string) {
	if l.record == true {
		l.persist = l.persist + str
	}

	if l.prefix == "stdout" {
		str = l.colorOkay + l.prefix + l.colorReset + " " + str
	} else if l.prefix == "stderr" {
		str = l.colorFail + l.prefix + l.colorReset + " " + str
	} else {
		str = l.prefix + str
	}

	l.Logger.Print(str)
}
<|endoftext|>


<commit_before>package bitswap

import (
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	pubsub "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/tuxychandru/pubsub"

	blocks "github.com/jbenet/go-ipfs/blocks"
	u "github.com/jbenet/go-ipfs/util"
)

type notifications struct {
	wrapped *pubsub.PubSub
}

func newNotifications() *notifications {
	const bufferSize = 16
	return &notifications{pubsub.New(bufferSize)}
}

func (ps *notifications) Publish(block *blocks.Block) {
	topic := string(block.Key())
	ps.wrapped.Pub(block, topic)
}

// Sub returns a one-time use |blockChannel|. |blockChannel| returns nil if the
// |ctx| times out or is cancelled
func (ps *notifications) Subscribe(ctx context.Context, k u.Key) <-chan *blocks.Block {
	topic := string(k)
	subChan := ps.wrapped.Sub(topic)
	blockChannel := make(chan *blocks.Block)
	go func() {
		defer close(blockChannel)
		select {
		case val := <-subChan:
			block, ok := val.(*blocks.Block)
			if ok {
				blockChannel <- block
			}
		case <-ctx.Done():
			ps.wrapped.Unsub(subChan, topic)
		}
	}()
	return blockChannel
}

func (ps *notifications) Shutdown() {
	ps.wrapped.Shutdown()
}
<commit_msg>fix(bitswap:notifications) close chan on Publish<commit_after>package bitswap

import (
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	pubsub "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/tuxychandru/pubsub"

	blocks "github.com/jbenet/go-ipfs/blocks"
	u "github.com/jbenet/go-ipfs/util"
)

type notifications struct {
	wrapped *pubsub.PubSub
}

func newNotifications() *notifications {
	const bufferSize = 16
	return &notifications{pubsub.New(bufferSize)}
}

func (ps *notifications) Publish(block *blocks.Block) {
	topic := string(block.Key())
	ps.wrapped.Pub(block, topic)
}

// Sub returns a one-time use |blockChannel|. |blockChannel| returns nil if the
// |ctx| times out or is cancelled
func (ps *notifications) Subscribe(ctx context.Context, k u.Key) <-chan *blocks.Block {
	topic := string(k)
	subChan := ps.wrapped.SubOnce(topic)
	blockChannel := make(chan *blocks.Block)
	go func() {
		defer close(blockChannel)
		select {
		case val := <-subChan:
			block, ok := val.(*blocks.Block)
			if ok {
				blockChannel <- block
			}
		case <-ctx.Done():
			ps.wrapped.Unsub(subChan, topic)
		}
	}()
	return blockChannel
}

func (ps *notifications) Shutdown() {
	ps.wrapped.Shutdown()
}
<|endoftext|>


<commit_before>package matriximage

import (
	"image"
	"image/color"
	_ "image/gif"
	_ "image/jpeg"
	"image/png"
	"math"
	"os"

	"github.com/mjibson/go-dsp/dsputils"
	"github.com/mjibson/go-dsp/fft"
)

const MaxUint = math.MaxUint16

type Image struct {
	image.Image
}

func (m Image) DFT() FourierImage {
	return FourierImage{Matrix: m.fftn()}
}

// Work with gray for now
// Returns a matrix with values scaled from 0.0 - 1.0
func (m Image) toGrayMatrix() *dsputils.Matrix {
	// Generate 0-based dimensions
	min, max := m.Bounds().Min, m.Bounds().Max
	lenY, lenX := max.Y-min.Y, max.X-min.X

	matrix := dsputils.MakeEmptyMatrix([]int{lenY, lenX})

	scale := 1.0

	for i := 0; i < lenX; i++ {
		for j := 0; j < lenY; j++ {

			v := scale * float64(m.Image.(*image.Gray16).Gray16At(i+min.X, j+min.Y).Y)

			matrix.SetValue(complex(v, 0), []int{j, i})
		}
	}

	return matrix
}

func (m Image) fftn() *dsputils.Matrix {
	matrix := m.toGrayMatrix()
	return fft.FFTN(matrix)
}

func FromFile(filename string) (*Image, error) {
	infile, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer infile.Close()

	// Decode will figure out what type of image is in the file on its own.
	// We just have to be sure all the image packages we want are imported.
	src, _, err := image.Decode(infile)
	if err != nil {
		return nil, err
	}

	grayImage := imageToGray(src)

	return &Image{Image: grayImage}, nil
}

func imageToGray(m image.Image) *image.Gray16 {
	b := m.Bounds()
	gray := image.NewGray16(b)

	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			gray.SetGray16(x, y, color.Gray16Model.Convert(m.At(x, y)).(color.Gray16))
		}
	}
	return gray
}

func (m Image) ToFile(named string) error {
	outfile, err := os.Create(named)
	if err != nil {
		return err
	}
	defer outfile.Close()

	return png.Encode(outfile, m.Image)
}
<commit_msg>Make the utility to convert to a grayscale image public.<commit_after>package matriximage

import (
	"image"
	"image/color"
	_ "image/gif"
	_ "image/jpeg"
	"image/png"
	"math"
	"os"

	"github.com/mjibson/go-dsp/dsputils"
	"github.com/mjibson/go-dsp/fft"
)

const MaxUint = math.MaxUint16

type Image struct {
	image.Image
}

func (m Image) DFT() FourierImage {
	return FourierImage{Matrix: m.fftn()}
}

// Work with gray for now
// Returns a matrix with values scaled from 0.0 - 1.0
func (m Image) toGrayMatrix() *dsputils.Matrix {
	// Generate 0-based dimensions
	min, max := m.Bounds().Min, m.Bounds().Max
	lenY, lenX := max.Y-min.Y, max.X-min.X

	matrix := dsputils.MakeEmptyMatrix([]int{lenY, lenX})

	scale := 1.0

	for i := 0; i < lenX; i++ {
		for j := 0; j < lenY; j++ {

			v := scale * float64(m.Image.(*image.Gray16).Gray16At(i+min.X, j+min.Y).Y)

			matrix.SetValue(complex(v, 0), []int{j, i})
		}
	}

	return matrix
}

func (m Image) fftn() *dsputils.Matrix {
	matrix := m.toGrayMatrix()
	return fft.FFTN(matrix)
}

func FromFile(filename string) (*Image, error) {
	infile, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer infile.Close()

	// Decode will figure out what type of image is in the file on its own.
	// We just have to be sure all the image packages we want are imported.
	src, _, err := image.Decode(infile)
	if err != nil {
		return nil, err
	}

	grayImage := ImageToGray(src)

	return &Image{Image: grayImage}, nil
}

func ImageToGray(m image.Image) *image.Gray16 {
	b := m.Bounds()
	gray := image.NewGray16(b)

	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			gray.SetGray16(x, y, color.Gray16Model.Convert(m.At(x, y)).(color.Gray16))
		}
	}
	return gray
}

func (m Image) ToFile(named string) error {
	outfile, err := os.Create(named)
	if err != nil {
		return err
	}
	defer outfile.Close()

	return png.Encode(outfile, m.Image)
}
<|endoftext|>


<commit_before>package gosudoc

import (
	"fmt"
	"reflect"
	"testing"
)

var mergedTests = []struct {
	Input    []string
	Expected map[string]string
}{
	{[]string{"071860576"}, map[string]string{"071860576": "007527446"}},
	{[]string{"071860576", "04770280X"}, map[string]string{"071860576": "007527446", "04770280X": "096729856"}},
	{[]string{"071860576", "04770XXXX"}, map[string]string{"071860576": "007527446"}},
	{[]string{"123456789"}, map[string]string{}},
	{[]string{}, map[string]string{}},
}

func TestGetMerged(t *testing.T) {
	for _, test := range mergedTests {
		actual, _ := GetMerged(test.Input)
		if reflect.DeepEqual(test.Expected, actual) {
			t.Logf("PASS: got %v", test.Expected)
		} else {
			t.Fatalf("FAIL for %s: expected %v, actual result was %v", test.Input, test.Expected, actual)
		}
	}
}

func ExampleGetMerged() {

	result, _ := GetMerged(mergedTests[0].Input)
	for k, v := range result {
		fmt.Println(k, v)
	}

	// Output:
	// 071860576 007527446
}

// TODO: test with a mock server rather than a live http.Get
<commit_msg>correct example<commit_after>package gosudoc

import (
	"fmt"
	"reflect"
	"testing"
)

var mergedTests = []struct {
	Input    []string
	Expected map[string]string
}{
	{[]string{"071860576"}, map[string]string{"071860576": "007527446"}},
	{[]string{"071860576", "04770280X"}, map[string]string{"071860576": "007527446", "04770280X": "096729856"}},
	{[]string{"071860576", "04770XXXX"}, map[string]string{"071860576": "007527446"}},
	{[]string{"123456789"}, map[string]string{}},
	{[]string{}, map[string]string{}},
}

func TestGetMerged(t *testing.T) {
	for _, test := range mergedTests {
		actual, _ := GetMerged(test.Input)
		if reflect.DeepEqual(test.Expected, actual) {
			t.Logf("PASS: got %v", test.Expected)
		} else {
			t.Fatalf("FAIL for %s: expected %v, actual result was %v", test.Input, test.Expected, actual)
		}
	}
}

func ExampleGetMerged() {
	input := []string{"071860576"}
	result, _ := GetMerged(input)
	for k, v := range result {
		fmt.Println(k, v)
	}

	// Output:
	// 071860576 007527446
}

// TODO: test with a mock server rather than a live http.Get
<|endoftext|>


<commit_before>package gogitlab

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"
)

type HookObjAttr struct {
	Id              int       `json:"id,omitempty"`
	Title           string    `json:"title,omitempty"`
	AssigneeId      int       `json:"assignee_id,omitempty"`
	AuthorId        int       `json:"author_id,omitempty"`
	ProjectId       int       `json:"project_id,omitempty"`
	CreatedAt       time.Time `json:"created_at,omitempty"`
	UpdatedAt       time.Time `json:"updated_at,omitempty"`
	Position        int       `json:"position,omitempty"`
	BranchName      string    `json:"branch_name,omitempty"`
	Description     string    `json:"description,omitempty"`
	MilestoneId     int       `json:"milestone_id,omitempty"`
	State           string    `json:"state,omitempty"`
	IId             int       `json:"iid,omitempty"`
	TargetBranch    string    `json:"target_branch,omitempty"`
	SourceBranch    string    `json:"source_branch,omitempty"`
	SourceProjectId int       `json:"source_project_id,omitempty"`
	StCommits       string    `json:"st_commits,omitempty"`
	StDiffs         string    `json:"st_diffs,omitempty"`
	MergeStatus     string    `json:"merge_status,omitempty"`
	TargetProjectId int       `json:"target_project_id,omitempty"`
}

type hRepository struct {
	Name        string `json:"name,omitempty"`
	URL         string `json:"url,omitempty"`
	Description string `json:"description,omitempty"`
	Homepage    string `json:"homepage,omitempty"`
}

type hCommit struct {
	Id        string    `json:"id,omitempty"`
	Message   string    `json:"message,omitempty"`
	Timestamp time.Time `json:"timestamp,omitempty"`
	URL       string    `json:"url,omitempty"`
	Author    *Person   `json:"author,omitempty"`
}

type HookPayload struct {
	Before            string       `json:"before,omitempty"`
	After             string       `json:"after,omitempty"`
	Ref               string       `json:"ref,omitempty"`
	UserId            int          `json:"user_id,omitempty"`
	UserName          string       `json:"user_name,omitempty"`
	ProjectId         int          `json:"project_id,omitempty"`
	Repository        *hRepository `json:"repository,omitempty"`
	Commits           []hCommit    `json:"commits,omitempty"`
	TotalCommitsCount int          `json:"total_commits_count,omitempty"`
	ObjectKind        string       `json:"object_kind,omitempty"`
	ObjectAttributes  *HookObjAttr `json:"object_attributes,omitempty"`
}

// ParseHook parses hook payload from GitLab
func ParseHook(payload []byte) (*HookPayload, error) {
	hp := HookPayload{}
	if err := json.Unmarshal(payload, &hp); err != nil {
		return nil, err
	}

	// Basic sanity check
	switch {
	case len(hp.ObjectKind) == 0:
		// Assume this is a post-receive within repository
		if len(hp.After) == 0 {
			return nil, fmt.Errorf("Invalid hook received, commit hash not found.")
		}
	case hp.ObjectKind == "issue":
		fallthrough
	case hp.ObjectKind == "merge_request":
		if hp.ObjectAttributes == nil {
			return nil, fmt.Errorf("Invalid hook received, attributes not found.")
		}
	default:
		return nil, fmt.Errorf("Invalid hook received, payload format not recognized.")
	}

	return &hp, nil
}

// Branch returns current branch for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Branch() string {
	return strings.TrimPrefix(h.Ref, "refs/heads/")
}

// Head returns the latest changeset for push event hook payload
func (h *HookPayload) Head() hCommit {
	c := hCommit{}
	for _, cm := range h.Commits {
		if h.After == cm.Id {
			return cm
		}
	}
	return c
}
<commit_msg>Add body elements for pipeline payloads<commit_after>package gogitlab

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"
)

type HookObjAttr struct {
	Id              int       `json:"id,omitempty"`
	Ref             string    `json:"ref,omitempty"`
	Tag             bool      `json:"tag,omitempty"`
	Sha             string    `json:"sha,omitempty"`
	BeforeSha       string    `json:"before_sha,omitempty"`
	Title           string    `json:"title,omitempty"`
	AssigneeId      int       `json:"assignee_id,omitempty"`
	AuthorId        int       `json:"author_id,omitempty"`
	ProjectId       int       `json:"project_id,omitempty"`
	Status          string    `json:"status,omitempty"`
	Stages          []string  `json:"stages,omitempty"`
	CreatedAt       time.Time `json:"created_at,omitempty"`
	UpdatedAt       time.Time `json:"updated_at,omitempty"`
	FinishedAt      time.Time `json:"finished_at,omitempty"`
	Duration        int       `json:"duration,omitempty"`
	Position        int       `json:"position,omitempty"`
	BranchName      string    `json:"branch_name,omitempty"`
	Description     string    `json:"description,omitempty"`
	MilestoneId     int       `json:"milestone_id,omitempty"`
	State           string    `json:"state,omitempty"`
	IId             int       `json:"iid,omitempty"`
	TargetBranch    string    `json:"target_branch,omitempty"`
	SourceBranch    string    `json:"source_branch,omitempty"`
	SourceProjectId int       `json:"source_project_id,omitempty"`
	StCommits       string    `json:"st_commits,omitempty"`
	StDiffs         string    `json:"st_diffs,omitempty"`
	MergeStatus     string    `json:"merge_status,omitempty"`
	TargetProjectId int       `json:"target_project_id,omitempty"`
}

type hRepository struct {
	Name        string `json:"name,omitempty"`
	URL         string `json:"url,omitempty"`
	Description string `json:"description,omitempty"`
	Homepage    string `json:"homepage,omitempty"`
}

type hProject struct {
	Project
	// Overwrite type *gogitlab.Namespace with type string,
	// otherwise the project hash passed for pipeline hooks is
	// identical
	Namespace string `json:"namespace,omitempty"`
}

type hCommit struct {
	Id        string    `json:"id,omitempty"`
	Message   string    `json:"message,omitempty"`
	Timestamp time.Time `json:"timestamp,omitempty"`
	URL       string    `json:"url,omitempty"`
	Author    *Person   `json:"author,omitempty"`
}

type HookPayload struct {
	Before            string       `json:"before,omitempty"`
	After             string       `json:"after,omitempty"`
	Ref               string       `json:"ref,omitempty"`
	UserId            int          `json:"user_id,omitempty"`
	UserName          string       `json:"user_name,omitempty"`
	ProjectId         int          `json:"project_id,omitempty"`
	Project           *hProject    `json:"project,omitempty"`
	Repository        *hRepository `json:"repository,omitempty"`
	Commits           []hCommit    `json:"commits,omitempty"`
	Commit            *hCommit     `json:"commit,omitempty"`
	TotalCommitsCount int          `json:"total_commits_count,omitempty"`
	ObjectKind        string       `json:"object_kind,omitempty"`
	ObjectAttributes  *HookObjAttr `json:"object_attributes,omitempty"`
}

// ParseHook parses hook payload from GitLab
func ParseHook(payload []byte) (*HookPayload, error) {
	hp := HookPayload{}
	if err := json.Unmarshal(payload, &hp); err != nil {
		return nil, err
	}

	// Basic sanity check
	switch {
	case len(hp.ObjectKind) == 0:
		// Assume this is a post-receive within repository
		if len(hp.After) == 0 {
			return nil, fmt.Errorf("Invalid hook received, commit hash not found.")
		}
	case hp.ObjectKind == "pipeline":
		fallthrough
	case hp.ObjectKind == "issue":
		fallthrough
	case hp.ObjectKind == "merge_request":
		if hp.ObjectAttributes == nil {
			return nil, fmt.Errorf("Invalid hook received, attributes not found.")
		}
	default:
		return nil, fmt.Errorf("Invalid hook received, payload format not recognized.")
	}

	return &hp, nil
}

// Branch returns current branch for push event hook payload
// This function returns empty string for any other events
func (h *HookPayload) Branch() string {
	return strings.TrimPrefix(h.Ref, "refs/heads/")
}

// Head returns the latest changeset for push event hook payload
func (h *HookPayload) Head() hCommit {
	c := hCommit{}
	for _, cm := range h.Commits {
		if h.After == cm.Id {
			return cm
		}
	}
	return c
}
<|endoftext|>


<commit_before>package GoSDK

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	_StatsPreamble  = "/api/v/3/platform/statistics"
	_DBConnPreamble = "/api/v/3/platform/dbconnections"
	_LogsPreamble   = "/api/v/3/platform/logs"
)

func (u *UserClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeStatistics(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeStatistics(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getStatistics(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _StatsPreamble+"`"+systemKey, q, creds, headers)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	mapResp, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	mapBody, ok := mapResp.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return mapBody, nil
}

func (u *UserClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeDBConnections(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeDBConnections(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getDBConnections(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _DBConnPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func (u *UserClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(d, systemKey, query, nil)
}

func (u *UserClient) GetConnectedEdgeLogs(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(u, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func (d *DevClient) GetConnectedEdgeLogs(edgeName, systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(d, systemKey, query, map[string][]string{"ClearBlade-Edge": []string{edgeName}})
}

func getLogs(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _LogsPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func dbQueryToReqQuery(query *Query) (map[string]string, error) {
	var qry map[string]string
	if query != nil {
		queryMap := query.serialize()
		queryBytes, err := json.Marshal(queryMap)
		if err != nil {
			return nil, err
		}
		qry = map[string]string{
			"query": url.QueryEscape(string(queryBytes)),
		}
	} else {
		qry = nil
	}
	return qry, nil
}
<commit_msg>remove edge specific calls<commit_after>package GoSDK

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	_StatsPreamble  = "/api/v/3/platform/statistics"
	_DBConnPreamble = "/api/v/3/platform/dbconnections"
	_LogsPreamble   = "/api/v/3/platform/logs"
)

func (u *UserClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformStatistics(systemKey string, query *Query) (map[string]interface{}, error) {
	return getStatistics(d, systemKey, query, nil)
}

func getStatistics(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _StatsPreamble+"/"+systemKey, q, creds, headers)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	mapResp, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	mapBody, ok := mapResp.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return mapBody, nil
}

func (u *UserClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformDBConnections(systemKey string, query *Query) (map[string]interface{}, error) {
	return getDBConnections(d, systemKey, query, nil)
}

func getDBConnections(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _DBConnPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func (u *UserClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(u, systemKey, query, nil)
}

func (d *DevClient) GetPlatformLogs(systemKey string, query *Query) (map[string]interface{}, error) {
	return getLogs(d, systemKey, query, nil)
}

func getLogs(c cbClient, systemKey string, query *Query, headers map[string][]string) (map[string]interface{}, error) {
	creds, err := c.credentials()
	if err != nil {
		return nil, err
	}
	q, err := dbQueryToReqQuery(query)
	if err != nil {
		return nil, err
	}
	resp, err := get(c, _LogsPreamble+"/"+systemKey, q, creds, nil)
	if err != nil {
		return nil, fmt.Errorf("Unable to communicate with platform, got %s", err)
	}
	respMap, err := mapResponse(resp, err)
	if err != nil {
		return nil, err
	}
	respBody, ok := respMap.Body.(map[string]interface{})
	if !ok {
		return nil, fmt.Errorf("Expected JSON response, got %#v", resp)
	}
	return respBody, nil
}

func dbQueryToReqQuery(query *Query) (map[string]string, error) {
	var qry map[string]string
	if query != nil {
		queryMap := query.serialize()
		queryBytes, err := json.Marshal(queryMap)
		if err != nil {
			return nil, err
		}
		qry = map[string]string{
			"query": url.QueryEscape(string(queryBytes)),
		}
	} else {
		qry = nil
	}
	return qry, nil
}
<|endoftext|>


<commit_before>// Package metrics provides a framework for application instrumentation. All
// metrics are safe for concurrent use. Considerable design influence has been
// taken from https://github.com/codahale/metrics and https://prometheus.io.
//
// This package contains the common interfaces. Your code should take these
// interfaces as parameters. Implementations are provided for different
// instrumentation systems in the various subdirectories.
//
// Usage
//
// Metrics are dependencies and should be passed to the components that need
// them in the same way you'd construct and pass a database handle, or reference
// to another component. So, create metrics in your func main, using whichever
// concrete implementation is appropriate for your organization.
//
//    latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
//        Namespace: "myteam",
//        Subsystem: "foosvc",
//        Name:      "request_latency_seconds",
//        Help:      "Incoming request latency in seconds."
//    }, []string{"method", "status_code"})
//
// Write your components to take the metrics they will use as parameters to
// their constructors. Use the interface types, not the concrete types. That is,
//
//    // NewAPI takes metrics.Histogram, not *prometheus.Summary
//    func NewAPI(s Store, logger log.Logger, latency metrics.Histogram) *API {
//        // ...
//    }
//
//    func (a *API) ServeFoo(w http.ResponseWriter, r *http.Request) {
//        begin := time.Now()
//        // ...
//        a.latency.Observe(time.Since(begin).Seconds())
//    }
//
// Finally, pass the metrics as dependencies when building your object graph.
// This should happen in func main, not in the global scope.
//
//    api := NewAPI(store, logger, latency)
//    http.ListenAndServe("/", api)
//
// Implementation details
//
// Each telemetry system has different semantics for label values, push vs.
// pull, support for histograms, etc. These properties influence the design of
// their respective packages. This table attempts to summarize the key points of
// distinction.
//
//    SYSTEM      DIM  COUNTERS               GAUGES                 HISTOGRAMS
//    dogstatsd   n    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    statsd      1    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    graphite    1    batch, push-aggregate  batch, push-aggregate  synthetic, batch, push-aggregate
//    expvar      1    atomic                 atomic                 synthetic, batch, in-place expose
//    influx      n    custom                 custom                 custom
//    prometheus  n    native                 native                 native
//    circonus    1    native                 native                 native
//    pcp         1    native                 native                 native
//
package metrics
<commit_msg>metrics: improve README<commit_after>// Package metrics provides a framework for application instrumentation. It's
// primarily designed to help you get started with good and robust
// instrumentation, and to help you migrate from a less-capable system like
// Graphite to a more-capable system like Prometheus. If your organization has
// already standardized on an instrumentation system like Prometheus, and has no
// plans to change, it probably makes sense to use that system's instrumentation
// library directly.
//
// This package provides three core metric abstractions (Counter, Gauge, and
// Histogram) and implementations for almost all common instrumentation
// backends. Each metric has an observation method (Add, Set, or Observe,
// respectively) used to record values, and a With method to "scope" the
// observation by various parameters. For example, you might have a Histogram to
// record request durations, parameterized by the method that's being called.
//
//    var requestDuration metrics.Histogram
//    // ...
//    requestDuration.With("method", "MyMethod").Observe(time.Since(begin))
//
// This allows a single high-level metrics object (requestDuration) to work with
// many code paths somewhat dynamically. The concept of With is fully supported
// in some backends like Prometheus, and not supported in other backends like
// Graphite. So, With may be a no-op, depending on the concrete implementation
// you choose.
//
// Usage
//
// Metrics are dependencies, and should be passed to the components that need
// them in the same way you'd construct and pass a database handle, or reference
// to another component. Metrics should *not* be created in the global scope.
// Instead, instantiate metrics in your func main, using whichever concrete
// implementation is appropriate for your organization.
//
//    latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
//        Namespace: "myteam",
//        Subsystem: "foosvc",
//        Name:      "request_latency_seconds",
//        Help:      "Incoming request latency in seconds."
//    }, []string{"method", "status_code"})
//
// Write your components to take the metrics they will use as parameters to
// their constructors. Use the interface types, not the concrete types. That is,
//
//    // NewAPI takes metrics.Histogram, not *prometheus.Summary
//    func NewAPI(s Store, logger log.Logger, latency metrics.Histogram) *API {
//        // ...
//    }
//
//    func (a *API) ServeFoo(w http.ResponseWriter, r *http.Request) {
//        begin := time.Now()
//        // ...
//        a.latency.Observe(time.Since(begin).Seconds())
//    }
//
// Finally, pass the metrics as dependencies when building your object graph.
// This should happen in func main, not in the global scope.
//
//    api := NewAPI(store, logger, latency)
//    http.ListenAndServe("/", api)
//
// Note that metrics are "write-only" interfaces.
//
// Implementation details
//
// All metrics are safe for concurrent use. Considerable design influence has
// been taken from https://github.com/codahale/metrics and
// https://prometheus.io.
//
// Each telemetry system has different semantics for label values, push vs.
// pull, support for histograms, etc. These properties influence the design of
// their respective packages. This table attempts to summarize the key points of
// distinction.
//
//    SYSTEM      DIM  COUNTERS               GAUGES                 HISTOGRAMS
//    dogstatsd   n    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    statsd      1    batch, push-aggregate  batch, push-aggregate  native, batch, push-each
//    graphite    1    batch, push-aggregate  batch, push-aggregate  synthetic, batch, push-aggregate
//    expvar      1    atomic                 atomic                 synthetic, batch, in-place expose
//    influx      n    custom                 custom                 custom
//    prometheus  n    native                 native                 native
//    circonus    1    native                 native                 native
//    pcp         1    native                 native                 native
//
package metrics
<|endoftext|>