<commit_before>package golib

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/gotoeveryone/golib/config"
)

func TestLoadConfigNoSuchFile(t *testing.T) {
	current, _ := filepath.Abs(".")

	if err := os.Remove(filepath.Join(current, "config.json")); err != nil {
		if !os.IsNotExist(err) {
			t.Errorf("Remove log error: %s", err)
		}
	}

	var config config.Config
	if err := LoadConfig(&config, ""); err != nil {
		if !strings.Contains(err.Error(), "Read file Error") {
			t.Error(err)
		}
	}
}

func TestLoadConfigParseError(t *testing.T) {
	current, _ := filepath.Abs(".")
	dir := filepath.Join(current, "hoge")

	if err := os.MkdirAll(dir, 0755); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(dir, "config.json"), []byte("hoge"), 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, dir); err != nil {
		if !strings.Contains(err.Error(), "Unmarshal error") {
			t.Error(err)
		}
	}

}

func TestLoadConfig(t *testing.T) {
	current, _ := filepath.Abs(".")

	f, err := ioutil.ReadFile(filepath.Join(current, "config.json.example"))
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(current, "config.json"), f, 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, ""); err != nil {
		t.Error(err)
	}
}

func TestLoadConfigCustomPath(t *testing.T) {
	current, _ := filepath.Abs(".")

	f, err := ioutil.ReadFile(filepath.Join(current, "config.json.example"))
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(current, "tmp", "config.json"), f, 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, filepath.Join(current, "tmp")); err != nil {
		t.Error(err)
	}
}
<commit_msg>modify: test file io path<commit_after>package golib

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/gotoeveryone/golib/config"
)

func TestLoadConfigNoSuchFile(t *testing.T) {
	current, _ := filepath.Abs(".")

	if err := os.Remove(filepath.Join(current, "config.json")); err != nil {
		if !os.IsNotExist(err) {
			t.Errorf("Remove log error: %s", err)
		}
	}

	var config config.Config
	if err := LoadConfig(&config, ""); err != nil {
		if !strings.Contains(err.Error(), "Read file Error") {
			t.Error(err)
		}
	}
}

func TestLoadConfigParseError(t *testing.T) {
	current, _ := filepath.Abs(".")
	dir := filepath.Join(current, "tmp")

	if err := os.MkdirAll(dir, 0755); err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(dir, "config.json"), []byte("hoge"), 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, dir); err != nil {
		if !strings.Contains(err.Error(), "Unmarshal error") {
			t.Error(err)
		}
	}
}

func TestLoadConfig(t *testing.T) {
	current, _ := filepath.Abs(".")

	f, err := ioutil.ReadFile(filepath.Join(current, "config.json.example"))
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(current, "config.json"), f, 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, current); err != nil {
		t.Error(err)
	}
}

func TestLoadConfigCustomPath(t *testing.T) {
	current, _ := filepath.Abs(".")

	f, err := ioutil.ReadFile(filepath.Join(current, "config.json.example"))
	if err != nil {
		t.Error(err)
	}

	if err := ioutil.WriteFile(filepath.Join(current, "tmp", "config.json"), f, 0755); err != nil {
		t.Error(err)
	}

	var config config.Config
	if err := LoadConfig(&config, filepath.Join(current, "tmp")); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package flickr

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sort"
)

const (
	endpoint        = "http://api.flickr.com/services/rest/?"
	uploadEndpoint  = "http://api.flickr.com/services/upload/"
	replaceEndpoint = "http://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() error { return nil }

type Error string

func (e Error) Error() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	// Remove api_sig
	delete(args, "api_sig")

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k := range args {
		sorted_keys[i] = k
		i++
	}
	sort.Strings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	delete(args, "api_key")
	delete(args, "method")

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum(nil))
}

func (request *Request) URL() string {
	args := request.Args

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)
	return s
}

func (request *Request) Execute() (response string, ret error) {
	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	s := request.URL()

	res, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + "=" + url.QueryEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url_ string, filename string, filetype string) (*http.Request, error) {
	real_url, _ := url.Parse(url_)

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	f_size := stat.Size()

	request.Args["api_key"] = request.ApiKey

	boundary, end := "----###---###--flickr-go-rules", "\r\n"

	// Build out all of POST body sans file
	header := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		header.WriteString("--" + boundary + end)
		header.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		header.WriteString(v + end)
	}
	header.WriteString("--" + boundary + end)
	header.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	header.WriteString("Content-Type: " + filetype + end + end)

	footer := bytes.NewBufferString(end + "--" + boundary + "--" + end)

	body_len := int64(header.Len()) + int64(footer.Len()) + f_size

	r, w := io.Pipe()
	go func() {
		pieces := []io.Reader{header, f, footer}

		for _, k := range pieces {
			_, err = io.Copy(w, k)
			if err != nil {
				w.CloseWithError(nil)
				return
			}
		}
		f.Close()
		w.Close()
	}()

	http_header := make(http.Header)
	http_header.Add("Content-Type", "multipart/form-data; boundary="+boundary)

	postRequest := &http.Request{
		Method:        "POST",
		URL:           real_url,
		Host:          apiHost,
		Header:        http_header,
		Body:          r,
		ContentLength: body_len,
	}
	return postRequest, nil
}

// Example:
// r.Upload("thumb.jpg", "image/jpeg")
func (request *Request) Upload(filename string, filetype string) (response string, err error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	client := &http.DefaultClient
	resp, err := client.Do(postRequest)

	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	return string(rawBody), nil
}
<commit_msg>Changed Endpoint urls to https as it is required to work by flickr<commit_after>package flickr

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"sort"
)

const (
	endpoint        = "https://api.flickr.com/services/rest/?"
	uploadEndpoint  = "https://api.flickr.com/services/upload/"
	replaceEndpoint = "https://api.flickr.com/services/replace/"
	apiHost         = "api.flickr.com"
)

type Request struct {
	ApiKey string
	Method string
	Args   map[string]string
}

type nopCloser struct {
	io.Reader
}

func (nopCloser) Close() error { return nil }

type Error string

func (e Error) Error() string {
	return string(e)
}

func (request *Request) Sign(secret string) {
	args := request.Args

	// Remove api_sig
	delete(args, "api_sig")

	sorted_keys := make([]string, len(args)+2)

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	// Sort array keys
	i := 0
	for k := range args {
		sorted_keys[i] = k
		i++
	}
	sort.Strings(sorted_keys)

	// Build out ordered key-value string prefixed by secret
	s := secret
	for _, key := range sorted_keys {
		if args[key] != "" {
			s += fmt.Sprintf("%s%s", key, args[key])
		}
	}

	// Since we're only adding two keys, it's easier
	// and more space-efficient to just delete them
	// them copy the whole map
	delete(args, "api_key")
	delete(args, "method")

	// Have the full string, now hash
	hash := md5.New()
	hash.Write([]byte(s))

	// Add api_sig as one of the args
	args["api_sig"] = fmt.Sprintf("%x", hash.Sum(nil))
}

func (request *Request) URL() string {
	args := request.Args

	args["api_key"] = request.ApiKey
	args["method"] = request.Method

	s := endpoint + encodeQuery(args)
	return s
}

func (request *Request) Execute() (response string, ret error) {
	if request.ApiKey == "" || request.Method == "" {
		return "", Error("Need both API key and method")
	}

	s := request.URL()

	res, err := http.Get(s)
	defer res.Body.Close()
	if err != nil {
		return "", err
	}

	body, _ := ioutil.ReadAll(res.Body)
	return string(body), nil
}

func encodeQuery(args map[string]string) string {
	i := 0
	s := bytes.NewBuffer(nil)
	for k, v := range args {
		if i != 0 {
			s.WriteString("&")
		}
		i++
		s.WriteString(k + "=" + url.QueryEscape(v))
	}
	return s.String()
}

func (request *Request) buildPost(url_ string, filename string, filetype string) (*http.Request, error) {
	real_url, _ := url.Parse(url_)

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}
	f_size := stat.Size()

	request.Args["api_key"] = request.ApiKey

	boundary, end := "----###---###--flickr-go-rules", "\r\n"

	// Build out all of POST body sans file
	header := bytes.NewBuffer(nil)
	for k, v := range request.Args {
		header.WriteString("--" + boundary + end)
		header.WriteString("Content-Disposition: form-data; name=\"" + k + "\"" + end + end)
		header.WriteString(v + end)
	}
	header.WriteString("--" + boundary + end)
	header.WriteString("Content-Disposition: form-data; name=\"photo\"; filename=\"photo.jpg\"" + end)
	header.WriteString("Content-Type: " + filetype + end + end)

	footer := bytes.NewBufferString(end + "--" + boundary + "--" + end)

	body_len := int64(header.Len()) + int64(footer.Len()) + f_size

	r, w := io.Pipe()
	go func() {
		pieces := []io.Reader{header, f, footer}

		for _, k := range pieces {
			_, err = io.Copy(w, k)
			if err != nil {
				w.CloseWithError(nil)
				return
			}
		}
		f.Close()
		w.Close()
	}()

	http_header := make(http.Header)
	http_header.Add("Content-Type", "multipart/form-data; boundary="+boundary)

	postRequest := &http.Request{
		Method:        "POST",
		URL:           real_url,
		Host:          apiHost,
		Header:        http_header,
		Body:          r,
		ContentLength: body_len,
	}
	return postRequest, nil
}

// Example:
// r.Upload("thumb.jpg", "image/jpeg")
func (request *Request) Upload(filename string, filetype string) (response string, err error) {
	postRequest, err := request.buildPost(uploadEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func (request *Request) Replace(filename string, filetype string) (response string, err error) {
	postRequest, err := request.buildPost(replaceEndpoint, filename, filetype)
	if err != nil {
		return "", err
	}
	return sendPost(postRequest)
}

func sendPost(postRequest *http.Request) (body string, err error) {
	// Create and use TCP connection (lifted mostly wholesale from http.send)
	client := &http.DefaultClient
	resp, err := client.Do(postRequest)

	if err != nil {
		return "", err
	}
	rawBody, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	return string(rawBody), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	_ "github.com/mattn/go-sqlite3"

	"github.com/Masterminds/squirrel"
	"github.com/biogo/biogo/io/featio"
	"github.com/biogo/biogo/io/featio/bed"
	"github.com/jmoiron/sqlx"
	"gopkg.in/alecthomas/kingpin.v2"
)

// CountBuilder is a squirrel select builder whose columns match Count fields.
var CountBuilder = squirrel.Select().
	Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
	Column(squirrel.Alias(squirrel.Expr("SUM(copy_number)"), "copyNum"))

// Count is a databases row with record count information.
type Count struct {
	Count   int `db:"count"`
	CopyNum int `db:"copyNum"`
}

const prog = "htsdb-count-reads-on-feats"
const version = "0.2"
const descr = `Print number of reads and read copies that are contained in
each feature of the input file. Currently only features in the BED6 format are
supported. Provided SQL filter will apply to all counts.`

var (
	app = kingpin.New(prog, descr)

	dbFile = app.Flag("db", "File to SQLite database.").
		PlaceHolder("<file>").Required().String()
	tab = app.Flag("table", "Database table name.").
		Default("sample").String()
	where = app.Flag("where", "SQL filter to inject in WHERE clause.").
		PlaceHolder("<SQL>").String()
	bed6 = app.Flag("bed6", "BED6 file with features.").
		PlaceHolder("<file>").Required().String()
	as = app.Flag("as", "Name to print describing the count/s.").
		Default("all").String()
	header = app.Flag("header", "Print header line.").
		Bool()
	useOri = app.Flag("use-ori", "Only report counts on the orientation of the feature.").
		Bool()
)

func main() {
	var err error
	var query string
	var db *sqlx.DB
	var stmt *sqlx.Stmt

	// read command line args and options
	app.HelpFlag.Short('h')
	app.Version(version)
	if _, err := app.Parse(os.Args[1:]); err != nil {
		kingpin.Fatalf("%s", err)
	}

	// assemble sqlx select builders
	countBuilder := CountBuilder.From(*tab).Where("rname = ? AND start BETWEEN ? AND ? AND stop BETWEEN ? AND ?")
	if *where != "" {
		countBuilder = countBuilder.Where(*where)
	}
	if *useOri == true {
		countBuilder = countBuilder.Where("strand = ?")
	}

	// open database connections.
	if db, err = sqlx.Connect("sqlite3", *dbFile); err != nil {
		panic(err)
	}

	// prepare statements.
	if query, _, err = countBuilder.ToSql(); err != nil {
		panic(err)
	}
	if stmt, err = db.Preparex(query); err != nil {
		panic(err)
	}

	// open BED6 scanner
	bedS, err := bed6Scanner(*bed6)
	if err != nil {
		panic(err)
	}

	// loop on the BED6 feats and count
	if *header == true {
		fmt.Printf("category\tfeat\tname\tcount\tcopyNumber\n")
	}
	for {
		if bedS.Next() == false {
			break
		}
		b, _ := bedS.Feat().(*bed.Bed6)
		start, stop, ori, chrom := b.Start(), b.End()-1, b.Orientation(), b.Location().Name()

		var c Count
		if *useOri == true {
			if err = stmt.Get(&c, chrom, start, stop, start, stop, ori); err != nil {
				panic(err)
			}
		} else {
			if err = stmt.Get(&c, chrom, start, stop, start, stop); err != nil {
				panic(err)
			}
		}
		fmt.Printf("%s\t%s:%d-%d:%d\t%s\t%d\t%d\n", *as, chrom, start, stop, ori, b.Name(), c.Count, c.CopyNum)
	}
	if err = bedS.Error(); err != nil {
		panic(err)
	}
}

func bed6Scanner(f string) (*featio.Scanner, error) {
	ioR, err := os.Open(*bed6)
	if err != nil {
		return nil, err
	}
	bedR, err := bed.NewReader(ioR, 6)
	if err != nil {
		return nil, err
	}
	return featio.NewScanner(bedR), nil
}
<commit_msg>htsdb-count-reads-on-feats: fix panic when no reads where found<commit_after>package main

import (
	"fmt"
	"os"

	_ "github.com/mattn/go-sqlite3"

	"github.com/Masterminds/squirrel"
	"github.com/biogo/biogo/io/featio"
	"github.com/biogo/biogo/io/featio/bed"
	"github.com/jmoiron/sqlx"
	"gopkg.in/alecthomas/kingpin.v2"
)

// CountBuilder is a squirrel select builder whose columns match Count fields.
var CountBuilder = squirrel.Select().
	Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
	Column(squirrel.Alias(squirrel.Expr("CAST(TOTAL(copy_number) AS INTEGER)"), "copyNum"))

// Count is a databases row with record count information.
type Count struct {
	Count   int `db:"count"`
	CopyNum int `db:"copyNum"`
}

const prog = "htsdb-count-reads-on-feats"
const version = "0.2"
const descr = `Print number of reads and read copies that are contained in
each feature of the input file. Currently only features in the BED6 format are
supported. Provided SQL filter will apply to all counts.`

var (
	app = kingpin.New(prog, descr)

	dbFile = app.Flag("db", "File to SQLite database.").
		PlaceHolder("<file>").Required().String()
	tab = app.Flag("table", "Database table name.").
		Default("sample").String()
	where = app.Flag("where", "SQL filter to inject in WHERE clause.").
		PlaceHolder("<SQL>").String()
	bed6 = app.Flag("bed6", "BED6 file with features.").
		PlaceHolder("<file>").Required().String()
	as = app.Flag("as", "Name to print describing the count/s.").
		Default("all").String()
	header = app.Flag("header", "Print header line.").
		Bool()
	useOri = app.Flag("use-ori", "Only report counts on the orientation of the feature.").
		Bool()
)

func main() {
	var err error
	var query string
	var db *sqlx.DB
	var stmt *sqlx.Stmt

	// read command line args and options
	app.HelpFlag.Short('h')
	app.Version(version)
	if _, err := app.Parse(os.Args[1:]); err != nil {
		kingpin.Fatalf("%s", err)
	}

	// assemble sqlx select builders
	countBuilder := CountBuilder.From(*tab).Where("rname = ? AND start BETWEEN ? AND ? AND stop BETWEEN ? AND ?")
	if *where != "" {
		countBuilder = countBuilder.Where(*where)
	}
	if *useOri == true {
		countBuilder = countBuilder.Where("strand = ?")
	}

	// open database connections.
	if db, err = sqlx.Connect("sqlite3", *dbFile); err != nil {
		panic(err)
	}

	// prepare statements.
	if query, _, err = countBuilder.ToSql(); err != nil {
		panic(err)
	}
	if stmt, err = db.Preparex(query); err != nil {
		panic(err)
	}

	// open BED6 scanner
	bedS, err := bed6Scanner(*bed6)
	if err != nil {
		panic(err)
	}

	// loop on the BED6 feats and count
	if *header == true {
		fmt.Printf("category\tfeat\tname\tcount\tcopyNumber\n")
	}
	for {
		if bedS.Next() == false {
			break
		}
		b, _ := bedS.Feat().(*bed.Bed6)
		start, stop, ori, chrom := b.Start(), b.End()-1, b.Orientation(), b.Location().Name()

		var c Count
		if *useOri == true {
			if err = stmt.Get(&c, chrom, start, stop, start, stop, ori); err != nil {
				panic(err)
			}
		} else {
			if err = stmt.Get(&c, chrom, start, stop, start, stop); err != nil {
				panic(err)
			}
		}
		fmt.Printf("%s\t%s:%d-%d:%d\t%s\t%d\t%d\n", *as, chrom, start, stop, ori, b.Name(), c.Count, c.CopyNum)
	}
	if err = bedS.Error(); err != nil {
		panic(err)
	}
}

func bed6Scanner(f string) (*featio.Scanner, error) {
	ioR, err := os.Open(*bed6)
	if err != nil {
		return nil, err
	}
	bedR, err := bed.NewReader(ioR, 6)
	if err != nil {
		return nil, err
	}
	return featio.NewScanner(bedR), nil
}
<|endoftext|>


<commit_before>/*
Package framed adds basic support for message framing over streams.

Messages contain an header and a body, both of which are length prefixed.

Here are the bytes (stored in little-endian byte order):

0-2: unsigned 16 bit int header length
2-4: unsinged 16 bit int body length
4+:  message content (header and body)

The use of a uint16 means that the maximum possible header and body lengths
are 65535 each.

Important Note for using with TCP -

When using framed with a TCPConn, we recommend calling
TCPConn.SetNoDelay(false).

framed was designed to maximize the chance of zero-copy ops being used to send
frames from one connection to another. Because of this, it writes a lot of
small pieces of data to the underlying stream (e.g. the length headers).  If
TCP is being used without Nagle's algorithm (i.e. setting
TCPConn.SetNoDelay(true), which is the default), then it can result in fairly
extreme packet fragmentation and consequently ballooning overhead.

Example Usage:

	package main

	import (
		"github.com/oxtoacart/framed"
		"io/ioutil"
		"log"
		"net"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := net.Dial("tcp", "host:port"); err == nil {
			framedConn = Framed{conn}
			if err := framedConn.WriteFrame([]byte("Header"), []byte("Hello World")); err == nil {
				if err, frame := framedConn.ReadInitial(); err == nil {
					// Note - Read is just like io.Reader.Read(), so we use ioutil.ReadAll
					if header, err := ioutil.ReadAll(framedConn.Header); err == nil {
						if body, err := ioutil.ReadAll(framedConn.Body); err == nil {
							nextFrame, err := frame.Next()
							// And so on
						}
					}
				}
			}
		}
	}
*/
package framed

import (
	"encoding/binary"
	"io"
	"io/ioutil"
)

var endianness = binary.LittleEndian

/*
A Framed enhances an io.ReadWriteCloser to provide methods that allow writing
and reading frames.

Although the underlying ReadWriteCloser may be safe to use from multiple
goroutines, a Framed is not.
*/
type Framed struct {
	raw            io.ReadWriteCloser // the raw underlying connection
	hasReadInitial bool
}

// AlreadyReadError is returned when someone has already read from a Frame
type AlreadyReadError string

func (err AlreadyReadError) Error() string {
	return string(err)
}

// Frame encapsulates a frame from a Framed
type Frame struct {
	framed         *Framed
	headerLength   uint16
	bodyLength     uint16
	bytesRemaining int
	header         *frameSection
	body           *frameSection
	doneReading    chan bool
}

// frameSection encapsulates a section of a frame (header or body)
type frameSection struct {
	frame          *Frame
	init           func() error
	bytesRemaining int
	startedReading bool
}

// NewFramed creates a new Framed on top of the given readWriteCloser
func NewFramed(readWriteCloser io.ReadWriteCloser) *Framed {
	return &Framed{readWriteCloser, false}
}

// Close() implements method Close() from io.Closer.
func (framed *Framed) Close() error {
	return framed.raw.Close()
}

/*
ReadInitial reads the initial frame from the framed.  It returns an
AlreadyReadError if the initial frame has already been read previously.
*/
func (framed *Framed) ReadInitial() (frame *Frame, err error) {
	if framed.hasReadInitial {
		return nil, AlreadyReadError("Initial Frame already read")
	}
	frame, err = framed.nextFrame()
	framed.hasReadInitial = true
	return
}

/*
Next returns the next frame from the Framed underlying the frame on which
it is called.  This method blocks until the previous frame has been consumed.
If there are no more frames, it returns io.EOF as an error.
*/
func (frame *Frame) Next() (nextFrame *Frame, err error) {
	<-frame.doneReading
	return frame.framed.nextFrame()
}

/*
WriteFrame writes the given header and body to the Framed.
Either or both can be nil.
*/
func (framed *Framed) WriteFrame(header []byte, body []byte) (err error) {
	headerLength := 0
	bodyLength := 0
	if header != nil {
		headerLength = len(header)
	}
	if body != nil {
		bodyLength = len(body)
	}

	if err = framed.WriteHeader(uint16(headerLength), uint16(bodyLength)); err != nil {
		return err
	}

	if header != nil {
		if _, err = framed.raw.Write(header); err != nil {
			return err
		}
	}

	if body != nil {
		_, err = framed.raw.Write(body)
	}

	return
}

// WriteHeader writes a frame header with the given lengths to the Framed.
func (framed *Framed) WriteHeader(headerLength uint16, bodyLength uint16) (err error) {
	return writeHeaderTo(framed.raw, headerLength, bodyLength)
}

/*
CopyTo copies the given Frame to the given Writer.  If reading of the Frame
has already started before CopyTo is called, CopyTo returns an
AlreadyReadError.
*/
func (frame *Frame) CopyTo(out io.Writer) (err error) {
	if frame.header.startedReading || frame.body.startedReading {
		return AlreadyReadError("Already read from frame, cannot copy")
	}
	if err = writeHeaderTo(out, frame.headerLength, frame.bodyLength); err != nil {
		return
	}
	var n int64
	n, err = io.CopyN(out, frame.framed.raw, int64(frame.headerLength+frame.bodyLength))
	frame.bytesRemaining -= int(n)
	frame.checkDone()
	return
}

/*
Read implements io.Reader.Read for a frameSection.  Just like the usual Read(),
this one may read incompletely, so make sure to call it until it returns EOF.
*/
func (section *frameSection) Read(p []byte) (n int, err error) {
	if section.bytesRemaining == 0 {
		return 0, err
	}
	if section.init != nil {
		if err = section.init(); err != nil {
			return 0, err
		}
	}
	section.startedReading = true
	if len(p) > section.bytesRemaining {
		p = p[0:section.bytesRemaining]
	}
	n, err = section.frame.framed.raw.Read(p)
	nint := int(n)
	section.frame.bytesRemaining -= nint
	section.bytesRemaining -= nint
	if section.bytesRemaining == 0 {
		err = io.EOF
	}
	section.frame.checkDone()
	return
}

func (frame *Frame) Discard() (err error) {
	var n int64
	n, err = io.CopyN(ioutil.Discard, frame.framed.raw, int64(frame.header.bytesRemaining+frame.body.bytesRemaining))
	frame.bytesRemaining -= int(n)
	frame.checkDone()
	return
}

func (section *frameSection) discard() (err error) {
	if section.init != nil {
		section.init()
	}
	if section.bytesRemaining > 0 {
		var n int64
		n, err = io.Copy(ioutil.Discard, section)
		nint := int(n)
		section.frame.bytesRemaining -= nint
		section.bytesRemaining -= nint
	}
	section.frame.checkDone()
	return
}

func (frame *Frame) HeaderLength() uint16 {
	return frame.headerLength
}

func (frame *Frame) BodyLength() uint16 {
	return frame.bodyLength
}

func (frame *Frame) Header() io.Reader {
	return frame.header
}

func (frame *Frame) Body() io.Reader {
	return frame.body
}

func (framed *Framed) nextFrame() (frame *Frame, err error) {
	frame = &Frame{framed: framed, doneReading: make(chan bool, 1)}
	frame.header = &frameSection{frame: frame}
	frame.body = &frameSection{frame: frame, init: frame.header.discard}
	if err = frame.readLengths(); err != nil {
		return
	}
	return
}

func (frame *Frame) readLengths() (err error) {
	if err = binary.Read(frame.framed.raw, endianness, &frame.headerLength); err != nil {
		return
	}
	if err = binary.Read(frame.framed.raw, endianness, &frame.bodyLength); err != nil {
		return
	}
	frame.header.bytesRemaining = int(frame.headerLength)
	frame.body.bytesRemaining = int(frame.bodyLength)
	frame.bytesRemaining = frame.header.bytesRemaining + frame.body.bytesRemaining
	return
}

func (frame *Frame) checkDone() {
	if frame.bytesRemaining == 0 {
		frame.doneReading <- true
	}
}

func writeHeaderTo(out io.Writer, headerLength uint16, bodyLength uint16) (err error) {
	if err = binary.Write(out, endianness, headerLength); err != nil {
		return
	}
	err = binary.Write(out, endianness, bodyLength)
	return
}
<commit_msg>Made stream a field<commit_after>/*
Package framed adds basic support for message framing over streams.

Messages contain an header and a body, both of which are length prefixed.

Here are the bytes (stored in little-endian byte order):

0-2: unsigned 16 bit int header length
2-4: unsinged 16 bit int body length
4+:  message content (header and body)

The use of a uint16 means that the maximum possible header and body lengths
are 65535 each.

Important Note for using with TCP -

When using framed with a TCPConn, we recommend calling
TCPConn.SetNoDelay(false).

framed was designed to maximize the chance of zero-copy ops being used to send
frames from one connection to another. Because of this, it writes a lot of
small pieces of data to the underlying stream (e.g. the length headers).  If
TCP is being used without Nagle's algorithm (i.e. setting
TCPConn.SetNoDelay(true), which is the default), then it can result in fairly
extreme packet fragmentation and consequently ballooning overhead.

Example Usage:

	package main

	import (
		"github.com/oxtoacart/framed"
		"io/ioutil"
		"log"
		"net"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := net.Dial("tcp", "host:port"); err == nil {
			framedConn = Framed{conn}
			if err := framedConn.WriteFrame([]byte("Header"), []byte("Hello World")); err == nil {
				if err, frame := framedConn.ReadInitial(); err == nil {
					// Note - Read is just like io.Reader.Read(), so we use ioutil.ReadAll
					if header, err := ioutil.ReadAll(framedConn.Header); err == nil {
						if body, err := ioutil.ReadAll(framedConn.Body); err == nil {
							nextFrame, err := frame.Next()
							// And so on
						}
					}
				}
			}
		}
	}
*/
package framed

import (
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
)

var endianness = binary.LittleEndian

/*
A Framed enhances an io.ReadWriteCloser to provide methods that allow writing
and reading frames.

Although the underlying ReadWriteCloser may be safe to use from multiple
goroutines, a Framed is not.
*/
type Framed struct {
	stream         io.ReadWriteCloser // the raw underlying connection
	hasReadInitial bool
}

// AlreadyReadError is returned when someone has already read from a Frame
type AlreadyReadError string

func (err AlreadyReadError) Error() string {
	return string(err)
}

// Frame encapsulates a frame from a Framed
type Frame struct {
	framed         *Framed
	headerLength   uint16
	bodyLength     uint16
	bytesRemaining int
	header         *frameSection
	body           *frameSection
	doneReading    chan bool
}

// frameSection encapsulates a section of a frame (header or body)
type frameSection struct {
	frame          *Frame
	init           func() error
	bytesRemaining int
	startedReading bool
}

// NewFramed creates a new Framed on top of the given readWriteCloser
func NewFramed(readWriteCloser io.ReadWriteCloser) *Framed {
	return &Framed{readWriteCloser, false}
}

// Close() implements method Close() from io.Closer.
func (framed *Framed) Close() error {
	return framed.stream.Close()
}

// Write() implements method Write() from io.Writer.
func (framed *Framed) Write(p []byte) (n int, err error) {
	return framed.stream.Write(p)
}

/*
ReadInitial reads the initial frame from the framed.  It returns an
AlreadyReadError if the initial frame has already been read previously.
*/
func (framed *Framed) ReadInitial() (frame *Frame, err error) {
	if framed.hasReadInitial {
		return nil, AlreadyReadError("Initial Frame already read")
	}
	frame, err = framed.nextFrame()
	framed.hasReadInitial = true
	return
}

/*
Next returns the next frame from the Framed underlying the frame on which
it is called.  This method blocks until the previous frame has been consumed.
If there are no more frames, it returns io.EOF as an error.
*/
func (frame *Frame) Next() (nextFrame *Frame, err error) {
	<-frame.doneReading
	return frame.framed.nextFrame()
}

/*
WriteFrame writes the given header and body to the Framed.
Either or both can be nil.
*/
func (framed *Framed) WriteFrame(header []byte, body []byte) (err error) {
	headerLength := 0
	bodyLength := 0
	if header != nil {
		headerLength = len(header)
	}
	if body != nil {
		bodyLength = len(body)
	}

	if err = framed.WriteHeader(uint16(headerLength), uint16(bodyLength)); err != nil {
		return err
	}

	if header != nil {
		if _, err = framed.stream.Write(header); err != nil {
			return err
		}
	}

	if body != nil {
		_, err = framed.stream.Write(body)
	}

	return
}

/*
StreamBody writes the body to the framed.
*/
func (framed *Framed) StreamBody(bodyLength uint16, body io.Reader) (err error) {

	bodyLength64 := int64(bodyLength)

	if body != nil {
		if n, err := io.CopyN(framed.stream, body, bodyLength64); err != nil {
			return err
		} else if n != bodyLength64 {
			return fmt.Errorf("Amount of body data written did not match bodyLength.  Expected %d, wrote %d.", bodyLength, n)
		}
	}

	return
}

// WriteHeader writes a frame header with the given lengths to the Framed.
func (framed *Framed) WriteHeader(headerLength uint16, bodyLength uint16) (err error) {
	return writeHeaderTo(framed.stream, headerLength, bodyLength)
}

/*
CopyTo copies the given Frame to the given Writer.  If reading of the Frame
has already started before CopyTo is called, CopyTo returns an
AlreadyReadError.
*/
func (frame *Frame) CopyTo(out io.Writer) (err error) {
	if frame.header.startedReading || frame.body.startedReading {
		return AlreadyReadError("Already read from frame, cannot copy")
	}
	if err = writeHeaderTo(out, frame.headerLength, frame.bodyLength); err != nil {
		return
	}
	var n int64
	n, err = io.CopyN(out, frame.framed.stream, int64(frame.headerLength+frame.bodyLength))
	frame.bytesRemaining -= int(n)
	frame.checkDone()
	return
}

/*
Read implements io.Reader.Read for a frameSection.  Just like the usual Read(),
this one may read incompletely, so make sure to call it until it returns EOF.
*/
func (section *frameSection) Read(p []byte) (n int, err error) {
	if section.bytesRemaining == 0 {
		return 0, err
	}
	if section.init != nil {
		if err = section.init(); err != nil {
			return 0, err
		}
	}
	section.startedReading = true
	if len(p) > section.bytesRemaining {
		p = p[0:section.bytesRemaining]
	}
	n, err = section.frame.framed.stream.Read(p)
	nint := int(n)
	section.frame.bytesRemaining -= nint
	section.bytesRemaining -= nint
	if section.bytesRemaining == 0 {
		err = io.EOF
	}
	section.frame.checkDone()
	return
}

func (frame *Frame) Discard() (err error) {
	var n int64
	n, err = io.CopyN(ioutil.Discard, frame.framed.stream, int64(frame.header.bytesRemaining+frame.body.bytesRemaining))
	frame.bytesRemaining -= int(n)
	frame.checkDone()
	return
}

func (section *frameSection) discard() (err error) {
	if section.init != nil {
		section.init()
	}
	if section.bytesRemaining > 0 {
		var n int64
		n, err = io.Copy(ioutil.Discard, section)
		nint := int(n)
		section.frame.bytesRemaining -= nint
		section.bytesRemaining -= nint
	}
	section.frame.checkDone()
	return
}

func (frame *Frame) HeaderLength() uint16 {
	return frame.headerLength
}

func (frame *Frame) BodyLength() uint16 {
	return frame.bodyLength
}

func (frame *Frame) Header() io.Reader {
	return frame.header
}

func (frame *Frame) Body() io.Reader {
	return frame.body
}

func (framed *Framed) nextFrame() (frame *Frame, err error) {
	frame = &Frame{framed: framed, doneReading: make(chan bool, 1)}
	frame.header = &frameSection{frame: frame}
	frame.body = &frameSection{frame: frame, init: frame.header.discard}
	if err = frame.readLengths(); err != nil {
		return nil, err
	}
	return
}

func (frame *Frame) readLengths() (err error) {
	if err = binary.Read(frame.framed.stream, endianness, &frame.headerLength); err != nil {
		return
	}
	if err = binary.Read(frame.framed.stream, endianness, &frame.bodyLength); err != nil {
		return
	}
	frame.header.bytesRemaining = int(frame.headerLength)
	frame.body.bytesRemaining = int(frame.bodyLength)
	frame.bytesRemaining = frame.header.bytesRemaining + frame.body.bytesRemaining
	return
}

func (frame *Frame) checkDone() {
	if frame.bytesRemaining == 0 {
		frame.doneReading <- true
	}
}

func writeHeaderTo(out io.Writer, headerLength uint16, bodyLength uint16) (err error) {
	if err = binary.Write(out, endianness, headerLength); err != nil {
		return
	}
	err = binary.Write(out, endianness, bodyLength)
	return
}
<|endoftext|>


<commit_before>package filters

import (
	"strings"

	"regexp"

	d "github.com/appPlant/alpinepass/src/data"
	"github.com/appPlant/alpinepass/src/util"
	"github.com/fatih/structs"
)

//PropertyFilter allows filtering a Config for the content of its properties.
type PropertyFilter struct {
	Slices []string
}

func (p PropertyFilter) filter(data d.Config) d.Config {
	verifyFlags(p.Slices) //TODO verify only once and not for every Config
	for i := 0; i < len(p.Slices); i++ {
		data = filterProperty(p.Slices[i], data)
	}
	return data
}

func filterProperty(property string, data d.Config) d.Config {
	var split = strings.Split(property, ":")
	var key = split[0]
	var value = split[1]
	var regex = regexp.MustCompile(value)

	//TODO rather use reflection?
	switch key {
	case "id":
		if !regex.MatchString(data.ID) {
			data.IsValid = false
		}
	case "title":
		if !regex.MatchString(data.Title) {
			data.IsValid = false
		}
	case "location":
		if !regex.MatchString(data.Location) {
			data.IsValid = false
		}
	case "environment":
		if !regex.MatchString(data.Environment) {
			data.IsValid = false
		}
	case "user":
		if !regex.MatchString(data.User) {
			data.IsValid = false
		}
	case "host":
		if !regex.MatchString(data.Host) {
			data.IsValid = false
		}
	}

	return data
}

//verifyFlags checks that the input flags are valid.
func verifyFlags(flags []string) {
	for i := 0; i < len(flags); i++ {
		flag := flags[i]

		//Do some simple checks.
		if !strings.Contains(flag, ":") {
			util.ThrowError("The filter does not contain ':'! Flag: " + flag)
		}
		if strings.Count(flag, ":") > 1 {
			util.ThrowError("The filter contains too many ':'! Flag: " + flag)
		}

		//check that the key part of a flag matches the fields available in struct Config.
		data := d.Config{}
		fieldNames := structs.Names(data)
		key := strings.ToLower(strings.Split(flag, ":")[0])
		isContained := false
		for j := 0; j < len(fieldNames); j++ {
			if key == strings.ToLower(fieldNames[j]) {
				isContained = true
			}
		}
		if !isContained {
			util.ThrowError("The filter type is not valid! Flag: " + flag)
		}
	}
}
<commit_msg>Use ‚=‚ for filter flags<commit_after>package filters

import (
	"strings"

	"regexp"

	d "github.com/appPlant/alpinepass/src/data"
	"github.com/appPlant/alpinepass/src/util"
	"github.com/fatih/structs"
)

//Separator is tha character used for separating filter keys and values.
const Separator string = "="

//PropertyFilter allows filtering a Config for the content of its properties.
type PropertyFilter struct {
	Slices []string
}

func (p PropertyFilter) filter(data d.Config) d.Config {
	verifyFlags(p.Slices) //TODO verify only once and not for every Config
	for i := 0; i < len(p.Slices); i++ {
		data = filterProperty(p.Slices[i], data)
	}
	return data
}

func filterProperty(property string, data d.Config) d.Config {
	var split = strings.Split(property, Separator)
	var key = split[0]
	var value = split[1]
	var regex = regexp.MustCompile(value)

	//TODO rather use reflection?
	switch key {
	case "id":
		if !regex.MatchString(data.ID) {
			data.IsValid = false
		}
	case "title":
		if !regex.MatchString(data.Title) {
			data.IsValid = false
		}
	case "location":
		if !regex.MatchString(data.Location) {
			data.IsValid = false
		}
	case "environment":
		if !regex.MatchString(data.Environment) {
			data.IsValid = false
		}
	case "user":
		if !regex.MatchString(data.User) {
			data.IsValid = false
		}
	case "host":
		if !regex.MatchString(data.Host) {
			data.IsValid = false
		}
	}

	return data
}

//verifyFlags checks that the input flags are valid.
func verifyFlags(flags []string) {
	for i := 0; i < len(flags); i++ {
		flag := flags[i]

		//Do some simple checks.
		if !strings.Contains(flag, Separator) {
			util.ThrowError("The filter does not contain '" + Separator + "'! Flag: " + flag)
		}
		if strings.Count(flag, Separator) > 1 {
			util.ThrowError("The filter contains too many '" + Separator + "'! Flag: " + flag)
		}

		//check that the key part of a flag matches the fields available in struct Config.
		data := d.Config{}
		fieldNames := structs.Names(data)
		key := strings.ToLower(strings.Split(flag, Separator)[0])
		isContained := false
		for j := 0; j < len(fieldNames); j++ {
			if key == strings.ToLower(fieldNames[j]) {
				isContained = true
			}
		}
		if !isContained {
			util.ThrowError("The filter type is not valid! Flag: " + flag)
		}
	}
}
<|endoftext|>


<commit_before>package fluidbackup

import "testing"
import "time"
import "io/ioutil"
import "math/rand"
import "os"
import "bytes"
import "strings"
import "fmt"

func TestBasic(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

// Attempts to kill the local peer
// and ensure that the file can be recovered.
func TestStable(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.Save()
	bMain.Stop()
	time.Sleep(5 * time.Second)
	bMain = MakeFluidBackup(mainPort)
	bMain.Load()

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestReread(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	time.Sleep(time.Second)
	bMain.Save()
	bMain.Stop()
	time.Sleep(2 * time.Second)
	bMain = MakeFluidBackup(mainPort)
	bMain.Load()

	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestFailVerify(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)
	files, _ := ioutil.ReadDir("store/")
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".obj") {
			os.Remove("store/" + f.Name())
		}
	}
	time.Sleep(25 * time.Second)
	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestBasicPeerSharing(t *testing.T) {
	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)

	// Make a giant centroid peer network first
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)
		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(time.Second)

	// Now, connect anoter peer to the centroid
	newDiscoverer := MakeFluidBackup(mainPort - 1)
	newDiscoverer.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
	// No need for next line: they are now autodiscovered
	// numPeers := newDiscoverer.peerList.FindNewPeers(len(bOther))

	time.Sleep(time.Second * 5)

	// check the number of discovered peers for one on centroid
	numPeers := newDiscoverer.peerList.NumPeers()
	desiredNum := PeerListDefaultDesiredNumPeers
	fmt.Printf("Discoverer Found %v peers!\n", numPeers)
	if numPeers < desiredNum {
		t.Fatalf("Num peers %v, expected at least %v", numPeers, desiredNum)
	}

	// check the number of discovered peers for each bOther
	for i := range bOther {
		fbOther := bOther[i]
		numPeers := fbOther.peerList.NumPeers()
		fmt.Printf("%v Found %v peers!\n", fbOther.protocol.port, numPeers)
		if numPeers < desiredNum {
			t.Fatalf("%v: Num peers %v, expected at least %v", fbOther.protocol.port, numPeers, desiredNum)
		}
	}

	// shut down fluidbackup instances
	bMain.Stop()
	newDiscoverer.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestBenchmark(t *testing.T) {

}
<commit_msg>A quick test for seeing if fluidbackup is able to shift storage off of untrustworthy peers. Next step: try storing additional information on the system and see if any of it gets propogated onto untrustworthy peers.<commit_after>package fluidbackup

import "testing"
import "time"
import "io/ioutil"
import "math/rand"
import "os"
import "bytes"
import "strings"
import "fmt"

func TestBasic(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

// Attempts to kill the local peer
// and ensure that the file can be recovered.
func TestStable(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.Save()
	bMain.Stop()
	time.Sleep(5 * time.Second)
	bMain = MakeFluidBackup(mainPort)
	bMain.Load()

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestReread(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	time.Sleep(time.Second)
	bMain.Save()
	bMain.Stop()
	time.Sleep(2 * time.Second)
	bMain = MakeFluidBackup(mainPort)
	bMain.Load()

	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)

	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestFailVerify(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bMain.fileStore.RegisterFile("orange.txt")
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)

		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(5 * time.Second)
	files, _ := ioutil.ReadDir("store/")
	for _, f := range files {
		if strings.HasSuffix(f.Name(), ".obj") {
			os.Remove("store/" + f.Name())
		}
	}
	time.Sleep(25 * time.Second)
	err = os.Remove("orange.txt")

	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	bMain.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestBasicPeerSharing(t *testing.T) {
	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)

	// Make a giant centroid peer network first
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)
		if i < 5 {
			bMain.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port})
		} else {
			bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
		}
	}

	time.Sleep(time.Second)

	// Now, connect anoter peer to the centroid
	newDiscoverer := MakeFluidBackup(mainPort - 1)
	newDiscoverer.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})
	// No need for next line: they are now autodiscovered
	// numPeers := newDiscoverer.peerList.FindNewPeers(len(bOther))

	time.Sleep(time.Second * 5)

	// check the number of discovered peers for one on centroid
	numPeers := newDiscoverer.peerList.NumPeers()
	desiredNum := PeerListDefaultDesiredNumPeers
	fmt.Printf("Discoverer Found %v peers!\n", numPeers)
	if numPeers < desiredNum {
		t.Fatalf("Num peers %v, expected at least %v", numPeers, desiredNum)
	}

	// check the number of discovered peers for each bOther
	for i := range bOther {
		fbOther := bOther[i]
		numPeers := fbOther.peerList.NumPeers()
		fmt.Printf("%v Found %v peers!\n", fbOther.protocol.port, numPeers)
		if numPeers < desiredNum {
			t.Fatalf("%v: Num peers %v, expected at least %v", fbOther.protocol.port, numPeers, desiredNum)
		}
	}

	// shut down fluidbackup instances
	bMain.Stop()
	newDiscoverer.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func killForSeconds(fb *FluidBackup, port int, seconds int) {
	// Kill our discoverer
	fb.Save()
	fb.Stop()
	time.Sleep(time.Duration(seconds) * time.Second)
	fb = MakeFluidBackup(port)
	fb.Load()
}

// Does a very simple test to see if fluidbackup
// shifts its shard storage in response to a change
// in trustworthiness of peers. This does not directly
// test the trust mechanism but may instead test the
// next available peer mechanism, but should pass.
func TestBasicTrustMechanism(t *testing.T) {
	fileContents := make([]byte, 1500*1024)
	for i := range fileContents {
		fileContents[i] = byte(rand.Int() % 256)
	}
	err := ioutil.WriteFile("orange.txt", fileContents, 0644)
	if err != nil {
		t.Error("failed to write random contents to orange.txt: " + err.Error())
	}

	// Build our initial network
	mainPort := 19838
	bMain := MakeFluidBackup(mainPort)
	bOther := make([]*FluidBackup, 11)
	for i := range bOther {
		port := 19839 + i
		bOther[i] = MakeFluidBackup(port)
		// build an initial ring structure
		bOther[i].peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: port - 1})
	}
	discoverer := MakeFluidBackup(mainPort - 1)
	discoverer.peerList.DiscoveredPeer(PeerId{Address: "127.0.0.1", Port: mainPort})

	discoverer.fileStore.RegisterFile("orange.txt")

	// Make some peers less trustworthy by killing them for a while
	for i := 0; i < 5; i++ {
		killForSeconds(bOther[i], 19839+i, 2)
	}

	time.Sleep(5 * time.Second)
	fmt.Printf("NUM PEERS: %v\n", discoverer.peerList.NumPeers())

	err = os.Remove("orange.txt")
	if err != nil {
		t.Error("Encountered error while removing file: " + err.Error())
		return
	}

	// wait for equilibrium
	time.Sleep(3 * time.Second)
	// Kill these less trustworthy peers permanently
	// the shards should be on the more trustworthy peers.
	for i := 0; i < 5; i++ {
		bOther[i].Stop()
	}

	time.Sleep(3 * time.Second)

	killForSeconds(discoverer, mainPort-1, 5)

	// Attempt to recover the file
	discoverer.fileStore.RecoverFile("orange.txt")
	newFileContents, err := ioutil.ReadFile("orange.txt")
	if err != nil {
		t.Error("failed to read from recovered orange.txt: " + err.Error())
	}

	if !bytes.Equal(fileContents, newFileContents) {
		t.Error("recovered file does not match original")
	}

	// shut down fluidbackup instances
	bMain.Stop()
	for _, inst := range bOther {
		inst.Stop()
	}
}

func TestBenchmark(t *testing.T) {

}
<|endoftext|>


<commit_before>package locket

import (
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket/models"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

type ClientLocketConfig struct {
	LocketAddress        string `json:"locket_address,omitempty" yaml:"locket_address,omitempty"`
	LocketCACertFile     string `json:"locket_ca_cert_file,omitempty" yaml:"locket_ca_cert_file,omitempty"`
	LocketClientCertFile string `json:"locket_client_cert_file,omitempty" yaml:"locket_client_cert_file,omitempty"`
	LocketClientKeyFile  string `json:"locket_client_key_file,omitempty" yaml:"locket_client_key_file,omitempty"`
}

func NewClient(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	locketTLSConfig, err := cfhttp.NewTLSConfig(config.LocketClientCertFile, config.LocketClientKeyFile, config.LocketCACertFile)
	if err != nil {
		logger.Error("failed-to-open-tls-config", err, lager.Data{"keypath": config.LocketClientKeyFile, "certpath": config.LocketClientCertFile, "capath": config.LocketCACertFile})
		return nil, err
	}

	conn, err := grpc.Dial(config.LocketAddress, grpc.WithTransportCredentials(credentials.NewTLS(locketTLSConfig)))
	if err != nil {
		return nil, err
	}
	return models.NewLocketClient(conn), nil
}
<commit_msg>Add NewClientSkipCertVerify<commit_after>package locket

import (
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/locket/models"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

type ClientLocketConfig struct {
	LocketAddress        string `json:"locket_address,omitempty" yaml:"locket_address,omitempty"`
	LocketCACertFile     string `json:"locket_ca_cert_file,omitempty" yaml:"locket_ca_cert_file,omitempty"`
	LocketClientCertFile string `json:"locket_client_cert_file,omitempty" yaml:"locket_client_cert_file,omitempty"`
	LocketClientKeyFile  string `json:"locket_client_key_file,omitempty" yaml:"locket_client_key_file,omitempty"`
}

func NewClientSkipCertVerify(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, true)
}

func NewClient(logger lager.Logger, config ClientLocketConfig) (models.LocketClient, error) {
	return newClientInternal(logger, config, false)
}

func newClientInternal(logger lager.Logger, config ClientLocketConfig, skipCertVerify bool) (models.LocketClient, error) {
	locketTLSConfig, err := cfhttp.NewTLSConfig(config.LocketClientCertFile, config.LocketClientKeyFile, config.LocketCACertFile)
	if err != nil {
		logger.Error("failed-to-open-tls-config", err, lager.Data{"keypath": config.LocketClientKeyFile, "certpath": config.LocketClientCertFile, "capath": config.LocketCACertFile})
		return nil, err
	}
	locketTLSConfig.InsecureSkipVerify = skipCertVerify

	conn, err := grpc.Dial(config.LocketAddress, grpc.WithTransportCredentials(credentials.NewTLS(locketTLSConfig)))
	if err != nil {
		return nil, err
	}
	return models.NewLocketClient(conn), nil
}
<|endoftext|>


<commit_before>package logger

import (
	"context"
	"os"
)

type Helper struct {
	logger Logger
}

func NewHelper(logger Logger) *Helper {
	return &Helper{logger: logger}
}

// Extract always returns valid Helper with logger from context or with DefaultLogger as fallback.
// Can be used in pair with function NewContext(ctx context.Context, l Logger) context.Context.
// (e.g. propagate RequestID to logger in service handler methods).
func Extract(ctx context.Context) *Helper {
	if l, ok := FromContext(ctx); ok {
		return NewHelper(l)
	}

	return NewHelper(DefaultLogger)
}

func (h *Helper) Log(level Level, args ...interface{}) {
	h.logger.Log(level, args...)
}

func (h *Helper) Logf(level Level, template string, args ...interface{}) {
	h.logger.Logf(level, template, args...)
}

func (h *Helper) Info(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(InfoLevel) {
		return
	}
	h.logger.Log(InfoLevel, args...)
}

func (h *Helper) Infof(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(InfoLevel) {
		return
	}
	h.logger.Logf(InfoLevel, template, args...)
}

func (h *Helper) Trace(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(TraceLevel) {
		return
	}
	h.logger.Log(TraceLevel, args...)
}

func (h *Helper) Tracef(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(TraceLevel) {
		return
	}
	h.logger.Logf(TraceLevel, template, args...)
}

func (h *Helper) Debug(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(DebugLevel) {
		return
	}
	h.logger.Log(DebugLevel, args...)
}

func (h *Helper) Debugf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(DebugLevel) {
		return
	}
	h.logger.Logf(DebugLevel, template, args...)
}

func (h *Helper) Warn(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(WarnLevel) {
		return
	}
	h.logger.Log(WarnLevel, args...)
}

func (h *Helper) Warnf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(WarnLevel) {
		return
	}
	h.logger.Logf(WarnLevel, template, args...)
}

func (h *Helper) Error(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(ErrorLevel) {
		return
	}
	h.logger.Log(ErrorLevel, args...)
}

func (h *Helper) Errorf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(ErrorLevel) {
		return
	}
	h.logger.Logf(ErrorLevel, template, args...)
}

func (h *Helper) Fatal(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(FatalLevel) {
		return
	}
	h.logger.Log(FatalLevel, args...)
	os.Exit(1)
}

func (h *Helper) Fatalf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(FatalLevel) {
		return
	}
	h.logger.Logf(FatalLevel, template, args...)
	os.Exit(1)
}

func (h *Helper) WithError(err error) *Helper {
	return &Helper{logger: h.logger.Fields(map[string]interface{}{"error": err})}
}

func (h *Helper) WithFields(fields map[string]interface{}) *Helper {
	return &Helper{logger: h.logger.Fields(fields)}
}
<commit_msg>logger helper: add "inject" method, to make a pair with "extract" (#2283)<commit_after>package logger

import (
	"context"
	"os"
)

type Helper struct {
	logger Logger
}

func NewHelper(logger Logger) *Helper {
	return &Helper{logger: logger}
}

// Extract always returns valid Helper with logger from context or with DefaultLogger as fallback.
// Can be used in pair with function Inject.
// Example: propagate RequestID to logger in service handler methods.
func Extract(ctx context.Context) *Helper {
	if l, ok := FromContext(ctx); ok {
		return NewHelper(l)
	}

	return NewHelper(DefaultLogger)
}

func (h *Helper) Inject(ctx context.Context) context.Context {
	return NewContext(ctx, h.logger)
}

func (h *Helper) Log(level Level, args ...interface{}) {
	h.logger.Log(level, args...)
}

func (h *Helper) Logf(level Level, template string, args ...interface{}) {
	h.logger.Logf(level, template, args...)
}

func (h *Helper) Info(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(InfoLevel) {
		return
	}
	h.logger.Log(InfoLevel, args...)
}

func (h *Helper) Infof(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(InfoLevel) {
		return
	}
	h.logger.Logf(InfoLevel, template, args...)
}

func (h *Helper) Trace(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(TraceLevel) {
		return
	}
	h.logger.Log(TraceLevel, args...)
}

func (h *Helper) Tracef(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(TraceLevel) {
		return
	}
	h.logger.Logf(TraceLevel, template, args...)
}

func (h *Helper) Debug(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(DebugLevel) {
		return
	}
	h.logger.Log(DebugLevel, args...)
}

func (h *Helper) Debugf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(DebugLevel) {
		return
	}
	h.logger.Logf(DebugLevel, template, args...)
}

func (h *Helper) Warn(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(WarnLevel) {
		return
	}
	h.logger.Log(WarnLevel, args...)
}

func (h *Helper) Warnf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(WarnLevel) {
		return
	}
	h.logger.Logf(WarnLevel, template, args...)
}

func (h *Helper) Error(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(ErrorLevel) {
		return
	}
	h.logger.Log(ErrorLevel, args...)
}

func (h *Helper) Errorf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(ErrorLevel) {
		return
	}
	h.logger.Logf(ErrorLevel, template, args...)
}

func (h *Helper) Fatal(args ...interface{}) {
	if !h.logger.Options().Level.Enabled(FatalLevel) {
		return
	}
	h.logger.Log(FatalLevel, args...)
	os.Exit(1)
}

func (h *Helper) Fatalf(template string, args ...interface{}) {
	if !h.logger.Options().Level.Enabled(FatalLevel) {
		return
	}
	h.logger.Logf(FatalLevel, template, args...)
	os.Exit(1)
}

func (h *Helper) WithError(err error) *Helper {
	return &Helper{logger: h.logger.Fields(map[string]interface{}{"error": err})}
}

func (h *Helper) WithFields(fields map[string]interface{}) *Helper {
	return &Helper{logger: h.logger.Fields(fields)}
}
<|endoftext|>


<commit_before>package logger

import (
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("gesclient")

func nillable(args []interface{}) []interface{} {
	out := make([]interface{}, len(args))
	for i, a := range args {
		switch a.(type) {
		case *int:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int))
			}
		case *int32:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int32))
			}
		case *int64:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int64))
			}
		default:
			out[i] = a
		}
	}
	return out
}

func Fatal(args ...interface{}) {
	if len(args) > 0 {
		log.Fatal(nillable(args)...)
	} else {
		log.Fatal()
	}
}

func Fatalf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Fatalf(format, nillable(args)...)
	} else {
		log.Fatalf(format)
	}
}

func Panic(args ...interface{}) {
	if len(args) > 0 {
		log.Panic(nillable(args)...)
	} else {
		log.Panic()
	}
}

func Panicf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Panicf(format, nillable(args)...)
	} else {
		log.Panicf(format)
	}
}

func Critical(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Critical(format, nillable(args)...)
	} else {
		log.Critical(format)
	}
}

func Error(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Error(format, nillable(args)...)
	} else {
		log.Error(format)
	}
}

func Errorf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Errorf(format, nillable(args)...)
	} else {
		log.Errorf(format)
	}
}

func Warning(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Warning(format, nillable(args)...)
	} else {
		log.Warning(format)
	}
}

func Warningf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Warningf(format, nillable(args)...)
	} else {
		log.Warningf(format)
	}
}

func Notice(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Notice(format, nillable(args)...)
	} else {
		log.Notice(format)
	}
}

func Noticef(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Noticef(format, nillable(args)...)
	} else {
		log.Noticef(format)
	}
}

func Info(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Info(format, nillable(args)...)
	} else {
		log.Info(format)
	}
}

func Infof(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Infof(format, nillable(args)...)
	} else {
		log.Infof(format)
	}
}

func Debug(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Debug(format, nillable(args)...)
	} else {
		log.Debug(format)
	}
}

func Debugf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Debugf(format, nillable(args)...)
	} else {
		log.Debugf(format)
	}
}
<commit_msg>Fix logger (#6)<commit_after>package logger

import (
	"github.com/op/go-logging"
)

var log = logging.MustGetLogger("gesclient")

func nillable(args []interface{}) []interface{} {
	out := make([]interface{}, len(args))
	for i, a := range args {
		switch a.(type) {
		case *int:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int))
			}
		case *int32:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int32))
			}
		case *int64:
			if a == nil {
				out[i] = "<nil>"
			} else {
				out[i] = *(a.(*int64))
			}
		default:
			out[i] = a
		}
	}
	return out
}

func Fatal(args ...interface{}) {
	if len(args) > 0 {
		log.Fatal(nillable(args)...)
	} else {
		log.Fatal()
	}
}

func Fatalf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Fatalf(format, nillable(args)...)
	} else {
		log.Fatalf(format)
	}
}

func Panic(args ...interface{}) {
	if len(args) > 0 {
		log.Panic(nillable(args)...)
	} else {
		log.Panic()
	}
}

func Panicf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Panicf(format, nillable(args)...)
	} else {
		log.Panicf(format)
	}
}

func Critical(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Critical(nillable(args)...)
	} else {
		log.Critical(format)
	}
}

func Error(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Error(nillable(args)...)
	} else {
		log.Error(format)
	}
}

func Errorf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Errorf(format, nillable(args)...)
	} else {
		log.Errorf(format)
	}
}

func Warning(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Warning(nillable(args)...)
	} else {
		log.Warning(format)
	}
}

func Warningf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Warningf(format, nillable(args)...)
	} else {
		log.Warningf(format)
	}
}

func Notice(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Notice(nillable(args)...)
	} else {
		log.Notice(format)
	}
}

func Noticef(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Noticef(format, nillable(args)...)
	} else {
		log.Noticef(format)
	}
}

func Info(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Info(nillable(args)...)
	} else {
		log.Info(format)
	}
}

func Infof(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Infof(format, nillable(args)...)
	} else {
		log.Infof(format)
	}
}

func Debug(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Debug(nillable(args)...)
	} else {
		log.Debug(format)
	}
}

func Debugf(format string, args ...interface{}) {
	if len(args) > 0 {
		log.Debugf(format, nillable(args)...)
	} else {
		log.Debugf(format)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009-2010 by Maurice Gilden. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 A collection of helper functions for console output.
*/
package logger

import "fmt"
import "os"

const (
	DEBUG   = -1 + iota // start with -1
	DEFAULT // should be 0
	WARN
	ERROR
)

var verbosity int

/*
 Sets the verbosity level. Possible values are DEBUG, DEFAULT, WARN and ERROR.
 DEFAULT is the standard value which will print everything but debug messages.
*/
func SetVerbosityLevel(level int) { verbosity = level }

/*
 Prints debug messages. Same syntax as fmt.Printf.
*/
func Debug(format string, v ...interface{}) {
	if verbosity <= DEBUG {
		fmt.Printf("DEBUG: ")
		fmt.Printf(format, v)
	}
}

/*
 Prints a debug message if enabled but without the "DEBUG: " prefix.
 Same syntax as fmt.Printf.
*/
func DebugContinue(format string, v ...interface{}) {
	if verbosity <= DEBUG {
		fmt.Printf("       ")
		fmt.Printf(format, v)
	}
}

/*
 Prints an info message if enabled. This is for general feedback about what
 gobuild is currently doing. Same syntax as fmt.Printf.
*/
func Info(format string, v ...interface{}) {
	if verbosity <= DEFAULT {
		fmt.Printf(format, v)
	}
}

/*
 Prints a warning if warnings are enabled. Same syntax as fmt.Printf.
*/
func Warn(format string, v ...interface{}) {
	if verbosity <= WARN {
		fmt.Print("WARNING: ")
		fmt.Printf(format, v)
	}
}

/*
 Prints a warning message if enabled but without the "WARNING: " prefix.
 Same syntax as fmt.Printf.
*/
func WarnContinue(format string, v ...interface{}) {
	if verbosity <= WARN {
		fmt.Print("         ")
		fmt.Printf(format, v)
	}
}

/*
 Prints an error message. Same syntax as fmt.Printf.
*/
func Error(format string, v ...interface{}) {
	if verbosity <= ERROR {
		fmt.Fprint(os.Stderr, "ERROR: ")
		fmt.Fprintf(os.Stderr, format, v)
	}
}

/*
 Prints an error message but without the "ERROR: " prefix.
 Same syntax as fmt.Printf.
*/
func ErrorContinue(format string, v ...interface{}) {
	if verbosity <= ERROR {
		fmt.Fprint(os.Stderr, "       ")
		fmt.Fprintf(os.Stderr, format, v)
	}
}
<commit_msg>Fix for latest go release.<commit_after>// Copyright 2009-2010 by Maurice Gilden. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
 A collection of helper functions for console output.
*/
package logger

import "fmt"
import "os"

const (
	DEBUG   = -1 + iota // start with -1
	DEFAULT // should be 0
	WARN
	ERROR
)

var verbosity int

/*
 Sets the verbosity level. Possible values are DEBUG, DEFAULT, WARN and ERROR.
 DEFAULT is the standard value which will print everything but debug messages.
*/
func SetVerbosityLevel(level int) { verbosity = level }

/*
 Prints debug messages. Same syntax as fmt.Printf.
*/
func Debug(format string, v ...interface{}) {
	if verbosity <= DEBUG {
		fmt.Printf("DEBUG: ")
		fmt.Printf(format, v...)
	}
}

/*
 Prints a debug message if enabled but without the "DEBUG: " prefix.
 Same syntax as fmt.Printf.
*/
func DebugContinue(format string, v ...interface{}) {
	if verbosity <= DEBUG {
		fmt.Printf("       ")
		fmt.Printf(format, v...)
	}
}

/*
 Prints an info message if enabled. This is for general feedback about what
 gobuild is currently doing. Same syntax as fmt.Printf.
*/
func Info(format string, v ...interface{}) {
	if verbosity <= DEFAULT {
		fmt.Printf(format, v...)
	}
}

/*
 Prints a warning if warnings are enabled. Same syntax as fmt.Printf.
*/
func Warn(format string, v ...interface{}) {
	if verbosity <= WARN {
		fmt.Print("WARNING: ")
		fmt.Printf(format, v...)
	}
}

/*
 Prints a warning message if enabled but without the "WARNING: " prefix.
 Same syntax as fmt.Printf.
*/
func WarnContinue(format string, v ...interface{}) {
	if verbosity <= WARN {
		fmt.Print("         ")
		fmt.Printf(format, v...)
	}
}

/*
 Prints an error message. Same syntax as fmt.Printf.
*/
func Error(format string, v ...interface{}) {
	if verbosity <= ERROR {
		fmt.Fprint(os.Stderr, "ERROR: ")
		fmt.Fprintf(os.Stderr, format, v...)
	}
}

/*
 Prints an error message but without the "ERROR: " prefix.
 Same syntax as fmt.Printf.
*/
func ErrorContinue(format string, v ...interface{}) {
	if verbosity <= ERROR {
		fmt.Fprint(os.Stderr, "       ")
		fmt.Fprintf(os.Stderr, format, v...)
	}
}
<|endoftext|>


<commit_before>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Based on the CholeskyDecomposition class from Jama 1.0.3.

package mat64

import (
	"math"
)

type CholeskyFactor struct {
	L   *Dense
	SPD bool
}

// CholeskyL returns the left Cholesky decomposition of the matrix a and whether
// the matrix is symmetric or positive definite, the returned matrix l is a lower
// triangular matrix such that a = l.l'.
func Cholesky(a *Dense) CholeskyFactor {
	// Initialize.
	m, n := a.Dims()
	spd := m == n
	l := NewDense(n, n, nil)

	// Main loop.
	for j := 0; j < n; j++ {
		lRowj := l.RowView(j)
		var d float64
		for k := 0; k < j; k++ {
			var s float64
			for i, v := range l.RowView(k)[:k] {
				s += v * lRowj[i]
			}
			s = (a.at(j, k) - s) / l.at(k, k)
			lRowj[k] = s
			d += s * s
			spd = spd && a.at(k, j) == a.at(j, k)
		}
		d = a.at(j, j) - d
		spd = spd && d > 0
		l.set(j, j, math.Sqrt(math.Max(d, 0)))
		for k := j + 1; k < n; k++ {
			l.set(j, k, 0)
		}
	}

	return CholeskyFactor{L: l, SPD: spd}
}

// CholeskySolve returns a matrix x that solves a.x = b where a = l.l'. The matrix b must
// have the same number of rows as a, and a must be symmetric and positive definite. The
// matrix b is overwritten by the operation.
func (f CholeskyFactor) Solve(b *Dense) (x *Dense) {
	if !f.SPD {
		panic("mat64: matrix not symmetric positive definite")
	}
	l := f.L

	m, n := l.Dims()
	bm, bn := b.Dims()
	if m != bm {
		panic(ErrShape)
	}

	nx := bn
	x = b

	// Solve L*Y = B;
	for k := 0; k < n; k++ {
		for j := 0; j < nx; j++ {
			for i := 0; i < k; i++ {
				x.set(k, j, x.at(k, j)-x.at(i, j)*l.at(k, i))
			}
			x.set(k, j, x.at(k, j)/l.at(k, k))
		}
	}

	// Solve L'*X = Y;
	for k := n - 1; k >= 0; k-- {
		for j := 0; j < nx; j++ {
			for i := k + 1; i < n; i++ {
				x.set(k, j, x.at(k, j)-x.at(i, j)*l.at(i, k))
			}
			x.set(k, j, x.at(k, j)/l.at(k, k))
		}
	}

	return x
}
<commit_msg>Fix comment for func Cholesky()<commit_after>// Copyright ©2013 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Based on the CholeskyDecomposition class from Jama 1.0.3.

package mat64

import (
	"math"
)

type CholeskyFactor struct {
	L   *Dense
	SPD bool
}

// CholeskyL returns the left Cholesky decomposition of the matrix a and whether
// the matrix is symmetric and positive definite. The returned matrix l is a lower
// triangular matrix such that a = l.l'.
func Cholesky(a *Dense) CholeskyFactor {
	// Initialize.
	m, n := a.Dims()
	spd := m == n
	l := NewDense(n, n, nil)

	// Main loop.
	for j := 0; j < n; j++ {
		lRowj := l.RowView(j)
		var d float64
		for k := 0; k < j; k++ {
			var s float64
			for i, v := range l.RowView(k)[:k] {
				s += v * lRowj[i]
			}
			s = (a.at(j, k) - s) / l.at(k, k)
			lRowj[k] = s
			d += s * s
			spd = spd && a.at(k, j) == a.at(j, k)
		}
		d = a.at(j, j) - d
		spd = spd && d > 0
		l.set(j, j, math.Sqrt(math.Max(d, 0)))
		for k := j + 1; k < n; k++ {
			l.set(j, k, 0)
		}
	}

	return CholeskyFactor{L: l, SPD: spd}
}

// CholeskySolve returns a matrix x that solves a.x = b where a = l.l'. The matrix b must
// have the same number of rows as a, and a must be symmetric and positive definite. The
// matrix b is overwritten by the operation.
func (f CholeskyFactor) Solve(b *Dense) (x *Dense) {
	if !f.SPD {
		panic("mat64: matrix not symmetric positive definite")
	}
	l := f.L

	m, n := l.Dims()
	bm, bn := b.Dims()
	if m != bm {
		panic(ErrShape)
	}

	nx := bn
	x = b

	// Solve L*Y = B;
	for k := 0; k < n; k++ {
		for j := 0; j < nx; j++ {
			for i := 0; i < k; i++ {
				x.set(k, j, x.at(k, j)-x.at(i, j)*l.at(k, i))
			}
			x.set(k, j, x.at(k, j)/l.at(k, k))
		}
	}

	// Solve L'*X = Y;
	for k := n - 1; k >= 0; k-- {
		for j := 0; j < nx; j++ {
			for i := k + 1; i < n; i++ {
				x.set(k, j, x.at(k, j)-x.at(i, j)*l.at(i, k))
			}
			x.set(k, j, x.at(k, j)/l.at(k, k))
		}
	}

	return x
}
<|endoftext|>


<commit_before>// Copyright 2013 The Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Note that this doc comment shares text with the Scheduler documentation.
// Please keep in sync.

// Package fsched implements a sequential scheduler for function callbacks.
// The scheduler keeps track of the current time (different from real time).
// Each call to CallNext fast-forwards to the timestamp on the earliest event,
// and executes the associated callback function. Note that the scheduler's
// time is arbitrary and must only be internally consistent; it is unrelated
// to any real sense of time (ie, clock cycles, seconds since epoch, etc).
//
// While the scheduler interface is generic, it is designed to be used for
// simulating highly-parallel processes where accurate time is important,
// but the simulation is too processor-intensive to be run in real time.
//
// Note that the scheduler is NOT thread-safe. The intended usage of the
// scheduler is to call CallNext sequentially. In particular, this
// allows scheduled callbacks to safely interact with the scheduler,
// for example to schedule more events.
package fsched

import (
	"container/heap"
	"errors"
	"time"
)

type event struct {
	f    func(time.Time) interface{}
	time time.Time
}

type eventHeap []event

func (e eventHeap) Len() int            { return len(e) }
func (e eventHeap) Less(i, j int) bool  { return e[i].time.Before(e[j].time) }
func (e eventHeap) Swap(i, j int)       { e[i], e[j] = e[j], e[i] }
func (e *eventHeap) Push(x interface{}) { *e = append(*e, x.(event)) }
func (e *eventHeap) Pop() interface{} {
	old := *e
	n := len(old)
	x := old[n-1]
	*e = old[0 : n-1]
	return x
}

var (
	ErrPast  = errors.New("Event scheduled in the past")
	ErrEmpty = errors.New("Empty")
)

// Note that this doc comment shares text with
// the package overview. Please keep in sync.

// Scheduler allows for time- and offset-based
// scheduling of function callbacks.
//
// Note that Scheduler is NOT thread-safe.
// The intended usage of Scheduler is to call
// CallNext sequentially. In particular, this
// allows scheduled callbacks to safely interact
// with the Scheduler, for example to schedule more events.
type Scheduler struct {
	heap *eventHeap
	now  time.Time
}

// Returns a new Scheduler whose
// internal clock is set to the
// zero value of time.Time.
func NewScheduler() *Scheduler {
	s := Scheduler{heap: new(eventHeap)}
	*s.heap = make([]event, 0)
	return &s
}

// Returns a new Scheduler whose
// internal clock is set to t.
func NewSchedulerTime(t time.Time) *Scheduler {
	s := Scheduler{new(eventHeap), t}
	*s.heap = make([]event, 0)
	return &s
}

// Returns the current value
// of the internal clock.
func (s *Scheduler) Now() time.Time {
	return s.now
}

// Returns whether there are
// 0 events scheduled.
func (s *Scheduler) Empty() bool {
	return s.heap.Len() < 1
}

// Schedule f to be called when
// the internal clock reaches t.
//
// Returns ErrPast if t is before
// s.Now().
func (s *Scheduler) Schedule(f func(time.Time) interface{}, t time.Time) error {
	if t.Before(s.now) {
		return ErrPast
	}
	heap.Push(s.heap, event{f, t})
	return nil
}

// Schedule f to be called when
// offset has elapsed.
//
// Returns ErrPast if offset is
// negative.
func (s *Scheduler) ScheduleOffset(f func(time.Time) interface{}, offset time.Duration) error {
	return s.Schedule(f, s.now.Add(offset))
}

// Returns the timestamp on the next
// scheduled event, or the zero value
// and ErrEmpty if no events are
// scheduled.
func (s Scheduler) PeekNext() (time.Time, error) {
	var t time.Time
	if s.Empty() {
		return t, ErrEmpty
	}
	return (*s.heap)[0].time, nil
}

// Fast-forward the internal clock
// to match the next scheduled event,
// and call the associated callback,
// passing the (now updated) time
// as the single argument. Return
// the value returned from this call.
//
// If there are no events scheduled,
// return a nil interface value and
// ErrEmpty.
//
// Note that CallNext does not modify
// s after calling the callback. Thus,
// it is safe to call methods on s
// from within the callback.
func (s *Scheduler) CallNext() (interface{}, error) {
	if s.Empty() {
		return nil, ErrEmpty
	}
	evt := heap.Pop(s.heap).(event)
	s.now = evt.time
	return evt.f(evt.time), nil
}

// Remove the next scheduled event
// from the Scheduler, but do not
// alter the internal clock.
func (s *Scheduler) RemoveNext() {
	if !s.Empty() {
		heap.Pop(s.heap)
	}
}

// Fast-forward the internal clock
// to match the next scheduled event,
// and remove the event from the
// Scheduler. If there are no events
// scheduled, do not alter the clock.
func (s *Scheduler) RemoveNextUpdate() {
	if !s.Empty() {
		evt := heap.Pop(s.heap).(event)
		s.now = evt.time
	}
}

// Remove all scheduled events from
// the Scheduler, but do not alter
// the internal clock.
func (s *Scheduler) RemoveAll() {
	*s.heap = make([]event, 0)
}

// Remove all scheduled events from
// the Scheduler, fast-forwarding
// the internal clock to match the
// latest scheduled event. If there
// are no events scheduled, do not
// alter the clock.
func (s *Scheduler) RemoveAllUpdate() {
	if !s.Empty() {
		latest := (*s.heap)[0].time
		for _, evt := range (*s.heap)[1:] {
			if evt.time.After(latest) {
				latest = evt.time
			}
		}
		s.now = latest
	}
	*s.heap = make([]event, 0)
}
<commit_msg>Updated PeekNext to take pointer receiver.<commit_after>// Copyright 2013 The Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Note that this doc comment shares text with the Scheduler documentation.
// Please keep in sync.

// Package fsched implements a sequential scheduler for function callbacks.
// The scheduler keeps track of the current time (different from real time).
// Each call to CallNext fast-forwards to the timestamp on the earliest event,
// and executes the associated callback function. Note that the scheduler's
// time is arbitrary and must only be internally consistent; it is unrelated
// to any real sense of time (ie, clock cycles, seconds since epoch, etc).
//
// While the scheduler interface is generic, it is designed to be used for
// simulating highly-parallel processes where accurate time is important,
// but the simulation is too processor-intensive to be run in real time.
//
// Note that the scheduler is NOT thread-safe. The intended usage of the
// scheduler is to call CallNext sequentially. In particular, this
// allows scheduled callbacks to safely interact with the scheduler,
// for example to schedule more events.
package fsched

import (
	"container/heap"
	"errors"
	"time"
)

type event struct {
	f    func(time.Time) interface{}
	time time.Time
}

type eventHeap []event

func (e eventHeap) Len() int            { return len(e) }
func (e eventHeap) Less(i, j int) bool  { return e[i].time.Before(e[j].time) }
func (e eventHeap) Swap(i, j int)       { e[i], e[j] = e[j], e[i] }
func (e *eventHeap) Push(x interface{}) { *e = append(*e, x.(event)) }
func (e *eventHeap) Pop() interface{} {
	old := *e
	n := len(old)
	x := old[n-1]
	*e = old[0 : n-1]
	return x
}

var (
	ErrPast  = errors.New("Event scheduled in the past")
	ErrEmpty = errors.New("Empty")
)

// Note that this doc comment shares text with
// the package overview. Please keep in sync.

// Scheduler allows for time- and offset-based
// scheduling of function callbacks.
//
// Note that Scheduler is NOT thread-safe.
// The intended usage of Scheduler is to call
// CallNext sequentially. In particular, this
// allows scheduled callbacks to safely interact
// with the Scheduler, for example to schedule more events.
type Scheduler struct {
	heap *eventHeap
	now  time.Time
}

// Returns a new Scheduler whose
// internal clock is set to the
// zero value of time.Time.
func NewScheduler() *Scheduler {
	s := Scheduler{heap: new(eventHeap)}
	*s.heap = make([]event, 0)
	return &s
}

// Returns a new Scheduler whose
// internal clock is set to t.
func NewSchedulerTime(t time.Time) *Scheduler {
	s := Scheduler{new(eventHeap), t}
	*s.heap = make([]event, 0)
	return &s
}

// Returns the current value
// of the internal clock.
func (s *Scheduler) Now() time.Time {
	return s.now
}

// Returns whether there are
// 0 events scheduled.
func (s *Scheduler) Empty() bool {
	return s.heap.Len() < 1
}

// Schedule f to be called when
// the internal clock reaches t.
//
// Returns ErrPast if t is before
// s.Now().
func (s *Scheduler) Schedule(f func(time.Time) interface{}, t time.Time) error {
	if t.Before(s.now) {
		return ErrPast
	}
	heap.Push(s.heap, event{f, t})
	return nil
}

// Schedule f to be called when
// offset has elapsed.
//
// Returns ErrPast if offset is
// negative.
func (s *Scheduler) ScheduleOffset(f func(time.Time) interface{}, offset time.Duration) error {
	return s.Schedule(f, s.now.Add(offset))
}

// Returns the timestamp on the next
// scheduled event, or the zero value
// and ErrEmpty if no events are
// scheduled.
func (s *Scheduler) PeekNext() (time.Time, error) {
	var t time.Time
	if s.Empty() {
		return t, ErrEmpty
	}
	return (*s.heap)[0].time, nil
}

// Fast-forward the internal clock
// to match the next scheduled event,
// and call the associated callback,
// passing the (now updated) time
// as the single argument. Return
// the value returned from this call.
//
// If there are no events scheduled,
// return a nil interface value and
// ErrEmpty.
//
// Note that CallNext does not modify
// s after calling the callback. Thus,
// it is safe to call methods on s
// from within the callback.
func (s *Scheduler) CallNext() (interface{}, error) {
	if s.Empty() {
		return nil, ErrEmpty
	}
	evt := heap.Pop(s.heap).(event)
	s.now = evt.time
	return evt.f(evt.time), nil
}

// Remove the next scheduled event
// from the Scheduler, but do not
// alter the internal clock.
func (s *Scheduler) RemoveNext() {
	if !s.Empty() {
		heap.Pop(s.heap)
	}
}

// Fast-forward the internal clock
// to match the next scheduled event,
// and remove the event from the
// Scheduler. If there are no events
// scheduled, do not alter the clock.
func (s *Scheduler) RemoveNextUpdate() {
	if !s.Empty() {
		evt := heap.Pop(s.heap).(event)
		s.now = evt.time
	}
}

// Remove all scheduled events from
// the Scheduler, but do not alter
// the internal clock.
func (s *Scheduler) RemoveAll() {
	*s.heap = make([]event, 0)
}

// Remove all scheduled events from
// the Scheduler, fast-forwarding
// the internal clock to match the
// latest scheduled event. If there
// are no events scheduled, do not
// alter the clock.
func (s *Scheduler) RemoveAllUpdate() {
	if !s.Empty() {
		latest := (*s.heap)[0].time
		for _, evt := range (*s.heap)[1:] {
			if evt.time.After(latest) {
				latest = evt.time
			}
		}
		s.now = latest
	}
	*s.heap = make([]event, 0)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/resources"
	"github.com/lxc/lxd/lxd/response"
)

var api10ResourcesCmd = APIEndpoint{
	Path: "resources",

	Get: APIEndpointAction{Handler: api10ResourcesGet, AccessHandler: AllowAuthenticated},
}

var storagePoolResourcesCmd = APIEndpoint{
	Path: "storage-pools/{name}/resources",

	Get: APIEndpointAction{Handler: storagePoolResourcesGet, AccessHandler: AllowAuthenticated},
}

// /1.0/resources
// Get system resources
func api10ResourcesGet(d *Daemon, r *http.Request) response.Response {
	// If a target was specified, forward the request to the relevant node.
	resp := ForwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Get the local resource usage
	res, err := resources.GetResources()
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, res)
}

// /1.0/storage-pools/{name}/resources
// Get resources for a specific storage pool
func storagePoolResourcesGet(d *Daemon, r *http.Request) response.Response {
	// If a target was specified, forward the request to the relevant node.
	resp := ForwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Get the existing storage pool
	poolName := mux.Vars(r)["name"]
	s, err := storagePoolInit(d.State(), poolName)
	if err != nil {
		return response.InternalError(err)
	}

	err = s.StoragePoolCheck()
	if err != nil {
		return response.InternalError(err)
	}

	res, err := s.StoragePoolResources()
	if err != nil {
		return response.InternalError(err)
	}

	return response.SyncResponse(true, &res)
}
<commit_msg>lxd/resources: Port to new storage API<commit_after>package main

import (
	"net/http"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/resources"
	"github.com/lxc/lxd/lxd/response"
	storagePools "github.com/lxc/lxd/lxd/storage"
	storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
)

var api10ResourcesCmd = APIEndpoint{
	Path: "resources",

	Get: APIEndpointAction{Handler: api10ResourcesGet, AccessHandler: AllowAuthenticated},
}

var storagePoolResourcesCmd = APIEndpoint{
	Path: "storage-pools/{name}/resources",

	Get: APIEndpointAction{Handler: storagePoolResourcesGet, AccessHandler: AllowAuthenticated},
}

// /1.0/resources
// Get system resources
func api10ResourcesGet(d *Daemon, r *http.Request) response.Response {
	// If a target was specified, forward the request to the relevant node.
	resp := ForwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Get the local resource usage
	res, err := resources.GetResources()
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponse(true, res)
}

// /1.0/storage-pools/{name}/resources
// Get resources for a specific storage pool
func storagePoolResourcesGet(d *Daemon, r *http.Request) response.Response {
	// If a target was specified, forward the request to the relevant node.
	resp := ForwardedResponseIfTargetIsRemote(d, r)
	if resp != nil {
		return resp
	}

	// Get the existing storage pool
	poolName := mux.Vars(r)["name"]
	var res *api.ResourcesStoragePool

	// Check if we can load new storage layer for pool driver type.
	pool, err := storagePools.GetPoolByName(d.State(), poolName)
	if err != storageDrivers.ErrUnknownDriver {
		if err != nil {
			return response.InternalError(err)
		}

		res, err = pool.GetResources()
		if err != nil {
			return response.InternalError(err)
		}
	} else { // Fallback to old storage layer.
		s, err := storagePoolInit(d.State(), poolName)
		if err != nil {
			return response.InternalError(err)
		}

		err = s.StoragePoolCheck()
		if err != nil {
			return response.InternalError(err)
		}

		res, err = s.StoragePoolResources()
		if err != nil {
			return response.InternalError(err)
		}
	}

	return response.SyncResponse(true, res)
}
<|endoftext|>