<commit_before>package cryptanalysis

import ()

func Chunk(data []byte, size int) [][]byte {
    var chunks [][]byte

    if len(data) % size != 0 {
        data = PadPkcs7(data, size)
    }

    for i:=0; i<len(data); i=i+size {
        chunks = append(chunks, data[i:i+size])
    }

    return chunks
}

func Transpose(data [][]byte) [][]byte {
    var transpose [][]byte

    for i, _ := range data[0] {
        var temp []byte

        for j, _ := range data {
            temp = append(temp, data[j][i])
        }

        transpose = append(transpose, temp)
    }

    return transpose
}

func PadPkcs7(data []byte, size int) []byte {
    if len(data) < size {
        pad := size - len(data)
        for i:=0; i<pad; i++ {
            data = append(data, byte(pad))
        }
    }

    return data
}
<commit_msg>Pad all data not just block_size chunks.<commit_after>package cryptanalysis

import ()

func Chunk(data []byte, size int) [][]byte {
    var chunks [][]byte
    data = PadPkcs7(data, size)

    for i:=0; i<len(data); i=i+size {
        chunks = append(chunks, data[i:i+size])
    }

    return chunks
}

func Transpose(data [][]byte) [][]byte {
    var transpose [][]byte

    for i, _ := range data[0] {
        var temp []byte

        for j, _ := range data {
            temp = append(temp, data[j][i])
        }

        transpose = append(transpose, temp)
    }

    return transpose
}

func PadPkcs7(data []byte, block_size int) []byte {
    if (len(data) % block_size) != 0 {
        pad := len(data) / block_size
        for i:=0; i<pad; i++ {
            data = append(data, byte(pad))
        }
    }

    return data
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"text/template"

	"github.com/gorilla/sessions"
	"github.com/josephspurrier/csrfbanana"
)

var (
	Store       *sessions.CookieStore
	SessionName = "banana"
)

var templateString string = `
<!DOCTYPE html>

{{ if .name }}

Your name: {{ .name }}

Tip: Try to reload the page...

{{ end }}

<form action="/" method="POST">
	Enter your name:
	<input type="text" name="name" id="name">

	<input type="hidden" name="token" value="{{.token}}">
	<input type="submit" value="Submit with Token" style="width: 160px;">
</form>

<form action="/" method="POST">
	Type in a number:
	<input type="text" name="num" id="num">

	<input type="submit" value="Submit without Token" style="width: 160px;">
</form>

`

// Compiled template
var templ = template.Must(template.New("t1").Parse(templateString))

// Login handles GET and POST
func routeLogin(w http.ResponseWriter, r *http.Request) {

	// Get session
	sess := Session(r, SessionName)

	// Create a map for the template
	vars := make(map[string]string)

	// Store the CSRF token
	vars["token"] = csrfbanana.Token(w, r, sess)

	// If a POST operation
	if r.Method == "POST" {
		// Store the name to a template variable
		vars["name"] = r.FormValue("name")
	}

	// Show the template
	templ.Execute(w, vars)
}

// InvalidToken handles CSRF attacks
func routeInvalidToken(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	w.WriteHeader(http.StatusForbidden)
	fmt.Fprint(w, `Your token <strong>expired</strong>, click <a href="javascript:void(0)" onclick="window.history.back()">here</a> to try again.`)
}

// Session returns a new session, never returns an error
func Session(r *http.Request, name string) *sessions.Session {
	session, _ := Store.Get(r, name)
	return session
}

func main() {
	// Create cookie store
	Store = sessions.NewCookieStore([]byte("This is super screen..."))
	Store.Options = &sessions.Options{
		//Domain:   "localhost", // Chrome doesn't work with localhost domain
		Path:     "/",
		MaxAge:   3600 * 8, // 8 hours
		HttpOnly: true,
	}

	// Default handler
	h := http.HandlerFunc(routeLogin)

	// Prevents CSRF
	cs := csrfbanana.New(h, Store, SessionName)

	// Set error page for CSRF
	cs.FailureHandler(http.HandlerFunc(routeInvalidToken))

	// Generate a new token after each check (also prevents double submits)
	cs.ClearAfterUsage(true)

	// Exclude /static/ from tokens (even though we don't have a static file handler...)
	cs.ExcludeRegexPaths([]string{"/static(.*)"})

	// Optional - set the token length
	csrfbanana.TokenLength = 32

	// Optional - set the token name used in the forms
	csrfbanana.TokenName = "token"

	fmt.Println("Listening on http://localhost:80/")
	http.ListenAndServe(":80", cs)
}
<commit_msg>initial<commit_after>package main

import (
	"fmt"
	"net/http"
	"text/template"

	"github.com/gorilla/sessions"
	"github.com/josephspurrier/csrfbanana"
)

var (
	Store       *sessions.CookieStore
	SessionName = "banana"
)

var templateString string = `
<!DOCTYPE html>

{{ if .name }}

Your name: {{ .name }}

Tip: Try to reload the page...

{{ end }}

<form action="/" method="POST">
	Enter your name:
	<input type="text" name="name" id="name">

	<input type="hidden" name="token" value="{{.token}}">
	<input type="submit" value="Submit with Token" style="width: 160px;">
</form>

<form action="/" method="POST">
	Type in a number:
	<input type="text" name="num" id="num">

	<input type="submit" value="Submit without Token" style="width: 160px;">
</form>

`

// Compiled template
var templ = template.Must(template.New("t1").Parse(templateString))

// Login handles GET and POST
func routeLogin(w http.ResponseWriter, r *http.Request) {

	// Get session
	sess := Session(r, SessionName)

	// Create a map for the template
	vars := make(map[string]string)

	// Store the CSRF token
	vars["token"] = csrfbanana.Token(w, r, sess)

	// If a POST operation
	if r.Method == "POST" {
		// Store the name to a template variable
		vars["name"] = r.FormValue("name")
	}

	// Show the template
	templ.Execute(w, vars)
}

// InvalidToken handles CSRF attacks
func routeInvalidToken(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")
	w.WriteHeader(http.StatusForbidden)
	fmt.Fprint(w, `Your token <strong>expired</strong>, click <a href="javascript:void(0)" onclick="window.history.back()">here</a> to try again.`)
}

// Session returns a new session, never returns an error
func Session(r *http.Request, name string) *sessions.Session {
	session, _ := Store.Get(r, name)
	return session
}

func main() {
	// Create cookie store
	Store = sessions.NewCookieStore([]byte("This is super screen..."))
	Store.Options = &sessions.Options{
		//Domain:   "localhost", // Chrome doesn't work with localhost domain
		Path:     "/",
		MaxAge:   3600 * 8, // 8 hours
		HttpOnly: true,
	}

	// Default handler
	h := http.HandlerFunc(routeLogin)

	// Prevents CSRF
	cs := csrfbanana.New(h, Store, SessionName)

	// Set error page for CSRF
	cs.FailureHandler(http.HandlerFunc(routeInvalidToken))

	// Generate a new token after each check (also prevents double submits)
	cs.ClearAfterUsage(true)

	// Exclude /static/ from tokens (even though we don't have a static file handler...)
	cs.ExcludeRegexPaths([]string{"/static(.*)"})

	// Optional - set the token length
	csrfbanana.TokenLength = 32

	// Optional - set the token name used in the forms
	csrfbanana.TokenName = "token"

	fmt.Println("Listening on http://localhost:80/")
	http.ListenAndServe(":8080", cs)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	consumergroup "go-consumergroup"
	"os"
	"os/signal"
	"syscall"
	"time"
)

func handleSignal(sig os.Signal, cg *consumergroup.ConsumerGroup) {
	switch sig {
	case syscall.SIGINT:
		cg.ExitGroup()
	case syscall.SIGTERM:
		cg.ExitGroup()
	default:
	}
}

func registerSignal(cg *consumergroup.ConsumerGroup) {
	go func() {
		c := make(chan os.Signal)
		sigs := []os.Signal{
			syscall.SIGINT,
			syscall.SIGTERM,
		}
		signal.Notify(c, sigs...)
		sig := <-c
		handleSignal(sig, cg)
	}()
}

func main() {
	conf := consumergroup.NewConfig()
	conf.ZkList = []string{"127.0.0.1:2181"}
	conf.ZkSessionTimeout = 6 * time.Second
	conf.TopicList = []string{"test"}
	conf.GroupID = "go-test-group-id"

	cg, err := consumergroup.NewConsumerGroup(conf)
	if err != nil {
		fmt.Println("Failed to create consumer group, err ", err.Error())
		os.Exit(1)
	}

	registerSignal(cg)

	err = cg.JoinGroup()
	if err != nil {
		fmt.Println("Failed to join group, err ", err.Error())
		os.Exit(1)
	}
	if messages, ok := cg.GetMessages("test"); ok {
		for message := range messages {
			fmt.Println(string(message.Value), message.Offset)
			time.Sleep(500 * time.Millisecond)
		}
	} else {
		fmt.Println("Topic was not found in consumergroup")
	}
}
<commit_msg>MOD: Drain topic error channel in example.go<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"

	consumergroup "github.com/meitu/go-consumergroup"
)

func handleSignal(sig os.Signal, cg *consumergroup.ConsumerGroup) {
	switch sig {
	case syscall.SIGINT:
		cg.ExitGroup()
	case syscall.SIGTERM:
		cg.ExitGroup()
	default:
	}
}

func registerSignal(cg *consumergroup.ConsumerGroup) {
	go func() {
		c := make(chan os.Signal)
		sigs := []os.Signal{
			syscall.SIGINT,
			syscall.SIGTERM,
		}
		signal.Notify(c, sigs...)
		sig := <-c
		handleSignal(sig, cg)
	}()
}

func main() {
	conf := consumergroup.NewConfig()
	conf.ZkList = []string{"127.0.0.1:2181"}
	conf.ZkSessionTimeout = 6 * time.Second
	conf.TopicList = []string{"test"}
	conf.GroupID = "go-test-group-id"

	cg, err := consumergroup.NewConsumerGroup(conf)
	if err != nil {
		fmt.Println("Failed to create consumer group, err ", err.Error())
		os.Exit(1)
	}

	registerSignal(cg)

	err = cg.JoinGroup()
	if err != nil {
		fmt.Println("Failed to join group, err ", err.Error())
		os.Exit(1)
	}

	// Retrieve the error and log
	go func() {
		if topicErrChan, ok := cg.GetErrors("test"); ok {
			for err := range topicErrChan {
				if err != nil {
					fmt.Println("toipic %s got err, %s", err)
				}
			}
		}
	}()

	if msgChan, ok := cg.GetMessages("test"); ok {
		for message := range msgChan {
			fmt.Println(string(message.Value), message.Offset)
			time.Sleep(500 * time.Millisecond)
		}
	} else {
		fmt.Println("Topic was not found in consumergroup")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/squiidz/fur/middle"
	"github.com/squiidz/pod"
)

func main() {
	p := pod.NewPod()

	p.Glob(GlobalMiddle, middle.Logger)

	http.Handle("/home", p.Fuse(HomeHandler).Add(HomeMiddle))

	http.ListenAndServe(":8080", nil)
}

func GlobalMiddle(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- GlobalMiddlware\n"))
}

func HomeHandler(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- HomeHandler\n"))
}

func HomeMiddle(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- HomeMiddleware\n"))
}
<commit_msg>Support two different middleware signature<commit_after>package main

import (
	"net/http"

	"github.com/squiidz/fur/middle"
	"github.com/squiidz/pod"
)

func main() {
	p := pod.NewPod()

	p.Glob(GlobalMiddle, middle.Logger)

	http.Handle("/home", p.Fuse(HomeHandler).Add(HomeMiddle, Middle))

	http.ListenAndServe(":8080", nil)
}

func GlobalMiddle(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- GlobalMiddlware\n"))
}

func HomeHandler(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- HomeHandler\n"))
}

func HomeMiddle(rw http.ResponseWriter, req *http.Request) {
	rw.Write([]byte("- HomeMiddleware\n"))
}

func Middle(next http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		rw.Write([]byte("- Middle\n"))
		next.ServeHTTP(rw, req)
	})
}
<|endoftext|>


<commit_before>// Package info implements the info command.
package info

import (
	"encoding/json"
	"encoding/pem"
	"fmt"

	"github.com/cloudflare/cfssl/api"
	"github.com/cloudflare/cfssl/api/client"
	"github.com/cloudflare/cfssl/cli"
	"github.com/cloudflare/cfssl/cli/sign"
	"github.com/cloudflare/cfssl/config"
	"github.com/cloudflare/cfssl/errors"
	"github.com/cloudflare/cfssl/helpers"

	goerr "errors"
)

var infoUsageTxt = `cfssl info -- get info about a remote signer

Usage:

Get info about a remote signer:
cfssl info -remote remote_host [-label label] [-profile profile] [-label label]

Flags:
`

var infoFlags = []string{"remote", "label", "profile", "config"}

func getInfoFromRemote(c cli.Config) (resp *client.InfoResp, err error) {

	req := new(client.InfoReq)
	req.Label = c.Label
	req.Profile = c.Profile

	serv := client.NewServer(c.Remote)

	reqJSON, _ := json.Marshal(req)
	resp, err = serv.Info(reqJSON)
	if err != nil {
		return
	}

	_, err = helpers.ParseCertificatePEM([]byte(resp.Certificate))
	if err != nil {
		return
	}

	return
}

func getInfoFromConfig(c cli.Config) (resp *client.InfoResp, err error) {
	s, err := sign.SignerFromConfig(c)
	if err != nil {
		return
	}

	cert, err := s.Certificate(c.Label, c.Profile)
	if err != nil {
		return
	}

	blk := pem.Block{
		Type:  "CERTIFICATE",
		Bytes: cert.Raw,
	}

	certPem := pem.EncodeToMemory(&blk)

	var profile *config.SigningProfile

	policy := s.Policy()

	if policy != nil && policy.Profiles != nil && c.Profile != "" {
		profile = policy.Profiles[c.Profile]
	}

	if profile == nil && policy != nil {
		profile = policy.Default
	}

	resp = &client.InfoResp{
		Certificate:  string(certPem),
		Usage:        profile.Usage,
		ExpiryString: profile.ExpiryString,
	}

	return
}

func infoMain(args []string, c cli.Config) (err error) {
	if len(args) > 0 {
		return goerr.New("argument is provided but not defined; please refer to the usage by flag -h.")
	}

	var resp *client.InfoResp

	if c.Remote != "" {
		resp, err = getInfoFromRemote(c)
		if err != nil {
			return
		}

	} else if c.CFG != nil {
		resp, err = getInfoFromConfig(c)
		if err != nil {
			return
		}
	} else {
		return goerr.New("Either -remote or -config must be given. Refer to cfssl info -h for usage.")
	}

	response := api.NewSuccessResponse(resp)
	respJSON, err := json.Marshal(response)
	if err != nil {
		return errors.NewBadRequest(err)
	}
	fmt.Print(string(respJSON))
	return nil
}

// Command defines the commmand-line procedure for info
var Command = &cli.Command{
	UsageText: infoUsageTxt,
	Flags:     infoFlags,
	Main:      infoMain,
}
<commit_msg>Print the result of the API response in info.<commit_after>// Package info implements the info command.
package info

import (
	"encoding/json"
	"encoding/pem"
	"fmt"

	"github.com/cloudflare/cfssl/api/client"
	"github.com/cloudflare/cfssl/cli"
	"github.com/cloudflare/cfssl/cli/sign"
	"github.com/cloudflare/cfssl/config"
	"github.com/cloudflare/cfssl/errors"
	"github.com/cloudflare/cfssl/helpers"

	goerr "errors"
)

var infoUsageTxt = `cfssl info -- get info about a remote signer

Usage:

Get info about a remote signer:
cfssl info -remote remote_host [-label label] [-profile profile] [-label label]

Flags:
`

var infoFlags = []string{"remote", "label", "profile", "config"}

func getInfoFromRemote(c cli.Config) (resp *client.InfoResp, err error) {

	req := new(client.InfoReq)
	req.Label = c.Label
	req.Profile = c.Profile

	serv := client.NewServer(c.Remote)

	reqJSON, _ := json.Marshal(req)
	resp, err = serv.Info(reqJSON)
	if err != nil {
		return
	}

	_, err = helpers.ParseCertificatePEM([]byte(resp.Certificate))
	if err != nil {
		return
	}

	return
}

func getInfoFromConfig(c cli.Config) (resp *client.InfoResp, err error) {
	s, err := sign.SignerFromConfig(c)
	if err != nil {
		return
	}

	cert, err := s.Certificate(c.Label, c.Profile)
	if err != nil {
		return
	}

	blk := pem.Block{
		Type:  "CERTIFICATE",
		Bytes: cert.Raw,
	}

	certPem := pem.EncodeToMemory(&blk)

	var profile *config.SigningProfile

	policy := s.Policy()

	if policy != nil && policy.Profiles != nil && c.Profile != "" {
		profile = policy.Profiles[c.Profile]
	}

	if profile == nil && policy != nil {
		profile = policy.Default
	}

	resp = &client.InfoResp{
		Certificate:  string(certPem),
		Usage:        profile.Usage,
		ExpiryString: profile.ExpiryString,
	}

	return
}

func infoMain(args []string, c cli.Config) (err error) {
	if len(args) > 0 {
		return goerr.New("argument is provided but not defined; please refer to the usage by flag -h.")
	}

	var resp *client.InfoResp

	if c.Remote != "" {
		resp, err = getInfoFromRemote(c)
		if err != nil {
			return
		}

	} else if c.CFG != nil {
		resp, err = getInfoFromConfig(c)
		if err != nil {
			return
		}
	} else {
		return goerr.New("Either -remote or -config must be given. Refer to cfssl info -h for usage.")
	}

	respJSON, err := json.Marshal(resp)
	if err != nil {
		return errors.NewBadRequest(err)
	}
	fmt.Print(string(respJSON))
	return nil
}

// Command defines the commmand-line procedure for info
var Command = &cli.Command{
	UsageText: infoUsageTxt,
	Flags:     infoFlags,
	Main:      infoMain,
}
<|endoftext|>


<commit_before>package unit_test

import (
	"flag"
	"github.com/timeredbull/tsuru/api/unit"
	. "launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { TestingT(t) }

type S struct{}

var _ = Suite(&S{})

var lxcEnabled = flag.Bool("juju", false, "enable unit tests that require juju")

func (s *S) SetUpSuite(c *C) {
	if !*lxcEnabled {
		c.Skip("unit tests need juju installed (-juju to enable)")
	}
}

func (s *S) TestCreateAndDestroy(c *C) {
	u := unit.Unit{Type: "django", Name: "myUnit"}

	err := u.Create()
	c.Assert(err, IsNil)

	err = u.Destroy()
	c.Assert(err, IsNil)
}
<commit_msg>fixed juju enabled flag nage<commit_after>package unit_test

import (
	"flag"
	"github.com/timeredbull/tsuru/api/unit"
	. "launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { TestingT(t) }

type S struct{}

var _ = Suite(&S{})

var jujuEnabled = flag.Bool("juju", false, "enable unit tests that require juju")

func (s *S) SetUpSuite(c *C) {
	if !*jujuEnabled {
		c.Skip("unit tests need juju installed (-juju to enable)")
	}
}

func (s *S) TestCreateAndDestroy(c *C) {
	u := unit.Unit{Type: "django", Name: "myUnit"}

	err := u.Create()
	c.Assert(err, IsNil)

	err = u.Destroy()
	c.Assert(err, IsNil)
}
<|endoftext|>


<commit_before>package utxodb

import (
	"container/heap"
	"errors"
	"sort"
	"sync"
	"time"

	"golang.org/x/net/context"

	"chain/fedchain/bc"
	"chain/log"
	"chain/metrics"
)

var (
	// ErrInsufficient indicates the account doesn't contain enough
	// units of the requested asset to satisfy the reservation.
	// New units must be deposited into the account in order to
	// satisfy the request; change will not be sufficient.
	ErrInsufficient = errors.New("reservation found insufficient funds")

	// ErrReserved indicates that a reservation could not be
	// satisfied because some of the outputs were already reserved.
	// When those reservations are finalized into a transaction
	// (and no other transaction spends funds from the account),
	// new change outputs will be created
	// in sufficient amounts to satisfy the request.
	ErrReserved = errors.New("reservation found outputs already reserved")
)

type (
	Reserver struct {
		db DB

		mu  sync.Mutex // protects the following
		tab map[key]*pool
	}

	key struct{ AccountID, AssetID string }

	// TODO(kr): see if we can avoid storing
	// AccountID and AssetID in UTXO

	// TODO(kr): try interning strings in UTXO

	UTXO struct {
		// Size of this struct matters.
		// We keep lots of them in memory.

		AccountID string
		AssetID   string
		Amount    uint64

		ResvExpires time.Time
		heapIndex   int
		reserved    uint64 // only valid if ResvExpires after now

		Outpoint  bc.Outpoint
		AddrIndex [2]uint32
	}

	Receiver struct {
		ManagerNodeID string   `json:"manager_node_id"`
		AccountID     string   `json:"account_id"`
		AddrIndex     []uint32 `json:"address_index"`
		IsChange      bool     `json:"is_change"`
	}

	// Change represents reserved units beyond what was asked for.
	// Total reservation is for Amount+Input.Amount.
	Change struct {
		Input  Input
		Amount uint64
	}

	Input struct {
		AssetID   string `json:"asset_id"`
		AccountID string `json:"account_id"`
		TxID      string `json:"transaction_id"`
		Amount    uint64
	}

	DB interface {
		// LoadUTXOs loads the set of UTXOs
		// available to reserve
		// for the given asset in the given account.
		LoadUTXOs(ctx context.Context, accountID, assetID string) ([]*UTXO, error)

		// SaveReservations stores the reservation expiration
		// time in the database for the given UTXOs.
		SaveReservations(ctx context.Context, u []*UTXO, expires time.Time) error

		// ApplyTx applies the Tx to the database,
		// deleteing spent outputs and inserting new UTXOs.
		// It returns the deleted and inserted outputs.
		ApplyTx(context.Context, *bc.Tx, []*Receiver) (deleted, inserted []*UTXO, err error)
	}
)

func New(db DB) *Reserver {
	return &Reserver{
		db:  db,
		tab: make(map[key]*pool),
	}
}

// pool returns the pool for the given account and asset,
// creating it if necessary.
func (rs *Reserver) pool(accountID, assetID string) *pool {
	rs.mu.Lock()
	defer rs.mu.Unlock()
	k := key{accountID, assetID}
	p, ok := rs.tab[k]
	if !ok {
		p = new(pool)
		rs.tab[k] = p
	}
	return p
}

func (rs *Reserver) Reserve(ctx context.Context, inputs []Input, ttl time.Duration) (u []*UTXO, c []Change, err error) {
	defer metrics.RecordElapsed(time.Now())

	var reserved []*UTXO
	var change []Change
	defer func() {
		if err != nil {
			u = nil
			c = nil
			rs.unreserve(reserved)
		}
	}()

	now := time.Now().UTC()
	exp := now.Add(ttl)

	sort.Sort(byKey(inputs))
	for _, in := range inputs {
		p := rs.pool(in.AccountID, in.AssetID)
		err := p.init(ctx, rs.db, key{in.AccountID, in.AssetID})
		if err != nil {
			return nil, nil, err
		}
		res, err := p.reserve(in.Amount, now, exp)
		if err != nil {
			return nil, nil, err
		}
		reserved = append(reserved, res...)
		if n := sum(res); n > in.Amount {
			change = append(change, Change{in, n - in.Amount})
		}
	}

	if ttl > 2*time.Minute {
		err = rs.db.SaveReservations(ctx, reserved, exp)
	}
	return reserved, change, err
}

// Cancel cancels the given reservations, if they still exist.
// If any do not exist (if they've already been consumed
// or canceled), it silently ignores them.
func (rs *Reserver) Cancel(ctx context.Context, outpoints []bc.Outpoint) {
	var utxos []*UTXO
	for _, op := range outpoints {
		if u := rs.findReservation(op); u != nil {
			utxos = append(utxos, u)
		}
	}
	rs.unreserve(utxos)
}

func (rs *Reserver) Apply(ctx context.Context, tx *bc.Tx, outRecs []*Receiver) error {
	defer metrics.RecordElapsed(time.Now())
	deleted, inserted, err := rs.db.ApplyTx(ctx, tx, outRecs)
	if err != nil {
		return err
	}
	rs.delete(deleted)
	internIDs(inserted)
	sort.Sort(byKeyUTXO(inserted))
	rs.insert(inserted)
	return nil
}

// findReservation does a linear scan through the set
// of pools in rs to find the UTXO that reserves op.
// If there is no such reservation, it returns nil.
func (rs *Reserver) findReservation(op bc.Outpoint) *UTXO {
	// TODO(kr): augment the SDK to include account ID and asset ID
	// for each reservation, so we can do this lookup faster.
	defer metrics.RecordElapsed(time.Now())
	var keys []key
	rs.mu.Lock()
	for k := range rs.tab {
		keys = append(keys, k)
	}
	rs.mu.Unlock()

	for _, k := range keys {
		p := rs.pool(k.AccountID, k.AssetID)
		if u := p.findReservation(op); u != nil {
			return u
		}
	}
	return nil
}

// mappool finds the pool for each element of utxos
// and calls f.
// It holds the pool's lock when it calls f,
// so f can modify the pool outputs list and u.
// f must preserve the heap invariant for p.outputs.
func (rs *Reserver) mappool(utxos []*UTXO, f func(*pool, *UTXO)) {
	var prev *pool
	for _, u := range utxos {
		p := rs.pool(u.AccountID, u.AssetID)
		if p != prev {
			p.mu.Lock()
			if prev != nil {
				prev.mu.Unlock()
			}
			prev = p
		}
		f(p, u)
	}
	if prev != nil {
		prev.mu.Unlock()
	}
}

// utxos must not already be in rs.
func (rs *Reserver) insert(utxos []*UTXO) {
	ctx := context.TODO()
	var i int64
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u is already in the pool.
		// If so, there's nothing to do here.
		if p.byOutpoint(u.Outpoint) != nil {
			return
		}
		heap.Push(&p.outputs, u)
		i++
		if i%1e6 == 0 {
			log.Messagef(ctx, "build utxo heaps: did %d so far", i)
		}
	})
}

func (rs *Reserver) unreserve(utxos []*UTXO) {
	sort.Sort(byKeyUTXO(utxos))
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u has been removed from the pool
		// before we got here, since we just took the lock
		// at the start of unreserve (in mappool).
		// If u is no longer in p, it has been deleted
		// and unreserve should be a no-op.
		if p.contains(u) {
			u.ResvExpires = time.Time{}
			heap.Fix(&p.outputs, u.heapIndex)
		}
	})
}

func (rs *Reserver) delete(utxos []*UTXO) {
	sort.Sort(byKeyUTXO(utxos))
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u has already been deleted.
		// Also, u might not be the same object stored
		// in p; it just has the same outpoint.
		// So we look up the actual pointer and
		// make sure it's contained in p.
		if u = p.byOutpoint(u.Outpoint); u != nil {
			heap.Remove(&p.outputs, u.heapIndex)
		}
	})

}

func sum(utxos []*UTXO) (total uint64) {
	for _, u := range utxos {
		total += u.Amount
	}
	return
}
<commit_msg>api/utxodb: change lock/unlock order in mappool<commit_after>package utxodb

import (
	"container/heap"
	"errors"
	"sort"
	"sync"
	"time"

	"golang.org/x/net/context"

	"chain/fedchain/bc"
	"chain/log"
	"chain/metrics"
)

var (
	// ErrInsufficient indicates the account doesn't contain enough
	// units of the requested asset to satisfy the reservation.
	// New units must be deposited into the account in order to
	// satisfy the request; change will not be sufficient.
	ErrInsufficient = errors.New("reservation found insufficient funds")

	// ErrReserved indicates that a reservation could not be
	// satisfied because some of the outputs were already reserved.
	// When those reservations are finalized into a transaction
	// (and no other transaction spends funds from the account),
	// new change outputs will be created
	// in sufficient amounts to satisfy the request.
	ErrReserved = errors.New("reservation found outputs already reserved")
)

type (
	Reserver struct {
		db DB

		mu  sync.Mutex // protects the following
		tab map[key]*pool
	}

	key struct{ AccountID, AssetID string }

	// TODO(kr): see if we can avoid storing
	// AccountID and AssetID in UTXO

	// TODO(kr): try interning strings in UTXO

	UTXO struct {
		// Size of this struct matters.
		// We keep lots of them in memory.

		AccountID string
		AssetID   string
		Amount    uint64

		ResvExpires time.Time
		heapIndex   int
		reserved    uint64 // only valid if ResvExpires after now

		Outpoint  bc.Outpoint
		AddrIndex [2]uint32
	}

	Receiver struct {
		ManagerNodeID string   `json:"manager_node_id"`
		AccountID     string   `json:"account_id"`
		AddrIndex     []uint32 `json:"address_index"`
		IsChange      bool     `json:"is_change"`
	}

	// Change represents reserved units beyond what was asked for.
	// Total reservation is for Amount+Input.Amount.
	Change struct {
		Input  Input
		Amount uint64
	}

	Input struct {
		AssetID   string `json:"asset_id"`
		AccountID string `json:"account_id"`
		TxID      string `json:"transaction_id"`
		Amount    uint64
	}

	DB interface {
		// LoadUTXOs loads the set of UTXOs
		// available to reserve
		// for the given asset in the given account.
		LoadUTXOs(ctx context.Context, accountID, assetID string) ([]*UTXO, error)

		// SaveReservations stores the reservation expiration
		// time in the database for the given UTXOs.
		SaveReservations(ctx context.Context, u []*UTXO, expires time.Time) error

		// ApplyTx applies the Tx to the database,
		// deleteing spent outputs and inserting new UTXOs.
		// It returns the deleted and inserted outputs.
		ApplyTx(context.Context, *bc.Tx, []*Receiver) (deleted, inserted []*UTXO, err error)
	}
)

func New(db DB) *Reserver {
	return &Reserver{
		db:  db,
		tab: make(map[key]*pool),
	}
}

// pool returns the pool for the given account and asset,
// creating it if necessary.
func (rs *Reserver) pool(accountID, assetID string) *pool {
	rs.mu.Lock()
	defer rs.mu.Unlock()
	k := key{accountID, assetID}
	p, ok := rs.tab[k]
	if !ok {
		p = new(pool)
		rs.tab[k] = p
	}
	return p
}

func (rs *Reserver) Reserve(ctx context.Context, inputs []Input, ttl time.Duration) (u []*UTXO, c []Change, err error) {
	defer metrics.RecordElapsed(time.Now())

	var reserved []*UTXO
	var change []Change
	defer func() {
		if err != nil {
			u = nil
			c = nil
			rs.unreserve(reserved)
		}
	}()

	now := time.Now().UTC()
	exp := now.Add(ttl)

	sort.Sort(byKey(inputs))
	for _, in := range inputs {
		p := rs.pool(in.AccountID, in.AssetID)
		err := p.init(ctx, rs.db, key{in.AccountID, in.AssetID})
		if err != nil {
			return nil, nil, err
		}
		res, err := p.reserve(in.Amount, now, exp)
		if err != nil {
			return nil, nil, err
		}
		reserved = append(reserved, res...)
		if n := sum(res); n > in.Amount {
			change = append(change, Change{in, n - in.Amount})
		}
	}

	if ttl > 2*time.Minute {
		err = rs.db.SaveReservations(ctx, reserved, exp)
	}
	return reserved, change, err
}

// Cancel cancels the given reservations, if they still exist.
// If any do not exist (if they've already been consumed
// or canceled), it silently ignores them.
func (rs *Reserver) Cancel(ctx context.Context, outpoints []bc.Outpoint) {
	var utxos []*UTXO
	for _, op := range outpoints {
		if u := rs.findReservation(op); u != nil {
			utxos = append(utxos, u)
		}
	}
	rs.unreserve(utxos)
}

func (rs *Reserver) Apply(ctx context.Context, tx *bc.Tx, outRecs []*Receiver) error {
	defer metrics.RecordElapsed(time.Now())
	deleted, inserted, err := rs.db.ApplyTx(ctx, tx, outRecs)
	if err != nil {
		return err
	}
	rs.delete(deleted)
	internIDs(inserted)
	sort.Sort(byKeyUTXO(inserted))
	rs.insert(inserted)
	return nil
}

// findReservation does a linear scan through the set
// of pools in rs to find the UTXO that reserves op.
// If there is no such reservation, it returns nil.
func (rs *Reserver) findReservation(op bc.Outpoint) *UTXO {
	// TODO(kr): augment the SDK to include account ID and asset ID
	// for each reservation, so we can do this lookup faster.
	defer metrics.RecordElapsed(time.Now())
	var keys []key
	rs.mu.Lock()
	for k := range rs.tab {
		keys = append(keys, k)
	}
	rs.mu.Unlock()

	for _, k := range keys {
		p := rs.pool(k.AccountID, k.AssetID)
		if u := p.findReservation(op); u != nil {
			return u
		}
	}
	return nil
}

// mappool finds the pool for each element of utxos
// and calls f.
// It holds the pool's lock when it calls f,
// so f can modify the pool outputs list and u.
// f must preserve the heap invariant for p.outputs.
func (rs *Reserver) mappool(utxos []*UTXO, f func(*pool, *UTXO)) {
	var prev *pool
	for _, u := range utxos {
		p := rs.pool(u.AccountID, u.AssetID)
		if p != prev {
			if prev != nil {
				prev.mu.Unlock()
			}
			p.mu.Lock()
			prev = p
		}
		f(p, u)
	}
	if prev != nil {
		prev.mu.Unlock()
	}
}

// utxos must not already be in rs.
func (rs *Reserver) insert(utxos []*UTXO) {
	ctx := context.TODO()
	var i int64
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u is already in the pool.
		// If so, there's nothing to do here.
		if p.byOutpoint(u.Outpoint) != nil {
			return
		}
		heap.Push(&p.outputs, u)
		i++
		if i%1e6 == 0 {
			log.Messagef(ctx, "build utxo heaps: did %d so far", i)
		}
	})
}

func (rs *Reserver) unreserve(utxos []*UTXO) {
	sort.Sort(byKeyUTXO(utxos))
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u has been removed from the pool
		// before we got here, since we just took the lock
		// at the start of unreserve (in mappool).
		// If u is no longer in p, it has been deleted
		// and unreserve should be a no-op.
		if p.contains(u) {
			u.ResvExpires = time.Time{}
			heap.Fix(&p.outputs, u.heapIndex)
		}
	})
}

func (rs *Reserver) delete(utxos []*UTXO) {
	sort.Sort(byKeyUTXO(utxos))
	rs.mappool(utxos, func(p *pool, u *UTXO) {
		// It's possible u has already been deleted.
		// Also, u might not be the same object stored
		// in p; it just has the same outpoint.
		// So we look up the actual pointer and
		// make sure it's contained in p.
		if u = p.byOutpoint(u.Outpoint); u != nil {
			heap.Remove(&p.outputs, u.heapIndex)
		}
	})

}

func sum(utxos []*UTXO) (total uint64) {
	for _, u := range utxos {
		total += u.Amount
	}
	return
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	retryBackOffFactory retryhttp.BackOffFactory
	nestedRoundTripper  http.RoundTripper

	givenHttpClient *http.Client
}

func New(apiURL string, nestedRoundTripper http.RoundTripper) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		retryBackOffFactory: retryhttp.NewExponentialBackOffFactory(60 * time.Minute),

		nestedRoundTripper: nestedRoundTripper,
	}
}

func NewWithHTTPClient(apiURL string, httpClient *http.Client) Client {
	return &client{
		givenHttpClient:  httpClient,
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	if c.givenHttpClient != nil {
		return c.givenHttpClient
	}
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:         logger.Session("retry-round-tripper"),
			BackOffFactory: c.retryBackOffFactory,
			RoundTripper:   c.nestedRoundTripper,
			Retryer:        &retryhttp.DefaultRetryer{},
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, handle string, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Handle:       handle,
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolumeAsync, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeFutureResponse baggageclaim.VolumeFutureResponse
	err = json.NewDecoder(response.Body).Decode(&volumeFutureResponse)
	if err != nil {
		return nil, err
	}

	volumeFuture := &volumeFuture{
		client: c,
		handle: volumeFutureResponse.Handle,
		logger: logger,
	}

	defer volumeFuture.Destroy()

	volume, err := volumeFuture.Wait()
	if err != nil {
		return nil, err
	}
	return volume, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {
	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	if apiVolume.TTLInSeconds == 0 {
		return volume, true
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) getVolumeStatsResponse(logger lager.Logger, handle string) (baggageclaim.VolumeStatsResponse, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolumeStats, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeStatsResponse{}, nil
		}
		return baggageclaim.VolumeStatsResponse{}, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeStatsResponse{}, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeStatsResponse baggageclaim.VolumeStatsResponse
	err = json.NewDecoder(response.Body).Decode(&volumeStatsResponse)
	if err != nil {
		return baggageclaim.VolumeStatsResponse{}, err
	}

	return volumeStatsResponse, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) destroy(logger lager.Logger, handle string) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.DestroyVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setPrivileged(logger lager.Logger, handle string, privileged bool) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PrivilegedRequest{
		Value: privileged,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetPrivileged, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<commit_msg>missed a spot<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math"
	"net/http"
	"net/url"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/baggageclaim"
	"github.com/concourse/baggageclaim/api"
	"github.com/concourse/retryhttp"
)

type Client interface {
	baggageclaim.Client
}

type client struct {
	requestGenerator *rata.RequestGenerator

	retryBackOffFactory retryhttp.BackOffFactory
	nestedRoundTripper  http.RoundTripper

	givenHttpClient *http.Client
}

func New(apiURL string, nestedRoundTripper http.RoundTripper) Client {
	return &client{
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),

		retryBackOffFactory: retryhttp.NewExponentialBackOffFactory(60 * time.Minute),

		nestedRoundTripper: nestedRoundTripper,
	}
}

func NewWithHTTPClient(apiURL string, httpClient *http.Client) Client {
	return &client{
		givenHttpClient:  httpClient,
		requestGenerator: rata.NewRequestGenerator(apiURL, baggageclaim.Routes),
	}
}

func (c *client) httpClient(logger lager.Logger) *http.Client {
	if c.givenHttpClient != nil {
		return c.givenHttpClient
	}
	return &http.Client{
		Transport: &retryhttp.RetryRoundTripper{
			Logger:         logger.Session("retry-round-tripper"),
			BackOffFactory: c.retryBackOffFactory,
			RoundTripper:   c.nestedRoundTripper,
			Retryer:        &retryhttp.DefaultRetryer{},
		},
	}
}

func (c *client) CreateVolume(logger lager.Logger, handle string, volumeSpec baggageclaim.VolumeSpec) (baggageclaim.Volume, error) {
	strategy := volumeSpec.Strategy
	if strategy == nil {
		strategy = baggageclaim.EmptyStrategy{}
	}

	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.VolumeRequest{
		Handle:       handle,
		Strategy:     strategy.Encode(),
		TTLInSeconds: uint(math.Ceil(volumeSpec.TTL.Seconds())),
		Properties:   volumeSpec.Properties,
		Privileged:   volumeSpec.Privileged,
	})

	request, _ := c.requestGenerator.CreateRequest(baggageclaim.CreateVolumeAsync, nil, buffer)
	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeFutureResponse baggageclaim.VolumeFutureResponse
	err = json.NewDecoder(response.Body).Decode(&volumeFutureResponse)
	if err != nil {
		return nil, err
	}

	volumeFuture := &volumeFuture{
		client: c,
		handle: volumeFutureResponse.Handle,
		logger: logger,
	}

	defer volumeFuture.Destroy()

	volume, err := volumeFuture.Wait()
	if err != nil {
		return nil, err
	}
	return volume, nil
}

func (c *client) ListVolumes(logger lager.Logger, properties baggageclaim.VolumeProperties) (baggageclaim.Volumes, error) {
	if properties == nil {
		properties = baggageclaim.VolumeProperties{}
	}

	request, err := c.requestGenerator.CreateRequest(baggageclaim.ListVolumes, nil, nil)
	if err != nil {
		return nil, err
	}

	queryString := request.URL.Query()
	for key, val := range properties {
		queryString.Add(key, val)
	}

	request.URL.RawQuery = queryString.Encode()

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	if response.StatusCode != 200 {
		return nil, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return nil, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumesResponse []baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumesResponse)
	if err != nil {
		return nil, err
	}

	var volumes baggageclaim.Volumes
	for _, vr := range volumesResponse {
		v, initialHeartbeatSuccess := c.newVolume(logger, vr)
		if initialHeartbeatSuccess {
			volumes = append(volumes, v)
		}
	}

	return volumes, nil
}

func (c *client) LookupVolume(logger lager.Logger, handle string) (baggageclaim.Volume, bool, error) {
	volumeResponse, found, err := c.getVolumeResponse(logger, handle)
	if err != nil {
		return nil, false, err
	}
	if !found {
		return nil, found, nil
	}

	v, initialHeartbeatSuccess := c.newVolume(logger, volumeResponse)
	if !initialHeartbeatSuccess {
		return nil, false, nil
	}
	return v, true, nil
}

func (c *client) newVolume(logger lager.Logger, apiVolume baggageclaim.VolumeResponse) (baggageclaim.Volume, bool) {
	volume := &clientVolume{
		logger: logger,

		handle: apiVolume.Handle,
		path:   apiVolume.Path,

		bcClient:     c,
		heartbeating: new(sync.WaitGroup),
		release:      make(chan *time.Duration, 1),
	}

	if apiVolume.TTLInSeconds == 0 {
		return volume, true
	}

	initialHeartbeatSuccess := volume.startHeartbeating(logger, time.Duration(apiVolume.TTLInSeconds)*time.Second)

	return volume, initialHeartbeatSuccess
}

func (c *client) streamIn(logger lager.Logger, destHandle string, path string, tarContent io.Reader) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamIn, rata.Params{
		"handle": destHandle,
	}, tarContent)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()
	if response.StatusCode == http.StatusNoContent {
		return nil
	}
	return getError(response)
}

func (c *client) streamOut(logger lager.Logger, srcHandle string, path string) (io.ReadCloser, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.StreamOut, rata.Params{
		"handle": srcHandle,
	}, nil)

	request.URL.RawQuery = url.Values{"path": []string{path}}.Encode()
	if err != nil {
		return nil, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, getError(response)
	}

	return response.Body, nil
}

func getError(response *http.Response) error {
	var errorResponse *api.ErrorResponse
	err := json.NewDecoder(response.Body).Decode(&errorResponse)
	if err != nil {
		return err
	}

	if errorResponse.Message == api.ErrStreamOutNotFound.Error() {
		return baggageclaim.ErrFileNotFound
	}

	if response.StatusCode == 404 {
		return baggageclaim.ErrVolumeNotFound
	}

	return errors.New(errorResponse.Message)
}

func (c *client) getVolumeResponse(logger lager.Logger, handle string) (baggageclaim.VolumeResponse, bool, error) {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.GetVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		if response.StatusCode == http.StatusNotFound {
			return baggageclaim.VolumeResponse{}, false, nil
		}

		return baggageclaim.VolumeResponse{}, false, getError(response)
	}

	if header := response.Header.Get("Content-Type"); header != "application/json" {
		return baggageclaim.VolumeResponse{}, false, fmt.Errorf("unexpected content-type of: %s", header)
	}

	var volumeResponse baggageclaim.VolumeResponse
	err = json.NewDecoder(response.Body).Decode(&volumeResponse)
	if err != nil {
		return baggageclaim.VolumeResponse{}, false, err
	}

	return volumeResponse, true, nil
}

func (c *client) setTTL(logger lager.Logger, handle string, ttl time.Duration) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.TTLRequest{
		Value: uint(math.Ceil(ttl.Seconds())),
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetTTL, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) destroy(logger lager.Logger, handle string) error {
	request, err := c.requestGenerator.CreateRequest(baggageclaim.DestroyVolume, rata.Params{
		"handle": handle,
	}, nil)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setPrivileged(logger lager.Logger, handle string, privileged bool) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PrivilegedRequest{
		Value: privileged,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetPrivileged, rata.Params{
		"handle": handle,
	}, buffer)
	if err != nil {
		return err
	}

	request.Header.Add("Content-type", "application/json")

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}

func (c *client) setProperty(logger lager.Logger, handle string, propertyName string, propertyValue string) error {
	buffer := &bytes.Buffer{}
	json.NewEncoder(buffer).Encode(baggageclaim.PropertyRequest{
		Value: propertyValue,
	})

	request, err := c.requestGenerator.CreateRequest(baggageclaim.SetProperty, rata.Params{
		"handle":   handle,
		"property": propertyName,
	}, buffer)
	if err != nil {
		return err
	}

	response, err := c.httpClient(logger).Do(request)
	if err != nil {
		return err
	}

	defer response.Body.Close()

	if response.StatusCode != 204 {
		return getError(response)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"github.com/simonz05/godis"
)

const HELP string = `
<!DOCTYPE html>

        <title>cdrv.ws</title>

cdrvws(1)                          CDRV.WS                          cdrvws(1)

NAME
    cdrvws: command line url shortener

SYNOPSIS
    <command> | curl -F 'rvw=<-' http://cdrv.ws

EXAMPLE
    ~$ echo "http://ebushpilot.com/images/polarbear_1.jpg" | curl -F 'rvw=<-' http://cdrv.ws
    http://cdrv.ws/2
    ~$ open http://cdrv.ws/2

SEE ALSO
    http://github.com/JustinTulloss/cdrvws

CREDITS
    Inspired by sprunge:
    http://github.com/rupa/sprunge

`

const CHARS string = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const BASE uint64 = uint64(len(CHARS))

var redis *godis.Client

func main() {
	connectToRedis()
	http.HandleFunc("/", route)
	startServer()
}

func encode(id uint64) string {
	encoded := ""
	for id > 0 {
		encoded += string(CHARS[id%BASE])
		id = id / BASE
	}
	return encoded
}

func createShortUrl(longurl string) (error, string) {
	shortid, err := redis.Incr("urlId")
	if err != nil {
		return err, ""
	}
	shorturl := encode(uint64(shortid))
	if err := redis.Set(shorturl, longurl); err != nil {
		return err, ""
	}
	return nil, shorturl
}

func expand(shorturl string) (error, string) {
	longurl, err := redis.Get(shorturl)
	if err != nil && err.Error() == "Nonexisting key" {
		return nil, ""
	} else if err != nil {
		return err, ""
	}
	return nil, longurl.String()
}

func connectToRedis() {
	rawurl := os.Getenv("REDISTOGO_URL")
	log.Printf("Redis to go url: %s\n", rawurl)
	redisurl := url.URL{
		Host: "localhost:6379",
		User: url.UserPassword("", ""),
	}
	parsedurl := &redisurl
	if rawurl != "" {
		var err error
		parsedurl, err = parsedurl.Parse(rawurl)
		if err != nil {
			log.Fatal("Could not parse redis url", err)
		}
	}
	password, _ := parsedurl.User.Password()
	log.Printf("Connecting to redis: '%s' with password '%s'\n", parsedurl.Host, password)
	redis = godis.New("tcp:" + parsedurl.Host, 0, password)
}

func startServer() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Printf("Starting on %s\n", port)
	err := http.ListenAndServe(":" + port, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func route(w http.ResponseWriter, req *http.Request) {
	if req.Method == "GET" {
		if req.URL.String() == "/" {
			handleHome(w, req)
		} else {
			handleExpand(w, req)
		}
	} else if req.Method == "POST" {
		handleShorten(w, req)
	}
}

func handleHome(w http.ResponseWriter, req *http.Request) {
	fmt.Fprintln(w, HELP)
}

func handleShorten(w http.ResponseWriter, req *http.Request) {
	err, shorturl := createShortUrl(req.FormValue("rvw"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fullurl := url.URL{
		Scheme: "http",
		Host:   req.Host,
		Path:   "/" + shorturl,
	}
	fmt.Fprintln(w, fullurl.String())
}

func handleExpand(w http.ResponseWriter, req *http.Request) {
	shorturl := strings.Trim(req.URL.String(), "/")
	err, longurl := expand(shorturl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else if longurl == "" {
		http.NotFound(w, req)
	} else {
		http.Redirect(w, req, longurl, http.StatusMovedPermanently)
	}
}
<commit_msg>manpage is just text<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"github.com/simonz05/godis"
)

const HELP string = `cdrvws(1)                          CDRV.WS                          cdrvws(1)

NAME
    cdrvws: command line url shortener

SYNOPSIS
    <command> | curl -F 'rvw=<-' http://cdrv.ws

EXAMPLE
    ~$ echo "http://ebushpilot.com/images/polarbear_1.jpg" | curl -F 'rvw=<-' http://cdrv.ws
    http://cdrv.ws/2
    ~$ open http://cdrv.ws/2

SEE ALSO
    http://github.com/JustinTulloss/cdrvws

CREDITS
    Inspired by sprunge: http://github.com/rupa/sprunge
`

const CHARS string = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const BASE uint64 = uint64(len(CHARS))

var redis *godis.Client

func main() {
	connectToRedis()
	http.HandleFunc("/", route)
	startServer()
}

func encode(id uint64) string {
	encoded := ""
	for id > 0 {
		encoded += string(CHARS[id%BASE])
		id = id / BASE
	}
	return encoded
}

func createShortUrl(longurl string) (error, string) {
	shortid, err := redis.Incr("urlId")
	if err != nil {
		return err, ""
	}
	shorturl := encode(uint64(shortid))
	if err := redis.Set(shorturl, longurl); err != nil {
		return err, ""
	}
	return nil, shorturl
}

func expand(shorturl string) (error, string) {
	longurl, err := redis.Get(shorturl)
	if err != nil && err.Error() == "Nonexisting key" {
		return nil, ""
	} else if err != nil {
		return err, ""
	}
	return nil, longurl.String()
}

func connectToRedis() {
	rawurl := os.Getenv("REDISTOGO_URL")
	log.Printf("Redis to go url: %s\n", rawurl)
	redisurl := url.URL{
		Host: "localhost:6379",
		User: url.UserPassword("", ""),
	}
	parsedurl := &redisurl
	if rawurl != "" {
		var err error
		parsedurl, err = parsedurl.Parse(rawurl)
		if err != nil {
			log.Fatal("Could not parse redis url", err)
		}
	}
	password, _ := parsedurl.User.Password()
	log.Printf("Connecting to redis: '%s' with password '%s'\n", parsedurl.Host, password)
	redis = godis.New("tcp:" + parsedurl.Host, 0, password)
}

func startServer() {
	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Printf("Starting on %s\n", port)
	err := http.ListenAndServe(":" + port, nil)
	if err != nil {
		log.Fatal("ListenAndServe:", err)
	}
}

func route(w http.ResponseWriter, req *http.Request) {
	if req.Method == "GET" {
		if req.URL.String() == "/" {
			handleHome(w, req)
		} else {
			handleExpand(w, req)
		}
	} else if req.Method == "POST" {
		handleShorten(w, req)
	}
}

func handleHome(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	fmt.Fprintln(w, HELP)
}

func handleShorten(w http.ResponseWriter, req *http.Request) {
	err, shorturl := createShortUrl(req.FormValue("rvw"))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fullurl := url.URL{
		Scheme: "http",
		Host:   req.Host,
		Path:   "/" + shorturl,
	}
	fmt.Fprintln(w, fullurl.String())
}

func handleExpand(w http.ResponseWriter, req *http.Request) {
	shorturl := strings.Trim(req.URL.String(), "/")
	err, longurl := expand(shorturl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else if longurl == "" {
		http.NotFound(w, req)
	} else {
		http.Redirect(w, req, longurl, http.StatusMovedPermanently)
	}
}
<|endoftext|>


<commit_before>package tmspcli

import (
	"bufio"
	"container/list"
	"errors"
	"fmt"
	"net"
	"sync"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/tmsp/types"
)

const maxResponseSize = 1048576 // 1MB TODO make configurable
const flushThrottleMS = 20      // Don't wait longer than...

type Callback func(*types.Request, *types.Response)

// This is goroutine-safe, but users should beware that
// the application in general is not meant to be interfaced
// with concurrent callers.
type TMSPClient struct {
	QuitService
	sync.Mutex // [EB]: is this even used?

	reqQueue   chan *reqRes
	flushTimer *ThrottleTimer

	mtx       sync.Mutex
	conn      net.Conn
	bufWriter *bufio.Writer
	err       error
	reqSent   *list.List
	resCb     func(*types.Request, *types.Response)
}

func NewTMSPClient(conn net.Conn, bufferSize int) *TMSPClient {
	cli := &TMSPClient{
		reqQueue:   make(chan *reqRes, bufferSize),
		flushTimer: NewThrottleTimer("TMSPClient", flushThrottleMS),

		conn:      conn,
		bufWriter: bufio.NewWriter(conn),
		reqSent:   list.New(),
		resCb:     nil,
	}
	cli.QuitService = *NewQuitService(nil, "TMSPClient", cli)
	cli.Start() // Just start it, it's confusing for callers to remember to start.
	return cli
}

func (cli *TMSPClient) OnStart() error {
	cli.QuitService.OnStart()
	go cli.sendRequestsRoutine()
	go cli.recvResponseRoutine()
	return nil
}

func (cli *TMSPClient) OnStop() {
	cli.QuitService.OnStop()
	cli.conn.Close()
}

// NOTE: callback may get internally generated flush responses.
func (cli *TMSPClient) SetResponseCallback(resCb Callback) {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	cli.resCb = resCb
}

func (cli *TMSPClient) StopForError(err error) {
	cli.mtx.Lock()
	// log.Error("Stopping TMSPClient for error.", "error", err)
	if cli.err == nil {
		cli.err = err
	}
	cli.mtx.Unlock()
	cli.Stop()
}

func (cli *TMSPClient) Error() error {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	return cli.err
}

//----------------------------------------

func (cli *TMSPClient) sendRequestsRoutine() {
	for {
		select {
		case <-cli.flushTimer.Ch:
			select {
			case cli.reqQueue <- newReqRes(types.RequestFlush()):
			default:
				// Probably will fill the buffer, or retry later.
			}
		case <-cli.QuitService.Quit:
			return
		case reqres := <-cli.reqQueue:
			cli.willSendReq(reqres)
			err := types.WriteMessage(reqres.Request, cli.bufWriter)
			if err != nil {
				cli.StopForError(err)
				return
			}
			// log.Debug("Sent request", "requestType", reflect.TypeOf(reqres.Request), "request", reqres.Request)
			if reqres.Request.Type == types.MessageType_Flush {
				err = cli.bufWriter.Flush()
				if err != nil {
					cli.StopForError(err)
					return
				}
			}
		}
	}
}

func (cli *TMSPClient) recvResponseRoutine() {
	r := bufio.NewReader(cli.conn) // Buffer reads
	for {
		var res = &types.Response{}
		err := types.ReadMessage(r, res)
		if err != nil {
			cli.StopForError(err)
			return
		}
		switch res.Type {
		case types.MessageType_Exception:
			// XXX After setting cli.err, release waiters (e.g. reqres.Done())
			cli.StopForError(errors.New(res.Error))
		default:
			// log.Debug("Received response", "responseType", reflect.TypeOf(res), "response", res)
			err := cli.didRecvResponse(res)
			if err != nil {
				cli.StopForError(err)
			}
		}
	}
}

func (cli *TMSPClient) willSendReq(reqres *reqRes) {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	cli.reqSent.PushBack(reqres)
}

func (cli *TMSPClient) didRecvResponse(res *types.Response) error {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()

	// Get the first reqRes
	next := cli.reqSent.Front()
	if next == nil {
		return fmt.Errorf("Unexpected result type %v when nothing expected", res.Type)
	}
	reqres := next.Value.(*reqRes)
	if !resMatchesReq(reqres.Request, res) {
		return fmt.Errorf("Unexpected result type %v when response to %v expected",
			res.Type, reqres.Request.Type)
	}

	reqres.Response = res    // Set response
	reqres.Done()            // Release waiters
	cli.reqSent.Remove(next) // Pop first item from linked list

	// Callback if there is a listener
	if cli.resCb != nil {
		cli.resCb(reqres.Request, res)
	}

	return nil
}

//----------------------------------------

func (cli *TMSPClient) EchoAsync(msg string) {
	cli.queueRequest(types.RequestEcho(msg))
}

func (cli *TMSPClient) FlushAsync() {
	cli.queueRequest(types.RequestFlush())
}

func (cli *TMSPClient) SetOptionAsync(key string, value string) {
	cli.queueRequest(types.RequestSetOption(key, value))
}

func (cli *TMSPClient) AppendTxAsync(tx []byte) {
	cli.queueRequest(types.RequestAppendTx(tx))
}

func (cli *TMSPClient) CheckTxAsync(tx []byte) {
	cli.queueRequest(types.RequestCheckTx(tx))
}

func (cli *TMSPClient) GetHashAsync() {
	cli.queueRequest(types.RequestGetHash())
}

func (cli *TMSPClient) QueryAsync(query []byte) {
	cli.queueRequest(types.RequestQuery(query))
}

//----------------------------------------

func (cli *TMSPClient) InfoSync() (info string, err error) {
	reqres := cli.queueRequest(types.RequestInfo())
	cli.FlushSync()
	if cli.err != nil {
		return "", cli.err
	}
	return string(reqres.Response.Data), nil
}

func (cli *TMSPClient) FlushSync() error {
	cli.queueRequest(types.RequestFlush()).Wait()
	return cli.err
}

func (cli *TMSPClient) AppendTxSync(tx []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestAppendTx(tx))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

func (cli *TMSPClient) CheckTxSync(tx []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestCheckTx(tx))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

func (cli *TMSPClient) GetHashSync() (hash []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestGetHash())
	cli.FlushSync()
	if cli.err != nil {
		return nil, "", cli.err
	}
	res := reqres.Response
	return res.Data, res.Log, nil
}

func (cli *TMSPClient) QuerySync(query []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestQuery(query))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

//----------------------------------------

func (cli *TMSPClient) queueRequest(req *types.Request) *reqRes {
	reqres := newReqRes(req)
	// TODO: set cli.err if reqQueue times out
	cli.reqQueue <- reqres

	// Maybe auto-flush, or unset auto-flush
	switch req.Type {
	case types.MessageType_Flush:
		cli.flushTimer.Unset()
	default:
		cli.flushTimer.Set()
	}

	return reqres
}

//----------------------------------------

func resMatchesReq(req *types.Request, res *types.Response) (ok bool) {
	return req.Type == res.Type
}

type reqRes struct {
	*types.Request
	*sync.WaitGroup
	*types.Response // Not set atomically, so be sure to use WaitGroup.
}

func newReqRes(req *types.Request) *reqRes {
	return &reqRes{
		Request:   req,
		WaitGroup: waitGroup1(),
		Response:  nil,
	}
}

func waitGroup1() (wg *sync.WaitGroup) {
	wg = &sync.WaitGroup{}
	wg.Add(1)
	return
}
<commit_msg>Callbacks for ReqRes<commit_after>package tmspcli

import (
	"bufio"
	"container/list"
	"errors"
	"fmt"
	"net"
	"sync"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/tmsp/types"
)

const reqQueueSize = 256        // TODO make configurable
const maxResponseSize = 1048576 // 1MB TODO make configurable
const flushThrottleMS = 20      // Don't wait longer than...

type Callback func(*types.Request, *types.Response)

// This is goroutine-safe, but users should beware that
// the application in general is not meant to be interfaced
// with concurrent callers.
type TMSPClient struct {
	QuitService
	sync.Mutex // [EB]: is this even used?

	reqQueue   chan *ReqRes
	flushTimer *ThrottleTimer

	mtx       sync.Mutex
	addr      string
	conn      net.Conn
	bufWriter *bufio.Writer
	err       error
	reqSent   *list.List
	resCb     func(*types.Request, *types.Response) // listens to all callbacks
}

func NewTMSPClient(addr string) (*TMSPClient, error) {
	conn, err := Connect(addr)
	if err != nil {
		return nil, err
	}
	cli := &TMSPClient{
		reqQueue:   make(chan *ReqRes, reqQueueSize),
		flushTimer: NewThrottleTimer("TMSPClient", flushThrottleMS),

		conn:      conn,
		bufWriter: bufio.NewWriter(conn),
		reqSent:   list.New(),
		resCb:     nil,
	}
	cli.QuitService = *NewQuitService(nil, "TMSPClient", cli)
	cli.Start() // Just start it, it's confusing for callers to remember to start.
	return cli, nil
}

func (cli *TMSPClient) OnStart() error {
	cli.QuitService.OnStart()
	go cli.sendRequestsRoutine()
	go cli.recvResponseRoutine()
	return nil
}

func (cli *TMSPClient) OnStop() {
	cli.QuitService.OnStop()
	cli.conn.Close()
}

// Set listener for all responses
// NOTE: callback may get internally generated flush responses.
func (cli *TMSPClient) SetResponseCallback(resCb Callback) {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	cli.resCb = resCb
}

func (cli *TMSPClient) StopForError(err error) {
	cli.mtx.Lock()
	// log.Error("Stopping TMSPClient for error.", "error", err)
	if cli.err == nil {
		cli.err = err
	}
	cli.mtx.Unlock()
	cli.Stop()
}

func (cli *TMSPClient) Error() error {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	return cli.err
}

//----------------------------------------

func (cli *TMSPClient) sendRequestsRoutine() {
	for {
		select {
		case <-cli.flushTimer.Ch:
			select {
			case cli.reqQueue <- newReqRes(types.RequestFlush()):
			default:
				// Probably will fill the buffer, or retry later.
			}
		case <-cli.QuitService.Quit:
			return
		case reqres := <-cli.reqQueue:
			cli.willSendReq(reqres)
			err := types.WriteMessage(reqres.Request, cli.bufWriter)
			if err != nil {
				cli.StopForError(err)
				return
			}
			// log.Debug("Sent request", "requestType", reflect.TypeOf(reqres.Request), "request", reqres.Request)
			if reqres.Request.Type == types.MessageType_Flush {
				err = cli.bufWriter.Flush()
				if err != nil {
					cli.StopForError(err)
					return
				}
			}
		}
	}
}

func (cli *TMSPClient) recvResponseRoutine() {
	r := bufio.NewReader(cli.conn) // Buffer reads
	for {
		var res = &types.Response{}
		err := types.ReadMessage(r, res)
		if err != nil {
			cli.StopForError(err)
			return
		}
		switch res.Type {
		case types.MessageType_Exception:
			// XXX After setting cli.err, release waiters (e.g. reqres.Done())
			cli.StopForError(errors.New(res.Error))
		default:
			// log.Debug("Received response", "responseType", reflect.TypeOf(res), "response", res)
			err := cli.didRecvResponse(res)
			if err != nil {
				cli.StopForError(err)
			}
		}
	}
}

func (cli *TMSPClient) willSendReq(reqres *ReqRes) {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()
	cli.reqSent.PushBack(reqres)
}

func (cli *TMSPClient) didRecvResponse(res *types.Response) error {
	cli.mtx.Lock()
	defer cli.mtx.Unlock()

	// Get the first ReqRes
	next := cli.reqSent.Front()
	if next == nil {
		return fmt.Errorf("Unexpected result type %v when nothing expected", res.Type)
	}
	reqres := next.Value.(*ReqRes)
	if !resMatchesReq(reqres.Request, res) {
		return fmt.Errorf("Unexpected result type %v when response to %v expected",
			res.Type, reqres.Request.Type)
	}

	reqres.Response = res    // Set response
	reqres.Done()            // Release waiters
	cli.reqSent.Remove(next) // Pop first item from linked list

	// Notify reqRes listener if set
	if cb := reqres.GetCallback(); cb != nil {
		cb(res)
	}

	// Notify client listener if set
	if cli.resCb != nil {
		cli.resCb(reqres.Request, res)
	}

	return nil
}

//----------------------------------------

func (cli *TMSPClient) EchoAsync(msg string) *ReqRes {
	return cli.queueRequest(types.RequestEcho(msg))
}

func (cli *TMSPClient) FlushAsync() *ReqRes {
	return cli.queueRequest(types.RequestFlush())
}

func (cli *TMSPClient) SetOptionAsync(key string, value string) *ReqRes {
	return cli.queueRequest(types.RequestSetOption(key, value))
}

func (cli *TMSPClient) AppendTxAsync(tx []byte) *ReqRes {
	return cli.queueRequest(types.RequestAppendTx(tx))
}

func (cli *TMSPClient) CheckTxAsync(tx []byte) *ReqRes {
	return cli.queueRequest(types.RequestCheckTx(tx))
}

func (cli *TMSPClient) GetHashAsync() *ReqRes {
	return cli.queueRequest(types.RequestGetHash())
}

func (cli *TMSPClient) QueryAsync(query []byte) *ReqRes {
	return cli.queueRequest(types.RequestQuery(query))
}

//----------------------------------------

func (cli *TMSPClient) InfoSync() (info string, err error) {
	reqres := cli.queueRequest(types.RequestInfo())
	cli.FlushSync()
	if cli.err != nil {
		return "", cli.err
	}
	return string(reqres.Response.Data), nil
}

func (cli *TMSPClient) FlushSync() error {
	cli.queueRequest(types.RequestFlush()).Wait()
	return cli.err
}

func (cli *TMSPClient) AppendTxSync(tx []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestAppendTx(tx))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

func (cli *TMSPClient) CheckTxSync(tx []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestCheckTx(tx))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

func (cli *TMSPClient) GetHashSync() (hash []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestGetHash())
	cli.FlushSync()
	if cli.err != nil {
		return nil, "", cli.err
	}
	res := reqres.Response
	return res.Data, res.Log, nil
}

func (cli *TMSPClient) QuerySync(query []byte) (code types.CodeType, result []byte, log string, err error) {
	reqres := cli.queueRequest(types.RequestQuery(query))
	cli.FlushSync()
	if cli.err != nil {
		return types.CodeType_InternalError, nil, "", cli.err
	}
	res := reqres.Response
	return res.Code, res.Data, res.Log, nil
}

//----------------------------------------

func (cli *TMSPClient) queueRequest(req *types.Request) *ReqRes {
	reqres := newReqRes(req)
	// TODO: set cli.err if reqQueue times out
	cli.reqQueue <- reqres

	// Maybe auto-flush, or unset auto-flush
	switch req.Type {
	case types.MessageType_Flush:
		cli.flushTimer.Unset()
	default:
		cli.flushTimer.Set()
	}

	return reqres
}

//----------------------------------------

func resMatchesReq(req *types.Request, res *types.Response) (ok bool) {
	return req.Type == res.Type
}

type ReqRes struct {
	*types.Request
	*sync.WaitGroup
	*types.Response // Not set atomically, so be sure to use WaitGroup.

	mtx  sync.Mutex
	done bool                  // Gets set to true once *after* WaitGroup.Done().
	cb   func(*types.Response) // A single callback that may be set.
}

func newReqRes(req *types.Request) *ReqRes {
	return &ReqRes{
		Request:   req,
		WaitGroup: waitGroup1(),
		Response:  nil,

		done: false,
		cb:   nil,
	}
}

// Sets the callback for this ReqRes atomically.
// If reqRes is already done, calls cb immediately.
// NOTE: reqRes.cb should not change if reqRes.done.
// NOTE: only one callback is supported.
func (reqRes *ReqRes) SetCallback(cb func(res *types.Response)) {
	reqRes.mtx.Lock()

	if reqRes.done {
		reqRes.mtx.Unlock()
		cb(reqRes.Response)
		return
	}

	defer reqRes.mtx.Unlock()
	reqRes.cb = cb
}

func (reqRes *ReqRes) GetCallback() func(*types.Response) {
	reqRes.mtx.Lock()
	defer reqRes.mtx.Unlock()
	return reqRes.cb
}

// NOTE: it should be safe to read reqRes.cb without locks after this.
func (reqRes *ReqRes) SetDone() {
	reqRes.mtx.Lock()
	reqRes.done = true
	reqRes.mtx.Unlock()
}

func waitGroup1() (wg *sync.WaitGroup) {
	wg = &sync.WaitGroup{}
	wg.Add(1)
	return
}
<|endoftext|>


<commit_before>package stick_test

import (
	"os"

	"fmt"

	"path/filepath"

	"github.com/tyler-sommer/stick"
)

// ExampleEnv_Execute_fs shows an example of using the provided
// FilesystemLoader.
//
// This example makes use of templates in the testdata folder. In
// particular, this example shows vertical (via extends) and horizontal
// reuse (via use).
func ExampleEnv_Execute_fs() {
	d, _ := os.Getwd()
	env := stick.NewEnv(stick.NewFilesystemLoader(filepath.Join(d, "testdata")))

	params := map[string]stick.Value{"name": "World"}
	err := env.Execute("main.txt.twig", os.Stdout, params)
	if err != nil {
		fmt.Println(err)
	}
	// Output:
	// This is a document.
	//
	// Hello
	//
	// An introduction to the topic.
	//
	// The body of this topic.
	//
	// Another section
	//
	// Some extra information.
	//
	// Still nobody knows.
	//
	// Some kind of footer.
}
<commit_msg>Another formatting fix.<commit_after>package stick_test

import (
	"os"

	"fmt"

	"path/filepath"

	"github.com/tyler-sommer/stick"
)

// An example showing the use of the provided FilesystemLoader.
//
// This example makes use of templates in the testdata folder. In
// particular, this example shows vertical (via extends) and horizontal
// reuse (via use).
func ExampleEnv_Execute_filesystemLoader() {
	d, _ := os.Getwd()
	env := stick.NewEnv(stick.NewFilesystemLoader(filepath.Join(d, "testdata")))

	params := map[string]stick.Value{"name": "World"}
	err := env.Execute("main.txt.twig", os.Stdout, params)
	if err != nil {
		fmt.Println(err)
	}
	// Output:
	// This is a document.
	//
	// Hello
	//
	// An introduction to the topic.
	//
	// The body of this topic.
	//
	// Another section
	//
	// Some extra information.
	//
	// Still nobody knows.
	//
	// Some kind of footer.
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	ctx "context"
	"io"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"
)

// signalHandler. Listens for incoming SIGTERM or SIGUSR1 signals. If we get
// SIGTERM, stop listening for new connections and gracefully terminate the
// process. If we get SIGUSR1, reload certificates.
func (context *Context) signalHandler(proxy *proxy, closeables []io.Closer) {
	signals := make(chan os.Signal)
	signal.Notify(signals, syscall.SIGUSR1, syscall.SIGTERM, syscall.SIGINT, syscall.SIGCHLD)
	defer signal.Stop(signals)

	for {
		// Wait for a signal
		select {
		case sig := <-signals:
			switch sig {
			case syscall.SIGINT, syscall.SIGTERM, syscall.SIGCHLD:
				logger.Printf("received %s, shutting down", sig.String())

				// Best-effort graceful shutdown of status listener
				if context.statusHTTP != nil {
					go context.statusHTTP.Shutdown(ctx.Background())
				}

				// Force-exit after timeout
				time.AfterFunc(*shutdownTimeout, func() {
					// Graceful shutdown timeout reached. If we can't drain connections
					// to exit gracefully after this timeout, let's just exit.
					logger.Printf("graceful shutdown timeout: forcing exit")
					exitFunc(1)
				})

				atomic.StoreInt32(&proxy.quit, 1)
				for _, closeable := range closeables {
					closeable.Close()
				}

				logger.Printf("shutdown proxy, waiting for drain")
				return

			case syscall.SIGUSR1:
				logger.Printf("received %s, reloading certificates", sig.String())
				context.status.Reloading()
				err := context.cert.reload()
				if err != nil {
					logger.Printf("error reloading certificates: %s", err)
				}
				logger.Printf("reloading complete")
				context.status.Listening()
			}
		case _ = <-context.watcher:
			context.status.Reloading()
			err := context.cert.reload()
			if err != nil {
				logger.Printf("error reloading certificates: %s", err)
			}
			logger.Printf("reloading complete")
			context.status.Listening()
		}
	}
}
<commit_msg>Channel for signal.Notify should be buffered (as per os/signal doc)<commit_after>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	ctx "context"
	"io"
	"os"
	"os/signal"
	"sync/atomic"
	"syscall"
	"time"
)

// signalHandler. Listens for incoming SIGTERM or SIGUSR1 signals. If we get
// SIGTERM, stop listening for new connections and gracefully terminate the
// process. If we get SIGUSR1, reload certificates.
func (context *Context) signalHandler(proxy *proxy, closeables []io.Closer) {
	signals := make(chan os.Signal, 3)
	signal.Notify(signals, syscall.SIGUSR1, syscall.SIGTERM, syscall.SIGINT)
	defer signal.Stop(signals)

	for {
		// Wait for a signal
		select {
		case sig := <-signals:
			switch sig {
			case syscall.SIGINT, syscall.SIGTERM, syscall.SIGCHLD:
				logger.Printf("received %s, shutting down", sig.String())

				// Best-effort graceful shutdown of status listener
				if context.statusHTTP != nil {
					go context.statusHTTP.Shutdown(ctx.Background())
				}

				// Force-exit after timeout
				time.AfterFunc(*shutdownTimeout, func() {
					// Graceful shutdown timeout reached. If we can't drain connections
					// to exit gracefully after this timeout, let's just exit.
					logger.Printf("graceful shutdown timeout: forcing exit")
					exitFunc(1)
				})

				atomic.StoreInt32(&proxy.quit, 1)
				for _, closeable := range closeables {
					closeable.Close()
				}

				logger.Printf("shutdown proxy, waiting for drain")
				return

			case syscall.SIGUSR1:
				logger.Printf("received %s, reloading certificates", sig.String())
				context.status.Reloading()
				err := context.cert.reload()
				if err != nil {
					logger.Printf("error reloading certificates: %s", err)
				}
				logger.Printf("reloading complete")
				context.status.Listening()
			}
		case <-context.watcher:
			context.status.Reloading()
			err := context.cert.reload()
			if err != nil {
				logger.Printf("error reloading certificates: %s", err)
			}
			logger.Printf("reloading complete")
			context.status.Listening()
		}
	}
}
<|endoftext|>