<commit_before>package topic

import (
	"container/list"
	"fmt"
	"io"

	"gopkg.in/logex.v1"

	"github.com/chzyer/mmq/internal/bitmap"
	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

const (
	MaxBenchSize = 100
)

var (
	ErrBenchSizeTooLarge = logex.Define("bench size is too large")
)

type Config struct {
	Root      string
	IndexName string
	ChunkBit  uint
}

func (c *Config) Path(name string) string {
	return fmt.Sprintf("%s/%s", c.Root, name)
}

type Instance struct {
	config *Config
	Name   string
	index  int
	file   *bitmap.File
	writer *utils.Writer

	// linked list for Waiters
	waiterList *list.List
	newMsgChan chan struct{}
}

func New(name string, config *Config) (t *Instance, err error) {
	t = &Instance{
		config:     config,
		Name:       name,
		newMsgChan: make(chan struct{}, 1),
	}
	path := config.Path(t.TopicPath())
	t.file, err = bitmap.NewFileEx(path, config.ChunkBit)
	if err != nil {
		return nil, logex.Trace(err)
	}
	// TODO: must read some metafile to determine the offset
	t.writer = &utils.Writer{t.file, 0}
	return t, nil
}

func (t *Instance) TopicPath() string {
	return utils.PathEncode(t.Name)
}

func (t *Instance) Put(m *mmq.Message) error {
	_, err := m.WriteTo(t.writer)
	select {
	case t.newMsgChan <- struct{}{}:
	default:
	}
	return logex.Trace(err)
}

func (t *Instance) get(offset int64, size int, msgChan chan<- []*mmq.Message) ([]*mmq.Message, error) {
	if size > MaxBenchSize {
		return nil, ErrBenchSizeTooLarge.Trace()
	}

	msgs := make([]*mmq.Message, size)
	var (
		msg *mmq.Message
		err error
	)

	var header mmq.HeaderBin

	// check offset
	r := &utils.Reader{t.file, offset}
	p := 0
	for i := 0; i < size; i++ {
		msg, err = mmq.ReadMessage(&header, r, mmq.RF_RESEEK_ON_FAULT)
		if logex.Equal(err, io.ErrUnexpectedEOF) || logex.Equal(err, io.EOF) {
			// use chan, to subscribe
			break
		}
		if err != nil {
			return msgs[:p], logex.Trace(err)
		}
		msgs[p] = msg
		p++
	}

	return msgs[:p], nil
}
<commit_msg>add ioloop for topic instance<commit_after>package topic

import (
	"container/list"
	"fmt"
	"io"

	"gopkg.in/logex.v1"

	"time"

	"github.com/chzyer/mmq/internal/bitmap"
	"github.com/chzyer/mmq/internal/utils"
	"github.com/chzyer/mmq/mmq"
)

const (
	MaxBenchSize = 100
)

var (
	ErrBenchSizeTooLarge = logex.Define("bench size is too large")
)

type Config struct {
	Root      string
	IndexName string
	ChunkBit  uint
}

func (c *Config) Path(name string) string {
	return fmt.Sprintf("%s/%s", c.Root, name)
}

type Instance struct {
	config *Config
	Name   string
	index  int
	file   *bitmap.File
	writer *utils.Writer

	// linked list for Waiters
	waiterList *list.List
	newMsgChan chan struct{}
	putChan    chan *putArgs
	getChan    chan *getArgs
}

func New(name string, config *Config) (t *Instance, err error) {
	t = &Instance{
		config:     config,
		Name:       name,
		newMsgChan: make(chan struct{}, 1),
		putChan:    make(chan *putArgs, 1<<3),
	}
	path := config.Path(t.TopicPath())
	t.file, err = bitmap.NewFileEx(path, config.ChunkBit)
	if err != nil {
		return nil, logex.Trace(err)
	}
	// TODO: must read some metafile to determine the offset
	t.writer = &utils.Writer{t.file, 0}
	go t.ioLoop()
	return t, nil
}

func (t *Instance) TopicPath() string {
	return utils.PathEncode(t.Name)
}

func (t *Instance) notifyNewMsg() {
	select {
	case t.newMsgChan <- struct{}{}:
	default:
	}
}

func (t *Instance) ioLoop() {
	var (
		put *putArgs
		get *getArgs
		ok  bool

		timer = time.NewTimer(0)
	)
	for {
		select {
		case put, ok = <-t.putChan:
			if !ok {
				goto exit
			}
			t.put(put, timer)
		case get, ok = <-t.getChan:
			if !ok {
				goto exit
			}
			t.getAsync(get, timer)
		}
	}
exit:
}

type putArgs struct {
	msgs  []*mmq.Message
	reply chan<- []error
}

func (t *Instance) Put(msgs []*mmq.Message, reply chan []error) {
	t.putChan <- &putArgs{msgs, reply}
}

func (t *Instance) put(arg *putArgs, timer *time.Timer) {
	errs := make([]error, len(arg.msgs))
	for i := 0; i < len(arg.msgs); i++ {
		_, errs[i] = arg.msgs[i].WriteTo(t.writer)
	}
	t.notifyNewMsg()
	timer.Reset(time.Second)
	select {
	case arg.reply <- errs:
	case <-timer.C:
		logex.Error("write reply timeout")
	}
}

type getArgs struct {
	offset int64
	size   int
	reply  chan<- []*mmq.Message
	err    chan<- error
}

func (t *Instance) Get(offset int64, size int, reply chan<- []*mmq.Message, err chan<- error) {
	t.getChan <- &getArgs{offset, size, reply, err}
}

func (t *Instance) getAsync(arg *getArgs, timer *time.Timer) {
	err := t.get(arg)
	timer.Reset(time.Second)
	select {
	case arg.err <- err:
	case <-timer.C:
		logex.Error("reply to get timeout")
	}
}

func (t *Instance) get(arg *getArgs) error {
	if arg.size > MaxBenchSize {
		return ErrBenchSizeTooLarge.Trace()
	}

	msgs := make([]*mmq.Message, arg.size)
	var (
		msg *mmq.Message
		err error
	)

	var header mmq.HeaderBin

	// check offset
	r := &utils.Reader{t.file, arg.offset}
	p := 0
	for i := 0; i < arg.size; i++ {
		msg, err = mmq.ReadMessage(&header, r, mmq.RF_RESEEK_ON_FAULT)
		if logex.Equal(err, io.ErrUnexpectedEOF) || logex.Equal(err, io.EOF) {
			// use chan, to subscribe
			break
		}
		if err != nil {
			break
		}
		msgs[p] = msg
		p++
	}

	select {
	case arg.reply <- msgs[:p]:
	}
	return logex.Trace(err)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/ziegfried/poker-player-go/player"
)

func main() {
	PORT, err := strconv.Atoi(os.Getenv("PORT"))
	if err != nil {
		PORT = 4711
	}

	http.HandleFunc("/", handleRequest)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", PORT), nil); err != nil {
		log.Fatal(err)
	}
}

func handleRequest(w http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Printf("Error parsing form data: %s", err)
		http.Error(w, "Internal Server Error", 500)
		return
	}
	action := request.FormValue("action")
	log.Printf("Request method=%s url=%s action=%s from client=%s\n", request.Method, request.URL, action, request.RemoteAddr)
	switch action {
	case "check":
		fmt.Fprint(w, "")
		return
	case "bet_request":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		result := player.BetRequest(gameState)
		fmt.Fprintf(w, "%d", result)
		return
	case "showdown":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		player.Showdown(gameState)
		fmt.Fprint(w, "")
		return
	case "version":
		fmt.Fprint(w, player.Version())
		return
	default:
		http.Error(w, "Invalid action", 400)
	}
}

func parseGameState(stateStr string) *player.GameState {
	stateBytes := []byte(stateStr)
	gameState := new(player.GameState)
	if err := json.Unmarshal(stateBytes, &gameState); err != nil {
		log.Printf("Error parsing game state: %s", err)
		return nil
	}
	return gameState
}
<commit_msg>Update import path<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"

	"github.com/ferki/poker-player-go/player"
)

func main() {
	PORT, err := strconv.Atoi(os.Getenv("PORT"))
	if err != nil {
		PORT = 4711
	}

	http.HandleFunc("/", handleRequest)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", PORT), nil); err != nil {
		log.Fatal(err)
	}
}

func handleRequest(w http.ResponseWriter, request *http.Request) {
	if err := request.ParseForm(); err != nil {
		log.Printf("Error parsing form data: %s", err)
		http.Error(w, "Internal Server Error", 500)
		return
	}
	action := request.FormValue("action")
	log.Printf("Request method=%s url=%s action=%s from client=%s\n", request.Method, request.URL, action, request.RemoteAddr)
	switch action {
	case "check":
		fmt.Fprint(w, "")
		return
	case "bet_request":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		result := player.BetRequest(gameState)
		fmt.Fprintf(w, "%d", result)
		return
	case "showdown":
		gameState := parseGameState(request.FormValue("game_state"))
		if gameState == nil {
			http.Error(w, "Internal Server Error", 500)
			return
		}
		player.Showdown(gameState)
		fmt.Fprint(w, "")
		return
	case "version":
		fmt.Fprint(w, player.Version())
		return
	default:
		http.Error(w, "Invalid action", 400)
	}
}

func parseGameState(stateStr string) *player.GameState {
	stateBytes := []byte(stateStr)
	gameState := new(player.GameState)
	if err := json.Unmarshal(stateBytes, &gameState); err != nil {
		log.Printf("Error parsing game state: %s", err)
		return nil
	}
	return gameState
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"

	"github.com/FactomProject/factom"
)

// newtx creates a new transaction in the wallet.
var newtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli newtx TXNAME"
	cmd.description = "Create a new transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.NewTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("newtx", cmd)
	return cmd
}()

// rmtx removes a transaction in the wallet.
var rmtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli rmtx TXNAME"
	cmd.description = "Remove a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.DeleteTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("rmtx", cmd)
	return cmd
}()

// listtxs lists the working transactions in the wallet.
var listtxs = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli listtxs"
	cmd.description = "List current working transactions in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		txs, err := factom.ListTransactions()
		if err != nil {
			errorln(err)
			return
		}
		for _, tx := range txs {
			fmt.Println("{")
			fmt.Println("	Name:", tx.Name)
			fmt.Println("	TxID:", tx.TxID)
			fmt.Println("	TotalInputs:", float64(tx.TotalInputs)/100000000)
			fmt.Println("	TotalOutputs:", float64(tx.TotalOutputs)/100000000)
			fmt.Println("	TotalECOutputs:", float64(tx.TotalECOutputs)/100000000)
			feesPaid := tx.TotalInputs - (tx.TotalOutputs + tx.TotalECOutputs)
			fmt.Println("	FeesPaid:", float64(feesPaid)/100000000)
			fmt.Println("	RawTransaction:", tx.RawTransaction)
			fmt.Println("}")
		}
	}
	help.Add("listtxs", cmd)
	return cmd
}()

// addtxinput adds a factoid input to a transaction in the wallet.
var addtxinput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxinput TXNAME ADDRESS AMOUNT"
	cmd.description = "Add a Factoid input to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}
		if err := factom.AddTransactionInput(args[0], args[1], amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxinput", cmd)
	return cmd
}()

// addtxoutput adds a factoid output to a transaction in the wallet.
var addtxoutput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxoutput TXNAME ADDRESS AMOUNT"
	cmd.description = "Add a Factoid output to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		var res = flag.Bool("r", false, "resolve dns address")
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}

		out := args[1]
		if *res {
			if f, _, err := factom.ResolveDnsName(args[1]); err != nil {
				errorln(err)
				return
			} else if f == "" {
				errorln("could not resolve factoid address")
			} else {
				out = f
			}
		}
		if err := factom.AddTransactionOutput(args[0], out, amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxoutput", cmd)
	return cmd
}()

// addtxecoutput adds an entry credit output to a transaction in the wallet.
var addtxecoutput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxecoutput TXNAME ADDRESS AMOUNT"
	cmd.description = "Add an Entry Credit output to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		var res = flag.Bool("r", false, "resolve dns address")
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}

		out := args[1]
		if *res {
			if _, e, err := factom.ResolveDnsName(args[1]); err != nil {
				errorln(err)
				return
			} else if e == "" {
				errorln("could not resolve entry credit address")
			} else {
				out = e
			}
		}
		if err := factom.AddTransactionECOutput(args[0], out, amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxecoutput", cmd)
	return cmd
}()

// addtxfee adds an entry credit output to a transaction in the wallet.
var addtxfee = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxfee TXNAME ADDRESS"
	cmd.description = "Add the transaction fee to an input of a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.AddTransactionFee(args[0], args[1]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxfee", cmd)
	return cmd
}()

// subtxfee adds an entry credit output to a transaction in the wallet.
var subtxfee = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli subtxfee TXNAME ADDRESS"
	cmd.description = "Subtract the transaction fee from an output of a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.SubTransactionFee(args[0], args[1]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("subtxfee", cmd)
	return cmd
}()

// signtx signs a transaction in the wallet.
var signtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli signtx TXNAME"
	cmd.description = "Sign a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.SignTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("signtx", cmd)
	return cmd
}()

// composetx composes the signed json rpc object to make a transaction against factomd
var composetx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli composetx TXNAME"
	cmd.description = "Compose a wallet transaction into a json rpc object"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		p, err := factom.ComposeTransaction(args[0])
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println(string(p))
	}
	help.Add("composetx", cmd)
	return cmd
}()

// sendtx composes and sends the signed transaction to factomd
var sendtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli sendtx TXNAME"
	cmd.description = "Send a Transaction to Factom"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		t, err := factom.SendTransaction(args[0])
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("sendtx", cmd)
	return cmd
}()

// sendfct sends factoids between 2 addresses
var sendfct = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli sendfct FROMADDRESS TOADDRESS AMMOUNT"
	cmd.description = "Send Factoids between 2 addresses"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}
		t, err := factom.SendFactoid(args[0], args[1], amt)
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("sendfct", cmd)
	return cmd
}()

// buyec sends factoids between 2 addresses
var buyec = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli buyec FCTADDRESS ECADDRESS ECAMMOUNT"
	cmd.description = "Buy entry credits"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.Atoi(args[2]); err != nil {
			errorln(err)
			return
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
			return
		} else {
			rate, err := factom.GetRate()
			if err != nil {
				errorln(err)
			}
			amt = uint64(i) * rate
		}
		t, err := factom.BuyEC(args[0], args[1], amt)
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("buyec", cmd)
	return cmd
}()
<commit_msg>help messages<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"

	"github.com/FactomProject/factom"
)

// newtx creates a new transaction in the wallet.
var newtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli newtx TXNAME"
	cmd.description = "Create a new transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.NewTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("newtx", cmd)
	return cmd
}()

// rmtx removes a transaction in the wallet.
var rmtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli rmtx TXNAME"
	cmd.description = "Remove a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.DeleteTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("rmtx", cmd)
	return cmd
}()

// listtxs lists the working transactions in the wallet.
var listtxs = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli listtxs"
	cmd.description = "List current working transactions in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		txs, err := factom.ListTransactions()
		if err != nil {
			errorln(err)
			return
		}
		for _, tx := range txs {
			fmt.Println("{")
			fmt.Println("	Name:", tx.Name)
			fmt.Println("	TxID:", tx.TxID)
			fmt.Println("	TotalInputs:", float64(tx.TotalInputs)/100000000)
			fmt.Println("	TotalOutputs:", float64(tx.TotalOutputs)/100000000)
			fmt.Println("	TotalECOutputs:", float64(tx.TotalECOutputs)/100000000)
			feesPaid := tx.TotalInputs - (tx.TotalOutputs + tx.TotalECOutputs)
			fmt.Println("	FeesPaid:", float64(feesPaid)/100000000)
			fmt.Println("	RawTransaction:", tx.RawTransaction)
			fmt.Println("}")
		}
	}
	help.Add("listtxs", cmd)
	return cmd
}()

// addtxinput adds a factoid input to a transaction in the wallet.
var addtxinput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxinput TXNAME ADDRESS AMOUNT"
	cmd.description = "Add a Factoid input to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}
		if err := factom.AddTransactionInput(args[0], args[1], amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxinput", cmd)
	return cmd
}()

// addtxoutput adds a factoid output to a transaction in the wallet.
var addtxoutput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxoutput [-r] TXNAME ADDRESS AMOUNT"
	cmd.description = "Add a Factoid output to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		var res = flag.Bool("r", false, "resolve dns address")
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}

		out := args[1]
		if *res {
			if f, _, err := factom.ResolveDnsName(args[1]); err != nil {
				errorln(err)
				return
			} else if f == "" {
				errorln("could not resolve factoid address")
			} else {
				out = f
			}
		}
		if err := factom.AddTransactionOutput(args[0], out, amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxoutput", cmd)
	return cmd
}()

// addtxecoutput adds an entry credit output to a transaction in the wallet.
var addtxecoutput = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxecoutput [-r] TXNAME ADDRESS AMOUNT"
	cmd.description = "Add an Entry Credit output to a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		var res = flag.Bool("r", false, "resolve dns address")
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}

		out := args[1]
		if *res {
			if _, e, err := factom.ResolveDnsName(args[1]); err != nil {
				errorln(err)
				return
			} else if e == "" {
				errorln("could not resolve entry credit address")
			} else {
				out = e
			}
		}
		if err := factom.AddTransactionECOutput(args[0], out, amt); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxecoutput", cmd)
	return cmd
}()

// addtxfee adds an entry credit output to a transaction in the wallet.
var addtxfee = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli addtxfee TXNAME ADDRESS"
	cmd.description = "Add the transaction fee to an input of a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.AddTransactionFee(args[0], args[1]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("addtxfee", cmd)
	return cmd
}()

// subtxfee adds an entry credit output to a transaction in the wallet.
var subtxfee = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli subtxfee TXNAME ADDRESS"
	cmd.description = "Subtract the transaction fee from an output of a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 2 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.SubTransactionFee(args[0], args[1]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("subtxfee", cmd)
	return cmd
}()

// signtx signs a transaction in the wallet.
var signtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli signtx TXNAME"
	cmd.description = "Sign a transaction in the wallet"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		if err := factom.SignTransaction(args[0]); err != nil {
			errorln(err)
			return
		}
	}
	help.Add("signtx", cmd)
	return cmd
}()

// composetx composes the signed json rpc object to make a transaction against factomd
var composetx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli composetx TXNAME"
	cmd.description = "Compose a wallet transaction into a json rpc object"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		p, err := factom.ComposeTransaction(args[0])
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println(string(p))
	}
	help.Add("composetx", cmd)
	return cmd
}()

// sendtx composes and sends the signed transaction to factomd
var sendtx = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli sendtx TXNAME"
	cmd.description = "Send a Transaction to Factom"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 1 {
			fmt.Println(cmd.helpMsg)
			return
		}
		t, err := factom.SendTransaction(args[0])
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("sendtx", cmd)
	return cmd
}()

// sendfct sends factoids between 2 addresses
var sendfct = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli sendfct FROMADDRESS TOADDRESS AMMOUNT"
	cmd.description = "Send Factoids between 2 addresses"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.ParseFloat(args[2], 64); err != nil {
			errorln(err)
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
		} else {
			amt = uint64(i * 1e8)
		}
		t, err := factom.SendFactoid(args[0], args[1], amt)
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("sendfct", cmd)
	return cmd
}()

// buyec sends factoids between 2 addresses
var buyec = func() *fctCmd {
	cmd := new(fctCmd)
	cmd.helpMsg = "factom-cli buyec FCTADDRESS ECADDRESS ECAMMOUNT"
	cmd.description = "Buy entry credits"
	cmd.execFunc = func(args []string) {
		os.Args = args
		flag.Parse()
		args = flag.Args()

		if len(args) != 3 {
			fmt.Println(cmd.helpMsg)
			return
		}
		var amt uint64
		if i, err := strconv.Atoi(args[2]); err != nil {
			errorln(err)
			return
		} else if i < 0 {
			errorln("AMMOUNT may not be less than 0")
			return
		} else {
			rate, err := factom.GetRate()
			if err != nil {
				errorln(err)
			}
			amt = uint64(i) * rate
		}
		t, err := factom.BuyEC(args[0], args[1], amt)
		if err != nil {
			errorln(err)
			return
		}
		fmt.Println("TxID:", t)
	}
	help.Add("buyec", cmd)
	return cmd
}()
<|endoftext|>


<commit_before>package writeaheadlog

import (
	"bytes"
	"encoding/binary"
	"io"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/errors"
	"golang.org/x/crypto/blake2b"
)

// Update defines a single update that can be sent to the WAL and saved
// atomically. Updates are sent to the wal in groups of one or more, and upon
// being signaled, will be saved to disk in a high-integrity, all- or-nothing
// fasion that follows ACID principles.
//
// The name and version are optional, however best-practice code will make use
// of these fields.
//
// When using the Update, it recommended that you typecast the Update type to
// another type which has methods on it for creating and applying the Update +
// instructions, including any special handling based on the version.
type Update struct {
	// The name of the update type. When the WAL is loaded after an unclean
	// shutdown, any un-committed changes will be passed as Updates back to the
	// caller instantiating the WAL. The caller should determine what code to
	// run on the the update based on the name and version.
	Name string

	// The version of the update type. Update types and implementations can be
	// tweaked over time, and a version field allows for easy compatibility
	// between releases, even if an upgrade occurs between an unclean shutdown
	// and a reload.
	Version string

	// The marshalled data directing the update. The data is an opaque set of
	// instructions to follow that implement and idempotent change to a set of
	// persistent files. A series of unclean shutdowns in rapid succession could
	// mean that these instructions get followed multiple times, which means
	// idempotent instructions are required.
	Instructions []byte
}

// Transaction defines a series of updates that are to be performed atomically.
// In the event of an unclean shutdown, either all updates will have been saved
// together at full integrity, or none of the updates in the transaction will
// have been saved at all.
//
// While multiple transactions can be safely open at the same time, multiple
// methods should not be called on the transaction at the same time - the WAL
// is thread-safe, but the transactions are not.
//
// A Transaction is created by calling NewTransaction. Afterwards, the
// transactions SignalSetupComplete has to be called which returns a channel
// that is closed once the transaction is committed. Finally
// SignalUpdatesApplied needs to be called after the transaction was committed
// to signal a successfull transaction and free used pages.
type Transaction struct {
	// setupComplete, commitComplete, and releaseComplete signal the progress of
	// the transaction, and should be set to 'true' in order.
	//
	// When setupComplete is set to true, it means that the creater of the
	// transaction is ready for the transaction to be committed.
	//
	// When commitComplete is set to true, it means that the WAL has
	// successfully and fully committed the transaction.
	//
	// releaseComplete is set to true when the caller has fully applied the
	// transaction, meaning the transaction can be over-written safely in the
	// WAL, and the on-disk pages can be reclaimed for future transactions.
	setupComplete   bool
	commitComplete  bool
	releaseComplete bool

	// firstPage is the first page of the transaction. It is the last page that
	// is written when finalizing a commit and when releasing a transaction.
	firstPage *page

	// Updates defines the set of updates that compose the transaction.
	Updates []Update

	// The wal that was used to create the transaction
	wal *WAL
	// initComplete is used to signal if initializing the transaction is complete
	initComplete chan struct{}
	// initErr stores possible errors that might have occured during
	// initialization
	initErr error
}

// checksum calculates the checksum of a transaction excluding the checksum
// field of each page
func (t Transaction) checksum() (c checksum) {
	h, _ := blake2b.New256(nil)
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		h.Write(b[checksumSize:]) // exclude checksum
	}
	copy(c[:], h.Sum(buf[:0]))
	return
}

// commit commits a transaction by setting the correct status and checksum
func (t *Transaction) commit(done chan error) {
	// Signal completion of the commit
	defer close(done)

	// Make sure that the initialization of the transaction finished
	<-t.initComplete
	if t.initErr != nil {
		done <- t.initErr
		return
	}

	// set the status of the first page first
	t.firstPage.pageStatus = pageStatusComitted

	// Set the transaction number of the first page and increase the transactionCounter of the wal
	t.firstPage.transactionNumber = atomic.AddUint64(&t.wal.atomicNextTxnNum, 1) - 1

	// calculate the checksum
	t.firstPage.transactionChecksum = t.checksum()

	// Finalize the commit by writing the first page with the updated status if
	// there have been no errors so far.
	if t.wal.deps.disrupt("CommitFail") {
		// Disk failure causes the commit to fail
		done <- errors.New("Write failed on purpose")
		return
	}
	if _, err := t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the first page failed", err)
		return
	}

	t.commitComplete = true
	t.wal.fSync()
}

// marshalUpdates marshals the updates of a transaction
func marshalUpdates(updates []Update) []byte {
	buffer := new(bytes.Buffer)
	for _, update := range updates {
		// Marshal name
		name := []byte(update.Name)
		_ = binary.Write(buffer, binary.LittleEndian, uint64(len(name)))
		_, _ = buffer.Write(name)

		// Marshal version
		version := []byte(update.Version)
		_ = binary.Write(buffer, binary.LittleEndian, uint64(len(version)))
		_, _ = buffer.Write(version)

		// Append instructions
		_ = binary.Write(buffer, binary.LittleEndian, uint64(len(update.Instructions)))
		_, _ = buffer.Write(update.Instructions)
	}
	return buffer.Bytes()
}

// unmarshalUpdates unmarshals the updates of a transaction
func unmarshalUpdates(data []byte) ([]Update, error) {
	buffer := bytes.NewBuffer(data)
	updates := make([]Update, 0)

	for {
		update := Update{}

		// Unmarshal name
		var nameLength uint64
		err1 := binary.Read(buffer, binary.LittleEndian, &nameLength)
		if err1 == io.EOF {
			// End of buffer reached
			break
		}

		name := make([]byte, nameLength)
		_, err2 := buffer.Read(name)

		// Unmarshal version
		var versionLength uint64
		err3 := binary.Read(buffer, binary.LittleEndian, &versionLength)

		version := make([]byte, versionLength)
		_, err4 := buffer.Read(version)

		// Unmarshal instructions
		var instructionsLength uint64
		err5 := binary.Read(buffer, binary.LittleEndian, &instructionsLength)

		instructions := make([]byte, instructionsLength)
		_, err6 := buffer.Read(instructions)

		// Check if any errors occured
		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to unmarshal updates"), err1, err2, err3, err4, err5, err6)
		}

		update.Name = string(name)
		update.Version = string(version)
		update.Instructions = instructions

		updates = append(updates, update)
	}

	return updates, nil
}

// threadedInitTransaction reserves pages of the wal, marshalls the
// transactions's updates into a payload and splits the payload equally among
// the pages. Once finished those pages are written to disk and the transaction
// is committed.
func initTransaction(t *Transaction) {
	defer close(t.initComplete)

	// Marshal all the updates to get their total length on disk
	data := marshalUpdates(t.Updates)

	// Get the pages from the wal and set the first page's status
	pages := t.wal.managedReservePages(data)
	pages[0].pageStatus = pageStatusWritten

	// Set the first page of the transaction
	t.firstPage = &pages[0]

	// write the pages to disk
	if err := t.writeToFile(); err != nil {
		t.initErr = build.ExtendErr("Couldn't write the page to file", err)
		return
	}
}

// validateChecksum checks if a transaction has been corrupted by computing a hash
// and comparing it to the one in the firstPage of the transaction
func (t Transaction) validateChecksum() error {
	if t.firstPage == nil {
		return errors.New("firstPage is nil")
	} else if t.checksum() != t.firstPage.transactionChecksum {
		return errors.New("checksum not valid")
	}
	return nil
}

// SignalUpdatesApplied  informs the WAL that it is safe to free the used pages to reuse them in a new transaction
func (t *Transaction) SignalUpdatesApplied() error {
	if !t.setupComplete || !t.commitComplete || t.releaseComplete {
		return errors.New("misuse of transaction - call each of the signaling methods exactly once, in serial, in order")
	}
	t.releaseComplete = true

	// Set the page status to applied
	t.firstPage.pageStatus = pageStatusApplied

	// Write the page to disk
	var err error
	if t.wal.deps.disrupt("ReleaseFail") {
		// Disk failure causes the commit to fail
		err = errors.New("Write failed on purpose")
	} else {
		_, err = t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset))
	}
	if err != nil {
		return build.ExtendErr("Couldn't write the page to file", err)
	}
	t.wal.fSync()

	// Update the wal's available pages
	t.wal.mu.Lock()
	for page := t.firstPage; page != nil; page = page.nextPage {
		// Append the index of the freed page
		t.wal.availablePages = append(t.wal.availablePages, page.offset)
	}
	t.wal.mu.Unlock()

	// Decrease the number of active transactions
	if atomic.AddInt64(&t.wal.atomicUnfinishedTxns, -1) < 0 {
		panic("Sanity check failed. atomicUnfinishedTxns should never be negative")
	}

	return nil
}

// append is a helper function to append updates to a transaction on which
// SignalSetupComplete hasn't been called yet
func (t *Transaction) append(updates []Update, done chan error) {
	defer close(done)

	// If there is nothing to append we are done
	if len(updates) == 0 {
		return
	}

	// Make sure that the initialization finished
	<-t.initComplete
	if t.initErr != nil {
		done <- t.initErr
		return
	}

	// Marshal the data
	data := marshalUpdates(updates)

	// Find last page to which we append and count the pages
	lastPage := t.firstPage
	for lastPage.nextPage != nil {
		lastPage = lastPage.nextPage
	}

	// Write as much data to the last page as possible
	lenDiff := MaxPayloadSize - len(lastPage.payload)
	if len(data) <= lenDiff {
		lastPage.payload = append(lastPage.payload, data...)
		data = nil
	} else {
		lastPage.payload = append(lastPage.payload, data[:lenDiff]...)
		data = data[lenDiff:]
	}

	// If there is no more data to write we are done
	if data == nil {
		return
	}

	// Get enough pages for the remaining data
	pages := t.wal.managedReservePages(data)
	lastPage.nextPage = &pages[0]

	// Write the new pages to disk and sync them
	buf := make([]byte, pageSize)
	for _, page := range pages {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			done <- build.ExtendErr("Writing the page to disk failed", err)
			return
		}
	}
	t.wal.fSync()

	// Link the new pages to the last one and sync the last page
	b := lastPage.appendTo(buf[:0])
	if _, err := t.wal.logFile.WriteAt(b, int64(lastPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the last page to disk failed", err)
		return
	}
	t.wal.fSync()

	// Append the updates to the transaction
	t.Updates = append(t.Updates, updates...)
}

// Append appends additional updates to a transaction
func (t *Transaction) Append(updates []Update) <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of trnasaction - can't append to transaction once it is committed/released")
		return done
	}

	go t.append(updates, done)
	return done
}

// SignalSetupComplete will signal to the WAL that any required setup has
// completed, and that the WAL can safely commit to the transaction being
// applied atomically.
func (t *Transaction) SignalSetupComplete() <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of transaction - call each of the signaling methods exactly ones, in serial, in order")
		return done
	}
	t.setupComplete = true

	// Commit the transaction non-blocking
	go t.commit(done)
	return done
}

// NewTransaction creates a transaction from a set of updates
func (w *WAL) NewTransaction(updates []Update) (*Transaction, error) {
	if !w.recoveryComplete {
		return nil, errors.New("can't call NewTransaction before recovery is complete")
	}
	// Check that there are updates for the transaction to process.
	if len(updates) == 0 {
		return nil, errors.New("cannot create a transaction without updates")
	}

	// Create new transaction
	newTransaction := Transaction{
		Updates:      updates,
		wal:          w,
		initComplete: make(chan struct{}),
	}

	// Initialize the transaction by splitting up the payload among free pages
	// and writing them to disk.
	go initTransaction(&newTransaction)

	// Increase the number of active transaction
	atomic.AddInt64(&w.atomicUnfinishedTxns, 1)

	return &newTransaction, nil
}

// writeToFile writes all the pages of the transaction to disk
func (t *Transaction) writeToFile() error {
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			return build.ExtendErr("Writing the page to disk failed", err)
		}
	}
	return nil
}
<commit_msg>more efficient marshalUpdates<commit_after>package writeaheadlog

import (
	"bytes"
	"encoding/binary"
	"io"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/errors"
	"golang.org/x/crypto/blake2b"
)

// Update defines a single update that can be sent to the WAL and saved
// atomically. Updates are sent to the wal in groups of one or more, and upon
// being signaled, will be saved to disk in a high-integrity, all- or-nothing
// fasion that follows ACID principles.
//
// The name and version are optional, however best-practice code will make use
// of these fields.
//
// When using the Update, it recommended that you typecast the Update type to
// another type which has methods on it for creating and applying the Update +
// instructions, including any special handling based on the version.
type Update struct {
	// The name of the update type. When the WAL is loaded after an unclean
	// shutdown, any un-committed changes will be passed as Updates back to the
	// caller instantiating the WAL. The caller should determine what code to
	// run on the the update based on the name and version.
	Name string

	// The version of the update type. Update types and implementations can be
	// tweaked over time, and a version field allows for easy compatibility
	// between releases, even if an upgrade occurs between an unclean shutdown
	// and a reload.
	Version string

	// The marshalled data directing the update. The data is an opaque set of
	// instructions to follow that implement and idempotent change to a set of
	// persistent files. A series of unclean shutdowns in rapid succession could
	// mean that these instructions get followed multiple times, which means
	// idempotent instructions are required.
	Instructions []byte
}

// Transaction defines a series of updates that are to be performed atomically.
// In the event of an unclean shutdown, either all updates will have been saved
// together at full integrity, or none of the updates in the transaction will
// have been saved at all.
//
// While multiple transactions can be safely open at the same time, multiple
// methods should not be called on the transaction at the same time - the WAL
// is thread-safe, but the transactions are not.
//
// A Transaction is created by calling NewTransaction. Afterwards, the
// transactions SignalSetupComplete has to be called which returns a channel
// that is closed once the transaction is committed. Finally
// SignalUpdatesApplied needs to be called after the transaction was committed
// to signal a successfull transaction and free used pages.
type Transaction struct {
	// setupComplete, commitComplete, and releaseComplete signal the progress of
	// the transaction, and should be set to 'true' in order.
	//
	// When setupComplete is set to true, it means that the creater of the
	// transaction is ready for the transaction to be committed.
	//
	// When commitComplete is set to true, it means that the WAL has
	// successfully and fully committed the transaction.
	//
	// releaseComplete is set to true when the caller has fully applied the
	// transaction, meaning the transaction can be over-written safely in the
	// WAL, and the on-disk pages can be reclaimed for future transactions.
	setupComplete   bool
	commitComplete  bool
	releaseComplete bool

	// firstPage is the first page of the transaction. It is the last page that
	// is written when finalizing a commit and when releasing a transaction.
	firstPage *page

	// Updates defines the set of updates that compose the transaction.
	Updates []Update

	// The wal that was used to create the transaction
	wal *WAL
	// initComplete is used to signal if initializing the transaction is complete
	initComplete chan struct{}
	// initErr stores possible errors that might have occured during
	// initialization
	initErr error
}

// checksum calculates the checksum of a transaction excluding the checksum
// field of each page
func (t Transaction) checksum() (c checksum) {
	h, _ := blake2b.New256(nil)
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		h.Write(b[checksumSize:]) // exclude checksum
	}
	copy(c[:], h.Sum(buf[:0]))
	return
}

// commit commits a transaction by setting the correct status and checksum
func (t *Transaction) commit(done chan error) {
	// Signal completion of the commit
	defer close(done)

	// Make sure that the initialization of the transaction finished
	<-t.initComplete
	if t.initErr != nil {
		done <- t.initErr
		return
	}

	// set the status of the first page first
	t.firstPage.pageStatus = pageStatusComitted

	// Set the transaction number of the first page and increase the transactionCounter of the wal
	t.firstPage.transactionNumber = atomic.AddUint64(&t.wal.atomicNextTxnNum, 1) - 1

	// calculate the checksum
	t.firstPage.transactionChecksum = t.checksum()

	// Finalize the commit by writing the first page with the updated status if
	// there have been no errors so far.
	if t.wal.deps.disrupt("CommitFail") {
		// Disk failure causes the commit to fail
		done <- errors.New("Write failed on purpose")
		return
	}
	if _, err := t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the first page failed", err)
		return
	}

	t.commitComplete = true
	t.wal.fSync()
}

// marshalUpdates marshals the updates of a transaction
func marshalUpdates(updates []Update) []byte {
	// preallocate buffer of appropriate size
	var size int
	for _, u := range updates {
		size += 8 + len(u.Name)
		size += 8 + len(u.Version)
		size += 8 + len(u.Instructions)
	}
	buf := make([]byte, size)

	var n int
	for _, u := range updates {
		// u.Name
		binary.LittleEndian.PutUint64(buf[n:], uint64(len(u.Name)))
		n += 8
		n += copy(buf[n:], u.Name)
		// u.Version
		binary.LittleEndian.PutUint64(buf[n:], uint64(len(u.Version)))
		n += 8
		n += copy(buf[n:], u.Version)
		// u.Instructions
		binary.LittleEndian.PutUint64(buf[n:], uint64(len(u.Instructions)))
		n += 8
		n += copy(buf[n:], u.Instructions)
	}
	return buf
}

// unmarshalUpdates unmarshals the updates of a transaction
func unmarshalUpdates(data []byte) ([]Update, error) {
	buffer := bytes.NewBuffer(data)
	updates := make([]Update, 0)

	for {
		update := Update{}

		// Unmarshal name
		var nameLength uint64
		err1 := binary.Read(buffer, binary.LittleEndian, &nameLength)
		if err1 == io.EOF {
			// End of buffer reached
			break
		}

		name := make([]byte, nameLength)
		_, err2 := buffer.Read(name)

		// Unmarshal version
		var versionLength uint64
		err3 := binary.Read(buffer, binary.LittleEndian, &versionLength)

		version := make([]byte, versionLength)
		_, err4 := buffer.Read(version)

		// Unmarshal instructions
		var instructionsLength uint64
		err5 := binary.Read(buffer, binary.LittleEndian, &instructionsLength)

		instructions := make([]byte, instructionsLength)
		_, err6 := buffer.Read(instructions)

		// Check if any errors occured
		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to unmarshal updates"), err1, err2, err3, err4, err5, err6)
		}

		update.Name = string(name)
		update.Version = string(version)
		update.Instructions = instructions

		updates = append(updates, update)
	}

	return updates, nil
}

// threadedInitTransaction reserves pages of the wal, marshalls the
// transactions's updates into a payload and splits the payload equally among
// the pages. Once finished those pages are written to disk and the transaction
// is committed.
func initTransaction(t *Transaction) {
	defer close(t.initComplete)

	// Marshal all the updates to get their total length on disk
	data := marshalUpdates(t.Updates)

	// Get the pages from the wal and set the first page's status
	pages := t.wal.managedReservePages(data)
	pages[0].pageStatus = pageStatusWritten

	// Set the first page of the transaction
	t.firstPage = &pages[0]

	// write the pages to disk
	if err := t.writeToFile(); err != nil {
		t.initErr = build.ExtendErr("Couldn't write the page to file", err)
		return
	}
}

// validateChecksum checks if a transaction has been corrupted by computing a hash
// and comparing it to the one in the firstPage of the transaction
func (t Transaction) validateChecksum() error {
	if t.firstPage == nil {
		return errors.New("firstPage is nil")
	} else if t.checksum() != t.firstPage.transactionChecksum {
		return errors.New("checksum not valid")
	}
	return nil
}

// SignalUpdatesApplied  informs the WAL that it is safe to free the used pages to reuse them in a new transaction
func (t *Transaction) SignalUpdatesApplied() error {
	if !t.setupComplete || !t.commitComplete || t.releaseComplete {
		return errors.New("misuse of transaction - call each of the signaling methods exactly once, in serial, in order")
	}
	t.releaseComplete = true

	// Set the page status to applied
	t.firstPage.pageStatus = pageStatusApplied

	// Write the page to disk
	var err error
	if t.wal.deps.disrupt("ReleaseFail") {
		// Disk failure causes the commit to fail
		err = errors.New("Write failed on purpose")
	} else {
		_, err = t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset))
	}
	if err != nil {
		return build.ExtendErr("Couldn't write the page to file", err)
	}
	t.wal.fSync()

	// Update the wal's available pages
	t.wal.mu.Lock()
	for page := t.firstPage; page != nil; page = page.nextPage {
		// Append the index of the freed page
		t.wal.availablePages = append(t.wal.availablePages, page.offset)
	}
	t.wal.mu.Unlock()

	// Decrease the number of active transactions
	if atomic.AddInt64(&t.wal.atomicUnfinishedTxns, -1) < 0 {
		panic("Sanity check failed. atomicUnfinishedTxns should never be negative")
	}

	return nil
}

// append is a helper function to append updates to a transaction on which
// SignalSetupComplete hasn't been called yet
func (t *Transaction) append(updates []Update, done chan error) {
	defer close(done)

	// If there is nothing to append we are done
	if len(updates) == 0 {
		return
	}

	// Make sure that the initialization finished
	<-t.initComplete
	if t.initErr != nil {
		done <- t.initErr
		return
	}

	// Marshal the data
	data := marshalUpdates(updates)

	// Find last page to which we append and count the pages
	lastPage := t.firstPage
	for lastPage.nextPage != nil {
		lastPage = lastPage.nextPage
	}

	// Write as much data to the last page as possible
	lenDiff := MaxPayloadSize - len(lastPage.payload)
	if len(data) <= lenDiff {
		lastPage.payload = append(lastPage.payload, data...)
		data = nil
	} else {
		lastPage.payload = append(lastPage.payload, data[:lenDiff]...)
		data = data[lenDiff:]
	}

	// If there is no more data to write we are done
	if data == nil {
		return
	}

	// Get enough pages for the remaining data
	pages := t.wal.managedReservePages(data)
	lastPage.nextPage = &pages[0]

	// Write the new pages to disk and sync them
	buf := make([]byte, pageSize)
	for _, page := range pages {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			done <- build.ExtendErr("Writing the page to disk failed", err)
			return
		}
	}
	t.wal.fSync()

	// Link the new pages to the last one and sync the last page
	b := lastPage.appendTo(buf[:0])
	if _, err := t.wal.logFile.WriteAt(b, int64(lastPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the last page to disk failed", err)
		return
	}
	t.wal.fSync()

	// Append the updates to the transaction
	t.Updates = append(t.Updates, updates...)
}

// Append appends additional updates to a transaction
func (t *Transaction) Append(updates []Update) <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of trnasaction - can't append to transaction once it is committed/released")
		return done
	}

	go t.append(updates, done)
	return done
}

// SignalSetupComplete will signal to the WAL that any required setup has
// completed, and that the WAL can safely commit to the transaction being
// applied atomically.
func (t *Transaction) SignalSetupComplete() <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of transaction - call each of the signaling methods exactly ones, in serial, in order")
		return done
	}
	t.setupComplete = true

	// Commit the transaction non-blocking
	go t.commit(done)
	return done
}

// NewTransaction creates a transaction from a set of updates
func (w *WAL) NewTransaction(updates []Update) (*Transaction, error) {
	if !w.recoveryComplete {
		return nil, errors.New("can't call NewTransaction before recovery is complete")
	}
	// Check that there are updates for the transaction to process.
	if len(updates) == 0 {
		return nil, errors.New("cannot create a transaction without updates")
	}

	// Create new transaction
	newTransaction := Transaction{
		Updates:      updates,
		wal:          w,
		initComplete: make(chan struct{}),
	}

	// Initialize the transaction by splitting up the payload among free pages
	// and writing them to disk.
	go initTransaction(&newTransaction)

	// Increase the number of active transaction
	atomic.AddInt64(&w.atomicUnfinishedTxns, 1)

	return &newTransaction, nil
}

// writeToFile writes all the pages of the transaction to disk
func (t *Transaction) writeToFile() error {
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			return build.ExtendErr("Writing the page to disk failed", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package seed

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

const (
	MaxBatchSize = 1000
)

type Transaction struct {
	Date        time.Time `json:"date"`
	Description string    `json:"description"`
	Amount      int64     `json:"amount"`
	Error       string    `json:"error"`
	Status      string    `json:"status"`
	Category    string    `json:"category"`
}

type TransactionsRequest struct {
	CheckingAccountID string
	Status            string
	From              time.Time
	To                time.Time
	Client            *Client
}

type TransactionsIterator struct {
	request   *TransactionsRequest
	response  *TransactionsResponse
	batchSize int
}

type TransactionsResponse struct {
	Errors  ErrorList     `json:"errors"`
	Results []Transaction `json:"results"`
	Pages   Pages         `json:"pages"`
}

func (t *TransactionsRequest) Get() ([]Transaction, error) {
	var resp *TransactionsResponse
	var err error
	if resp, err = t.get(nil); err != nil {
		return []Transaction{}, err
	}

	if len(resp.Errors) > 0 {
		return resp.Results, resp.Errors
	}

	return resp.Results, nil
}

func (t *TransactionsRequest) get(paginationParams *PaginationParams) (*TransactionsResponse, error) {
	var err error
	var req *http.Request
	var response TransactionsResponse

	params := &url.Values{}
	if t.CheckingAccountID != "" {
		params.Set("checking_account_id", t.CheckingAccountID)
	}
	if t.Status != "" {
		params.Set("status", t.Status)
	}
	dateLayout := "2006-01-02"
	if !t.From.IsZero() {
		params.Set("from", t.From.Format(dateLayout))
	}
	if !t.To.IsZero() {
		params.Set("to", t.To.Format(dateLayout))
	}

	var url *url.URL
	if url, err = url.Parse(fmt.Sprintf("%s/%s", ApiBase, "transactions")); err != nil {
		return nil, err
	}

	if paginationParams != nil {
		url.RawQuery = fmt.Sprintf("%s&%s", params.Encode(), paginationParams.Encode())
	}

	if req, err = http.NewRequest("GET", url.String(), nil); err != nil {
		return nil, err
	}
	var resp *http.Response

	if resp, err = t.Client.do(req); err != nil {
		return nil, err
	}

	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, err
	}
	return &response, nil
}

func (r *TransactionsRequest) Iterator() TransactionsIterator {
	return TransactionsIterator{
		request:   r,
		batchSize: MaxBatchSize,
	}
}

func (t *TransactionsIterator) SetBatchSize(n int) {
	if n < MaxBatchSize {
		t.batchSize = n
	}
}

// Next will retrieve the next batch of transactions. It returns a slice of Transactions, and any http errors
func (t *TransactionsIterator) Next() ([]Transaction, error) {
	var err error
	var params PaginationParams
	if t.response != nil {
		params = t.response.Pages.Next
	} else {
		params = PaginationParams{Limit: t.batchSize}
	}

	if t.response, err = t.request.get(&params); err != nil {
		return []Transaction{}, fmt.Errorf("error when sending the request to seed: %v", err)
	}

	if len(t.response.Errors) > 0 {
		return t.response.Results, t.response.Errors
	}

	return t.response.Results, nil
}

// Previous will retrieve the previous batch of transactions. It returns a slice of Transactions, and any errors that happen
func (t *TransactionsIterator) Previous() ([]Transaction, error) {
	var err error
	var params PaginationParams
	if t.response != nil {
		params = t.response.Pages.Previous
	} else {
		params = PaginationParams{Limit: t.batchSize}
	}

	if t.response, err = t.request.get(&params); err != nil {
		return []Transaction{}, fmt.Errorf("error when sending the request to seed: %v", err)
	}
	if len(t.response.Errors) > 0 {
		return t.response.Results, t.response.Errors
	}

	return t.response.Results, nil
}
<commit_msg>fix start and end date parameter names<commit_after>package seed

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

const (
	MaxBatchSize = 1000
)

type Transaction struct {
	Date        time.Time `json:"date"`
	Description string    `json:"description"`
	Amount      int64     `json:"amount"`
	Error       string    `json:"error"`
	Status      string    `json:"status"`
	Category    string    `json:"category"`
}

type TransactionsRequest struct {
	CheckingAccountID string
	Status            string
	From              time.Time
	To                time.Time
	Client            *Client
}

type TransactionsIterator struct {
	request   *TransactionsRequest
	response  *TransactionsResponse
	batchSize int
}

type TransactionsResponse struct {
	Errors  ErrorList     `json:"errors"`
	Results []Transaction `json:"results"`
	Pages   Pages         `json:"pages"`
}

func (t *TransactionsRequest) Get() ([]Transaction, error) {
	var resp *TransactionsResponse
	var err error
	if resp, err = t.get(nil); err != nil {
		return []Transaction{}, err
	}

	if len(resp.Errors) > 0 {
		return resp.Results, resp.Errors
	}

	return resp.Results, nil
}

func (t *TransactionsRequest) get(paginationParams *PaginationParams) (*TransactionsResponse, error) {
	var err error
	var req *http.Request
	var response TransactionsResponse

	params := &url.Values{}
	if t.CheckingAccountID != "" {
		params.Set("checking_account_id", t.CheckingAccountID)
	}
	if t.Status != "" {
		params.Set("status", t.Status)
	}
	dateLayout := "2006-01-02"
	if !t.From.IsZero() {
		params.Set("start_date", t.From.Format(dateLayout))
	}
	if !t.To.IsZero() {
		params.Set("end_date", t.To.Format(dateLayout))
	}

	var url *url.URL
	if url, err = url.Parse(fmt.Sprintf("%s/%s", ApiBase, "transactions")); err != nil {
		return nil, err
	}

	if paginationParams != nil {
		url.RawQuery = fmt.Sprintf("%s&%s", params.Encode(), paginationParams.Encode())
	}

	if req, err = http.NewRequest("GET", url.String(), nil); err != nil {
		return nil, err
	}
	var resp *http.Response

	if resp, err = t.Client.do(req); err != nil {
		return nil, err
	}

	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return nil, err
	}
	return &response, nil
}

func (r *TransactionsRequest) Iterator() TransactionsIterator {
	return TransactionsIterator{
		request:   r,
		batchSize: MaxBatchSize,
	}
}

func (t *TransactionsIterator) SetBatchSize(n int) {
	if n < MaxBatchSize {
		t.batchSize = n
	}
}

// Next will retrieve the next batch of transactions. It returns a slice of Transactions, and any http errors
func (t *TransactionsIterator) Next() ([]Transaction, error) {
	var err error
	var params PaginationParams
	if t.response != nil {
		params = t.response.Pages.Next
	} else {
		params = PaginationParams{Limit: t.batchSize}
	}

	if t.response, err = t.request.get(&params); err != nil {
		return []Transaction{}, fmt.Errorf("error when sending the request to seed: %v", err)
	}

	if len(t.response.Errors) > 0 {
		return t.response.Results, t.response.Errors
	}

	return t.response.Results, nil
}

// Previous will retrieve the previous batch of transactions. It returns a slice of Transactions, and any errors that happen
func (t *TransactionsIterator) Previous() ([]Transaction, error) {
	var err error
	var params PaginationParams
	if t.response != nil {
		params = t.response.Pages.Previous
	} else {
		params = PaginationParams{Limit: t.batchSize}
	}

	if t.response, err = t.request.get(&params); err != nil {
		return []Transaction{}, fmt.Errorf("error when sending the request to seed: %v", err)
	}
	if len(t.response.Errors) > 0 {
		return t.response.Results, t.response.Errors
	}

	return t.response.Results, nil
}
<|endoftext|>


<commit_before>package plugins

import (
	"fmt"
	"math/rand"
	"regexp"
	"time"

	"github.com/matrix-org/gomatrix"
)

// Thanks responds to thanks
type Thanks struct {
}

// Descr describes this plugin
func (h *Thanks) Descr() string {
	return "Bots should be respectful. Respond to thanks."
}

// Re checks for various forms of thanks
func (h *Thanks) Re() string {
	return `(?i)^thank you|thank you$|^thanks|thanks$|^ty|ty$`
}

// Match determins if we are being thanked
func (h *Thanks) Match(user, msg string) bool {
	re := regexp.MustCompile(h.Re())
	return re.MatchString(msg) && ToMe(user, msg)
}

// SetStore we don't need a store here
func (h *Thanks) SetStore(s PluginStore) {}

// RespondText to welcome back events
func (h *Thanks) RespondText(c *gomatrix.Client, ev *gomatrix.Event, user, post string) {
	s := NameRE.ReplaceAllString(ev.Sender, "$1")
	a := []string{
		fmt.Sprintf("welcome %s", s),
		"welcome",
		"you're welcome",
		"np!",
		fmt.Sprintf("you'r welcome, %s", s),
	}

	rand.Seed(time.Now().Unix())

	SendText(c, ev.RoomID, a[rand.Intn(len(a))])
}

// Name Thanks
func (h *Thanks) Name() string {
	return "Thanks"
}
<commit_msg>fix typo<commit_after>package plugins

import (
	"fmt"
	"math/rand"
	"regexp"
	"time"

	"github.com/matrix-org/gomatrix"
)

// Thanks responds to thanks
type Thanks struct {
}

// Descr describes this plugin
func (h *Thanks) Descr() string {
	return "Bots should be respectful. Respond to thanks."
}

// Re checks for various forms of thanks
func (h *Thanks) Re() string {
	return `(?i)^thank you|thank you$|^thanks|thanks$|^ty|ty$`
}

// Match determins if we are being thanked
func (h *Thanks) Match(user, msg string) bool {
	re := regexp.MustCompile(h.Re())
	return re.MatchString(msg) && ToMe(user, msg)
}

// SetStore we don't need a store here
func (h *Thanks) SetStore(s PluginStore) {}

// RespondText to welcome back events
func (h *Thanks) RespondText(c *gomatrix.Client, ev *gomatrix.Event, user, post string) {
	s := NameRE.ReplaceAllString(ev.Sender, "$1")
	a := []string{
		fmt.Sprintf("welcome %s", s),
		"welcome",
		"you're welcome",
		"np!",
		fmt.Sprintf("you're welcome, %s", s),
	}

	rand.Seed(time.Now().Unix())

	SendText(c, ev.RoomID, a[rand.Intn(len(a))])
}

// Name Thanks
func (h *Thanks) Name() string {
	return "Thanks"
}
<|endoftext|>


<commit_before>package mapper

import (
	"fmt"

	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/encoders/base62"
)

type SourceMapper struct {
	ServerConfiguration *core.ServerConfiguration
}

// RemoteImageURL returns a URL string for original image
func (m *SourceMapper) RemoteImageURL(ic *core.ImageConfiguration) string {
	if ic.Source != "" {
		return ic.Source
	}
	url := ic.ServerConfiguration.SourceDomain + "/" + m.imageDirectory(ic) + "/original.jpg"
	return url
}

func (m *SourceMapper) imageDirectory(ic *core.ImageConfiguration) string {
	id := base62.Decode("ofrA")
	// fmt.Printf("Decoded %s to %d", ic.ID, id)
	return fmt.Sprintf("%s/%d", m.ServerConfiguration.NamespaceMappings[ic.Namespace], id)
}
<commit_msg>Use real image id, instead of placeholder base62 string, oops!<commit_after>package mapper

import (
	"fmt"

	"github.com/wanelo/image-server/core"
	"github.com/wanelo/image-server/encoders/base62"
)

type SourceMapper struct {
	ServerConfiguration *core.ServerConfiguration
}

// RemoteImageURL returns a URL string for original image
func (m *SourceMapper) RemoteImageURL(ic *core.ImageConfiguration) string {
	if ic.Source != "" {
		return ic.Source
	}
	url := ic.ServerConfiguration.SourceDomain + "/" + m.imageDirectory(ic) + "/original.jpg"
	return url
}

func (m *SourceMapper) imageDirectory(ic *core.ImageConfiguration) string {
	id := base62.Decode(ic.ID)
	return fmt.Sprintf("%s/%d", m.ServerConfiguration.NamespaceMappings[ic.Namespace], id)
}
<|endoftext|>


<commit_before>package gtcp

import (
	"fmt"
	"sync"
)

type (
	Statistics interface {
		AddConnStats(Conn)
		Reset()
		String() string
	}

	TrafficStatistics struct {
		mu       sync.RWMutex
		inBytes  int64
		outBytes int64
	}
)

func (ts *TrafficStatistics) AddConnStats(conn Conn) {
	in, out := conn.Stats()
	ts.mu.Lock()
	ts.inBytes += in
	ts.outBytes += out
	ts.mu.Unlock()
}

func (ts *TrafficStatistics) Reset() {
	ts.mu.Lock()
	ts.inBytes, ts.outBytes = 0, 0
	ts.mu.Unlock()
}

func (ts *TrafficStatistics) String() (str string) {
	ts.mu.RLock()
	str = fmt.Sprintf(`{"in_bytes": %d, "out_bytes": %d}`, ts.inBytes, ts.outBytes)
	ts.mu.RUnlock()
	return
}
<commit_msg>Add comments for statistics.go<commit_after>package gtcp

import (
	"fmt"
	"sync"
)

type (
	// Statistics is the interface that wraps operations for accumulating Conn statistics.
	Statistics interface {
		// AddConnStats adds a Conn statistics.
		AddConnStats(Conn)
		// Reset clears statistics holden now.
		Reset()
		// String returns a string that represents the current statistics.
		String() string
	}

	// TrafficStatistics implements Statistics to hold the in/out traffic on a gtcp server.
	TrafficStatistics struct {
		mu       sync.RWMutex
		inBytes  int64
		outBytes int64
	}
)

// AddConnStats ingests inBytes and outBytes from conn.
// You need to use StatsConn in gtcp.NewConn if you want in/out traffic.
func (ts *TrafficStatistics) AddConnStats(conn Conn) {
	in, out := conn.Stats()
	ts.mu.Lock()
	ts.inBytes += in
	ts.outBytes += out
	ts.mu.Unlock()
}

// Reset clears statistics holden now.
func (ts *TrafficStatistics) Reset() {
	ts.mu.Lock()
	ts.inBytes, ts.outBytes = 0, 0
	ts.mu.Unlock()
}

// String returns the in/out traffic on a gtcp server as a json string.
func (ts *TrafficStatistics) String() (str string) {
	ts.mu.RLock()
	str = fmt.Sprintf(`{"in_bytes": %d, "out_bytes": %d}`, ts.inBytes, ts.outBytes)
	ts.mu.RUnlock()
	return
}
<|endoftext|>


<commit_before>package actors

import (
	"context"
	"fmt"
	"strings"

	integration "github.com/cloudfoundry/bosh-bootloader/integration-test"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
)

type GCP struct {
	service   *compute.Service
	projectID string
	region    string
}

func NewGCP(config integration.Config) GCP {
	googleConfig, err := google.JWTConfigFromJSON([]byte(config.GCPServiceAccountKeyPath), "https://www.googleapis.com/auth/compute")
	if err != nil {
		panic(err)
	}

	service, err := compute.New(googleConfig.Client(context.Background()))
	if err != nil {
		panic(err)
	}

	return GCP{
		service:   service,
		projectID: config.GCPProjectID,
		region:    config.GCPRegion,
	}
}

func (g GCP) SSHKey() (string, error) {
	project, err := g.service.Projects.Get(g.projectID).Do()
	if err != nil {
		return "", err
	}

	for _, item := range project.CommonInstanceMetadata.Items {
		if item.Key == "sshKeys" {
			return *item.Value, nil
		}
	}

	return "", nil
}

func (g GCP) RemoveSSHKey(sshKey string) error {
	project, err := g.service.Projects.Get(g.projectID).Do()
	if err != nil {
		return err
	}

	for i, item := range project.CommonInstanceMetadata.Items {
		if item.Key == "sshKeys" {
			newSSHKeys := []string{}

			for _, keyFromGCP := range strings.Split(*item.Value, "\n") {
				if keyFromGCP != sshKey {
					newSSHKeys = append(newSSHKeys, keyFromGCP)
				}
			}

			newValue := strings.Join(newSSHKeys, "\n")
			project.CommonInstanceMetadata.Items[i].Value = &newValue
			break
		}
	}

	_, err = g.service.Projects.SetCommonInstanceMetadata(g.projectID, project.CommonInstanceMetadata).Do()
	if err != nil {
		return err
	}

	return nil
}

func (g GCP) GetNetwork(networkName string) (*compute.Network, error) {
	return g.service.Networks.Get(g.projectID, networkName).Do()
}

func (g GCP) GetSubnet(subnetName string) (*compute.Subnetwork, error) {
	return g.service.Subnetworks.Get(g.projectID, g.region, subnetName).Do()
}

func (g GCP) GetAddress(addressName string) (*compute.Address, error) {
	aggregatedList, err := g.service.Addresses.AggregatedList(g.projectID).Filter(fmt.Sprintf("name eq %s", addressName)).Do()
	if err != nil {
		return nil, err
	}

	items, ok := aggregatedList.Items["regions/"+g.region]
	if !ok {
		return nil, nil
	}

	if len(items.Addresses) == 0 {
		return nil, nil
	}

	return items.Addresses[0], nil
}

func (g GCP) GetFirewallRule(firewallRuleName string) (*compute.Firewall, error) {
	return g.service.Firewalls.Get(g.projectID, firewallRuleName).Do()
}

func (g GCP) GetTargetPool(targetPoolName string) (*compute.TargetPool, error) {
	return g.service.TargetPools.Get(g.projectID, g.region, targetPoolName).Do()
}

func (g GCP) GetTargetHTTPSProxy(name string) (*compute.TargetHttpsProxy, error) {
	return g.service.TargetHttpsProxies.Get(g.projectID, name).Do()
}

func (g GCP) GetHealthCheck(healthCheckName string) (*compute.HttpHealthCheck, error) {
	return g.service.HttpHealthChecks.Get(g.projectID, healthCheckName).Do()
}
<commit_msg>Support json or file path for GCP Account Key.<commit_after>package actors

import (
	"context"
	"fmt"
	"io/ioutil"
	"strings"

	integration "github.com/cloudfoundry/bosh-bootloader/integration-test"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
)

type GCP struct {
	service   *compute.Service
	projectID string
	region    string
}

func NewGCP(config integration.Config) GCP {
	rawServiceAccountKey, err := ioutil.ReadFile(config.GCPServiceAccountKeyPath)
	if err != nil {
		rawServiceAccountKey = []byte(config.GCPServiceAccountKeyPath)
	}

	googleConfig, err := google.JWTConfigFromJSON(rawServiceAccountKey, "https://www.googleapis.com/auth/compute")
	if err != nil {
		panic(err)
	}

	service, err := compute.New(googleConfig.Client(context.Background()))
	if err != nil {
		panic(err)
	}

	return GCP{
		service:   service,
		projectID: config.GCPProjectID,
		region:    config.GCPRegion,
	}
}

func (g GCP) SSHKey() (string, error) {
	project, err := g.service.Projects.Get(g.projectID).Do()
	if err != nil {
		return "", err
	}

	for _, item := range project.CommonInstanceMetadata.Items {
		if item.Key == "sshKeys" {
			return *item.Value, nil
		}
	}

	return "", nil
}

func (g GCP) RemoveSSHKey(sshKey string) error {
	project, err := g.service.Projects.Get(g.projectID).Do()
	if err != nil {
		return err
	}

	for i, item := range project.CommonInstanceMetadata.Items {
		if item.Key == "sshKeys" {
			newSSHKeys := []string{}

			for _, keyFromGCP := range strings.Split(*item.Value, "\n") {
				if keyFromGCP != sshKey {
					newSSHKeys = append(newSSHKeys, keyFromGCP)
				}
			}

			newValue := strings.Join(newSSHKeys, "\n")
			project.CommonInstanceMetadata.Items[i].Value = &newValue
			break
		}
	}

	_, err = g.service.Projects.SetCommonInstanceMetadata(g.projectID, project.CommonInstanceMetadata).Do()
	if err != nil {
		return err
	}

	return nil
}

func (g GCP) GetNetwork(networkName string) (*compute.Network, error) {
	return g.service.Networks.Get(g.projectID, networkName).Do()
}

func (g GCP) GetSubnet(subnetName string) (*compute.Subnetwork, error) {
	return g.service.Subnetworks.Get(g.projectID, g.region, subnetName).Do()
}

func (g GCP) GetAddress(addressName string) (*compute.Address, error) {
	aggregatedList, err := g.service.Addresses.AggregatedList(g.projectID).Filter(fmt.Sprintf("name eq %s", addressName)).Do()
	if err != nil {
		return nil, err
	}

	items, ok := aggregatedList.Items["regions/"+g.region]
	if !ok {
		return nil, nil
	}

	if len(items.Addresses) == 0 {
		return nil, nil
	}

	return items.Addresses[0], nil
}

func (g GCP) GetFirewallRule(firewallRuleName string) (*compute.Firewall, error) {
	return g.service.Firewalls.Get(g.projectID, firewallRuleName).Do()
}

func (g GCP) GetTargetPool(targetPoolName string) (*compute.TargetPool, error) {
	return g.service.TargetPools.Get(g.projectID, g.region, targetPoolName).Do()
}

func (g GCP) GetTargetHTTPSProxy(name string) (*compute.TargetHttpsProxy, error) {
	return g.service.TargetHttpsProxies.Get(g.projectID, name).Do()
}

func (g GCP) GetHealthCheck(healthCheckName string) (*compute.HttpHealthCheck, error) {
	return g.service.HttpHealthChecks.Get(g.projectID, healthCheckName).Do()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsx

import (
	"fmt"
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// We will skip forwards in a GCS response at most this many bytes.
// About 6 MB of data is buffered anyway, so 8 MB seems like a good round number.
const skipForwardThreshold = 8 * 1 << 20

// An object that knows how to read ranges within a particular generation of a
// particular GCS object. Optimised for (large) sequential reads.
//
// Not safe for concurrent access.
type RandomReader interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Matches the semantics of io.ReaderAt, with the addition of context
	// support.
	ReadAt(ctx context.Context, p []byte, offset int64) (n int, err error)

	// Return the record for the object to which the reader is bound.
	Object() (o *gcs.Object)

	// Clean up any resources associated with the reader, which must not be used
	// again.
	Destroy()
}

// Create a random reader for the supplied object record that reads using the
// given bucket.
func NewRandomReader(
	o *gcs.Object,
	bucket gcs.Bucket) (rr RandomReader, err error) {
	rr = &randomReader{
		object: o,
		bucket: bucket,
		start:  -1,
		limit:  -1,
	}

	return
}

type randomReader struct {
	object *gcs.Object
	bucket gcs.Bucket

	// If non-nil, an in-flight read request and a function for cancelling it.
	//
	// INVARIANT: (reader == nil) == (cancel == nil)
	reader io.ReadCloser
	cancel func()

	// The range of the object that we expect reader to yield, when reader is
	// non-nil. When reader is nil, limit is the limit of the previous read
	// operation, or -1 if there has never been one.
	//
	// INVARIANT: start <= limit
	// INVARIANT: limit < 0 implies reader != nil
	start int64
	limit int64
}

func (rr *randomReader) CheckInvariants() {
	// INVARIANT: (reader == nil) == (cancel == nil)
	if (rr.reader == nil) != (rr.cancel == nil) {
		panic(fmt.Sprintf("Mismatch: %v vs. %v", rr.reader == nil, rr.cancel == nil))
	}

	// INVARIANT: start <= limit
	if !(rr.start <= rr.limit) {
		panic(fmt.Sprintf("Unexpected range: [%d, %d)", rr.start, rr.limit))
	}

	// INVARIANT: limit < 0 implies reader != nil
	if rr.limit < 0 && rr.reader != nil {
		panic(fmt.Sprintf("Unexpected non-nil reader with limit == %d", rr.limit))
	}
}

func (rr *randomReader) ReadAt(
	ctx context.Context,
	p []byte,
	offset int64) (n int, err error) {
	for len(p) > 0 {
		// Have we blown past the end of the object?
		if offset >= int64(rr.object.Size) {
			err = io.EOF
			return
		}

		// When the offset is AFTER the reader position, try to seek forward, within reason.
		// This happens when the kernel page cache serves some data. It's very common for
		// concurrent reads, often by only a few 128kB fuse read requests. The aim is to
		// re-use GCS connection and avoid throwing away already read data.
		// For parallel sequential reads to a single file, not throwing away the connections
		// is a 15x improvement in throughput.
		if rr.reader != nil && rr.start < offset && offset - rr.start < skipForwardThreshold {
		        bytes_to_skip := int64(offset - rr.start)
		        p := make([]byte, bytes_to_skip)
		        n, _ := rr.reader.Read(p)
		        rr.start += int64(n)
		}

		// If we have an existing reader but it's positioned at the wrong place,
		// clean it up and throw it away.
		if rr.reader != nil && rr.start != offset {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// If we don't have a reader, start a read operation.
		if rr.reader == nil {
			err = rr.startRead(offset, int64(len(p)))
			if err != nil {
				err = fmt.Errorf("startRead: %v", err)
				return
			}
		}

		// Now we have a reader positioned at the correct place. Consume as much from
		// it as possible.
		var tmp int
		tmp, err = rr.readFull(ctx, p)

		n += tmp
		p = p[tmp:]
		rr.start += int64(tmp)
		offset += int64(tmp)

		// Sanity check.
		if rr.start > rr.limit {
			err = fmt.Errorf("Reader returned %d too many bytes", rr.start-rr.limit)

			// Don't attempt to reuse the reader when it's behaving wackily.
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
			rr.start = -1
			rr.limit = -1

			return
		}

		// Are we finished with this reader now?
		if rr.start == rr.limit {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// Handle errors.
		switch {
		case err == io.EOF || err == io.ErrUnexpectedEOF:
			// For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only
			// if the reader peters out early. That's fine, but it means we should
			// have hit the limit above.
			if rr.reader != nil {
				err = fmt.Errorf("Reader returned %d too few bytes", rr.limit-rr.start)
				return
			}

			err = nil

		case err != nil:
			// Propagate other errors.
			err = fmt.Errorf("readFull: %v", err)
			return
		}
	}

	return
}

func (rr *randomReader) Object() (o *gcs.Object) {
	o = rr.object
	return
}

func (rr *randomReader) Destroy() {
	// Close out the reader, if we have one.
	if rr.reader != nil {
		rr.reader.Close()
		rr.reader = nil
		rr.cancel = nil
	}
}

// Like io.ReadFull, but deals with the cancellation issues.
//
// REQUIRES: rr.reader != nil
func (rr *randomReader) readFull(
	ctx context.Context,
	p []byte) (n int, err error) {
	// Start a goroutine that will cancel the read operation we block on below if
	// the calling context is cancelled, but only if this method has not already
	// returned (to avoid souring the reader for the next read if this one is
	// successful, since the calling context will eventually be cancelled).
	readDone := make(chan struct{})
	defer close(readDone)

	go func() {
		select {
		case <-readDone:
			return

		case <-ctx.Done():
			select {
			case <-readDone:
				return

			default:
				rr.cancel()
			}
		}
	}()

	// Call through.
	n, err = io.ReadFull(rr.reader, p)

	return
}

// Ensure that rr.reader is set up for a range for which [start, start+size) is
// a prefix.
func (rr *randomReader) startRead(
	start int64,
	size int64) (err error) {
	// Make sure start and size are legal.
	if start < 0 || uint64(start) > rr.object.Size || size < 0 {
		err = fmt.Errorf(
			"Range [%d, %d) is illegal for %d-byte object",
			start,
			start+size,
			rr.object.Size)
		return
	}

	// GCS requests are expensive. Always issue read requests to the end of
	// the object. Sequential reads will simply sip from the fire house
	// with each call to ReadAt. In practice, GCS will fill the TCP buffers
	// with about 6 MB of data. Requests from outside GCP will be charged
	// about 6MB of egress data, even if less data is read. Inside GCP
	// regions, GCS egress is free. This logic should limit the number of
	// GCS read requests, which are not free.

	// TODO: Investigate impact on random read workloads.

	// Begin the read.
	ctx, cancel := context.WithCancel(context.Background())
	rc, err := rr.bucket.NewReader(
		ctx,
		&gcs.ReadObjectRequest{
			Name:       rr.object.Name,
			Generation: rr.object.Generation,
			Range: &gcs.ByteRange{
				Start: uint64(start),
				Limit: uint64(rr.object.Size),
			},
		})

	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	rr.reader = rc
	rr.cancel = cancel
	rr.start = start
	rr.limit = int64(rr.object.Size)

	return
}
<commit_msg>Add logic to detect and optimise random reads.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsx

import (
	"fmt"
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

const MB = 1 << 20

// Min read size in bytes for random reads.
// We will not send a request to GCS for less than this many bytes (unless the
// end of the object comes first).
const minReadSize = MB

// Max read size in bytes for random reads.
// If the average read size (between seeks) is below this number, reads will
// optimised for random access.
// We will skip forwards in a GCS response at most this many bytes.
// About 6 MB of data is buffered anyway, so 8 MB seems like a good round number.
const maxReadSize = 8 * MB

// Minimum number of seeks before evaluating if the read pattern is random.
const minSeeksForRandom = 2

// An object that knows how to read ranges within a particular generation of a
// particular GCS object. Optimised for (large) sequential reads.
//
// Not safe for concurrent access.
type RandomReader interface {
	// Panic if any internal invariants are violated.
	CheckInvariants()

	// Matches the semantics of io.ReaderAt, with the addition of context
	// support.
	ReadAt(ctx context.Context, p []byte, offset int64) (n int, err error)

	// Return the record for the object to which the reader is bound.
	Object() (o *gcs.Object)

	// Clean up any resources associated with the reader, which must not be used
	// again.
	Destroy()
}

// Create a random reader for the supplied object record that reads using the
// given bucket.
func NewRandomReader(
	o *gcs.Object,
	bucket gcs.Bucket) (rr RandomReader, err error) {
	rr = &randomReader{
		object: o,
		bucket: bucket,
		start:  -1,
		limit:  -1,
		seeks: 0,
		totalReadBytes: 0,
	}

	return
}

type randomReader struct {
	object *gcs.Object
	bucket gcs.Bucket

	// If non-nil, an in-flight read request and a function for cancelling it.
	//
	// INVARIANT: (reader == nil) == (cancel == nil)
	reader io.ReadCloser
	cancel func()

	// The range of the object that we expect reader to yield, when reader is
	// non-nil. When reader is nil, limit is the limit of the previous read
	// operation, or -1 if there has never been one.
	//
	// INVARIANT: start <= limit
	// INVARIANT: limit < 0 implies reader != nil
	start int64
	limit int64
	seeks uint64
	totalReadBytes uint64
}

func (rr *randomReader) CheckInvariants() {
	// INVARIANT: (reader == nil) == (cancel == nil)
	if (rr.reader == nil) != (rr.cancel == nil) {
		panic(fmt.Sprintf("Mismatch: %v vs. %v", rr.reader == nil, rr.cancel == nil))
	}

	// INVARIANT: start <= limit
	if !(rr.start <= rr.limit) {
		panic(fmt.Sprintf("Unexpected range: [%d, %d)", rr.start, rr.limit))
	}

	// INVARIANT: limit < 0 implies reader != nil
	if rr.limit < 0 && rr.reader != nil {
		panic(fmt.Sprintf("Unexpected non-nil reader with limit == %d", rr.limit))
	}
}

func (rr *randomReader) ReadAt(
	ctx context.Context,
	p []byte,
	offset int64) (n int, err error) {
	for len(p) > 0 {
		// Have we blown past the end of the object?
		if offset >= int64(rr.object.Size) {
			err = io.EOF
			return
		}

		// When the offset is AFTER the reader position, try to seek forward, within reason.
		// This happens when the kernel page cache serves some data. It's very common for
		// concurrent reads, often by only a few 128kB fuse read requests. The aim is to
		// re-use GCS connection and avoid throwing away already read data.
		// For parallel sequential reads to a single file, not throwing away the connections
		// is a 15-20x improvement in throughput: 150-200 MB/s instead of 10 MB/s.
		if rr.reader != nil && rr.start < offset && offset - rr.start < maxReadSize {
			bytes_to_skip := int64(offset - rr.start)
			p := make([]byte, bytes_to_skip)
			n, _ := rr.reader.Read(p)
			rr.start += int64(n)
		}

		// If we have an existing reader but it's positioned at the wrong place,
		// clean it up and throw it away.
		if rr.reader != nil && rr.start != offset {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
			rr.seeks += 1
		}

		// If we don't have a reader, start a read operation.
		if rr.reader == nil {
			err = rr.startRead(offset, int64(len(p)))
			if err != nil {
				err = fmt.Errorf("startRead: %v", err)
				return
			}
		}

		// Now we have a reader positioned at the correct place. Consume as much from
		// it as possible.
		var tmp int
		tmp, err = rr.readFull(ctx, p)

		n += tmp
		p = p[tmp:]
		rr.start += int64(tmp)
		offset += int64(tmp)
		rr.totalReadBytes += uint64(tmp)

		// Sanity check.
		if rr.start > rr.limit {
			err = fmt.Errorf("Reader returned %d too many bytes", rr.start-rr.limit)

			// Don't attempt to reuse the reader when it's behaving wackily.
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
			rr.start = -1
			rr.limit = -1

			return
		}

		// Are we finished with this reader now?
		if rr.start == rr.limit {
			rr.reader.Close()
			rr.reader = nil
			rr.cancel = nil
		}

		// Handle errors.
		switch {
		case err == io.EOF || err == io.ErrUnexpectedEOF:
			// For a non-empty buffer, ReadFull returns EOF or ErrUnexpectedEOF only
			// if the reader peters out early. That's fine, but it means we should
			// have hit the limit above.
			if rr.reader != nil {
				err = fmt.Errorf("Reader returned %d too few bytes", rr.limit-rr.start)
				return
			}

			err = nil

		case err != nil:
			// Propagate other errors.
			err = fmt.Errorf("readFull: %v", err)
			return
		}
	}

	return
}

func (rr *randomReader) Object() (o *gcs.Object) {
	o = rr.object
	return
}

func (rr *randomReader) Destroy() {
	// Close out the reader, if we have one.
	if rr.reader != nil {
		rr.reader.Close()
		rr.reader = nil
		rr.cancel = nil
	}
}

// Like io.ReadFull, but deals with the cancellation issues.
//
// REQUIRES: rr.reader != nil
func (rr *randomReader) readFull(
	ctx context.Context,
	p []byte) (n int, err error) {
	// Start a goroutine that will cancel the read operation we block on below if
	// the calling context is cancelled, but only if this method has not already
	// returned (to avoid souring the reader for the next read if this one is
	// successful, since the calling context will eventually be cancelled).
	readDone := make(chan struct{})
	defer close(readDone)

	go func() {
		select {
		case <-readDone:
			return

		case <-ctx.Done():
			select {
			case <-readDone:
				return

			default:
				rr.cancel()
			}
		}
	}()

	// Call through.
	n, err = io.ReadFull(rr.reader, p)

	return
}

// Ensure that rr.reader is set up for a range for which [start, start+size) is
// a prefix.
func (rr *randomReader) startRead(
	start int64,
	size int64) (err error) {
	// Make sure start and size are legal.
	if start < 0 || uint64(start) > rr.object.Size || size < 0 {
		err = fmt.Errorf(
			"Range [%d, %d) is illegal for %d-byte object",
			start,
			start+size,
			rr.object.Size)
		return
	}

	// GCS requests are expensive. Prefer to issue read requests to the end of
	// the object. Sequential reads will simply sip from the fire house
	// with each call to ReadAt. In practice, GCS will fill the TCP buffers
	// with about 6 MB of data. Requests from outside GCP will be charged
	// about 6MB of egress data, even if less data is read. Inside GCP
	// regions, GCS egress is free. This logic should limit the number of
	// GCS read requests, which are not free.

	// But if we notice random read patterns after a minimum number of seeks,
	// optimise for random reads. Random reads will read data in chunks of
	// (average read size in bytes rounded up to the next MB).
	end := int64(rr.object.Size)
	if rr.seeks >= minSeeksForRandom {
		averageReadBytes := rr.totalReadBytes / rr.seeks
		if averageReadBytes < maxReadSize {
			randomReadSize := int64(((averageReadBytes / MB) + 1 ) * MB)
			if randomReadSize < minReadSize {
			   randomReadSize = minReadSize
			}
			if randomReadSize > maxReadSize {
			   randomReadSize = maxReadSize
			}
			end = start + randomReadSize
		 }
	}

	// Begin the read.
	ctx, cancel := context.WithCancel(context.Background())
	rc, err := rr.bucket.NewReader(
		ctx,
		&gcs.ReadObjectRequest{
			Name:       rr.object.Name,
			Generation: rr.object.Generation,
			Range: &gcs.ByteRange{
				Start: uint64(start),
				Limit: uint64(end),
			},
		})

	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	rr.reader = rc
	rr.cancel = cancel
	rr.start = start
	rr.limit = end

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
)

type Substitute struct {
	searchPattern  string
	replacePattern string
	paths          []string
}

func (s *Substitute) Run() ([]byte, error) {
	grep := s.grep()
	sed := s.sed()
	grepOut, _ := grep.StdoutPipe()
	grep.Start()
	sed.Stdin = grepOut
	return sed.CombinedOutput()
}

func (s *Substitute) grep() *exec.Cmd {
	grepArgs := []string{"grep", "-El", s.searchPattern}
	if len(s.paths) > 0 {
		grepArgs = append(grepArgs, s.paths...)
	}
	return exec.Command("git", grepArgs...)
}

func (s *Substitute) sed() *exec.Cmd {
	search := fmt.Sprintf("s/%s/%s/g", s.searchPattern, s.replacePattern)
	return exec.Command("xargs", "sed", "-ri", search)
}
<commit_msg>use verbose versions of command options<commit_after>package main

import (
	"fmt"
	"os/exec"
)

type Substitute struct {
	searchPattern  string
	replacePattern string
	paths          []string
}

func (s *Substitute) Run() ([]byte, error) {
	grep := s.grep()
	sed := s.sed()
	grepOut, _ := grep.StdoutPipe()
	grep.Start()
	sed.Stdin = grepOut
	return sed.CombinedOutput()
}

func (s *Substitute) grep() *exec.Cmd {
	grepArgs := []string{"grep", "--extended-regexp", "--files-with-matches", s.searchPattern}
	if len(s.paths) > 0 {
		grepArgs = append(grepArgs, s.paths...)
	}
	return exec.Command("git", grepArgs...)
}

func (s *Substitute) sed() *exec.Cmd {
	search := fmt.Sprintf("s/%s/%s/g", s.searchPattern, s.replacePattern)
	return exec.Command("xargs", "sed", "--regexp-extended", "--in-place", search)
}
<|endoftext|>


<commit_before>package hrd

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"testing"
	"time"
	. "github.com/101loops/bdd"
	"appengine/aetest"
	"appengine/memcache"
)

var (
	ctx   aetest.Context
	store *Store
)

func TestSuite(t *testing.T) {
	var err error
	ctx, err = aetest.NewContext(nil)
	Check(err, IsNil)
	defer ctx.Close()

	store = NewStore(ctx)

	RegisterEntityMust(SimpleModel{})
	RegisterEntityMust(InvalidModel{})
	RegisterEntityMust(ComplexModel{})

	RunSpecs(t, "HRD Suite")
}

func randomColl() *Collection {
	var n int32
	binary.Read(rand.Reader, binary.LittleEndian, &n)
	return store.Coll(fmt.Sprintf("coll_%v", n))
}

func clearCache() {
	store.ClearCache()
	memcache.Flush(ctx)
}

type InvalidModel struct{}

type SimpleModel struct {
	id        int64
	Ignore    string    `datastore:"-"`
	Num       int64     `datastore:"num"`
	Data      []byte    `datastore:",index"`
	Text      string    `datastore:"html,index"`
	Time      time.Time `datastore:"timing,index,omitempty"`
	lifecycle string
	updatedAt time.Time
	createdAt time.Time
}

func (mdl *SimpleModel) ID() int64 {
	return mdl.id
}

func (mdl *SimpleModel) SetID(id int64) {
	mdl.id = id
}

func (mdl *SimpleModel) BeforeLoad() error {
	mdl.lifecycle = "before-load"
	return nil
}

func (mdl *SimpleModel) AfterLoad() error {
	mdl.lifecycle = "after-load"
	return nil
}

func (mdl *SimpleModel) BeforeSave() error {
	mdl.lifecycle = "before-save"
	return nil
}

func (mdl *SimpleModel) AfterSave() error {
	mdl.lifecycle = "after-save"
	return nil
}

func (mdl *SimpleModel) SetCreatedAt(t time.Time) {
	mdl.createdAt = t
}

func (mdl *SimpleModel) SetUpdatedAt(t time.Time) {
	mdl.updatedAt = t
}

type ComplexModel struct {
	Pair Pair `datastore:"tag"`
	//PairPtr  *Pair   `datastore:"pair"`
	Pairs []Pair `datastore:"tags"`
	//PairPtrs []*Pair `datastore:"pairs"`
	lifecycle string `datastore:"-"`
}

func (mdl *ComplexModel) BeforeLoad() error {
	mdl.lifecycle = "before-load"
	return nil
}

func (mdl *ComplexModel) AfterLoad() error {
	mdl.lifecycle = "after-load"
	return nil
}

func (mdl *ComplexModel) BeforeSave() error {
	mdl.lifecycle = "before-save"
	return nil
}

func (mdl *ComplexModel) AfterSave() error {
	mdl.lifecycle = "after-save"
	return nil
}

type Pair struct {
	Key string `datastore:"key,index,omitempty"`
	Val string
}
<commit_msg>fix test bug<commit_after>package hrd

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"testing"
	"time"
	. "github.com/101loops/bdd"
	"appengine/aetest"
	"appengine/memcache"
)

var (
	ctx   aetest.Context
	store *Store
)

func TestSuite(t *testing.T) {
	var err error
	ctx, err = aetest.NewContext(nil)
	if err != nil {
		panic(err)
	}
	defer ctx.Close()

	store = NewStore(ctx)

	RegisterEntityMust(SimpleModel{})
	RegisterEntityMust(InvalidModel{})
	RegisterEntityMust(ComplexModel{})

	RunSpecs(t, "HRD Suite")
}

func randomColl() *Collection {
	var n int32
	binary.Read(rand.Reader, binary.LittleEndian, &n)
	return store.Coll(fmt.Sprintf("coll_%v", n))
}

func clearCache() {
	store.ClearCache()
	memcache.Flush(ctx)
}

type InvalidModel struct{}

type SimpleModel struct {
	id        int64
	Ignore    string    `datastore:"-"`
	Num       int64     `datastore:"num"`
	Data      []byte    `datastore:",index"`
	Text      string    `datastore:"html,index"`
	Time      time.Time `datastore:"timing,index,omitempty"`
	lifecycle string
	updatedAt time.Time
	createdAt time.Time
}

func (mdl *SimpleModel) ID() int64 {
	return mdl.id
}

func (mdl *SimpleModel) SetID(id int64) {
	mdl.id = id
}

func (mdl *SimpleModel) BeforeLoad() error {
	mdl.lifecycle = "before-load"
	return nil
}

func (mdl *SimpleModel) AfterLoad() error {
	mdl.lifecycle = "after-load"
	return nil
}

func (mdl *SimpleModel) BeforeSave() error {
	mdl.lifecycle = "before-save"
	return nil
}

func (mdl *SimpleModel) AfterSave() error {
	mdl.lifecycle = "after-save"
	return nil
}

func (mdl *SimpleModel) SetCreatedAt(t time.Time) {
	mdl.createdAt = t
}

func (mdl *SimpleModel) SetUpdatedAt(t time.Time) {
	mdl.updatedAt = t
}

type ComplexModel struct {
	Pair Pair `datastore:"tag"`
	//PairPtr  *Pair   `datastore:"pair"`
	Pairs []Pair `datastore:"tags"`
	//PairPtrs []*Pair `datastore:"pairs"`
	lifecycle string `datastore:"-"`
}

func (mdl *ComplexModel) BeforeLoad() error {
	mdl.lifecycle = "before-load"
	return nil
}

func (mdl *ComplexModel) AfterLoad() error {
	mdl.lifecycle = "after-load"
	return nil
}

func (mdl *ComplexModel) BeforeSave() error {
	mdl.lifecycle = "before-save"
	return nil
}

func (mdl *ComplexModel) AfterSave() error {
	mdl.lifecycle = "after-save"
	return nil
}

type Pair struct {
	Key string `datastore:"key,index,omitempty"`
	Val string
}
<|endoftext|>