<commit_before>package main

/*
import (
	"bytes"
	"fmt"
	"testing"

	"golang.org/x/net/context"

	pb "github.com/creiht/formic/proto"
	"github.com/gogo/protobuf/proto"
)

// Minimal FileService for testing
type TestFS struct {
	writes [][]byte
	reads  [][]byte
}

func NewTestFS() *TestFS {
	return &TestFS{
		writes: make([][]byte, 0),
		reads:  make([][]byte, 0),
	}
}

func (fs *TestFS) GetChunk(ctx context.Context, id []byte) ([]byte, error) {
	if len(fs.reads) > 0 {
		chunk := fs.reads[0]
		fs.reads = fs.reads[1:]
		return chunk, nil
	} else {
		return []byte(""), nil
	}
}

func (fs *TestFS) WriteChunk(ctx context.Context, id, data []byte) error {
	fs.writes = append(fs.writes, data)
	return nil
}

func (fs *TestFS) clearwrites() {
	fs.writes = make([][]byte, 0)
}

func (fs *TestFS) addread(d []byte) {
	fs.reads = append(fs.reads, d)
}

func (ds *TestFS) GetAttr(ctx context.Context, id []byte) (*pb.Attr, error) {
	return &pb.Attr{}, nil
}

func (ds *TestFS) SetAttr(ctx context.Context, id []byte, attr *pb.Attr, valid uint32) (*pb.Attr, error) {
	return &pb.Attr{}, nil
}

func (ds *TestFS) Create(ctx context.Context, parent, id []byte, inode uint64, name string, attr *pb.Attr, isdir bool) (string, *pb.Attr, error) {
	return name, attr, nil
}

func (ds *TestFS) Lookup(ctx context.Context, parent []byte, name string) (string, *pb.Attr, error) {
	return "", &pb.Attr{}, nil
}

func (ds *TestFS) ReadDirAll(ctx context.Context, id []byte) (*pb.ReadDirAllResponse, error) {
	return &pb.ReadDirAllResponse{}, nil
}

func (ds *TestFS) Remove(ctx context.Context, parent []byte, name string) (int32, error) {
	return 1, nil
}

func (ds *TestFS) Update(ctx context.Context, id []byte, block, blocksize, size uint64, mtime int64) error {
	return nil
}

func (ds *TestFS) Symlink(ctx context.Context, parent, id []byte, name string, target string, attr *pb.Attr, inode uint64) (*pb.SymlinkResponse, error) {
	return &pb.SymlinkResponse{}, nil
}

func (ds *TestFS) Readlink(ctx context.Context, id []byte) (*pb.ReadlinkResponse, error) {
	return &pb.ReadlinkResponse{}, nil
}

func (ds *TestFS) Getxattr(ctx context.Context, id []byte, name string) (*pb.GetxattrResponse, error) {
	return &pb.GetxattrResponse{}, nil
}

func (ds *TestFS) Setxattr(ctx context.Context, id []byte, name string, value []byte) (*pb.SetxattrResponse, error) {
	return &pb.SetxattrResponse{}, nil
}

func (ds *TestFS) Listxattr(ctx context.Context, id []byte) (*pb.ListxattrResponse, error) {
	return &pb.ListxattrResponse{}, nil
}

func (ds *TestFS) Removexattr(ctx context.Context, id []byte, name string) (*pb.RemovexattrResponse, error) {
	return &pb.RemovexattrResponse{}, nil
}

func (ds *TestFS) Rename(ctx context.Context, oldParent, newParent []byte, oldName, newName string) (*pb.RenameResponse, error) {
	return &pb.RenameResponse{}, nil
}
func TestGetID(t *testing.T) {
	id1 := GetID(uint64(11), uint64(1), uint64(1), uint64(1))
	id2 := GetID(uint64(11), uint64(1), uint64(1), uint64(1))
	if !bytes.Equal(id1, id2) {
		t.Errorf("Generated IDs not equal")
	}
	id3 := GetID(uint64(11), uint64(1), uint64(1), uint64(2))
	if bytes.Equal(id1, id3) {
		t.Errorf("Generated IDs were equal")
	}
}

func TestCreate(t *testing.T) {
	api := NewApiServer(NewTestFS())
	_, err := api.Create(context.Background(), &pb.CreateRequest{Parent: 1, Name: "Test", Attr: &pb.Attr{Gid: 1001, Uid: 1001}})
	if err != nil {
		t.Error("Create Failed: ", err)
	}

}

func TestWrite_Basic(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 10
	chunk := pb.WriteRequest{
		Inode:   0,
		Offset:  0,
		Payload: []byte("1234567890"),
	}
	r, err := api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0])
	}
	chunk.Payload = []byte("1")
	fs.clearwrites()
	r, err = api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0])
	}

}

func TestWrite_Chunk(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 5
	chunk := pb.WriteRequest{
		Inode:   0,
		Offset:  0,
		Payload: []byte("1234567890"),
	}
	r, err := api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload[:5], fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload[:5], fs.writes[0])
	}
	if !bytes.Equal(chunk.Payload[5:], fs.writes[1]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload[5:], fs.writes[1])
	}
}

func TestWrite_Offset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 10
	chunk := pb.WriteRequest{
		Offset:  5,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err := api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][5:]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}
	fs.clearwrites()
	chunk = pb.WriteRequest{
		Offset:  2,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err = api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][2:7]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}

}

func TestWrite_MultiOffset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 20
	chunk := pb.WriteRequest{
		Offset:  5,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err := api.Write(context.Background(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][5:]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}
}

func TestRead_Basic(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 10
	write := []byte("0123456789")
	fs.addread(write)
	data, err := api.Read(context.Background(), &pb.ReadRequest{Inode: 0, Offset: 0, Size: 10})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload, write) {
		t.Errorf("Expected read: '%s' received: '%s'", write, data)
	}
}

func TestRead_Offset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 10
	write := []byte("0123456789")
	fs.addread(write)
	data, err := api.Read(context.Background(), &pb.ReadRequest{Inode: 0, Offset: 5, Size: 5})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload, write[5:]) {
		t.Errorf("Expected read: '%s' received: '%s'", write[5:], data)
	}
}

func TestRead_Chunk(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs)
	api.blocksize = 10
	write1 := []byte("0123456789")
	write2 := []byte("9876543210")
	fs.addread(write1)
	fs.addread(write2)
	data, err := api.Read(context.Background(), &pb.ReadRequest{Inode: 0, Offset: 0, Size: 20})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload[:10], write1) {
		t.Errorf("Expected read: '%s' received: '%s'", write1, data.Payload[:10])
	}
	if !bytes.Equal(data.Payload[10:], write2) {
		t.Errorf("Expected read: '%s' received: '%s'", write2, data.Payload[10:])
	}
}

func TestProtoWriteSize(t *testing.T) {
	data := make([]byte, 64*1024)
	block := &pb.FileBlock{
		Data:     data,
		Checksum: uint32(8),
	}
	b, _ := proto.Marshal(block)
	fmt.Printf("Storing 64K and checksum in protobufs takes %d bytes.", len(b))
}
*/
<commit_msg>Fixed tests to work again<commit_after>package main

import (
	"bytes"
	"fmt"
	"testing"
	"time"

	"google.golang.org/grpc/metadata"

	"golang.org/x/net/context"

	"github.com/creiht/formic"
	pb "github.com/creiht/formic/proto"
	"github.com/gogo/protobuf/proto"
	"github.com/satori/go.uuid"
)

// Minimal FileService for testing
type TestFS struct {
	writes [][]byte
	reads  [][]byte
}

func NewTestFS() *TestFS {
	return &TestFS{
		writes: make([][]byte, 0),
		reads:  make([][]byte, 0),
	}
}

func (fs *TestFS) InitFs(ctx context.Context, fsid []byte) error {
	return nil
}

func (fs *TestFS) GetDirent(ctx context.Context, parent []byte, name string) (*pb.DirEntry, error) {
	return nil, nil
}

func (fs *TestFS) GetInode(ctx context.Context, id []byte) (*pb.InodeEntry, error) {
	return nil, nil
}

func (fs *TestFS) GetChunk(ctx context.Context, id []byte) ([]byte, error) {
	if len(fs.reads) > 0 {
		chunk := fs.reads[0]
		fs.reads = fs.reads[1:]
		return chunk, nil
	} else {
		return []byte(""), nil
	}
}

func (fs *TestFS) WriteChunk(ctx context.Context, id, data []byte) error {
	fs.writes = append(fs.writes, data)
	return nil
}

func (fs *TestFS) DeleteChunk(ctx context.Context, id []byte, tsm int64) error {
	return nil
}

func (fs *TestFS) DeleteListing(ctx context.Context, parent []byte, name string, tsm int64) error {
	return nil
}

func (fs *TestFS) clearwrites() {
	fs.writes = make([][]byte, 0)
}

func (fs *TestFS) addread(d []byte) {
	fs.reads = append(fs.reads, d)
}

func (ds *TestFS) GetAttr(ctx context.Context, id []byte) (*pb.Attr, error) {
	return &pb.Attr{}, nil
}

func (ds *TestFS) SetAttr(ctx context.Context, id []byte, attr *pb.Attr, valid uint32) (*pb.Attr, error) {
	return &pb.Attr{}, nil
}

func (ds *TestFS) Create(ctx context.Context, parent, id []byte, inode uint64, name string, attr *pb.Attr, isdir bool) (string, *pb.Attr, error) {
	return name, attr, nil
}

func (ds *TestFS) Lookup(ctx context.Context, parent []byte, name string) (string, *pb.Attr, error) {
	return "", &pb.Attr{}, nil
}

func (ds *TestFS) ReadDirAll(ctx context.Context, id []byte) (*pb.ReadDirAllResponse, error) {
	return &pb.ReadDirAllResponse{}, nil
}

func (ds *TestFS) Remove(ctx context.Context, parent []byte, name string) (int32, error) {
	return 1, nil
}

func (ds *TestFS) Update(ctx context.Context, id []byte, block, blocksize, size uint64, mtime int64) error {
	return nil
}

func (ds *TestFS) Symlink(ctx context.Context, parent, id []byte, name string, target string, attr *pb.Attr, inode uint64) (*pb.SymlinkResponse, error) {
	return &pb.SymlinkResponse{}, nil
}

func (ds *TestFS) Readlink(ctx context.Context, id []byte) (*pb.ReadlinkResponse, error) {
	return &pb.ReadlinkResponse{}, nil
}

func (ds *TestFS) Getxattr(ctx context.Context, id []byte, name string) (*pb.GetxattrResponse, error) {
	return &pb.GetxattrResponse{}, nil
}

func (ds *TestFS) Setxattr(ctx context.Context, id []byte, name string, value []byte) (*pb.SetxattrResponse, error) {
	return &pb.SetxattrResponse{}, nil
}

func (ds *TestFS) Listxattr(ctx context.Context, id []byte) (*pb.ListxattrResponse, error) {
	return &pb.ListxattrResponse{}, nil
}

func (ds *TestFS) Removexattr(ctx context.Context, id []byte, name string) (*pb.RemovexattrResponse, error) {
	return &pb.RemovexattrResponse{}, nil
}

func (ds *TestFS) Rename(ctx context.Context, oldParent, newParent []byte, oldName, newName string) (*pb.RenameResponse, error) {
	return &pb.RenameResponse{}, nil
}

func getContext() context.Context {
	fsid := uuid.NewV4()
	c, _ := context.WithTimeout(context.Background(), 5*time.Second)
	c = metadata.NewContext(
		c,
		metadata.Pairs("fsid", fsid.String()),
	)
	return c
}

func TestGetID(t *testing.T) {
	id1 := formic.GetID([]byte("1"), uint64(1), uint64(1))
	id2 := formic.GetID([]byte("1"), uint64(1), uint64(1))
	if !bytes.Equal(id1, id2) {
		t.Errorf("Generated IDs not equal")
	}
	id3 := formic.GetID([]byte("1"), uint64(1), uint64(2))
	if bytes.Equal(id1, id3) {
		t.Errorf("Generated IDs were equal")
	}
}

func TestCreate(t *testing.T) {
	api := NewApiServer(NewTestFS(), 1)
	_, err := api.Create(getContext(), &pb.CreateRequest{Parent: 1, Name: "Test", Attr: &pb.Attr{Gid: 1001, Uid: 1001}})
	if err != nil {
		t.Error("Create Failed: ", err)
	}

}

func TestWrite_Basic(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 10
	chunk := pb.WriteRequest{
		Inode:   0,
		Offset:  0,
		Payload: []byte("1234567890"),
	}
	r, err := api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0])
	}
	chunk.Payload = []byte("1")
	fs.clearwrites()
	r, err = api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0])
	}

}

func TestWrite_Chunk(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 5
	chunk := pb.WriteRequest{
		Inode:   0,
		Offset:  0,
		Payload: []byte("1234567890"),
	}
	r, err := api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload[:5], fs.writes[0]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload[:5], fs.writes[0])
	}
	if !bytes.Equal(chunk.Payload[5:], fs.writes[1]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload[5:], fs.writes[1])
	}
}

func TestWrite_Offset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 10
	chunk := pb.WriteRequest{
		Offset:  5,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err := api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][5:]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}
	fs.clearwrites()
	chunk = pb.WriteRequest{
		Offset:  2,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err = api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][2:7]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}

}

func TestWrite_MultiOffset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 20
	chunk := pb.WriteRequest{
		Offset:  5,
		Payload: []byte("12345"),
		Inode:   0,
	}
	r, err := api.Write(getContext(), &chunk)
	if err != nil {
		t.Error("Write Failed: ", err)
	}
	if r.Status != 0 {
		t.Error("Write status expected: 0, received: ", r.Status)
	}
	if !bytes.Equal(chunk.Payload, fs.writes[0][5:]) {
		fmt.Println(fs.writes)
		t.Errorf("Expected write: '%s' recieved: '%s'", chunk.Payload, fs.writes[0][5:])
	}
}

func TestRead_Basic(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 10
	write := []byte("0123456789")
	fs.addread(write)
	data, err := api.Read(getContext(), &pb.ReadRequest{Inode: 0, Offset: 0, Size: 10})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload, write) {
		t.Errorf("Expected read: '%s' received: '%s'", write, data)
	}
}

func TestRead_Offset(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 10
	write := []byte("0123456789")
	fs.addread(write)
	data, err := api.Read(getContext(), &pb.ReadRequest{Inode: 0, Offset: 5, Size: 5})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload, write[5:]) {
		t.Errorf("Expected read: '%s' received: '%s'", write[5:], data)
	}
}

func TestRead_Chunk(t *testing.T) {
	fs := NewTestFS()
	api := NewApiServer(fs, 1)
	api.blocksize = 10
	write1 := []byte("0123456789")
	write2 := []byte("9876543210")
	fs.addread(write1)
	fs.addread(write2)
	data, err := api.Read(getContext(), &pb.ReadRequest{Inode: 0, Offset: 0, Size: 20})
	if err != nil {
		t.Error("Read Failed: ", err)
	}
	if !bytes.Equal(data.Payload[:10], write1) {
		t.Errorf("Expected read: '%s' received: '%s'", write1, data.Payload[:10])
	}
	if !bytes.Equal(data.Payload[10:], write2) {
		t.Errorf("Expected read: '%s' received: '%s'", write2, data.Payload[10:])
	}
}

func TestProtoWriteSize(t *testing.T) {
	data := make([]byte, 64*1024)
	block := &pb.FileBlock{
		Data:     data,
		Checksum: uint32(8),
	}
	b, _ := proto.Marshal(block)
	fmt.Printf("Storing 64K and checksum in protobufs takes %d bytes.", len(b))
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

var (
	VERSION     = "0.10.0"
	SPACES      = regexp.MustCompile("\\s+")
	INVALID_POS = errors.New("invalid position")

	OK              = 0
	MATCH_FOUND     = 100
	MATCH_NOT_FOUND = 101
)

type Pos struct {
	Start, End *int
}

func (p Pos) String() (result string) {
	if p.Start != nil {
		result = strconv.Itoa(*p.Start)
	}

	result += ":"

	if p.End != nil {
		result += strconv.Itoa(*p.End)
	}

	return
}

func (p *Pos) Set(s string) error {
	p.Start = nil
	p.End = nil

	parts := strings.Split(s, ":")
	if len(parts) < 1 || len(parts) > 2 {
		return INVALID_POS
	}

	if len(parts[0]) > 0 {
		v, err := strconv.Atoi(parts[0])
		if err != nil {
			return err
		}

		p.Start = &v
	}

	if len(parts) == 1 {
		// not a slice
		// note: same pointer (to distinguish from *p.End == *p.Start that returns an empty slice)
		p.End = p.Start
	} else if len(parts[1]) > 0 {
		v, err := strconv.Atoi(parts[1])
		if err != nil {
			return err
		}

		p.End = &v
	}

	return nil
}

func Slice(source []string, p Pos) []string {
	var start, end int

	if p.Start == nil {
		start = 0
	} else if *p.Start >= len(source) {
		return source[0:0]
	} else if *p.Start < 0 {
		start = len(source) + *p.Start

		if start < 0 {
			start = 0
		}
	} else {
		start = *p.Start
	}

	if p.End == p.Start {
		// this should return source[start]
		end = start + 1
	} else if p.End == nil || *p.End >= len(source) {
		return source[start:]
	} else if *p.End < 0 {
		end = len(source) + *p.End
	} else {
		end = *p.End
	}

	if end < start {
		end = start
	}

	return source[start:end]
}

func Quote(a []string) []string {
	q := make([]string, len(a))
	for i, s := range a {
		q[i] = fmt.Sprintf("%q", s)
	}

	return q
}

func Unquote(a []string) []string {
	q := make([]string, len(a))
	for i, s := range a {
		q[i] = strings.Trim(s, `"'`)
	}

	return q
}

func Print(format string, a []string) {
	printable := make([]interface{}, len(a))

	for i, v := range a {
		printable[i] = v
	}

	fmt.Printf(format, printable...)
}

func main() {
	version := flag.Bool("version", false, "print version and exit")
	quote := flag.Bool("quote", false, "quote returned fields")
	unquote := flag.Bool("unquote", false, "quote returned fields")
	ifs := flag.String("ifs", " ", "input field separator")
	ofs := flag.String("ofs", " ", "input field separator")
	format := flag.String("printf", "", "output is formatted according to specified format")
	matches := flag.String("matches", "", "return status code 100 if any line matches the specified pattern, 101 otherwise")

	flag.Parse()

	if *version {
		fmt.Printf("%s version %s\n", path.Base(os.Args[0]), VERSION)
		return
	}

	pos := make([]Pos, len(flag.Args()))

	for i, arg := range flag.Args() {
		pos[i].Set(arg)
	}

	if len(*format) > 0 && !strings.HasSuffix(*format, "\n") {
		*format += "\n"
	}

	var match_pattern *regexp.Regexp
	status_code := OK

	if len(*matches) > 0 {
		match_pattern = regexp.MustCompile(*matches)
		status_code = MATCH_NOT_FOUND
	}

	scanner := bufio.NewScanner(os.Stdin)

	for scanner.Scan() {
		if scanner.Err() != nil {
			log.Fatal(scanner.Err())
		}

		line := scanner.Text()
		fields := []string{line} // $0 is the full line

		// split the line according to input field separator
		if *ifs == " " {
			fields = append(fields, SPACES.Split(strings.TrimSpace(line), -1)...)
		} else {
			fields = append(fields, strings.Split(line, *ifs)...)
		}

		var result []string

		// do some processing
		if len(pos) > 0 {
			result = make([]string, 0)

			for _, p := range pos {
				val := strings.Join(Slice(fields, p), *ifs)
				result = append(result, val)
			}
		} else {
			result = fields[0:1]
		}

		if *unquote {
			result = Unquote(result)
		}

		if *quote {
			result = Quote(result)
		}

		if len(*format) > 0 {
			Print(*format, result)
		} else {
			// join the result according to output field separator
			fmt.Println(strings.Join(result, *ofs))
		}

		if match_pattern != nil && match_pattern.MatchString(line) {
			status_code = MATCH_FOUND
		}
	}

	os.Exit(status_code)
}
<commit_msg>Added "--re='split-pattern'" option. The value should be a regular expression with groups, in which case lines are matched against it (via regexp.FindSubmatch) and match and groups are returned as output fields.<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
)

var (
	VERSION     = "0.10.0"
	SPACES      = regexp.MustCompile("\\s+")
	INVALID_POS = errors.New("invalid position")

	OK              = 0
	MATCH_FOUND     = 100
	MATCH_NOT_FOUND = 101
)

type Pos struct {
	Start, End *int
}

func (p Pos) String() (result string) {
	if p.Start != nil {
		result = strconv.Itoa(*p.Start)
	}

	result += ":"

	if p.End != nil {
		result += strconv.Itoa(*p.End)
	}

	return
}

func (p *Pos) Set(s string) error {
	p.Start = nil
	p.End = nil

	parts := strings.Split(s, ":")
	if len(parts) < 1 || len(parts) > 2 {
		return INVALID_POS
	}

	if len(parts[0]) > 0 {
		v, err := strconv.Atoi(parts[0])
		if err != nil {
			return err
		}

		p.Start = &v
	}

	if len(parts) == 1 {
		// not a slice
		// note: same pointer (to distinguish from *p.End == *p.Start that returns an empty slice)
		p.End = p.Start
	} else if len(parts[1]) > 0 {
		v, err := strconv.Atoi(parts[1])
		if err != nil {
			return err
		}

		p.End = &v
	}

	return nil
}

func Slice(source []string, p Pos) []string {
	var start, end int

	if p.Start == nil {
		start = 0
	} else if *p.Start >= len(source) {
		return source[0:0]
	} else if *p.Start < 0 {
		start = len(source) + *p.Start

		if start < 0 {
			start = 0
		}
	} else {
		start = *p.Start
	}

	if p.End == p.Start {
		// this should return source[start]
		end = start + 1
	} else if p.End == nil || *p.End >= len(source) {
		return source[start:]
	} else if *p.End < 0 {
		end = len(source) + *p.End
	} else {
		end = *p.End
	}

	if end < start {
		end = start
	}

	return source[start:end]
}

func Quote(a []string) []string {
	q := make([]string, len(a))
	for i, s := range a {
		q[i] = fmt.Sprintf("%q", s)
	}

	return q
}

func Unquote(a []string) []string {
	q := make([]string, len(a))
	for i, s := range a {
		q[i] = strings.Trim(s, `"'`)
	}

	return q
}

func Print(format string, a []string) {
	printable := make([]interface{}, len(a))

	for i, v := range a {
		printable[i] = v
	}

	fmt.Printf(format, printable...)
}

func main() {
	version := flag.Bool("version", false, "print version and exit")
	quote := flag.Bool("quote", false, "quote returned fields")
	unquote := flag.Bool("unquote", false, "quote returned fields")
	ifs := flag.String("ifs", " ", "input field separator")
	ofs := flag.String("ofs", " ", "input field separator")
        re := flag.String("re", "", "regular expression for parsing input")
	format := flag.String("printf", "", "output is formatted according to specified format")
	matches := flag.String("matches", "", "return status code 100 if any line matches the specified pattern, 101 otherwise")

	flag.Parse()

	if *version {
		fmt.Printf("%s version %s\n", path.Base(os.Args[0]), VERSION)
		return
	}

	pos := make([]Pos, len(flag.Args()))

	for i, arg := range flag.Args() {
		pos[i].Set(arg)
	}

	if len(*format) > 0 && !strings.HasSuffix(*format, "\n") {
		*format += "\n"
	}

        var split_pattern *regexp.Regexp
	var match_pattern *regexp.Regexp
	status_code := OK

	if len(*matches) > 0 {
		match_pattern = regexp.MustCompile(*matches)
		status_code = MATCH_NOT_FOUND
	}

        if len(*re) > 0 {
            split_pattern = regexp.MustCompile(*re)
        }

	scanner := bufio.NewScanner(os.Stdin)

	for scanner.Scan() {
		if scanner.Err() != nil {
			log.Fatal(scanner.Err())
		}

		line := scanner.Text()
		fields := []string{line} // $0 is the full line

		// split the line according to input field separator
                if (split_pattern != nil) {
                        if matches := split_pattern.FindStringSubmatch(line); matches != nil {
                            fields = matches
                        }
		} else if *ifs == " " {
			fields = append(fields, SPACES.Split(strings.TrimSpace(line), -1)...)
		} else {
			fields = append(fields, strings.Split(line, *ifs)...)
		}

		var result []string

		// do some processing
		if len(pos) > 0 {
			result = make([]string, 0)

			for _, p := range pos {
				val := strings.Join(Slice(fields, p), *ifs)
				result = append(result, val)
			}
		} else {
			result = fields[0:1]
		}

		if *unquote {
			result = Unquote(result)
		}

		if *quote {
			result = Quote(result)
		}

		if len(*format) > 0 {
			Print(*format, result)
		} else {
			// join the result according to output field separator
			fmt.Println(strings.Join(result, *ofs))
		}

		if match_pattern != nil && match_pattern.MatchString(line) {
			status_code = MATCH_FOUND
		}
	}

	os.Exit(status_code)
}
<|endoftext|>


<commit_before>package gobf

import (
	"fmt"
	"io"
)

type Program struct {
}

func NewProgram(r io.Reader) *Program {
	return &Program{}
}

func (p *Program) Run() error {
	return fmt.Errorf("Not implemented")
}
<commit_msg>add main loop<commit_after>package gobf

import (
	"fmt"
	"io"
	"io/ioutil"
)

// Program represents brainfuck programm
type Program struct {
	code []byte
}

// NewProgram initialize new program
func NewProgram(r io.Reader) *Program {
	code, err := ioutil.ReadAll(r)
	if err != nil {
		panic(fmt.Sprintf("Failed to read program code: %v", err))
	}
	return &Program{
		code: code,
	}
}

// Run runs brainfuck program
func (p *Program) Run() error {
	for p.nextCmd() {
		err := p.runCmd()
		if err != nil {
			return err
		}
	}
	return fmt.Errorf("Not implemented")
}

func (p *Program) nextCmd() bool {
	return false
}

func (p *Program) runCmd() error {
	return fmt.Errorf("Bad cmd symbol: %v", p.cmd)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"errors"
	"fmt"
	"net/http"
	"reflect"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

// PutComplete is like Put, but errors if a key is incomplete.
func (g *Goon) PutComplete(src interface{}) (*datastore.Key, error) {
	k, err := g.getStructKey(src)
	if err != nil {
		return nil, err
	}
	if k.Incomplete() {
		err := fmt.Errorf("goon: incomplete key: %v", k)
		g.error(err)
		return nil, err
	}
	return g.Put(src)
}

// PutMultiComplete is like PutMulti, but errors if a key is incomplete.
func (g *Goon) PutMultiComplete(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}
	for i, k := range keys {
		if k.Incomplete() {
			err := fmt.Errorf("goon: incomplete key (%dth index): %v", i, k)
			g.error(err)
			return nil, err
		}
	}
	return g.PutMulti(src)
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

const getMultiLimit = 1000

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}
	if len(memkeys) == 0 {
		return nil
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)
	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}
	if len(dskeys) == 0 {
		return nil
	}

	multiErr := make(appengine.MultiError, len(keys))
	var toCache []interface{}
	var ret error
	for i := 0; i <= len(dskeys)/getMultiLimit; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(dskeys) {
			hi = len(dskeys)
		}
		gmerr := datastore.GetMulti(g.context, dskeys[lo:hi], dsdst[lo:hi])
		if gmerr != nil {
			merr, ok := gmerr.(appengine.MultiError)
			if !ok {
				g.error(gmerr)
				return gmerr
			}
			for i, idx := range dixs[lo:hi] {
				multiErr[idx] = merr[i]
				if merr[i] == nil {
					toCache = append(toCache, dsdst[lo+i])
				}
			}
			ret = multiErr
		} else {
			toCache = append(toCache, dsdst[lo:hi]...)
		}
	}

	if len(toCache) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

const deleteMultiLimit = 500

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	for i := 0; i <= len(keys)/deleteMultiLimit; i++ {
		lo := i * deleteMultiLimit
		hi := (i + 1) * deleteMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if err := datastore.DeleteMulti(g.context, keys[lo:hi]); err != nil {
			return err
		}
	}
	return nil
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<commit_msg>Disable memory cache for now<commit_after>/*
 * Copyright (c) 2012 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goon

import (
	"errors"
	"fmt"
	"net/http"
	"reflect"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

var (
	// LogErrors issues appengine.Context.Errorf on any error.
	LogErrors bool = true
)

// Goon holds the app engine context and request memory cache.
type Goon struct {
	context       appengine.Context
	cache         map[string]interface{}
	inTransaction bool
	toSet         map[string]interface{}
	toDelete      []string
}

func memkey(k *datastore.Key) string {
	return k.Encode()
}

// NewGoon creates a new Goon object from the given request.
func NewGoon(r *http.Request) *Goon {
	return FromContext(appengine.NewContext(r))
}

// FromContext creates a new Goon object from the given appengine Context.
func FromContext(c appengine.Context) *Goon {
	return &Goon{
		context: c,
		cache:   make(map[string]interface{}),
	}
}

func (g *Goon) error(err error) {
	if LogErrors {
		g.context.Errorf("goon: %v", err.Error())
	}
}

func (g *Goon) extractKeys(src interface{}) ([]*datastore.Key, error) {
	v := reflect.Indirect(reflect.ValueOf(src))
	if v.Kind() != reflect.Slice {
		return nil, errors.New("goon: value must be a slice or pointer-to-slice")
	}
	l := v.Len()

	keys := make([]*datastore.Key, l)
	for i := 0; i < l; i++ {
		vi := v.Index(i)
		key, err := g.getStructKey(vi.Interface())
		if err != nil {
			return nil, err
		}
		keys[i] = key
	}
	return keys, nil
}

// Key is the same as KeyError, except nil is returned on error.
func (g *Goon) Key(src interface{}) *datastore.Key {
	if k, err := g.KeyError(src); err == nil {
		return k
	}
	return nil
}

// Key returns the key of src based on its properties.
func (g *Goon) KeyError(src interface{}) (*datastore.Key, error) {
	return g.getStructKey(src)
}

// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
	var ng *Goon
	err := datastore.RunInTransaction(g.context, func(tc appengine.Context) error {
		ng = &Goon{
			context:       tc,
			inTransaction: true,
			toSet:         make(map[string]interface{}),
		}
		return f(ng)
	}, opts)

	if err == nil {
		for k, v := range ng.toSet {
			g.cache[k] = v
		}

		for _, k := range ng.toDelete {
			delete(g.cache, k)
		}
	} else {
		g.error(err)
	}

	return err
}

// Put saves the entity src into the datastore based on src's key k. If k
// is an incomplete key, the returned key will be a unique key generated by
// the datastore.
func (g *Goon) Put(src interface{}) (*datastore.Key, error) {
	ks, err := g.PutMulti([]interface{}{src})
	if len(ks) == 1 {
		return ks[0], err
	}
	return nil, err
}

// PutMany is a wrapper around PutMulti.
func (g *Goon) PutMany(srcs ...interface{}) ([]*datastore.Key, error) {
	return g.PutMulti(srcs)
}

const putMultiLimit = 500

// PutMulti is a batch version of Put.
//
// src must satisfy the same conditions as the dst argument to GetMulti.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}

	var memkeys []string
	for _, key := range keys {
		if !key.Incomplete() {
			memkeys = append(memkeys, memkey(key))
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i <= len(keys)/putMultiLimit; i++ {
		lo := i * putMultiLimit
		hi := (i + 1) * putMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		rkeys, err := datastore.PutMulti(g.context, keys[lo:hi], v.Slice(lo, hi).Interface())
		if err != nil {
			g.error(err)
			return nil, err
		}

		for i, key := range keys[lo:hi] {
			vi := v.Index(lo + i).Interface()
			if key.Incomplete() {
				setStructKey(vi, rkeys[i])
				keys[i] = rkeys[i]
			}
			if g.inTransaction {
				g.toSet[memkey(rkeys[i])] = vi
			}
		}
	}

	if !g.inTransaction {
		g.putMemoryMulti(src)
	}

	return keys, nil
}

// PutComplete is like Put, but errors if a key is incomplete.
func (g *Goon) PutComplete(src interface{}) (*datastore.Key, error) {
	k, err := g.getStructKey(src)
	if err != nil {
		return nil, err
	}
	if k.Incomplete() {
		err := fmt.Errorf("goon: incomplete key: %v", k)
		g.error(err)
		return nil, err
	}
	return g.Put(src)
}

// PutMultiComplete is like PutMulti, but errors if a key is incomplete.
func (g *Goon) PutMultiComplete(src interface{}) ([]*datastore.Key, error) {
	keys, err := g.extractKeys(src)
	if err != nil {
		return nil, err
	}
	for i, k := range keys {
		if k.Incomplete() {
			err := fmt.Errorf("goon: incomplete key (%dth index): %v", i, k)
			g.error(err)
			return nil, err
		}
	}
	return g.PutMulti(src)
}

func (g *Goon) putMemoryMulti(src interface{}) {
	v := reflect.Indirect(reflect.ValueOf(src))
	for i := 0; i < v.Len(); i++ {
		g.putMemory(v.Index(i).Interface())
	}
}

func (g *Goon) putMemory(src interface{}) {
	key, _ := g.getStructKey(src)
	g.cache[memkey(key)] = src
}

func (g *Goon) putMemcache(srcs []interface{}) error {
	items := make([]*memcache.Item, len(srcs))

	for i, src := range srcs {
		gob, err := toGob(src)
		if err != nil {
			g.error(err)
			return err
		}
		key, err := g.getStructKey(src)

		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: gob,
		}
	}

	memcache.SetMulti(g.context, items)
	g.putMemoryMulti(srcs)
	return nil
}

// Get loads the entity based on dst's key into dst
// If there is no such entity for the key, Get returns
// datastore.ErrNoSuchEntity.
func (g *Goon) Get(dst interface{}) error {
	dsts := []interface{}{dst}
	if err := g.GetMulti(dsts); err != nil {
		// Look for an embedded error if it's multi
		if me, ok := err.(appengine.MultiError); ok {
			for i, merr := range me {
				if i == 0 {
					return merr
				}
			}
		}
		// Not multi, normal error
		return err
	}
	return nil
}

const getMultiLimit = 1000

// GetMulti is a batch version of Get.
//
// dst has similar constraints as datastore.GetMulti.
func (g *Goon) GetMulti(dst interface{}) error {
	keys, err := g.extractKeys(dst)
	if err != nil {
		return err
	}

	if g.inTransaction {
		// todo: support getMultiLimit in transactions
		return datastore.GetMulti(g.context, keys, dst)
	}

	var dskeys []*datastore.Key
	var dsdst []interface{}
	var dixs []int

	var memkeys []string
	var mixs []int

	v := reflect.Indirect(reflect.ValueOf(dst))
	for i, key := range keys {
		m := memkey(key)
		if s, present := g.cache[m]; present && false {
			vi := v.Index(i)
			vi.Set(reflect.ValueOf(s))
		} else {
			memkeys = append(memkeys, m)
			mixs = append(mixs, i)
		}
	}
	if len(memkeys) == 0 {
		return nil
	}

	memvalues, _ := memcache.GetMulti(g.context, memkeys)
	for i, m := range memkeys {
		d := v.Index(mixs[i]).Interface()
		if s, present := memvalues[m]; present {
			err := fromGob(d, s.Value)
			if err != nil {
				g.error(err)
				return err
			}

			g.putMemory(d)
		} else {
			key, err := g.getStructKey(d)
			if err != nil {
				g.error(err)
				return err
			}
			dskeys = append(dskeys, key)
			dsdst = append(dsdst, d)
			dixs = append(dixs, mixs[i])
		}
	}
	if len(dskeys) == 0 {
		return nil
	}

	multiErr := make(appengine.MultiError, len(keys))
	var toCache []interface{}
	var ret error
	for i := 0; i <= len(dskeys)/getMultiLimit; i++ {
		lo := i * getMultiLimit
		hi := (i + 1) * getMultiLimit
		if hi > len(dskeys) {
			hi = len(dskeys)
		}
		gmerr := datastore.GetMulti(g.context, dskeys[lo:hi], dsdst[lo:hi])
		if gmerr != nil {
			merr, ok := gmerr.(appengine.MultiError)
			if !ok {
				g.error(gmerr)
				return gmerr
			}
			for i, idx := range dixs[lo:hi] {
				multiErr[idx] = merr[i]
				if merr[i] == nil {
					toCache = append(toCache, dsdst[lo+i])
				}
			}
			ret = multiErr
		} else {
			toCache = append(toCache, dsdst[lo:hi]...)
		}
	}

	if len(toCache) > 0 {
		if err := g.putMemcache(toCache); err != nil {
			g.error(err)
			return err
		}
	}

	return ret
}

// Delete deletes the entity for the given key.
func (g *Goon) Delete(key *datastore.Key) error {
	keys := []*datastore.Key{key}
	return g.DeleteMulti(keys)
}

const deleteMultiLimit = 500

// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
	memkeys := make([]string, len(keys))
	for i, k := range keys {
		mk := memkey(k)
		memkeys[i] = mk

		if g.inTransaction {
			g.toDelete = append(g.toDelete, mk)
		} else {
			delete(g.cache, mk)
		}
	}

	// Memcache needs to be updated after the datastore to prevent a common race condition
	defer memcache.DeleteMulti(g.context, memkeys)

	for i := 0; i <= len(keys)/deleteMultiLimit; i++ {
		lo := i * deleteMultiLimit
		hi := (i + 1) * deleteMultiLimit
		if hi > len(keys) {
			hi = len(keys)
		}
		if err := datastore.DeleteMulti(g.context, keys[lo:hi]); err != nil {
			return err
		}
	}
	return nil
}

// NotFound returns true if err is an appengine.MultiError and err[idx] is a datastore.ErrNoSuchEntity.
func NotFound(err error, idx int) bool {
	if merr, ok := err.(appengine.MultiError); ok {
		return idx < len(merr) && merr[idx] == datastore.ErrNoSuchEntity
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program gops is a tool to list currently running Go processes.
package main

import (
	"fmt"
	"log"

	"hello/gops/internal/objfile"

	ps "github.com/keybase/go-ps"
)

func main() {
	pss, err := ps.Processes()
	if err != nil {
		log.Fatal(err)
	}

	var undetermined int
	for _, pr := range pss {
		name, err := pr.Path()
		if err != nil {
			undetermined++
			continue
		}

		ok, err := isGo(name)
		if err != nil {
			// TODO(jbd): worth to report the number?
			continue
		}

		if ok {
			fmt.Printf("%d\t%v\t(%v)\n", pr.Pid(), pr.Executable(), name)
		}
	}

	if undetermined > 0 {
		fmt.Printf("\n%d processes left undetermined\n", undetermined)
	}
}

func isGo(filename string) (ok bool, err error) {
	obj, err := objfile.Open(filename)
	if err != nil {
		return false, err
	}
	defer obj.Close()

	symbols, err := obj.Symbols()
	if err != nil {
		return false, err
	}

	// TODO(jbd): find a faster way to determine Go programs
	// looping over the symbols is a joke.
	for _, s := range symbols {
		if s.Name == "runtime.buildVersion" {
			return true, nil
		}
	}
	return false, nil
}
<commit_msg>fix style<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program gops is a tool to list currently running Go processes.
package main

import (
	"fmt"
	"log"

	"hello/gops/internal/objfile"

	ps "github.com/keybase/go-ps"
)

func main() {
	pss, err := ps.Processes()
	if err != nil {
		log.Fatal(err)
	}

	var undetermined int
	for _, pr := range pss {
		name, err := pr.Path()
		if err != nil {
			undetermined++
			continue
		}
		ok, err := isGo(name)
		if err != nil {
			// TODO(jbd): worth to report the number?
			continue
		}
		if ok {
			fmt.Printf("%d\t%v\t(%v)\n", pr.Pid(), pr.Executable(), name)
		}
	}

	if undetermined > 0 {
		fmt.Printf("\n%d processes left undetermined\n", undetermined)
	}
}

func isGo(filename string) (ok bool, err error) {
	obj, err := objfile.Open(filename)
	if err != nil {
		return false, err
	}
	defer obj.Close()

	symbols, err := obj.Symbols()
	if err != nil {
		return false, err
	}

	// TODO(jbd): find a faster way to determine Go programs
	// looping over the symbols is a joke.
	for _, s := range symbols {
		if s.Name == "runtime.buildVersion" {
			return true, nil
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>package gseg

import (
	//	"fmt"
	"github.com/l2x/gseg/trie"
	"strings"
)

var (
	MaxWordLen = 9
)

type Seg struct {
	dict *trie.Trie
}

type Cache struct {
	Ind int
	C   []Cache
}

func New() Seg {
	return Seg{}
}

//Simple 匹配方法
func (s *Seg) Simple(words string) []string {
	var w []string = wordsInit(words)
	var res []string = []string{}
	var start, end, i, max int
	max = len(w)

	for start < max {
		_, i = s.dict.GetMax(w[start:])
		end = start + i
		res = append(res, strings.Join(w[start:end], ""))
		start = end
	}

	return res
}

//complex 匹配方法
func (s *Seg) Complex(words string) []string {
	var w []string = wordsInit(words)

	segment := []string{}
	start := 0
	end := 0
	max := len(w)

	for start < max {
		cache := searchWords(s, w[start:])

		//如果第一个词只有一个, 那就把这个词作为第一个词
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end
			continue
		}

		//maximum matching
		cache = maxMatch(cache)

		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//largest average word length
		cache = largestAvg(cache)
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//smallest variance of word lengths
		cache = smallestVariance(cache)
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//TODO
		//largest sum of degree of morphemic freedom of one-character words
		end = start + cache[0][0] + 1
		segment = append(segment, strings.Join(w[start:end], ""))
		start = end

	}

	return segment
}
<commit_msg>update<commit_after>package gseg

import (
	//	"fmt"
	"github.com/l2x/gseg/trie"
	"strings"
)

var (
	MaxWordLen = 9
)

type Seg struct {
	dict *trie.Trie
}

type Cache struct {
	Ind int
	C   []Cache
}

func New() Seg {
	return Seg{}
}

//Simple 匹配方法
func (s *Seg) Simple(words string) []string {
	var w []string = wordsInit(words)
	var res []string = []string{}
	var start, end, i, max int
	max = len(w)

	for start < max {
		_, i = s.dict.GetMax(w[start:])
		end = start + i
		res = append(res, strings.Join(w[start:end], ""))
		start = end
	}

	return res
}

//complex 匹配方法
func (s *Seg) Complex(words string) []string {
	var w []string = wordsInit(words)

	segment := []string{}
	start := 0
	end := 0
	max := len(w)

	for start < max {
		cache := searchWords(s, w[start:])

		//如果第一个词只有一个, 那就把这个词作为第一个词
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end
			continue
		}

		//maximum matching
		cache = maxMatch(cache)

		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//largest average word length
		cache = largestAvg(cache)
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//smallest variance of word lengths
		cache = smallestVariance(cache)
		if len(cache) == 1 {
			end = start + cache[0][0] + 1
			segment = append(segment, strings.Join(w[start:end], ""))
			start = end

			continue
		}

		//TODO
		//largest sum of degree of morphemic freedom of one-character words

		end = start + cache[0][0] + 1
		segment = append(segment, strings.Join(w[start:end], ""))
		start = end
	}

	return segment
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pat

import (
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
)

// New returns a new router.
func New() *Router {
	return &Router{}
}

// Router is a request router that implements a pat-like API.
//
// pat docs: http://gopkgdoc.appspot.com/pkg/github.com/bmizerany/pat
type Router struct {
	mux.Router
}

// Add registers a pattern with a handler for the given request method.
func (r *Router) Add(meth, pat string, h http.Handler) *mux.Route {
	return r.NewRoute().PathPrefix(pat).Handler(h).Methods(meth)
}

// Delete registers a pattern with a handler for DELETE requests.
func (r *Router) Delete(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("DELETE", pat, h)
}

// Get registers a pattern with a handler for GET requests.
func (r *Router) Get(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("GET", pat, h)
}

// Post registers a pattern with a handler for POST requests.
func (r *Router) Post(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("POST", pat, h)
}

// Put registers a pattern with a handler for PUT requests.
func (r *Router) Put(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("PUT", pat, h)
}

// ServeHTTP dispatches the handler registered in the matched route.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match mux.RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		registerVars(req, match.Vars)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	defer context.Clear(req)
	handler.ServeHTTP(w, req)
}

// registerVars adds the matched route variables to the URL query.
func registerVars(r *http.Request, vars map[string]string) {
	parts, i := make([]string, len(vars)), 0
	for key, value := range vars {
		parts[i] = url.QueryEscape(":"+key) + "=" + url.QueryEscape(value)
		i++
	}
	q := strings.Join(parts, "&")
	if r.URL.RawQuery == "" {
		r.URL.RawQuery = q
	} else {
		r.URL.RawQuery += "&" + q
	}
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<commit_msg>Update URL to github.com/bmizerany/pat docs<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pat

import (
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/gorilla/context"
	"github.com/gorilla/mux"
)

// New returns a new router.
func New() *Router {
	return &Router{}
}

// Router is a request router that implements a pat-like API.
//
// pat docs: http://godoc.org/github.com/bmizerany/pat
type Router struct {
	mux.Router
}

// Add registers a pattern with a handler for the given request method.
func (r *Router) Add(meth, pat string, h http.Handler) *mux.Route {
	return r.NewRoute().PathPrefix(pat).Handler(h).Methods(meth)
}

// Delete registers a pattern with a handler for DELETE requests.
func (r *Router) Delete(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("DELETE", pat, h)
}

// Get registers a pattern with a handler for GET requests.
func (r *Router) Get(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("GET", pat, h)
}

// Post registers a pattern with a handler for POST requests.
func (r *Router) Post(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("POST", pat, h)
}

// Put registers a pattern with a handler for PUT requests.
func (r *Router) Put(pat string, h http.HandlerFunc) *mux.Route {
	return r.Add("PUT", pat, h)
}

// ServeHTTP dispatches the handler registered in the matched route.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Clean path to canonical form and redirect.
	if p := cleanPath(req.URL.Path); p != req.URL.Path {
		w.Header().Set("Location", p)
		w.WriteHeader(http.StatusMovedPermanently)
		return
	}
	var match mux.RouteMatch
	var handler http.Handler
	if matched := r.Match(req, &match); matched {
		handler = match.Handler
		registerVars(req, match.Vars)
	}
	if handler == nil {
		if r.NotFoundHandler == nil {
			r.NotFoundHandler = http.NotFoundHandler()
		}
		handler = r.NotFoundHandler
	}
	defer context.Clear(req)
	handler.ServeHTTP(w, req)
}

// registerVars adds the matched route variables to the URL query.
func registerVars(r *http.Request, vars map[string]string) {
	parts, i := make([]string, len(vars)), 0
	for key, value := range vars {
		parts[i] = url.QueryEscape(":"+key) + "=" + url.QueryEscape(value)
		i++
	}
	q := strings.Join(parts, "&")
	if r.URL.RawQuery == "" {
		r.URL.RawQuery = q
	} else {
		r.URL.RawQuery += "&" + q
	}
}

// cleanPath returns the canonical path for p, eliminating . and .. elements.
// Borrowed from the net/http package.
func cleanPath(p string) string {
	if p == "" {
		return "/"
	}
	if p[0] != '/' {
		p = "/" + p
	}
	np := path.Clean(p)
	// path.Clean removes trailing slash except for root;
	// put the trailing slash back if necessary.
	if p[len(p)-1] == '/' && np != "/" {
		np += "/"
	}
	return np
}
<|endoftext|>


<commit_before>// Package findb contains the persistence layer for the fin package.
package findb

import (
  "errors"
  "github.com/keep94/appcommon/db"
  "github.com/keep94/finance/fin"
  "github.com/keep94/gofunctional3/consume"
  "github.com/keep94/gofunctional3/functional"
  "time"
)

var (
  ConcurrentUpdate = errors.New("findb: Concurrent update.")
  NoSuchId = errors.New("findb: No Such Id.")
  NoPermission = errors.New("findb: Insufficient permission.")
)

type AccountByIdRunner interface {
  // AccountById fetches an account by Id.
  AccountById(t db.Transaction, acctId int64, account *fin.Account) error
}

type AccountsRunner interface {
  // Accounts fetches all accounts.
  Accounts(t db.Transaction, consumer functional.Consumer) error
}

type ActiveAccountsRunner interface {
  // ActiveAccounts fetches all active accounts sorted by name.
  ActiveAccounts(t db.Transaction) (accounts []*fin.Account, err error)
}

type AddAccountRunner interface {
  // AddAccount adds a new account.
  AddAccount(t db.Transaction, Account *fin.Account) error
}

type UpdateAccountImportSDRunner interface {
  // UpdateAccountImportSD updates the import start date of an account.
  UpdateAccountImportSD(
      t db.Transaction, accountId int64, date time.Time) error
}

type UpdateAccountRunner interface {
  // UpdateAccount updates an account.
  UpdateAccount(
      t db.Transaction, account *fin.Account) error
}

type RemoveAccountRunner interface {
  // RemoveAccount removes an account.
  RemoveAccount(t db.Transaction, accountId int64) error
}

type DoEntryChangesRunner interface {
  // DoEntryChanges adds, updates, and deletes entries in bulk.
  DoEntryChanges(t db.Transaction, changes *EntryChanges) error
}

type EntriesRunner interface {
  // Entries gets entries from most to least recent.
  // options is additional options for getting entries, may be nil;
  // consumer consumes the Stream of fetched entries.
  Entries(t db.Transaction, options *EntryListOptions,
      consumer functional.Consumer) error
}

type EntriesByAccountIdRunner interface {
  // EntryByAccountId gets entries by account from most to least recent.
  // acctId is the account ID; account is where
  // Account object is stored; consumer consumes the Stream of EntryBalance
  // values.
  EntriesByAccountId(t db.Transaction, acctId int64,
      account *fin.Account, consumer functional.Consumer) error
}

type EntryByIdRunner interface {
  // EntryById fetches an Entry by id.
  EntryById(t db.Transaction, id int64, entry *fin.Entry) error
}

type UnreconciledEntriesRunner interface {
  // UnreconciledEntries gets unreconciled entries by account from most to least
  // recent.
  // acctId is the account ID; account, which can be nil, is where
  // Account object is stored; consumer consumes the Stream of Entry values
  UnreconciledEntries(t db.Transaction, acctId int64,
      account *fin.Account, consumer functional.Consumer) error
}

type AddRecurringEntryRunner interface {
  // AddRecurringEntry adds a new recurring entry.
  AddRecurringEntry(t db.Transaction, entry *fin.RecurringEntry) error
}

type UpdateRecurringEntryRunner interface {
  // UpdateRecurringEntry updates a recurring entry.
  UpdateRecurringEntry(t db.Transaction, entry *fin.RecurringEntry) error
}

type RecurringEntryByIdRunner interface {
  // RecurringEntryById gets a recurring entry by id.
  RecurringEntryById(
      t db.Transaction, id int64, entry *fin.RecurringEntry) error
}

type RecurringEntriesRunner interface {
  // RecurringEntries gets all the recurring entries sorted by id
  // in descending order.
  RecurringEntries(t db.Transaction, consumer functional.Consumer) error
}

type RemoveRecurringEntryByIdRunner interface {
  // RemoveRecurringEntryById removes a recurring entry by id.
  RemoveRecurringEntryById(t db.Transaction, id int64) error
}

type AddUserRunner interface {
  // AddUser adds a new user.
  AddUser(t db.Transaction, user *fin.User) error
}

type UpdateUserRunner interface {
  // UpdateUser updates a user.
  UpdateUser(t db.Transaction, user *fin.User) error
}

type UserByIdRunner interface {
  // UserById gets a user by id.
  UserById(t db.Transaction, id int64, user *fin.User) error
}

type UserByNameRunner interface {
  // UserByName gets a user by name.
  UserByName(t db.Transaction, name string, user *fin.User) error
}

type UsersRunner interface {
  //Users gets all the users sorted by user name.
  Users(t db.Transaction, consumer functional.Consumer) error
}

type RemoveUserByNameRunner interface {
  // RemoveUserByName removes a user by name.
  RemoveUserByName(t db.Transaction, name string) error
}

// EntryChanges represents changes to entries.
type EntryChanges struct {
  // Adds is entries to add
  Adds []*fin.Entry
  // The key is the entry id; the value does the update in-place.
  Updates map[int64]functional.Filterer
  // Deletes is the ids of the entries to delete.
  Deletes []int64
  // Etags contains the etags of the entries being updated.
  // It is used to detect concurrent updates.
  // The key is the entry id; the value is the etag of the original entry.
  // This field is optional, but if present it must contain the etag of
  // each entry being updated.
  Etags map[int64]uint32
}

// EntryListOptions represents options to list entries.
type EntryListOptions struct {
  // If set, entries listed are on or after this date.
  Start *time.Time
  // If set, entries listed are before this date
  End *time.Time
  // If true, show only unreviewed entries
  Unreviewed bool
}

// NoPermissionStore always returns NoPermissionError
type NoPermissionStore struct {
}

func (n NoPermissionStore) AccountById(
    t db.Transaction, acctId int64, account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) Accounts(
    t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) ActiveAccounts(
    t db.Transaction) (accounts []*fin.Account, err error) {
  return nil, NoPermission
}

func (n NoPermissionStore) AddAccount(
    t db.Transaction, Account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateAccountImportSD(
      t db.Transaction, accountId int64, date time.Time) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateAccount(
      t db.Transaction, account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveAccount(
    t db.Transaction, accountId int64) error {
  return NoPermission
}

func (n NoPermissionStore) DoEntryChanges(
    t db.Transaction, changes *EntryChanges) error {
  return NoPermission
}

func (n NoPermissionStore) Entries( t db.Transaction, options *EntryListOptions,
    consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) EntriesByAccountId(t db.Transaction, acctId int64,
    account *fin.Account, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) EntryById(
    t db.Transaction, id int64, entry *fin.Entry) error {
  return NoPermission
}

func (n NoPermissionStore) UnreconciledEntries(
    t db.Transaction, acctId int64, account *fin.Account,
    consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) AddRecurringEntry(
    t db.Transaction, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateRecurringEntry(
    t db.Transaction, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) RecurringEntryById(
    t db.Transaction, id int64, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) RecurringEntries(
    t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveRecurringEntryById(
    t db.Transaction, id int64) error {
  return NoPermission
}

func (n NoPermissionStore) AddUser(t db.Transaction, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateUser(t db.Transaction, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UserById(t db.Transaction, id int64, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UserByName(t db.Transaction, name string, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) Users(t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveUserByName(t db.Transaction, name string) error {
  return NoPermission
}

type RecurringEntriesApplier interface {
  DoEntryChangesRunner
  UpdateRecurringEntryRunner
  RecurringEntriesRunner
}

// ApplyRecurringEntriesDryRun returns out how many new entries would be
// added to the database if ApplyRecurringEntries were run.
// t is the database transaction.
// store is the database store.
// currentDate is the current date.
func ApplyRecurringEntriesDryRun(
    t db.Transaction,
    store RecurringEntriesRunner,
    currentDate time.Time) (int, error) {
  _, entriesToAdd, err := applyRecurringEntriesDryRun(t, store, currentDate)
  return len(entriesToAdd), err
}

// ApplyRecurringEntries applies all outstanding recurring entries.
// If there are no outstanding recurring entries, this function does
// nothing. Note that ApplyRecurringEntries is idempotent.
// t is the database transaction and must be non-nil.
// store is the database store.
// currentDate is the current date.
func ApplyRecurringEntries(
    t db.Transaction,
    store RecurringEntriesApplier,
    currentDate time.Time) (int, error) {
  if t == nil {
    panic("non nil transaction required.")
  }
  recurringEntries, entries, err := applyRecurringEntriesDryRun(
      t, store, currentDate)
  if err != nil {
    return 0, err
  }
  for i := range recurringEntries {
    if err := store.UpdateRecurringEntry(
        t, recurringEntries[i]); err != nil {
      return 0, err
    }
  }
  changes := &EntryChanges{Adds: entries}
  if err := store.DoEntryChanges(t, changes); err != nil {
    return 0, err
  }
  return len(entries), nil
}

func applyRecurringEntriesDryRun(
    t db.Transaction,
    store RecurringEntriesRunner,
    currentDate time.Time) (
        recurringEntriesToUpdate []*fin.RecurringEntry,
        entriesToAdd []*fin.Entry,
        err error) {
  if err = store.RecurringEntries(
      t, consume.AppendPtrsTo(&recurringEntriesToUpdate, nil)); err != nil {
    return
  }
  idx := 0
  for i := range recurringEntriesToUpdate {
    if recurringEntriesToUpdate[i].Advance(currentDate, &entriesToAdd) {
      recurringEntriesToUpdate[idx] = recurringEntriesToUpdate[i]
      idx++
    }
  }
  recurringEntriesToUpdate = recurringEntriesToUpdate[:idx]
  return
}
<commit_msg>Improve documentation on findb.ApplyRecurringEntries.<commit_after>// Package findb contains the persistence layer for the fin package.
package findb

import (
  "errors"
  "github.com/keep94/appcommon/db"
  "github.com/keep94/finance/fin"
  "github.com/keep94/gofunctional3/consume"
  "github.com/keep94/gofunctional3/functional"
  "time"
)

var (
  ConcurrentUpdate = errors.New("findb: Concurrent update.")
  NoSuchId = errors.New("findb: No Such Id.")
  NoPermission = errors.New("findb: Insufficient permission.")
)

type AccountByIdRunner interface {
  // AccountById fetches an account by Id.
  AccountById(t db.Transaction, acctId int64, account *fin.Account) error
}

type AccountsRunner interface {
  // Accounts fetches all accounts.
  Accounts(t db.Transaction, consumer functional.Consumer) error
}

type ActiveAccountsRunner interface {
  // ActiveAccounts fetches all active accounts sorted by name.
  ActiveAccounts(t db.Transaction) (accounts []*fin.Account, err error)
}

type AddAccountRunner interface {
  // AddAccount adds a new account.
  AddAccount(t db.Transaction, Account *fin.Account) error
}

type UpdateAccountImportSDRunner interface {
  // UpdateAccountImportSD updates the import start date of an account.
  UpdateAccountImportSD(
      t db.Transaction, accountId int64, date time.Time) error
}

type UpdateAccountRunner interface {
  // UpdateAccount updates an account.
  UpdateAccount(
      t db.Transaction, account *fin.Account) error
}

type RemoveAccountRunner interface {
  // RemoveAccount removes an account.
  RemoveAccount(t db.Transaction, accountId int64) error
}

type DoEntryChangesRunner interface {
  // DoEntryChanges adds, updates, and deletes entries in bulk.
  DoEntryChanges(t db.Transaction, changes *EntryChanges) error
}

type EntriesRunner interface {
  // Entries gets entries from most to least recent.
  // options is additional options for getting entries, may be nil;
  // consumer consumes the Stream of fetched entries.
  Entries(t db.Transaction, options *EntryListOptions,
      consumer functional.Consumer) error
}

type EntriesByAccountIdRunner interface {
  // EntryByAccountId gets entries by account from most to least recent.
  // acctId is the account ID; account is where
  // Account object is stored; consumer consumes the Stream of EntryBalance
  // values.
  EntriesByAccountId(t db.Transaction, acctId int64,
      account *fin.Account, consumer functional.Consumer) error
}

type EntryByIdRunner interface {
  // EntryById fetches an Entry by id.
  EntryById(t db.Transaction, id int64, entry *fin.Entry) error
}

type UnreconciledEntriesRunner interface {
  // UnreconciledEntries gets unreconciled entries by account from most to least
  // recent.
  // acctId is the account ID; account, which can be nil, is where
  // Account object is stored; consumer consumes the Stream of Entry values
  UnreconciledEntries(t db.Transaction, acctId int64,
      account *fin.Account, consumer functional.Consumer) error
}

type AddRecurringEntryRunner interface {
  // AddRecurringEntry adds a new recurring entry.
  AddRecurringEntry(t db.Transaction, entry *fin.RecurringEntry) error
}

type UpdateRecurringEntryRunner interface {
  // UpdateRecurringEntry updates a recurring entry.
  UpdateRecurringEntry(t db.Transaction, entry *fin.RecurringEntry) error
}

type RecurringEntryByIdRunner interface {
  // RecurringEntryById gets a recurring entry by id.
  RecurringEntryById(
      t db.Transaction, id int64, entry *fin.RecurringEntry) error
}

type RecurringEntriesRunner interface {
  // RecurringEntries gets all the recurring entries sorted by id
  // in descending order.
  RecurringEntries(t db.Transaction, consumer functional.Consumer) error
}

type RemoveRecurringEntryByIdRunner interface {
  // RemoveRecurringEntryById removes a recurring entry by id.
  RemoveRecurringEntryById(t db.Transaction, id int64) error
}

type AddUserRunner interface {
  // AddUser adds a new user.
  AddUser(t db.Transaction, user *fin.User) error
}

type UpdateUserRunner interface {
  // UpdateUser updates a user.
  UpdateUser(t db.Transaction, user *fin.User) error
}

type UserByIdRunner interface {
  // UserById gets a user by id.
  UserById(t db.Transaction, id int64, user *fin.User) error
}

type UserByNameRunner interface {
  // UserByName gets a user by name.
  UserByName(t db.Transaction, name string, user *fin.User) error
}

type UsersRunner interface {
  //Users gets all the users sorted by user name.
  Users(t db.Transaction, consumer functional.Consumer) error
}

type RemoveUserByNameRunner interface {
  // RemoveUserByName removes a user by name.
  RemoveUserByName(t db.Transaction, name string) error
}

// EntryChanges represents changes to entries.
type EntryChanges struct {
  // Adds is entries to add
  Adds []*fin.Entry
  // The key is the entry id; the value does the update in-place.
  Updates map[int64]functional.Filterer
  // Deletes is the ids of the entries to delete.
  Deletes []int64
  // Etags contains the etags of the entries being updated.
  // It is used to detect concurrent updates.
  // The key is the entry id; the value is the etag of the original entry.
  // This field is optional, but if present it must contain the etag of
  // each entry being updated.
  Etags map[int64]uint32
}

// EntryListOptions represents options to list entries.
type EntryListOptions struct {
  // If set, entries listed are on or after this date.
  Start *time.Time
  // If set, entries listed are before this date
  End *time.Time
  // If true, show only unreviewed entries
  Unreviewed bool
}

// NoPermissionStore always returns NoPermissionError
type NoPermissionStore struct {
}

func (n NoPermissionStore) AccountById(
    t db.Transaction, acctId int64, account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) Accounts(
    t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) ActiveAccounts(
    t db.Transaction) (accounts []*fin.Account, err error) {
  return nil, NoPermission
}

func (n NoPermissionStore) AddAccount(
    t db.Transaction, Account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateAccountImportSD(
      t db.Transaction, accountId int64, date time.Time) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateAccount(
      t db.Transaction, account *fin.Account) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveAccount(
    t db.Transaction, accountId int64) error {
  return NoPermission
}

func (n NoPermissionStore) DoEntryChanges(
    t db.Transaction, changes *EntryChanges) error {
  return NoPermission
}

func (n NoPermissionStore) Entries( t db.Transaction, options *EntryListOptions,
    consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) EntriesByAccountId(t db.Transaction, acctId int64,
    account *fin.Account, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) EntryById(
    t db.Transaction, id int64, entry *fin.Entry) error {
  return NoPermission
}

func (n NoPermissionStore) UnreconciledEntries(
    t db.Transaction, acctId int64, account *fin.Account,
    consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) AddRecurringEntry(
    t db.Transaction, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateRecurringEntry(
    t db.Transaction, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) RecurringEntryById(
    t db.Transaction, id int64, entry *fin.RecurringEntry) error {
  return NoPermission
}

func (n NoPermissionStore) RecurringEntries(
    t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveRecurringEntryById(
    t db.Transaction, id int64) error {
  return NoPermission
}

func (n NoPermissionStore) AddUser(t db.Transaction, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UpdateUser(t db.Transaction, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UserById(t db.Transaction, id int64, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) UserByName(t db.Transaction, name string, user *fin.User) error {
  return NoPermission
}

func (n NoPermissionStore) Users(t db.Transaction, consumer functional.Consumer) error {
  return NoPermission
}

func (n NoPermissionStore) RemoveUserByName(t db.Transaction, name string) error {
  return NoPermission
}

type RecurringEntriesApplier interface {
  DoEntryChangesRunner
  UpdateRecurringEntryRunner
  RecurringEntriesRunner
}

// ApplyRecurringEntriesDryRun returns out how many new entries would be
// added to the database if ApplyRecurringEntries were run.
// t is the database transaction.
// store is the database store.
// currentDate is the current date.
func ApplyRecurringEntriesDryRun(
    t db.Transaction,
    store RecurringEntriesRunner,
    currentDate time.Time) (int, error) {
  _, entriesToAdd, err := applyRecurringEntriesDryRun(t, store, currentDate)
  return len(entriesToAdd), err
}

// ApplyRecurringEntries applies all outstanding recurring entries
// and returns how many new entries were added to the database as a result.
// If there are no outstanding recurring entries, this function does
// nothing and returns 0. Note that ApplyRecurringEntries is idempotent.
// t is the database transaction and must be non-nil.
// store is the database store.
// currentDate is the current date.
func ApplyRecurringEntries(
    t db.Transaction,
    store RecurringEntriesApplier,
    currentDate time.Time) (int, error) {
  if t == nil {
    panic("non nil transaction required.")
  }
  recurringEntries, entries, err := applyRecurringEntriesDryRun(
      t, store, currentDate)
  if err != nil {
    return 0, err
  }
  for i := range recurringEntries {
    if err := store.UpdateRecurringEntry(
        t, recurringEntries[i]); err != nil {
      return 0, err
    }
  }
  changes := &EntryChanges{Adds: entries}
  if err := store.DoEntryChanges(t, changes); err != nil {
    return 0, err
  }
  return len(entries), nil
}

func applyRecurringEntriesDryRun(
    t db.Transaction,
    store RecurringEntriesRunner,
    currentDate time.Time) (
        recurringEntriesToUpdate []*fin.RecurringEntry,
        entriesToAdd []*fin.Entry,
        err error) {
  if err = store.RecurringEntries(
      t, consume.AppendPtrsTo(&recurringEntriesToUpdate, nil)); err != nil {
    return
  }
  idx := 0
  for i := range recurringEntriesToUpdate {
    if recurringEntriesToUpdate[i].Advance(currentDate, &entriesToAdd) {
      recurringEntriesToUpdate[idx] = recurringEntriesToUpdate[i]
      idx++
    }
  }
  recurringEntriesToUpdate = recurringEntriesToUpdate[:idx]
  return
}
<|endoftext|>


<commit_before>package cioutil

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", fmt.Sprintf("%x", md5.Sum([]byte(val))))
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new " + method + " request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod": method,
				"url":        cioURL,
				"statusCode": fmt.Sprintf("%d", statusCode),
				"payload":    responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from " + method + " to: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload: " + responseBody)
		}
	}
}
<commit_msg>replace passwords, secrets, with the word "redacted"<commit_after>package cioutil

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/Sirupsen/logrus"
	"github.com/garyburd/go-oauth/oauth"
)

// ClientRequest defines information that can be used to make a request
type ClientRequest struct {
	Method      string
	Path        string
	FormValues  interface{}
	QueryValues interface{}
}

// DoFormRequest makes the actual request
func (cio Cio) DoFormRequest(request ClientRequest, result interface{}) error {

	// Construct the url
	cioURL := cio.Host + request.Path + QueryString(request.QueryValues)

	// Construct the body
	var bodyReader io.Reader
	bodyValues := FormValues(request.FormValues)
	bodyString := bodyValues.Encode()
	if len(bodyString) > 0 {
		bodyReader = bytes.NewReader([]byte(bodyString))
	}
	logRequest(cio.Log, request.Method, cioURL, bodyValues)

	// Construct the request
	httpReq, err := cio.createRequest(request, cioURL, bodyReader, bodyValues)
	if err != nil {
		return err
	}

	// Send the request
	return cio.sendRequest(httpReq, result, cioURL)
}

// createRequest creates the *http.Request object
func (cio Cio) createRequest(request ClientRequest, cioURL string, bodyReader io.Reader, bodyValues url.Values) (*http.Request, error) {
	// Construct the request
	httpReq, err := http.NewRequest(request.Method, cioURL, bodyReader)
	if err != nil {
		return httpReq, fmt.Errorf("Could not create request: %s", err)
	}

	// oAuth signature
	var client oauth.Client
	client.Credentials = oauth.Credentials{Token: cio.apiKey, Secret: cio.apiSecret}

	// Add headers
	httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	httpReq.Header.Set("Accept", "application/json")
	httpReq.Header.Set("Accept-Charset", "utf-8")
	httpReq.Header.Set("User-Agent", "Golang CIO Library")
	httpReq.Header.Set("Authorization", client.AuthorizationHeader(nil, request.Method, httpReq.URL, bodyValues))

	return httpReq, nil
}

// sendRequest sends the *http.Request
func (cio Cio) sendRequest(httpReq *http.Request, result interface{}, cioURL string) error {
	// Create the HTTP client
	httpClient := &http.Client{
		Transport: http.DefaultTransport,
		Timeout:   cio.RequestTimeout,
	}

	// Make the request
	res, err := httpClient.Do(httpReq)
	if err != nil {
		return fmt.Errorf("Failed to make request: %s", err)
	}

	// Parse the response
	defer func() {
		if closeErr := res.Body.Close(); closeErr != nil {
			logBodyCloseError(cio.Log, closeErr)
		}
	}()

	resBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return fmt.Errorf("Could not read response: %s", err)
	}
	resBodyString := string(resBody)

	// Unmarshal result
	err = json.Unmarshal(resBody, &result)

	// Log the response
	logResponse(cio.Log, httpReq.Method, cioURL, res.StatusCode, resBodyString, err)

	// Return special error if Status Code >= 400
	if res.StatusCode >= 400 {
		return fmt.Errorf("%d Status Code with Payload %s", res.StatusCode, resBodyString)
	}

	// Return Unmarshal error (if any) if Status Code is < 400
	return err
}

// logRequest logs the request about to be made to CIO, redacting sensitive information in the body
func logRequest(log Logger, method string, cioURL string, bodyValues url.Values) {
	if log != nil {

		// Copy url.Values
		redactedValues := url.Values{}
		for k, v := range bodyValues {
			redactedValues[k] = v
		}

		// Redact sensitive information
		if val := redactedValues.Get("password"); len(val) > 0 {
			redactedValues.Set("password", "redacted")
		}
		if val := redactedValues.Get("provider_refresh_token"); len(val) > 0 {
			redactedValues.Set("provider_refresh_token", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_key"); len(val) > 0 {
			redactedValues.Set("provider_consumer_key", "redacted")
		}
		if val := redactedValues.Get("provider_consumer_secret"); len(val) > 0 {
			redactedValues.Set("provider_consumer_secret", "redacted")
		}

		// Actually log
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithFields(logrus.Fields{"httpMethod": method, "url": cioURL, "payload": redactedValues.Encode()}).Debug("Creating new request to CIO")
		} else {
			// Else just log with Println
			log.Println("Creating new " + method + " request to: " + cioURL + " with payload: " + redactedValues.Encode())
		}
	}
}

// logBodyCloseError logs any error that happens when trying to close the *http.Response.Body
func logBodyCloseError(log Logger, closeError error) {
	if log != nil {
		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logrusLogger.WithError(closeError).Warn("Unable to close response body from CIO")
		} else {
			// Else just log with Println
			log.Println("Unable to close response body from CIO, with error: " + closeError.Error())
		}
	}
}

// logResponse logs the response from CIO, if any logger is set
func logResponse(log Logger, method string, cioURL string, statusCode int, responseBody string, unmarshalError error) {
	if log != nil {

		// TODO: redact access_token and access_token_secret before logging (only occurs with 3-legged oauth [rare])

		if logrusLogger, ok := log.(*logrus.Logger); ok {
			// If logrus, use structured fields
			logEntry := logrusLogger.WithFields(logrus.Fields{
				"httpMethod": method,
				"url":        cioURL,
				"statusCode": fmt.Sprintf("%d", statusCode),
				"payload":    responseBody})
			if unmarshalError != nil || statusCode >= 400 {
				logEntry.Warn("Received response from CIO")
			} else {
				logEntry.Debug("Received response from CIO")
			}

		} else {
			// Else just log with Println
			log.Println("Received response from " + method + " to: " + cioURL +
				" with status code: " + fmt.Sprintf("%d", statusCode) +
				" and payload: " + responseBody)
		}
	}
}
<|endoftext|>