<commit_before>package fetch

import (
	"bytes"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"

	"github.com/daddye/vips"
	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
)

func needsRotation(src io.Reader) int {
	metadata, err := exif.Decode(src)
	if err != nil {
		return 0
	}

	orientation, err := metadata.Get(exif.Orientation)
	if err != nil {
		return 0
	}

	switch orientation.String() {
	case "6":
		return 270
	case "3":
		return 180
	case "8":
		return 90
	default:
		return 0
	}

}

func GetRotatedImage(src io.Reader) (image.Image, string, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, "", err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		return nil, "", err
	}

	if _, err := data.Seek(0, 0); err != nil {
		return nil, "", err
	}

	angle := needsRotation(data)
	switch angle {
	case 90:
		image = imaging.Rotate90(image)
	case 180:
		image = imaging.Rotate180(image)
	case 270:
		image = imaging.Rotate270(image)
	}

	return image, format, nil
}

func Resize(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}

	img, _, err := image.Decode(bytes.NewReader(raw))
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	factor := float64(c.Width) / float64(img.Bounds().Size().X)
	height := int(float64(img.Bounds().Size().Y) * factor)

	options := vips.Options{
		Width:        c.Width,
		Height:       height,
		Crop:         false,
		Extend:       vips.EXTEND_WHITE,
		Interpolator: vips.BILINEAR,
		Gravity:      vips.CENTRE,
		Quality:      80,
	}

	res, err := vips.Resize(raw, options)
	if err != nil {
		return nil, err
	}

	return bytes.NewBuffer(res), err
}

func CenterCrop(src io.Reader, c *CacheContext) (io.Reader, error) {
	image, format, err := image.Decode(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	height := image.Bounds().Size().Y
	width := image.Bounds().Size().X

	if width < height {
		image = imaging.CropCenter(image, width, width)
	} else if width > height {
		image = imaging.CropCenter(image, height, height)
	} else {
		image = imaging.CropCenter(image, width, height)
	}

	buf := new(bytes.Buffer)

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}
<commit_msg>Fix image size off by one<commit_after>package fetch

import (
	"bytes"
	"fmt"
	"image"
	"image/jpeg"
	"image/png"
	"io"
	"io/ioutil"

	"github.com/daddye/vips"
	"github.com/disintegration/imaging"
	"github.com/rwcarlsen/goexif/exif"
)

func needsRotation(src io.Reader) int {
	metadata, err := exif.Decode(src)
	if err != nil {
		return 0
	}

	orientation, err := metadata.Get(exif.Orientation)
	if err != nil {
		return 0
	}

	switch orientation.String() {
	case "6":
		return 270
	case "3":
		return 180
	case "8":
		return 90
	default:
		return 0
	}

}

func GetRotatedImage(src io.Reader) (image.Image, string, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, "", err
	}

	data := bytes.NewReader(raw)

	image, format, err := image.Decode(data)
	if err != nil {
		return nil, "", err
	}

	if _, err := data.Seek(0, 0); err != nil {
		return nil, "", err
	}

	angle := needsRotation(data)
	switch angle {
	case 90:
		image = imaging.Rotate90(image)
	case 180:
		image = imaging.Rotate180(image)
	case 270:
		image = imaging.Rotate270(image)
	}

	return image, format, nil
}

func Resize(src io.Reader, c *CacheContext) (io.Reader, error) {
	raw, err := ioutil.ReadAll(src)
	if err != nil {
		return nil, err
	}

	options := vips.Options{
		Width:        c.Width,
		Crop:         true,
		Extend:       vips.EXTEND_WHITE,
		Interpolator: vips.BILINEAR,
		Gravity:      vips.CENTRE,
		Quality:      80,
	}

	res, err := vips.Resize(raw, options)
	if err != nil {
		return nil, err
	}

	return bytes.NewBuffer(res), err
}

func CenterCrop(src io.Reader, c *CacheContext) (io.Reader, error) {
	image, format, err := image.Decode(src)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}

	height := image.Bounds().Size().Y
	width := image.Bounds().Size().X

	if width < height {
		image = imaging.CropCenter(image, width, width)
	} else if width > height {
		image = imaging.CropCenter(image, height, height)
	} else {
		image = imaging.CropCenter(image, width, height)
	}

	buf := new(bytes.Buffer)

	switch format {
	case "jpeg":
		err = jpeg.Encode(buf, image, nil)
	case "png":
		err = png.Encode(buf, image)
	}

	return buf, err
}
<|endoftext|>


<commit_before>package hamt

import (
	"os"
	"testing"

	"github.com/lleo/util"
)

type entry struct {
	hashPath []uint8
	hash     uint64
	key      []byte
	val      interface{}
}

var TEST_SET_1 = []entry{
	{hash: path2hash([]uint8{1, 2, 3}), key: []byte("foo"), val: 1},
	{hash: path2hash([]uint8{1, 2, 1}), key: []byte("foo"), val: 1},
}

//func printHash(hash uint64) {
//	top4 := ^uint64(1<<60 - 1)
//}

//func addToHashPath(hashPath uint64, depth int, val uint8) uint64 {
//	return hashPath & uint64(val << (depth * NBITS))
//}

func path2hash(path []uint8) uint64 {
	var hashPath uint64
	for depth, val := range path {
		hashPath &= uint64(val << (uint(depth) * NBITS))
	}
	return hashPath
}

var midNumEnts []keyVal

func TestMain(m *testing.M) {
	// SETUP

	midNumEnts = make([]keyVal, 0, 32) //binary growth
	var s = util.Str("")
	//nEnts := 10000 //ten thousand
	nEnts := 1000
	for i := 0; i < nEnts; i++ {
		s = s.Inc(1) //get off "" first
		var key = []byte(s)
		var val = i + 1
		midNumEnts = append(midNumEnts, keyVal{key, val})
	}

	// RUN
	xit := m.Run()

	// TEARDOW

	os.Exit(xit)
}

func TestEmptyPutOnceGetOnce(t *testing.T) {
	key := []byte("foo")

	h, _ := EMPTY.Put(key, 1)

	val, ok := h.Get(key)

	if !ok {
		t.Fatal("failed to retrieve \"foo\"")
	}

	if val != 1 {
		t.Fatal("failed to rerieve the correct val for \"foo\"")
	}
}

func TestEmptyPutThriceFlatGetThrice(t *testing.T) {
	var keys = [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
	var vals = []int{1, 2, 3}

	var h *Hamt = &EMPTY

	for i := range keys {
		h, _ = h.Put(keys[i], vals[i])
	}

	t.Logf("h.root =\n%s", h.root.LongString(""))

	for i := range vals {
		var val, found = h.Get(keys[i])

		if !found {
			t.Fatalf("failed to get key \"%s\" from h", keys[i])
		}
		if val != vals[i] {
			t.Fatalf("failed to get val for \"%s\" val,%d != vals[%d],%d from h", keys[i], val, i, vals[i])
		}
	}
}

// "d":4 && "aa":27 collide at depth 0 & 1
func TestPutGetTwoTableDeepCollision(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("d"), 4)
	h, _ = h.Put([]byte("aa"), 27)

	t.Log("h.root =\n%s", h.root.LongString(""))

	var val interface{}
	var found bool
	val, found = h.Get([]byte("d"))
	if !found {
		t.Error("failed to find val for key=\"d\"")
	}
	if val != 4 {
		t.Error("h.Get(\"d\") failed to retrieve val = 4")
	}

	val, found = h.Get([]byte("aa"))
	if !found {
		t.Error("failed to find val for key=\"aa\"")
	}
	if val != 27 {
		t.Error("h.Get(\"d\") failed to retrieve val = 27")
	}

	return
}

// Where Many == 64
func TestEmptyPutManyGetMany(t *testing.T) {
	var h = &EMPTY

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var val = midNumEnts[i].val
		h, _ = h.Put(key, val)
	}

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var expected_val = midNumEnts[i].val

		var val, found = h.Get(key)
		if !found {
			t.Errorf("Did NOT find val for key=\"%s\"", key)
		}
		if val != expected_val {
			t.Errorf("val,%d != expected_val,%d", val, expected_val)
		}
	}
}

func TestEmptyPutOnceDelOnce(t *testing.T) {
	var h = &EMPTY

	var key = []byte("a")
	var val interface{} = 1

	h, _ = h.Put(key, val)

	var v interface{}
	var deleted, found bool

	h, v, deleted = h.Del(key)
	if !deleted {
		t.Fatalf("key=%q not deleted from h.", key)
	}
	if v != val {
		t.Fatalf("Returned deleted value val,%d != v,%d .", val, v)
	}

	v, found = h.Get(key)
	if found {
		t.Fatalf("h.Get(%q) retrived a value v=%v.", key, v)
	}
}

func TestEmptyPutOnceDelOnceIsEmpty(t *testing.T) {
	var h = &EMPTY

	var key = []byte("a")
	var val interface{} = 1

	h, _ = h.Put(key, val)

	var v interface{}
	var deleted, found bool

	h, v, deleted = h.Del(key)
	if !deleted {
		t.Fatalf("key=%q not deleted from h.", key)
	}
	if v != val {
		t.Fatalf("Returned deleted value val,%d != v,%d .", val, v)
	}

	v, found = h.Get(key)
	if found {
		t.Fatalf("h.Get(%q) retrived a value v=%v.", key, v)
	}

	if !h.IsEmpty() {
		t.Fatal("NOT h.IsEmpty()")
	}
}

func TestEmptyPutThriceFlatDelThriceIsEmpty(t *testing.T) {
	var keys = [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
	var vals = []int{1, 2, 3}

	var h *Hamt = &EMPTY

	for i := range keys {
		h, _ = h.Put(keys[i], vals[i])
	}

	for i := range vals {
		var val interface{}
		var deleted bool
		h, val, deleted = h.Del(keys[i])

		if !deleted {
			t.Fatalf("failed to delete key \"%s\" from h", keys[i])
		}
		if val != vals[i] {
			t.Fatalf("deleted val for \"%s\" val,%d != vals[%d],%d from h", keys[i], val, i, vals[i])
		}

	}

	if !h.IsEmpty() {
		t.Fatal("h is NOT empty")
	}
}

// "c":3 && "fg":38 at depth 1
func TestPutDelOneTableDeepCollisionIsEmpty(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("c"), 3)
	h, _ = h.Put([]byte("fg"), 38)

	var val interface{}
	var deleted bool

	h, val, deleted = h.Del([]byte("c"))

	if !deleted {
		t.Error("failed to delete for key=\"c\"")
	}
	if val != 3 {
		t.Error("h.Get(\"c\") failed to retrieve val = 3")
	}

	h, val, deleted = h.Del([]byte("fg"))

	if !deleted {
		t.Error("failed to delete for key=\"fg\"")
	}
	if val != 38 {
		t.Error("h.Get(\"fg\") failed to retrieve val = 38")
	}

	if !h.IsEmpty() {
		t.Error("h is NOT empty")
	}
}

// "d":4 && "aa":27 collide at depth 2
func TestPutDelTwoTableDeepCollisionIsEmpty(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("d"), 4)
	h, _ = h.Put([]byte("aa"), 27)

	t.Logf("h =\n%s", h.LongString(""))

	var val interface{}
	var deleted bool
	h, val, deleted = h.Del([]byte("d"))
	if !deleted {
		t.Error("failed to delete for key=\"d\"")
	}
	if val != 4 {
		t.Error("h.Get(\"d\") failed to retrieve val = 4")
	}

	t.Logf("After h.Del(%q): h =\n%s", "d", h.LongString(""))

	h, val, deleted = h.Del([]byte("aa"))
	if !deleted {
		t.Error("failed to delete for key=\"aa\"")
	}
	if val != 27 {
		t.Error("h.Get(\"d\") failed to retrieve val = 27")
	}

	t.Logf("After h.Del(%q): h =\n%s", "aa", h.LongString(""))

	if !h.IsEmpty() {
		t.Error("h is NOT empty")
	}

	return
}

// Where Many == 64
func TestEmptyPutManyDelManyIsEmpty(t *testing.T) {
	var h = &EMPTY

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var val = midNumEnts[i].val
		h, _ = h.Put(key, val)
	}

	t.Log("h.root =\n", h.root.LongString(""))

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var expected_val = midNumEnts[i].val

		var val interface{}
		var deleted bool
		h, val, deleted = h.Del(key)
		if !deleted {
			t.Errorf("Did NOT find&delete for key=\"%s\"", key)
		}
		if val != expected_val {
			t.Errorf("val,%d != expected_val,%d", val, expected_val)
		}

		if h.root == nil {
			t.Log("h.root == nil")
		} else {
			t.Log("h.root ==\n", h.root.LongString(""))
		}
	}
	t.Log("### Testing compressedTable Shrinkage ###")

	t.Log(h)

	if !h.IsEmpty() {
		t.Fatal("NOT h.IsEmpty()")
	}
}
<commit_msg>created a large & crazy list & test<commit_after>package hamt

import (
	"log"
	"os"
	"testing"

	"github.com/lleo/util"
)

type entry struct {
	hashPath []uint8
	hash     uint64
	key      []byte
	val      interface{}
}

var TEST_SET_1 = []entry{
	{hash: path2hash([]uint8{1, 2, 3}), key: []byte("foo"), val: 1},
	{hash: path2hash([]uint8{1, 2, 1}), key: []byte("foo"), val: 1},
}

//func printHash(hash uint64) {
//	top4 := ^uint64(1<<60 - 1)
//}

//func addToHashPath(hashPath uint64, depth int, val uint8) uint64 {
//	return hashPath & uint64(val << (depth * NBITS))
//}

func path2hash(path []uint8) uint64 {
	var hashPath uint64
	for depth, val := range path {
		hashPath &= uint64(val << (uint(depth) * NBITS))
	}
	return hashPath
}

var midNumEnts []keyVal
var hugeNumEnts []keyVal

func TestMain(m *testing.M) {
	// SETUP

	var logFile, err = os.OpenFile("test.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0755)
	if err != nil {
		log.Fatal(err)
	}
	defer logFile.Close()
	Lgr.SetOutput(logFile)

	midNumEnts = make([]keyVal, 0, 32)
	var s0 = util.Str("")
	//nEnts := 10000 //ten thousand
	var midEnts = 1000
	for i := 0; i < midEnts; i++ {
		s0 = s0.Inc(1) //get off "" first
		var key = []byte(s0)
		var val = i + 1
		midNumEnts = append(midNumEnts, keyVal{key, val})
	}

	hugeNumEnts = make([]keyVal, 0, 32)
	var s1 = util.Str("")
	//var hugeEnts = 1024
	var hugeEnts = 32 * 1024
	//var hugeEnts = 256 * 1024 * 1024 //256 MB
	for i := 0; i < hugeEnts; i++ {
		s1 = s1.Inc(1)
		var key = []byte(s1)
		var val = i + 1
		hugeNumEnts = append(hugeNumEnts, keyVal{key, val})
	}

	// RUN
	xit := m.Run()

	// TEARDOW

	os.Exit(xit)
}

func dTestEmptyPutDelCrazy(t *testing.T) {
	var key = []byte("aaaaaaaaaaaaaaaaaaaaaabbcdefghijkl")
	var val = 14126
	var h = &EMPTY

	h, _ = h.Put(key, val)

	var v interface{}
	var d bool
	h, v, d = h.Del(key)
	if !d {
		t.Fatalf("failed to retrieve %q", key)
	}
	if v != val {
		t.Fatalf("failed to retrieve the correct val,%d for %q", val, key)
	}

	if !h.IsEmpty() {
		t.Fatal("hash is not Empty")
	}
}

func TestEmptyPutOnceGetOnce(t *testing.T) {
	key := []byte("foo")

	h, _ := EMPTY.Put(key, 1)

	val, ok := h.Get(key)

	if !ok {
		t.Fatal("failed to retrieve \"foo\"")
	}

	if val != 1 {
		t.Fatal("failed to rerieve the correct val for \"foo\"")
	}
}

func TestEmptyPutThriceFlatGetThrice(t *testing.T) {
	var keys = [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
	var vals = []int{1, 2, 3}

	var h *Hamt = &EMPTY

	for i := range keys {
		h, _ = h.Put(keys[i], vals[i])
	}

	t.Logf("h.root =\n%s", h.root.LongString(""))

	for i := range vals {
		var val, found = h.Get(keys[i])

		if !found {
			t.Fatalf("failed to get key \"%s\" from h", keys[i])
		}
		if val != vals[i] {
			t.Fatalf("failed to get val for \"%s\" val,%d != vals[%d],%d from h", keys[i], val, i, vals[i])
		}
	}
}

// "d":4 && "aa":27 collide at depth 0 & 1
func TestPutGetTwoTableDeepCollision(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("d"), 4)
	h, _ = h.Put([]byte("aa"), 27)

	t.Log("h.root =\n%s", h.root.LongString(""))

	var val interface{}
	var found bool
	val, found = h.Get([]byte("d"))
	if !found {
		t.Error("failed to find val for key=\"d\"")
	}
	if val != 4 {
		t.Error("h.Get(\"d\") failed to retrieve val = 4")
	}

	val, found = h.Get([]byte("aa"))
	if !found {
		t.Error("failed to find val for key=\"aa\"")
	}
	if val != 27 {
		t.Error("h.Get(\"d\") failed to retrieve val = 27")
	}

	return
}

// Where Many == 64
func TestEmptyPutManyGetMany(t *testing.T) {
	var h = &EMPTY

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var val = midNumEnts[i].val
		h, _ = h.Put(key, val)
	}

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var expected_val = midNumEnts[i].val

		var val, found = h.Get(key)
		if !found {
			t.Errorf("Did NOT find val for key=\"%s\"", key)
		}
		if val != expected_val {
			t.Errorf("val,%d != expected_val,%d", val, expected_val)
		}
	}
}

func TestEmptyPutOnceDelOnce(t *testing.T) {
	var h = &EMPTY

	var key = []byte("a")
	var val interface{} = 1

	h, _ = h.Put(key, val)

	var v interface{}
	var deleted, found bool

	h, v, deleted = h.Del(key)
	if !deleted {
		t.Fatalf("key=%q not deleted from h.", key)
	}
	if v != val {
		t.Fatalf("Returned deleted value val,%d != v,%d .", val, v)
	}

	v, found = h.Get(key)
	if found {
		t.Fatalf("h.Get(%q) retrived a value v=%v.", key, v)
	}
}

func TestEmptyPutOnceDelOnceIsEmpty(t *testing.T) {
	var h = &EMPTY

	var key = []byte("a")
	var val interface{} = 1

	h, _ = h.Put(key, val)

	var v interface{}
	var deleted, found bool

	h, v, deleted = h.Del(key)
	if !deleted {
		t.Fatalf("key=%q not deleted from h.", key)
	}
	if v != val {
		t.Fatalf("Returned deleted value val,%d != v,%d .", val, v)
	}

	v, found = h.Get(key)
	if found {
		t.Fatalf("h.Get(%q) retrived a value v=%v.", key, v)
	}

	if !h.IsEmpty() {
		t.Fatal("NOT h.IsEmpty()")
	}
}

func TestEmptyPutThriceFlatDelThriceIsEmpty(t *testing.T) {
	var keys = [][]byte{[]byte("foo"), []byte("bar"), []byte("baz")}
	var vals = []int{1, 2, 3}

	var h *Hamt = &EMPTY

	for i := range keys {
		h, _ = h.Put(keys[i], vals[i])
	}

	for i := range vals {
		var val interface{}
		var deleted bool
		h, val, deleted = h.Del(keys[i])

		if !deleted {
			t.Fatalf("failed to delete key \"%s\" from h", keys[i])
		}
		if val != vals[i] {
			t.Fatalf("deleted val for \"%s\" val,%d != vals[%d],%d from h", keys[i], val, i, vals[i])
		}

	}

	if !h.IsEmpty() {
		t.Fatal("h is NOT empty")
	}
}

// "c":3 && "fg":38 at depth 1
func TestPutDelOneTableDeepCollisionIsEmpty(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("c"), 3)
	h, _ = h.Put([]byte("fg"), 38)

	var val interface{}
	var deleted bool

	h, val, deleted = h.Del([]byte("c"))

	if !deleted {
		t.Error("failed to delete for key=\"c\"")
	}
	if val != 3 {
		t.Error("h.Get(\"c\") failed to retrieve val = 3")
	}

	h, val, deleted = h.Del([]byte("fg"))

	if !deleted {
		t.Error("failed to delete for key=\"fg\"")
	}
	if val != 38 {
		t.Error("h.Get(\"fg\") failed to retrieve val = 38")
	}

	if !h.IsEmpty() {
		t.Error("h is NOT empty")
	}
}

// "d":4 && "aa":27 collide at depth 2
func TestPutDelTwoTableDeepCollisionIsEmpty(t *testing.T) {
	var h = &EMPTY

	h, _ = h.Put([]byte("d"), 4)
	h, _ = h.Put([]byte("aa"), 27)

	t.Logf("h =\n%s", h.LongString(""))

	var val interface{}
	var deleted bool
	h, val, deleted = h.Del([]byte("d"))
	if !deleted {
		t.Error("failed to delete for key=\"d\"")
	}
	if val != 4 {
		t.Error("h.Get(\"d\") failed to retrieve val = 4")
	}

	t.Logf("After h.Del(%q): h =\n%s", "d", h.LongString(""))

	h, val, deleted = h.Del([]byte("aa"))
	if !deleted {
		t.Error("failed to delete for key=\"aa\"")
	}
	if val != 27 {
		t.Error("h.Get(\"d\") failed to retrieve val = 27")
	}

	t.Logf("After h.Del(%q): h =\n%s", "aa", h.LongString(""))

	if !h.IsEmpty() {
		t.Error("h is NOT empty")
	}

	return
}

// Where Many == 64
func TestEmptyPutManyDelManyIsEmpty(t *testing.T) {
	var h = &EMPTY

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var val = midNumEnts[i].val
		h, _ = h.Put(key, val)
	}

	t.Log("h.root =\n", h.root.LongString(""))

	for i := 0; i < 64; i++ {
		var key = midNumEnts[i].key
		var expected_val = midNumEnts[i].val

		var val interface{}
		var deleted bool
		h, val, deleted = h.Del(key)
		if !deleted {
			t.Errorf("Did NOT find&delete for key=\"%s\"", key)
		}
		if val != expected_val {
			t.Errorf("val,%d != expected_val,%d", val, expected_val)
		}

		if h.root == nil {
			t.Log("h.root == nil")
		} else {
			t.Log("h.root ==\n", h.root.LongString(""))
		}
	}
	t.Log("### Testing compressedTable Shrinkage ###")

	t.Log(h)

	if !h.IsEmpty() {
		t.Fatal("NOT h.IsEmpty()")
	}
}

func TestEmptyPutDelTrumpIsEmpty(t *testing.T) {
	var h = &EMPTY

	for i := 0; i < len(hugeNumEnts); i++ {
		h, _ = h.Put(hugeNumEnts[i].key, hugeNumEnts[i].val)
	}

	Lgr.Println("h.root =")
	Lgr.Println(h.root.LongString(""))

	for i := 0; i < len(hugeNumEnts); i++ {
		var key = hugeNumEnts[i].key
		var expected_val = hugeNumEnts[i].val

		var val interface{}
		var deleted bool
		h, val, deleted = h.Del(key)
		if !deleted {
			t.Errorf("Did NOT find&delete for key=\"%s\"", key)
		}
		if val != expected_val {
			t.Errorf("val,%d != expected_val,%d", val, expected_val)
		}
	}
	//t.Log("### Testing compressedTable Shrinkage ###")

	if !h.IsEmpty() {
		Lgr.Println(h.LongString(""))
		t.Fatal("NOT h.IsEmpty()")
	}
}

// collided depth 3:
//     "b",2 & "rstuvvw",670
//     "gg",39 & "yzz",152     <=== I like this one
//     "mm",51 & "efggh",283
//     "stt",169 & "abcddefgh",940
<|endoftext|>


<commit_before>package timber

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
)

type FilenameFields struct {
	Hostname string
	Date     time.Time
	Pid      int
}

func GetFilenameFields() *FilenameFields {
	h, _ := os.Hostname()
	return &FilenameFields{
		Hostname: h,
		Date:     time.Now(),
		Pid:      os.Getpid(),
	}
}

func preprocessFilename(name string) string {
	t := template.Must(template.New("filename").Parse(name))
	buf := new(bytes.Buffer)
	t.Execute(buf, GetFilenameFields())
	return buf.String()
}

type FileWriter struct {
	wr              *BufferedWriter
	BaseFilename    string
	currentFilename string
	mutex           *sync.RWMutex
	RotateChan      chan string // defaults to nil.  receives previous filename on rotate

	rotateTicker *time.Ticker
	rotateReset  chan int
}

// This writer has a buffer that I don't ever bother to flush, so it may take a while
// to see messages.  Filenames ending in .gz will automatically be compressed on write.
// Filename string is proccessed through the template library using the FilenameFields
// struct.
func NewFileWriter(name string) (*FileWriter, error) {
	w := &FileWriter{
		BaseFilename: name,
		mutex:        new(sync.RWMutex),
	}
	if err := w.open(); err != nil {
		return nil, err
	}
	return w, nil
}

func (w *FileWriter) open() error {
	name := preprocessFilename(w.BaseFilename)
	file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("TIMBER! Can't open %v: %v", name, err)
	}

	var output io.WriteCloser = file
	// Wrap in gz writer
	if strings.HasSuffix(name, ".gz") {
		output = &gzFileWriter{
			gzip.NewWriter(output),
			output,
		}
	}

	w.mutex.Lock()
	defer w.mutex.Unlock()
	if w.wr != nil {
		w.wr.Close()
		// send previous filename on rotate chan
		if c := w.RotateChan; c != nil {
			c <- w.currentFilename
		}
	}
	w.currentFilename = name
	w.wr, _ = NewBufferedWriter(output)

	return nil
}

func (w *FileWriter) LogWrite(m string) {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	w.wr.LogWrite(m)
}

func (w *FileWriter) Flush() error {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	return w.wr.Flush()
}

// Close and re-open the file.
// You should use the timestamp in the filename if you're going to use rotation
func (w *FileWriter) Rotate() error {
	return w.open()
}

// Automatically rotate every `d`
func (w *FileWriter) RotateEvery(d time.Duration) {
	// reset ticker
	w.mutex.Lock()
	if w.rotateTicker != nil {
		w.rotateTicker.Stop()
		w.rotateReset <- 1
	}
	w.rotateTicker = time.NewTicker(d)
	w.mutex.Unlock()

	// trigger a rotate every X
	go func() {
		for {
			select {
			case <-w.rotateReset:
				return
			case <-w.rotateTicker.C:
				w.Rotate()
			}
		}
	}()
}

func (w *FileWriter) Close() {
	w.mutex.Lock()
	defer w.mutex.Unlock()
	w.wr.Close()
	w.wr = nil
}

type gzFileWriter struct {
	*gzip.Writer // the compressor
	file         io.WriteCloser
}

func (w *gzFileWriter) Close() error {
	w.Writer.Close()
	return w.file.Close()
}
<commit_msg>added max file size before rotation<commit_after>package timber

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
)

type FilenameFields struct {
	Hostname string
	Date     time.Time
	Pid      int
}

func GetFilenameFields() *FilenameFields {
	h, _ := os.Hostname()
	return &FilenameFields{
		Hostname: h,
		Date:     time.Now(),
		Pid:      os.Getpid(),
	}
}

func preprocessFilename(name string) string {
	t := template.Must(template.New("filename").Parse(name))
	buf := new(bytes.Buffer)
	t.Execute(buf, GetFilenameFields())
	return buf.String()
}

type FileWriter struct {
	wr              *BufferedWriter
	cwr             *countingWriter
	BaseFilename    string
	currentFilename string
	mutex           *sync.RWMutex
	RotateChan      chan string // defaults to nil.  receives previous filename on rotate
	RotateSize      int64       // rotate after RotateSize bytes have been written to the file

	rotateTicker *time.Ticker
	rotateReset  chan int
}

// This writer has a buffer that I don't ever bother to flush, so it may take a while
// to see messages.  Filenames ending in .gz will automatically be compressed on write.
// Filename string is proccessed through the template library using the FilenameFields
// struct.
func NewFileWriter(name string) (*FileWriter, error) {
	w := &FileWriter{
		BaseFilename: name,
		mutex:        new(sync.RWMutex),
	}
	if err := w.open(); err != nil {
		return nil, err
	}
	return w, nil
}

func (w *FileWriter) open() error {
	// No lock here
	name := preprocessFilename(w.BaseFilename)
	file, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		return fmt.Errorf("TIMBER! Can't open %v: %v", name, err)
	}

	var cwr = &countingWriter{file, 0}
	var output io.WriteCloser = cwr
	// Wrap in gz writer
	if strings.HasSuffix(name, ".gz") {
		output = &gzFileWriter{
			gzip.NewWriter(output),
			output,
		}
	}

	// Locked from here
	w.mutex.Lock()
	defer w.mutex.Unlock()
	if w.wr != nil {
		w.wr.Close()
		// send previous filename on rotate chan
		if c := w.RotateChan; c != nil {
			c <- w.currentFilename
		}
	}
	w.currentFilename = name
	w.cwr = cwr
	w.wr, _ = NewBufferedWriter(output)

	return nil
}

func (w *FileWriter) LogWrite(m string) {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	w.wr.LogWrite(m)
	w.checkSize()
}

func (w *FileWriter) Flush() error {
	w.mutex.RLock()
	defer w.mutex.RUnlock()
	e := w.wr.Flush()
	w.checkSize()
	return e
}

// Close and re-open the file.
// You should use the timestamp in the filename if you're going to use rotation
func (w *FileWriter) Rotate() error {
	return w.open()
}

// Automatically rotate every `d`
func (w *FileWriter) RotateEvery(d time.Duration) {
	// reset ticker
	w.mutex.Lock()
	if w.rotateTicker != nil {
		w.rotateTicker.Stop()
		w.rotateReset <- 1
	}
	w.rotateTicker = time.NewTicker(d)
	w.mutex.Unlock()

	// trigger a rotate every X
	go func() {
		for {
			select {
			case <-w.rotateReset:
				return
			case <-w.rotateTicker.C:
				w.Rotate()
			}
		}
	}()
}

func (w *FileWriter) checkSize() {
	if w.RotateSize > 0 && w.cwr.bytesWritten >= w.RotateSize {
		go w.Rotate()
	}
}

func (w *FileWriter) Close() {
	w.mutex.Lock()
	defer w.mutex.Unlock()
	w.wr.Close()
	w.wr = nil
}

type countingWriter struct {
	io.WriteCloser
	bytesWritten int64
}

func (w *countingWriter) Write(b []byte) (int, error) {
	i, e := w.WriteCloser.Write(b)
	w.bytesWritten += int64(i)
	return i, e
}

type gzFileWriter struct {
	*gzip.Writer // the compressor
	file         io.WriteCloser
}

func (w *gzFileWriter) Close() error {
	w.Writer.Close()
	return w.file.Close()
}
<|endoftext|>


<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 单例对象管理.
// 框架内置了一些核心对象获取方法，并且可以通过Set和Get方法实现IoC以及对内置核心对象的自定义替换
package gins

import (
    "gitee.com/johng/gf/g/os/gcfg"
    "gitee.com/johng/gf/g/os/gcmd"
    "gitee.com/johng/gf/g/os/genv"
    "gitee.com/johng/gf/g/os/glog"
    "gitee.com/johng/gf/g/os/gview"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/util/gconv"
    "gitee.com/johng/gf/g/database/gdb"
    "gitee.com/johng/gf/g/os/gfsnotify"
    "fmt"
    "gitee.com/johng/gf/g/database/gredis"
    "gitee.com/johng/gf/g/util/gregex"
)

const (
    gFRAME_CORE_COMPONENT_NAME_VIEW       = "gf.core.component.view"
    gFRAME_CORE_COMPONENT_NAME_CONFIG     = "gf.core.component.config"
    gFRAME_CORE_COMPONENT_NAME_REDIS      = "gf.core.component.redis"
    gFRAME_CORE_COMPONENT_NAME_DATABASE   = "gf.core.component.database"
)

// 单例对象存储器
var instances = gmap.NewStringInterfaceMap()

// 获取单例对象
func Get(key string) interface{} {
    return instances.Get(key)
}

// 设置单例对象
func Set(key string, value interface{}) {
    instances.Set(key, key)
}

// 当键名存在时返回其键值，否则写入指定的键值
func GetOrSet(key string, value interface{}) interface{} {
    return instances.GetOrSet(key, value)
}

// 当键名存在时返回其键值，否则写入指定的键值，键值由指定的函数生成
func GetOrSetFunc(key string, f func() interface{}) interface{} {
    return instances.GetOrSetFunc(key, f)
}

// 与GetOrSetFunc不同的是，f是在写锁机制内执行
func GetOrSetFuncLock(key string, f func() interface{}) interface{} {
    return instances.GetOrSetFuncLock(key, f)
}

// 当键名不存在时写入，并返回true；否则返回false。
func SetIfNotExist(key string, value interface{}) bool {
    return instances.SetIfNotExist(key, value)
}

// 核心对象：View
func View(name...string) *gview.View {
    group := "default"
    if len(name) > 0 {
        group = name[0]
    }
    key := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_VIEW, group)
    return instances.GetOrSetFuncLock(key, func() interface{} {
        path := gcmd.Option.Get("gf.viewpath")
        if path == "" {
            path = genv.Get("GF_VIEWPATH")
            if path == "" {
                path = gfile.SelfDir()
            }
        }
        view := gview.Get(path)
        // 添加基于源码的搜索目录检索地址，常用于开发环境调试，只添加入口文件目录
        if p := gfile.MainPkgPath(); p != "" && gfile.Exists(p) {
            view.AddPath(p)
        }
        // 框架内置函数
        view.BindFunc("config", funcConfig)
        return view
    }).(*gview.View)
}

// 核心对象：Config
// 配置文件目录查找依次为：启动参数cfgpath、当前程序运行目录
func Config(file...string) *gcfg.Config {
    configFile := gcfg.DEFAULT_CONFIG_FILE
    if len(file) > 0 {
        configFile = file[0]
    }
    return instances.GetOrSetFuncLock(fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_CONFIG, configFile),
        func() interface{} {
            path := gcmd.Option.Get("gf.cfgpath")
            if path == "" {
                path = genv.Get("GF_CFGPATH")
                if path == "" {
                    path = gfile.SelfDir()
                }
            }
            config := gcfg.New(path, configFile)
            // 添加基于源码的搜索目录检索地址，常用于开发环境调试，只添加入口文件目录
            if p := gfile.MainPkgPath(); p != "" && gfile.Exists(p) {
                config.AddPath(p)
            }
            return config
    }).(*gcfg.Config)
}

// 数据库操作对象，使用了连接池
func Database(name...string) *gdb.Db {
    config := Config()
    group  := gdb.DEFAULT_GROUP_NAME
    if len(name) > 0 {
        group = name[0]
    }
    key := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group)
    db  := instances.GetOrSetFuncLock(key, func() interface{} {
        m := config.GetMap("database")
        if m == nil {
            glog.Errorfln(`incomplete configuration for database: "database" node not found in config file "%s"`, config.GetFilePath())
        }
        for group, v := range m {
            cg := gdb.ConfigGroup{}
            if list, ok := v.([]interface{}); ok {
                for _, nodev := range list {
                    node  := gdb.ConfigNode{}
                    nodem := nodev.(map[string]interface{})
                    if value, ok := nodem["host"]; ok {
                        node.Host = gconv.String(value)
                    }
                    if value, ok := nodem["port"]; ok {
                        node.Port = gconv.String(value)
                    }
                    if value, ok := nodem["user"]; ok {
                        node.User = gconv.String(value)
                    }
                    if value, ok := nodem["pass"]; ok {
                        node.Pass = gconv.String(value)
                    }
                    if value, ok := nodem["name"]; ok {
                        node.Name = gconv.String(value)
                    }
                    if value, ok := nodem["type"]; ok {
                        node.Type = gconv.String(value)
                    }
                    if value, ok := nodem["role"]; ok {
                        node.Role = gconv.String(value)
                    }
                    if value, ok := nodem["charset"]; ok {
                        node.Charset = gconv.String(value)
                    }
                    if value, ok := nodem["priority"]; ok {
                        node.Priority = gconv.Int(value)
                    }
                    if value, ok := nodem["linkinfo"]; ok {
                        node.Linkinfo = gconv.String(value)
                    }
                    if value, ok := nodem["max-idle"]; ok {
                        node.MaxIdleConnCount = gconv.Int(value)
                    }
                    if value, ok := nodem["max-open"]; ok {
                        node.MaxOpenConnCount = gconv.Int(value)
                    }
                    if value, ok := nodem["max-lifetime"]; ok {
                        node.MaxConnLifetime = gconv.Int(value)
                    }
                    cg = append(cg, node)
                }
            }
            gdb.AddConfigGroup(group, cg)
        }
        // 使用gfsnotify进行文件监控，当配置文件有任何变化时，清空数据库配置缓存
        gfsnotify.Add(config.GetFilePath(), func(event *gfsnotify.Event) {
            instances.Remove(key)
        })
        if db, err := gdb.New(name...); err == nil {
            return db
        } else {
            glog.Error(err)
        }
        return nil
    })
    if db != nil {
        return db.(*gdb.Db)
    }
    return nil
}

// Redis操作对象，使用了连接池
func Redis(name...string) *gredis.Redis {
    config := Config()
    group  := "default"
    if len(name) > 0 {
        group = name[0]
    }
    key    := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group)
    result := instances.GetOrSetFuncLock(key, func() interface{} {
        if m := config.GetMap("redis"); m != nil {
            // host:port[,db[,pass]]
            if v, ok := m[group]; ok {
                line     := gconv.String(v)
                array, _ := gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)`, line)
                if len(array) > 4 {
                    return gredis.New(gredis.Config{
                        Host : array[1],
                        Port : gconv.Int(array[2]),
                        Db   : gconv.Int(array[3]),
                        Pass : array[4],
                    })
                } else {
                    glog.Errorfln(`invalid redis node configuration: "%s"`, line)
                }
            } else {
                glog.Errorfln(`configuration for redis not found for group "%s"`, group)
            }
        } else {
            glog.Errorfln(`incomplete configuration for redis: "redis" node not found in config file "%s"`, config.GetFilePath())
        }
        return nil
    })
    if result != nil {
        return result.(*gredis.Redis)
    }
    return nil
}

// 模板内置方法：config
func funcConfig(pattern string, file...string) string {
    return Config().GetString(pattern, file...)
}

<commit_msg>修复gins.Set问题<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 单例对象管理.
// 框架内置了一些核心对象获取方法，并且可以通过Set和Get方法实现IoC以及对内置核心对象的自定义替换
package gins

import (
    "gitee.com/johng/gf/g/os/gcfg"
    "gitee.com/johng/gf/g/os/gcmd"
    "gitee.com/johng/gf/g/os/genv"
    "gitee.com/johng/gf/g/os/glog"
    "gitee.com/johng/gf/g/os/gview"
    "gitee.com/johng/gf/g/os/gfile"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/util/gconv"
    "gitee.com/johng/gf/g/database/gdb"
    "gitee.com/johng/gf/g/os/gfsnotify"
    "fmt"
    "gitee.com/johng/gf/g/database/gredis"
    "gitee.com/johng/gf/g/util/gregex"
)

const (
    gFRAME_CORE_COMPONENT_NAME_VIEW       = "gf.core.component.view"
    gFRAME_CORE_COMPONENT_NAME_CONFIG     = "gf.core.component.config"
    gFRAME_CORE_COMPONENT_NAME_REDIS      = "gf.core.component.redis"
    gFRAME_CORE_COMPONENT_NAME_DATABASE   = "gf.core.component.database"
)

// 单例对象存储器
var instances = gmap.NewStringInterfaceMap()

// 获取单例对象
func Get(key string) interface{} {
    return instances.Get(key)
}

// 设置单例对象
func Set(key string, value interface{}) {
    instances.Set(key, value)
}

// 当键名存在时返回其键值，否则写入指定的键值
func GetOrSet(key string, value interface{}) interface{} {
    return instances.GetOrSet(key, value)
}

// 当键名存在时返回其键值，否则写入指定的键值，键值由指定的函数生成
func GetOrSetFunc(key string, f func() interface{}) interface{} {
    return instances.GetOrSetFunc(key, f)
}

// 与GetOrSetFunc不同的是，f是在写锁机制内执行
func GetOrSetFuncLock(key string, f func() interface{}) interface{} {
    return instances.GetOrSetFuncLock(key, f)
}

// 当键名不存在时写入，并返回true；否则返回false。
func SetIfNotExist(key string, value interface{}) bool {
    return instances.SetIfNotExist(key, value)
}

// 核心对象：View
func View(name...string) *gview.View {
    group := "default"
    if len(name) > 0 {
        group = name[0]
    }
    key := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_VIEW, group)
    return instances.GetOrSetFuncLock(key, func() interface{} {
        path := gcmd.Option.Get("gf.viewpath")
        if path == "" {
            path = genv.Get("GF_VIEWPATH")
            if path == "" {
                path = gfile.SelfDir()
            }
        }
        view := gview.Get(path)
        // 添加基于源码的搜索目录检索地址，常用于开发环境调试，只添加入口文件目录
        if p := gfile.MainPkgPath(); p != "" && gfile.Exists(p) {
            view.AddPath(p)
        }
        // 框架内置函数
        view.BindFunc("config", funcConfig)
        return view
    }).(*gview.View)
}

// 核心对象：Config
// 配置文件目录查找依次为：启动参数cfgpath、当前程序运行目录
func Config(file...string) *gcfg.Config {
    configFile := gcfg.DEFAULT_CONFIG_FILE
    if len(file) > 0 {
        configFile = file[0]
    }
    return instances.GetOrSetFuncLock(fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_CONFIG, configFile),
        func() interface{} {
            path := gcmd.Option.Get("gf.cfgpath")
            if path == "" {
                path = genv.Get("GF_CFGPATH")
                if path == "" {
                    path = gfile.SelfDir()
                }
            }
            config := gcfg.New(path, configFile)
            // 添加基于源码的搜索目录检索地址，常用于开发环境调试，只添加入口文件目录
            if p := gfile.MainPkgPath(); p != "" && gfile.Exists(p) {
                config.AddPath(p)
            }
            return config
    }).(*gcfg.Config)
}

// 数据库操作对象，使用了连接池
func Database(name...string) *gdb.Db {
    config := Config()
    group  := gdb.DEFAULT_GROUP_NAME
    if len(name) > 0 {
        group = name[0]
    }
    key := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group)
    db  := instances.GetOrSetFuncLock(key, func() interface{} {
        m := config.GetMap("database")
        if m == nil {
            glog.Errorfln(`incomplete configuration for database: "database" node not found in config file "%s"`, config.GetFilePath())
        }
        for group, v := range m {
            cg := gdb.ConfigGroup{}
            if list, ok := v.([]interface{}); ok {
                for _, nodev := range list {
                    node  := gdb.ConfigNode{}
                    nodem := nodev.(map[string]interface{})
                    if value, ok := nodem["host"]; ok {
                        node.Host = gconv.String(value)
                    }
                    if value, ok := nodem["port"]; ok {
                        node.Port = gconv.String(value)
                    }
                    if value, ok := nodem["user"]; ok {
                        node.User = gconv.String(value)
                    }
                    if value, ok := nodem["pass"]; ok {
                        node.Pass = gconv.String(value)
                    }
                    if value, ok := nodem["name"]; ok {
                        node.Name = gconv.String(value)
                    }
                    if value, ok := nodem["type"]; ok {
                        node.Type = gconv.String(value)
                    }
                    if value, ok := nodem["role"]; ok {
                        node.Role = gconv.String(value)
                    }
                    if value, ok := nodem["charset"]; ok {
                        node.Charset = gconv.String(value)
                    }
                    if value, ok := nodem["priority"]; ok {
                        node.Priority = gconv.Int(value)
                    }
                    if value, ok := nodem["linkinfo"]; ok {
                        node.Linkinfo = gconv.String(value)
                    }
                    if value, ok := nodem["max-idle"]; ok {
                        node.MaxIdleConnCount = gconv.Int(value)
                    }
                    if value, ok := nodem["max-open"]; ok {
                        node.MaxOpenConnCount = gconv.Int(value)
                    }
                    if value, ok := nodem["max-lifetime"]; ok {
                        node.MaxConnLifetime = gconv.Int(value)
                    }
                    cg = append(cg, node)
                }
            }
            gdb.AddConfigGroup(group, cg)
        }
        // 使用gfsnotify进行文件监控，当配置文件有任何变化时，清空数据库配置缓存
        gfsnotify.Add(config.GetFilePath(), func(event *gfsnotify.Event) {
            instances.Remove(key)
        })
        if db, err := gdb.New(name...); err == nil {
            return db
        } else {
            glog.Error(err)
        }
        return nil
    })
    if db != nil {
        return db.(*gdb.Db)
    }
    return nil
}

// Redis操作对象，使用了连接池
func Redis(name...string) *gredis.Redis {
    config := Config()
    group  := "default"
    if len(name) > 0 {
        group = name[0]
    }
    key    := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_REDIS, group)
    result := instances.GetOrSetFuncLock(key, func() interface{} {
        if m := config.GetMap("redis"); m != nil {
            // host:port[,db[,pass]]
            if v, ok := m[group]; ok {
                line     := gconv.String(v)
                array, _ := gregex.MatchString(`(.+):(\d+),{0,1}(\d*),{0,1}(.*)`, line)
                if len(array) > 4 {
                    return gredis.New(gredis.Config{
                        Host : array[1],
                        Port : gconv.Int(array[2]),
                        Db   : gconv.Int(array[3]),
                        Pass : array[4],
                    })
                } else {
                    glog.Errorfln(`invalid redis node configuration: "%s"`, line)
                }
            } else {
                glog.Errorfln(`configuration for redis not found for group "%s"`, group)
            }
        } else {
            glog.Errorfln(`incomplete configuration for redis: "redis" node not found in config file "%s"`, config.GetFilePath())
        }
        return nil
    })
    if result != nil {
        return result.(*gredis.Redis)
    }
    return nil
}

// 模板内置方法：config
func funcConfig(pattern string, file...string) string {
    return Config().GetString(pattern, file...)
}

<|endoftext|>


<commit_before>package objectstore

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/metadata"
	"github.com/rancher/convoy/util"
	"io"
	"os"
	"path/filepath"

	. "github.com/rancher/convoy/logging"
)

type BlockMapping struct {
	Offset        int64
	BlockChecksum string
}

type DeltaBlockBackupOperations interface {
	HasSnapshot(id, volumeID string) bool
	CompareSnapshot(id, compareID, volumeID string) (*metadata.Mappings, error)
	OpenSnapshot(id, volumeID string) error
	ReadSnapshot(id, volumeID string, start int64, data []byte) error
	CloseSnapshot(id, volumeID string) error
}

const (
	DEFAULT_BLOCK_SIZE = 2097152

	BLOCKS_DIRECTORY      = "blocks"
	BLOCK_SEPARATE_LAYER1 = 2
	BLOCK_SEPARATE_LAYER2 = 4
)

func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, deltaOps DeltaBlockBackupOperations) (string, error) {
	if deltaOps == nil {
		return "", fmt.Errorf("Missing DeltaBlockBackupOperations")
	}

	bsDriver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, bsDriver); err != nil {
		return "", err
	}

	// Update volume from objectstore
	volume, err = loadVolume(volume.Name, bsDriver)
	if err != nil {
		return "", err
	}

	lastBackupName := volume.LastBackupName

	var lastSnapshotName string
	var lastBackup *Backup
	if lastBackupName != "" {
		lastBackup, err = loadBackup(lastBackupName, volume.Name, bsDriver)
		if err != nil {
			return "", err
		}

		lastSnapshotName = lastBackup.SnapshotName
		if lastSnapshotName == snapshot.Name {
			//Generate full snapshot if the snapshot has been backed up last time
			lastSnapshotName = ""
			log.Debug("Would create full snapshot metadata")
		} else if !deltaOps.HasSnapshot(lastSnapshotName, volume.Name) {
			// It's possible that the snapshot in objectstore doesn't exist
			// in local storage
			lastSnapshotName = ""
			log.WithFields(logrus.Fields{
				LOG_FIELD_REASON:   LOG_REASON_FALLBACK,
				LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
				LOG_FIELD_SNAPSHOT: lastSnapshotName,
				LOG_FIELD_VOLUME:   volume.Name,
			}).Debug("Cannot find last snapshot in local storage, would process with full backup")
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_START,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generating snapshot changed blocks metadata")

	if err := deltaOps.OpenSnapshot(snapshot.Name, volume.Name); err != nil {
		return "", err
	}
	defer deltaOps.CloseSnapshot(snapshot.Name, volume.Name)

	delta, err := deltaOps.CompareSnapshot(snapshot.Name, lastSnapshotName, volume.Name)
	if err != nil {
		return "", err
	}
	if delta.BlockSize != DEFAULT_BLOCK_SIZE {
		return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_COMPLETE,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generated snapshot changed blocks metadata")

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Creating backup")

	deltaBackup := &Backup{
		Name:         util.GenerateName("backup"),
		VolumeName:   volume.Name,
		SnapshotName: snapshot.Name,
		Blocks:       []BlockMapping{},
	}
	mCounts := len(delta.Mappings)
	for m, d := range delta.Mappings {
		block := make([]byte, DEFAULT_BLOCK_SIZE)
		blkCounts := d.Size / delta.BlockSize
		for i := int64(0); i < blkCounts; i++ {
			offset := d.Offset + i*delta.BlockSize
			log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.Name, m+1, mCounts, i+1, blkCounts)
			err := deltaOps.ReadSnapshot(snapshot.Name, volume.Name, offset, block)
			if err != nil {
				return "", err
			}
			checksum := util.GetChecksum(block)
			blkFile := getBlockFilePath(volume.Name, checksum)
			if bsDriver.FileSize(blkFile) >= 0 {
				blockMapping := BlockMapping{
					Offset:        offset,
					BlockChecksum: checksum,
				}
				deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
				log.Debugf("Found existed block match at %v", blkFile)
				continue
			}

			rs, err := util.CompressData(block)
			if err != nil {
				return "", err
			}

			if err := bsDriver.Write(blkFile, rs); err != nil {
				return "", err
			}
			log.Debugf("Created new block file at %v", blkFile)

			blockMapping := BlockMapping{
				Offset:        offset,
				BlockChecksum: checksum,
			}
			deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Created snapshot changed blocks")

	backup := mergeSnapshotMap(deltaBackup, lastBackup)
	backup.SnapshotName = snapshot.Name
	backup.SnapshotCreatedAt = snapshot.CreatedTime
	backup.CreatedTime = util.Now()

	if err := saveBackup(backup, bsDriver); err != nil {
		return "", err
	}

	volume.LastBackupName = backup.Name
	if err := saveVolume(volume, bsDriver); err != nil {
		return "", err
	}

	return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}

func mergeSnapshotMap(deltaBackup, lastBackup *Backup) *Backup {
	if lastBackup == nil {
		return deltaBackup
	}
	backup := &Backup{
		Name:         deltaBackup.Name,
		VolumeName:   deltaBackup.VolumeName,
		SnapshotName: deltaBackup.SnapshotName,
		Blocks:       []BlockMapping{},
	}
	var d, l int
	for d, l = 0, 0; d < len(deltaBackup.Blocks) && l < len(lastBackup.Blocks); {
		dB := deltaBackup.Blocks[d]
		lB := lastBackup.Blocks[l]
		if dB.Offset == lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
			l++
		} else if dB.Offset < lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
		} else {
			//dB.Offset > lB.offset
			backup.Blocks = append(backup.Blocks, lB)
			l++
		}
	}

	if d == len(deltaBackup.Blocks) {
		backup.Blocks = append(backup.Blocks, lastBackup.Blocks[l:]...)
	} else {
		backup.Blocks = append(backup.Blocks, deltaBackup.Blocks[d:]...)
	}

	return backup
}

func RestoreDeltaBlockBackup(backupURL, volDevName string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	srcBackupName, srcVolumeName, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	if _, err := loadVolume(srcVolumeName, bsDriver); err != nil {
		return generateError(logrus.Fields{
			LOG_FIELD_VOLUME:     srcVolumeName,
			LOG_FIELD_BACKUP_URL: backupURL,
		}, "Volume doesn't exist in objectstore: %v", err)
	}

	volDev, err := os.Create(volDevName)
	if err != nil {
		return err
	}
	defer volDev.Close()

	backup, err := loadBackup(srcBackupName, srcVolumeName, bsDriver)
	if err != nil {
		return err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:      LOG_REASON_START,
		LOG_FIELD_EVENT:       LOG_EVENT_RESTORE,
		LOG_FIELD_OBJECT:      LOG_FIELD_SNAPSHOT,
		LOG_FIELD_SNAPSHOT:    srcBackupName,
		LOG_FIELD_ORIN_VOLUME: srcVolumeName,
		LOG_FIELD_VOLUME_DEV:  volDevName,
		LOG_FIELD_BACKUP_URL:  backupURL,
	}).Debug()
	blkCounts := len(backup.Blocks)
	for i, block := range backup.Blocks {
		log.Debugf("Restore for %v: block %v, %v/%v", volDevName, block.BlockChecksum, i+1, blkCounts)
		blkFile := getBlockFilePath(srcVolumeName, block.BlockChecksum)
		rc, err := bsDriver.Read(blkFile)
		if err != nil {
			return err
		}
		r, err := util.DecompressAndVerify(rc, block.BlockChecksum)
		rc.Close()
		if err != nil {
			return err
		}
		if _, err := volDev.Seek(block.Offset, 0); err != nil {
			return err
		}
		if _, err := io.CopyN(volDev, r, DEFAULT_BLOCK_SIZE); err != nil {
			return err
		}
	}

	return nil
}

func DeleteDeltaBlockBackup(backupURL string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	backupName, volumeName, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	v, err := loadVolume(volumeName, bsDriver)
	if err != nil {
		return fmt.Errorf("Cannot find volume %v in objectstore", volumeName, err)
	}

	backup, err := loadBackup(backupName, volumeName, bsDriver)
	if err != nil {
		return err
	}
	discardBlockSet := make(map[string]bool)
	for _, blk := range backup.Blocks {
		discardBlockSet[blk.BlockChecksum] = true
	}
	discardBlockCounts := len(discardBlockSet)

	if err := removeBackup(backup, bsDriver); err != nil {
		return err
	}

	if backup.Name == v.LastBackupName {
		v.LastBackupName = ""
		if err := saveVolume(v, bsDriver); err != nil {
			return err
		}
	}

	backupNames, err := getBackupNamesForVolume(volumeName, bsDriver)
	if err != nil {
		return err
	}
	if len(backupNames) == 0 {
		log.Debugf("No snapshot existed for the volume %v, removing volume", volumeName)
		if err := removeVolume(volumeName, bsDriver); err != nil {
			log.Warningf("Failed to remove volume %v due to: %v", volumeName, err.Error())
		}
		return nil
	}

	log.Debug("GC started")
	for _, backupName := range backupNames {
		backup, err := loadBackup(backupName, volumeName, bsDriver)
		if err != nil {
			return err
		}
		for _, blk := range backup.Blocks {
			if _, exists := discardBlockSet[blk.BlockChecksum]; exists {
				delete(discardBlockSet, blk.BlockChecksum)
				discardBlockCounts--
				if discardBlockCounts == 0 {
					break
				}
			}
		}
		if discardBlockCounts == 0 {
			break
		}
	}

	var blkFileList []string
	for blk := range discardBlockSet {
		blkFileList = append(blkFileList, getBlockFilePath(volumeName, blk))
		log.Debugf("Found unused blocks %v for volume %v", blk, volumeName)
	}
	if err := bsDriver.Remove(blkFileList...); err != nil {
		return err
	}
	log.Debug("Removed unused blocks for volume ", volumeName)

	log.Debug("GC completed")
	log.Debug("Removed objectstore backup ", backupName)

	return nil
}

func getBlockPath(volumeName string) string {
	return filepath.Join(getVolumePath(volumeName), BLOCKS_DIRECTORY) + "/"
}

func getBlockFilePath(volumeName, checksum string) string {
	blockSubDirLayer1 := checksum[0:BLOCK_SEPARATE_LAYER1]
	blockSubDirLayer2 := checksum[BLOCK_SEPARATE_LAYER1:BLOCK_SEPARATE_LAYER2]
	path := filepath.Join(getBlockPath(volumeName), blockSubDirLayer1, blockSubDirLayer2)
	fileName := checksum + ".blk"

	return filepath.Join(path, fileName)
}
<commit_msg>objectstore: Validate block mapping's size<commit_after>package objectstore

import (
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/rancher/convoy/metadata"
	"github.com/rancher/convoy/util"
	"io"
	"os"
	"path/filepath"

	. "github.com/rancher/convoy/logging"
)

type BlockMapping struct {
	Offset        int64
	BlockChecksum string
}

type DeltaBlockBackupOperations interface {
	HasSnapshot(id, volumeID string) bool
	CompareSnapshot(id, compareID, volumeID string) (*metadata.Mappings, error)
	OpenSnapshot(id, volumeID string) error
	ReadSnapshot(id, volumeID string, start int64, data []byte) error
	CloseSnapshot(id, volumeID string) error
}

const (
	DEFAULT_BLOCK_SIZE = 2097152

	BLOCKS_DIRECTORY      = "blocks"
	BLOCK_SEPARATE_LAYER1 = 2
	BLOCK_SEPARATE_LAYER2 = 4
)

func CreateDeltaBlockBackup(volume *Volume, snapshot *Snapshot, destURL string, deltaOps DeltaBlockBackupOperations) (string, error) {
	if deltaOps == nil {
		return "", fmt.Errorf("Missing DeltaBlockBackupOperations")
	}

	bsDriver, err := GetObjectStoreDriver(destURL)
	if err != nil {
		return "", err
	}

	if err := addVolume(volume, bsDriver); err != nil {
		return "", err
	}

	// Update volume from objectstore
	volume, err = loadVolume(volume.Name, bsDriver)
	if err != nil {
		return "", err
	}

	lastBackupName := volume.LastBackupName

	var lastSnapshotName string
	var lastBackup *Backup
	if lastBackupName != "" {
		lastBackup, err = loadBackup(lastBackupName, volume.Name, bsDriver)
		if err != nil {
			return "", err
		}

		lastSnapshotName = lastBackup.SnapshotName
		if lastSnapshotName == snapshot.Name {
			//Generate full snapshot if the snapshot has been backed up last time
			lastSnapshotName = ""
			log.Debug("Would create full snapshot metadata")
		} else if !deltaOps.HasSnapshot(lastSnapshotName, volume.Name) {
			// It's possible that the snapshot in objectstore doesn't exist
			// in local storage
			lastSnapshotName = ""
			log.WithFields(logrus.Fields{
				LOG_FIELD_REASON:   LOG_REASON_FALLBACK,
				LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
				LOG_FIELD_SNAPSHOT: lastSnapshotName,
				LOG_FIELD_VOLUME:   volume.Name,
			}).Debug("Cannot find last snapshot in local storage, would process with full backup")
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_START,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generating snapshot changed blocks metadata")

	if err := deltaOps.OpenSnapshot(snapshot.Name, volume.Name); err != nil {
		return "", err
	}
	defer deltaOps.CloseSnapshot(snapshot.Name, volume.Name)

	delta, err := deltaOps.CompareSnapshot(snapshot.Name, lastSnapshotName, volume.Name)
	if err != nil {
		return "", err
	}
	if delta.BlockSize != DEFAULT_BLOCK_SIZE {
		return "", fmt.Errorf("Currently doesn't support different block sizes driver other than %v", DEFAULT_BLOCK_SIZE)
	}
	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:        LOG_REASON_COMPLETE,
		LOG_FIELD_OBJECT:        LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_EVENT:         LOG_EVENT_COMPARE,
		LOG_FIELD_SNAPSHOT:      snapshot.Name,
		LOG_FIELD_LAST_SNAPSHOT: lastSnapshotName,
	}).Debug("Generated snapshot changed blocks metadata")

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_START,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Creating backup")

	deltaBackup := &Backup{
		Name:         util.GenerateName("backup"),
		VolumeName:   volume.Name,
		SnapshotName: snapshot.Name,
		Blocks:       []BlockMapping{},
	}
	mCounts := len(delta.Mappings)
	for m, d := range delta.Mappings {
		if d.Size%delta.BlockSize != 0 {
			return "", fmt.Errorf("Mapping's size %v is not multiples of backup block size %v",
				d.Size, delta.BlockSize)
		}
		block := make([]byte, DEFAULT_BLOCK_SIZE)
		blkCounts := d.Size / delta.BlockSize
		for i := int64(0); i < blkCounts; i++ {
			offset := d.Offset + i*delta.BlockSize
			log.Debugf("Backup for %v: segment %v/%v, blocks %v/%v", snapshot.Name, m+1, mCounts, i+1, blkCounts)
			err := deltaOps.ReadSnapshot(snapshot.Name, volume.Name, offset, block)
			if err != nil {
				return "", err
			}
			checksum := util.GetChecksum(block)
			blkFile := getBlockFilePath(volume.Name, checksum)
			if bsDriver.FileSize(blkFile) >= 0 {
				blockMapping := BlockMapping{
					Offset:        offset,
					BlockChecksum: checksum,
				}
				deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
				log.Debugf("Found existed block match at %v", blkFile)
				continue
			}

			rs, err := util.CompressData(block)
			if err != nil {
				return "", err
			}

			if err := bsDriver.Write(blkFile, rs); err != nil {
				return "", err
			}
			log.Debugf("Created new block file at %v", blkFile)

			blockMapping := BlockMapping{
				Offset:        offset,
				BlockChecksum: checksum,
			}
			deltaBackup.Blocks = append(deltaBackup.Blocks, blockMapping)
		}
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:   LOG_REASON_COMPLETE,
		LOG_FIELD_EVENT:    LOG_EVENT_BACKUP,
		LOG_FIELD_OBJECT:   LOG_OBJECT_SNAPSHOT,
		LOG_FIELD_SNAPSHOT: snapshot.Name,
	}).Debug("Created snapshot changed blocks")

	backup := mergeSnapshotMap(deltaBackup, lastBackup)
	backup.SnapshotName = snapshot.Name
	backup.SnapshotCreatedAt = snapshot.CreatedTime
	backup.CreatedTime = util.Now()

	if err := saveBackup(backup, bsDriver); err != nil {
		return "", err
	}

	volume.LastBackupName = backup.Name
	if err := saveVolume(volume, bsDriver); err != nil {
		return "", err
	}

	return encodeBackupURL(backup.Name, volume.Name, destURL), nil
}

func mergeSnapshotMap(deltaBackup, lastBackup *Backup) *Backup {
	if lastBackup == nil {
		return deltaBackup
	}
	backup := &Backup{
		Name:         deltaBackup.Name,
		VolumeName:   deltaBackup.VolumeName,
		SnapshotName: deltaBackup.SnapshotName,
		Blocks:       []BlockMapping{},
	}
	var d, l int
	for d, l = 0, 0; d < len(deltaBackup.Blocks) && l < len(lastBackup.Blocks); {
		dB := deltaBackup.Blocks[d]
		lB := lastBackup.Blocks[l]
		if dB.Offset == lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
			l++
		} else if dB.Offset < lB.Offset {
			backup.Blocks = append(backup.Blocks, dB)
			d++
		} else {
			//dB.Offset > lB.offset
			backup.Blocks = append(backup.Blocks, lB)
			l++
		}
	}

	if d == len(deltaBackup.Blocks) {
		backup.Blocks = append(backup.Blocks, lastBackup.Blocks[l:]...)
	} else {
		backup.Blocks = append(backup.Blocks, deltaBackup.Blocks[d:]...)
	}

	return backup
}

func RestoreDeltaBlockBackup(backupURL, volDevName string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	srcBackupName, srcVolumeName, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	if _, err := loadVolume(srcVolumeName, bsDriver); err != nil {
		return generateError(logrus.Fields{
			LOG_FIELD_VOLUME:     srcVolumeName,
			LOG_FIELD_BACKUP_URL: backupURL,
		}, "Volume doesn't exist in objectstore: %v", err)
	}

	volDev, err := os.Create(volDevName)
	if err != nil {
		return err
	}
	defer volDev.Close()

	backup, err := loadBackup(srcBackupName, srcVolumeName, bsDriver)
	if err != nil {
		return err
	}

	log.WithFields(logrus.Fields{
		LOG_FIELD_REASON:      LOG_REASON_START,
		LOG_FIELD_EVENT:       LOG_EVENT_RESTORE,
		LOG_FIELD_OBJECT:      LOG_FIELD_SNAPSHOT,
		LOG_FIELD_SNAPSHOT:    srcBackupName,
		LOG_FIELD_ORIN_VOLUME: srcVolumeName,
		LOG_FIELD_VOLUME_DEV:  volDevName,
		LOG_FIELD_BACKUP_URL:  backupURL,
	}).Debug()
	blkCounts := len(backup.Blocks)
	for i, block := range backup.Blocks {
		log.Debugf("Restore for %v: block %v, %v/%v", volDevName, block.BlockChecksum, i+1, blkCounts)
		blkFile := getBlockFilePath(srcVolumeName, block.BlockChecksum)
		rc, err := bsDriver.Read(blkFile)
		if err != nil {
			return err
		}
		r, err := util.DecompressAndVerify(rc, block.BlockChecksum)
		rc.Close()
		if err != nil {
			return err
		}
		if _, err := volDev.Seek(block.Offset, 0); err != nil {
			return err
		}
		if _, err := io.CopyN(volDev, r, DEFAULT_BLOCK_SIZE); err != nil {
			return err
		}
	}

	return nil
}

func DeleteDeltaBlockBackup(backupURL string) error {
	bsDriver, err := GetObjectStoreDriver(backupURL)
	if err != nil {
		return err
	}

	backupName, volumeName, err := decodeBackupURL(backupURL)
	if err != nil {
		return err
	}

	v, err := loadVolume(volumeName, bsDriver)
	if err != nil {
		return fmt.Errorf("Cannot find volume %v in objectstore", volumeName, err)
	}

	backup, err := loadBackup(backupName, volumeName, bsDriver)
	if err != nil {
		return err
	}
	discardBlockSet := make(map[string]bool)
	for _, blk := range backup.Blocks {
		discardBlockSet[blk.BlockChecksum] = true
	}
	discardBlockCounts := len(discardBlockSet)

	if err := removeBackup(backup, bsDriver); err != nil {
		return err
	}

	if backup.Name == v.LastBackupName {
		v.LastBackupName = ""
		if err := saveVolume(v, bsDriver); err != nil {
			return err
		}
	}

	backupNames, err := getBackupNamesForVolume(volumeName, bsDriver)
	if err != nil {
		return err
	}
	if len(backupNames) == 0 {
		log.Debugf("No snapshot existed for the volume %v, removing volume", volumeName)
		if err := removeVolume(volumeName, bsDriver); err != nil {
			log.Warningf("Failed to remove volume %v due to: %v", volumeName, err.Error())
		}
		return nil
	}

	log.Debug("GC started")
	for _, backupName := range backupNames {
		backup, err := loadBackup(backupName, volumeName, bsDriver)
		if err != nil {
			return err
		}
		for _, blk := range backup.Blocks {
			if _, exists := discardBlockSet[blk.BlockChecksum]; exists {
				delete(discardBlockSet, blk.BlockChecksum)
				discardBlockCounts--
				if discardBlockCounts == 0 {
					break
				}
			}
		}
		if discardBlockCounts == 0 {
			break
		}
	}

	var blkFileList []string
	for blk := range discardBlockSet {
		blkFileList = append(blkFileList, getBlockFilePath(volumeName, blk))
		log.Debugf("Found unused blocks %v for volume %v", blk, volumeName)
	}
	if err := bsDriver.Remove(blkFileList...); err != nil {
		return err
	}
	log.Debug("Removed unused blocks for volume ", volumeName)

	log.Debug("GC completed")
	log.Debug("Removed objectstore backup ", backupName)

	return nil
}

func getBlockPath(volumeName string) string {
	return filepath.Join(getVolumePath(volumeName), BLOCKS_DIRECTORY) + "/"
}

func getBlockFilePath(volumeName, checksum string) string {
	blockSubDirLayer1 := checksum[0:BLOCK_SEPARATE_LAYER1]
	blockSubDirLayer2 := checksum[BLOCK_SEPARATE_LAYER1:BLOCK_SEPARATE_LAYER2]
	path := filepath.Join(getBlockPath(volumeName), blockSubDirLayer1, blockSubDirLayer2)
	fileName := checksum + ".blk"

	return filepath.Join(path, fileName)
}
<|endoftext|>


<commit_before>package task

import "code.google.com/p/go-uuid/uuid"

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued  = "queued"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed  = "failed"
)

type Task struct {
	// Command to be executed by the minion
	Command string `json:"command"`

	// Command arguments
	Args []string `json:"args"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique identifier
	TaskID uuid.UUID `json:"taskId"`

	// Result of task after processing
	Result string `json:"result"`

	// If true this task can run concurrently with other tasks
	IsConcurrent bool `json:"isConcurrent"`

	// Task error, if any
	Error string `json:"error"`

	// Task state
	State string `json:"state"`
}

func New(command string, args ...string) *Task {
	t := &Task{
		Command: command,
		Args:    args,
		TaskID:  uuid.NewRandom(),
		State:   TaskStateUnknown,
	}

	return t
}
<commit_msg>Add task state for tasks that are being processed<commit_after>package task

import "code.google.com/p/go-uuid/uuid"

// Task states
const (
	// Unknown state of the task
	// This is the default state of a task
	// when new task is initially created
	TaskStateUnknown    = "unknown"

	// Task has been received by the
	// minion and is queued for execution
	TaskStateQueued     = "queued"

	// Task is being processed
	TaskStateProcessing = "processing"

	// Task has been processed by the
	// minion and was flagged as successful
	TaskStateSuccess    = "success"

	// Task has been processed by the
	// minion and was flagged as failed
	TaskStateFailed     = "failed"
)

type Task struct {
	// Command to be executed by the minion
	Command string `json:"command"`

	// Command arguments
	Args []string `json:"args"`

	// Time when the command was sent for processing
	TimeReceived int64 `json:"timeReceived"`

	// Time when the command was processed
	TimeProcessed int64 `json:"timeProcessed"`

	// Task unique identifier
	TaskID uuid.UUID `json:"taskId"`

	// Result of task after processing
	Result string `json:"result"`

	// If true this task can run concurrently with other tasks
	IsConcurrent bool `json:"isConcurrent"`

	// Task error, if any
	Error string `json:"error"`

	// Task state
	State string `json:"state"`
}

func New(command string, args ...string) *Task {
	t := &Task{
		Command: command,
		Args:    args,
		TaskID:  uuid.NewRandom(),
		State:   TaskStateUnknown,
	}

	return t
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

var (
	_method         = flag.String("m", "GET", "HTTP Metod")
	_url            = flag.String("u", "http://localhost:3001/", "URL")
	_connection     = flag.Int("c", 100, "Connections count")
	_threads        = flag.Int("t", 12, "Threads count")
	_mrq            = flag.Int("mrq", -1, "Max request per second")
	_source         = flag.String("s", "", "POST/PUT Body source file with \"\\n\" delimeter or URLs on GET/DELETE")
	_duration       = flag.Duration("d", time.Duration(30)*time.Second, "Test duration")
	_reconnect      = flag.Bool("reconnect", false, "Reconnect for every request")
	_verbose        = flag.Bool("v", false, "Live stats view")
	_excludeSeconds = flag.Duration("es", time.Duration(0)*time.Second, "Exclude first seconds from stats")
	_help           = flag.Bool("h", false, "Help")
)

type RequestStats struct {
	ResponseCode int
	Duration     time.Duration
	ReadError    error
	WriteError   error
	NetIn        int64
	NetOut       int64
}

type Config struct {
	Method            string
	Url               *url.URL
	Connections       int
	Threads           int
	MRQ               int
	Reconnect         bool
	Verbose           bool
	ExcludeSeconds    time.Duration
	Source            *Source
	Duration          time.Duration
	ConnectionManager *ConnectionManager
	WorkerQuit        chan bool
	WorkerQuited      chan bool
	StatsQuit         chan bool
	RequestStats      chan *RequestStats
}

func main() {
	flag.Parse()

	if *_help {
		flag.Usage()
		return
	}

	var (
		sourceData *Source
		err        error
	)

	*_method = strings.ToUpper(*_method)

	if *_method == "POST" || *_method == "PUT" || (len(*_source) > 0 && FileExists(*_source)) {
		sourceData, err = LoadSource(*_source, "\n")
		if err != nil {
			fmt.Printf("ERROR: Can not load source file %s\n", *_source)
			return
		}
	} else {
		sourceData = &Source{}
	}

	URL, err := url.Parse(*_url)
	if err != nil {
		fmt.Printf("ERROR: URL is broken %s\n", *_url)
		return
	}

	config := &Config{
		Method:         *_method,
		Url:            URL,
		Connections:    *_connection,
		Threads:        *_threads,
		MRQ:            *_mrq,
		Reconnect:      *_reconnect,
		Verbose:        *_verbose,
		ExcludeSeconds: *_excludeSeconds,
		Source:         sourceData,
		Duration:       *_duration,
		WorkerQuit:     make(chan bool, *_threads),
		WorkerQuited:   make(chan bool, *_threads),
		StatsQuit:      make(chan bool, 2),
		RequestStats:   make(chan *RequestStats),
	}

	logUrl := config.Url.String()
	if *_method != "POST" && *_method == "PUT" {
		logUrl = config.Url.Host
	}

	if config.MRQ == -1 {
		fmt.Printf("Running test threads: %d, connections: %d in %v %s %s", config.Threads, config.Connections, config.Duration, config.Method, logUrl)
	} else {
		fmt.Printf("Running test threads: %d, connections: %d, max req/sec: %d, in %v %s %s", config.Threads, config.Connections, config.MRQ, config.Duration, config.Method, logUrl)
	}
	if config.Reconnect {
		fmt.Printf(" with reconnect")
	}
	fmt.Print("\n")

	config.ConnectionManager = NewConnectionManager(config)

	go StartStatsAggregator(config)

	for i := 0; i < config.Threads; i++ {
		go NewThread(config)
	}

	//Start Ctr+C listen
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM)

	//Wait timers or SIGTERM
	endTime := time.After(config.Duration)
	select {
	case <-endTime:
		for i := 0; i < config.Threads; i++ {
			config.WorkerQuit <- true
		}
	case <-signalChan:
		for i := 0; i < config.Threads; i++ {
			config.WorkerQuit <- true
		}
	}
	//Wait for threads complete
	for i := 0; i < config.Threads; i++ {
		<-config.WorkerQuited
	}

	//Stop stats aggregator
	config.StatsQuit <- true
	//Close connections
	for i := 0; i < config.Connections; i++ {
		connection := config.ConnectionManager.Get()
		if !connection.IsConnected() {
			continue
		}
		connection.conn.Close()
	}
	//Wait stats aggregator complete
	<-config.StatsQuit
	//Print result
	PrintStats(os.Stdout, config)
}

func FileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<commit_msg>perfect code<commit_after>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"
)

var (
	_method         = flag.String("m", "GET", "HTTP Metod")
	_url            = flag.String("u", "http://localhost:3001/", "URL")
	_connection     = flag.Int("c", 100, "Connections count")
	_threads        = flag.Int("t", 12, "Threads count")
	_mrq            = flag.Int("mrq", -1, "Max request per second")
	_source         = flag.String("s", "", "POST/PUT Body source file with \"\\n\" delimeter or URLs on GET/DELETE")
	_duration       = flag.Duration("d", time.Duration(30)*time.Second, "Test duration")
	_reconnect      = flag.Bool("reconnect", false, "Reconnect for every request")
	_verbose        = flag.Bool("v", false, "Live stats view")
	_excludeSeconds = flag.Duration("es", time.Duration(0)*time.Second, "Exclude first seconds from stats")
	_help           = flag.Bool("h", false, "Help")
)

type RequestStats struct {
	ResponseCode int
	Duration     time.Duration
	ReadError    error
	WriteError   error
	NetIn        int64
	NetOut       int64
}

type Config struct {
	Method            string
	Url               *url.URL
	Connections       int
	Threads           int
	MRQ               int
	Reconnect         bool
	Verbose           bool
	ExcludeSeconds    time.Duration
	Source            *Source
	Duration          time.Duration
	ConnectionManager *ConnectionManager
	WorkerQuit        chan bool
	WorkerQuited      chan bool
	StatsQuit         chan bool
	RequestStats      chan *RequestStats
}

func main() {
	flag.Parse()

	if *_help {
		flag.Usage()
		return
	}

	var (
		sourceData *Source
		err        error
	)

	*_method = strings.ToUpper(*_method)

	if *_method == "POST" || *_method == "PUT" || (len(*_source) > 0 && FileExists(*_source)) {
		sourceData, err = LoadSource(*_source, "\n")
		if err != nil {
			fmt.Printf("ERROR: Can not load source file %s\n", *_source)
			return
		}
	} else {
		sourceData = &Source{}
	}

	URL, err := url.Parse(*_url)
	if err != nil {
		fmt.Printf("ERROR: URL is broken %s\n", *_url)
		return
	}

	config := &Config{
		Method:         *_method,
		Url:            URL,
		Connections:    *_connection,
		Threads:        *_threads,
		MRQ:            *_mrq,
		Reconnect:      *_reconnect,
		Verbose:        *_verbose,
		ExcludeSeconds: *_excludeSeconds,
		Source:         sourceData,
		Duration:       *_duration,
		WorkerQuit:     make(chan bool, *_threads),
		WorkerQuited:   make(chan bool, *_threads),
		StatsQuit:      make(chan bool, 2),
		RequestStats:   make(chan *RequestStats),
	}

	logUrl := config.Url.String()
	if *_method != "POST" && *_method == "PUT" {
		logUrl = config.Url.Host
	}

	if config.MRQ == -1 {
		fmt.Printf("Running test threads: %d, connections: %d in %v %s %s", config.Threads, config.Connections, config.Duration, config.Method, logUrl)
	} else {
		fmt.Printf("Running test threads: %d, connections: %d, max req/sec: %d, in %v %s %s", config.Threads, config.Connections, config.MRQ, config.Duration, config.Method, logUrl)
	}
	if config.Reconnect {
		fmt.Printf(" with reconnect")
	}
	fmt.Print("\n")

	config.ConnectionManager = NewConnectionManager(config)

	go StartStatsAggregator(config)

	for i := 0; i < config.Threads; i++ {
		go NewThread(config)
	}

	//Start SIGTERM listen
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM)

	//Wait timers or SIGTERM
	select {
	case <-time.After(config.Duration):
	case <-signalChan:
	}
	for i := 0; i < config.Threads; i++ {
		config.WorkerQuit <- true
	}
	//Wait for threads complete
	for i := 0; i < config.Threads; i++ {
		<-config.WorkerQuited
	}

	//Stop stats aggregator
	config.StatsQuit <- true
	//Close connections
	for i := 0; i < config.Connections; i++ {
		connection := config.ConnectionManager.Get()
		if !connection.IsConnected() {
			continue
		}
		connection.conn.Close()
	}
	//Wait stats aggregator complete
	<-config.StatsQuit
	//Print result
	PrintStats(os.Stdout, config)
}

func FileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"fmt"
	"math"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const uid = 123
const gid = 456

const fileInodeID = 17
const fileInodeName = "foo/bar"
const fileMode os.FileMode = 0641

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	leaser lease.FileLeaser
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.leaser = lease.NewFileLeaser("", math.MaxInt32, math.MaxInt64)
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		[]byte(t.initialContents))

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		fuseops.InodeAttributes{
			Uid:  uid,
			Gid:  gid,
			Mode: fileMode,
		},
		math.MaxUint64, // GCS chunk size
		t.bucket,
		t.leaser,
		gcsproxy.NewObjectSyncer(
			1, // Append threshold
			".gcsfuse_tmp/",
			t.bucket),
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(uid, attrs.Uid)
	ExpectEq(gid, attrs.Gid)
	ExpectEq(fileMode, attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	AssertEq("taco", t.initialContents)

	// Make several reads, checking the expected contents. We should never get an
	// EOF error, since fuseops.ReadFileOp is not supposed to see those.
	testCases := []struct {
		offset   int64
		size     int
		expected string
	}{
		{0, 1, "t"},
		{0, 2, "ta"},
		{0, 3, "tac"},
		{0, 4, "taco"},
		{0, 5, "taco"},

		{1, 1, "a"},
		{1, 2, "ac"},
		{1, 3, "aco"},
		{1, 4, "aco"},

		{3, 1, "o"},
		{3, 2, "o"},

		// Empty ranges
		{0, 0, ""},
		{3, 0, ""},
		{4, 0, ""},
		{4, 1, ""},
		{5, 0, ""},
		{5, 1, ""},
		{5, 2, ""},
	}

	for _, tc := range testCases {
		desc := fmt.Sprintf("offset: %d, size: %d", tc.offset, tc.size)

		data := make([]byte, tc.size)
		n, err := t.in.Read(t.ctx, data, tc.offset)
		data = data[:n]

		AssertEq(nil, err, "%s", desc)
		ExpectEq(tc.expected, string(data), "%s", desc)
	}
}

func (t *FileTest) Write() {
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	// Add some data at the end.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()

	err = t.in.Write(t.ctx, []byte("burrito"), 4)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read back the content.
	var buf [1024]byte
	n, err := t.in.Read(t.ctx, buf[:], 0)
	AssertEq(nil, err)
	ExpectEq("pacoburrito", string(buf[:n]))

	// Check attributes.
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("pacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime))
}

func (t *FileTest) Truncate() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read the contents.
	var buf [1024]byte
	n, err := t.in.Read(t.ctx, buf[:], 0)
	AssertEq(nil, err)
	ExpectEq("ta", string(buf[:n]))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("ta"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) WriteThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(len("paco"), o.Size)

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("paco", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("paco"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) AppendThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Append some data.
	err = t.in.Write(t.ctx, []byte("burrito"), int64(len("taco")))
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(len("tacoburrito"), o.Size)

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("tacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) TruncateDownwardThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(2, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(2, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) TruncateUpwardThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq(4, len(t.initialContents))

	// Truncate upward.
	err = t.in.Truncate(t.ctx, 6)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(6, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(6, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) Sync_Clobbered() {
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Clobber the backing object.
	newObj, err := gcsutil.CreateObject(t.ctx, t.bucket, t.in.Name(), []byte("burrito"))
	AssertEq(nil, err)

	// Sync. The call should succeed, but nothing should change.
	err = t.in.Sync(t.ctx)

	AssertEq(nil, err)
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())

	// The object in the bucket should not have been changed.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(newObj.Generation, o.Generation)
	ExpectEq(newObj.Size, o.Size)
}
<commit_msg>Fixed test errors.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode_test

import (
	"fmt"
	"os"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/googlecloudplatform/gcsfuse/internal/gcsx"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
	"github.com/jacobsa/timeutil"
)

func TestFile(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

const uid = 123
const gid = 456

const fileInodeID = 17
const fileInodeName = "foo/bar"
const fileMode os.FileMode = 0641

type FileTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	clock  timeutil.SimulatedClock

	initialContents string
	backingObj      *gcs.Object

	in *inode.FileInode
}

var _ SetUpInterface = &FileTest{}
var _ TearDownInterface = &FileTest{}

func init() { RegisterTestSuite(&FileTest{}) }

func (t *FileTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.bucket = gcsfake.NewFakeBucket(&t.clock, "some_bucket")

	// Set up the backing object.
	var err error

	t.initialContents = "taco"
	t.backingObj, err = gcsutil.CreateObject(
		t.ctx,
		t.bucket,
		fileInodeName,
		[]byte(t.initialContents))

	AssertEq(nil, err)

	// Create the inode.
	t.in = inode.NewFileInode(
		fileInodeID,
		t.backingObj,
		fuseops.InodeAttributes{
			Uid:  uid,
			Gid:  gid,
			Mode: fileMode,
		},
		t.bucket,
		gcsx.NewSyncer(
			1, // Append threshold
			".gcsfuse_tmp/",
			t.bucket),
		&t.clock)

	t.in.Lock()
}

func (t *FileTest) TearDown() {
	t.in.Unlock()
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileTest) ID() {
	ExpectEq(fileInodeID, t.in.ID())
}

func (t *FileTest) Name() {
	ExpectEq(fileInodeName, t.in.Name())
}

func (t *FileTest) InitialSourceGeneration() {
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())
}

func (t *FileTest) InitialAttributes() {
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len(t.initialContents), attrs.Size)
	ExpectEq(1, attrs.Nlink)
	ExpectEq(uid, attrs.Uid)
	ExpectEq(gid, attrs.Gid)
	ExpectEq(fileMode, attrs.Mode)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(t.backingObj.Updated))
}

func (t *FileTest) Read() {
	AssertEq("taco", t.initialContents)

	// Make several reads, checking the expected contents. We should never get an
	// EOF error, since fuseops.ReadFileOp is not supposed to see those.
	testCases := []struct {
		offset   int64
		size     int
		expected string
	}{
		{0, 1, "t"},
		{0, 2, "ta"},
		{0, 3, "tac"},
		{0, 4, "taco"},
		{0, 5, "taco"},

		{1, 1, "a"},
		{1, 2, "ac"},
		{1, 3, "aco"},
		{1, 4, "aco"},

		{3, 1, "o"},
		{3, 2, "o"},

		// Empty ranges
		{0, 0, ""},
		{3, 0, ""},
		{4, 0, ""},
		{4, 1, ""},
		{5, 0, ""},
		{5, 1, ""},
		{5, 2, ""},
	}

	for _, tc := range testCases {
		desc := fmt.Sprintf("offset: %d, size: %d", tc.offset, tc.size)

		data := make([]byte, tc.size)
		n, err := t.in.Read(t.ctx, data, tc.offset)
		data = data[:n]

		AssertEq(nil, err, "%s", desc)
		ExpectEq(tc.expected, string(data), "%s", desc)
	}
}

func (t *FileTest) Write() {
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	// Add some data at the end.
	t.clock.AdvanceTime(time.Second)
	writeTime := t.clock.Now()

	err = t.in.Write(t.ctx, []byte("burrito"), 4)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read back the content.
	var buf [1024]byte
	n, err := t.in.Read(t.ctx, buf[:], 0)
	AssertEq(nil, err)
	ExpectEq("pacoburrito", string(buf[:n]))

	// Check attributes.
	attrs, err := t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("pacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(writeTime))
}

func (t *FileTest) Truncate() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Truncate downward.
	t.clock.AdvanceTime(time.Second)
	truncateTime := t.clock.Now()

	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Read the contents.
	var buf [1024]byte
	n, err := t.in.Read(t.ctx, buf[:], 0)
	AssertEq(nil, err)
	ExpectEq("ta", string(buf[:n]))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("ta"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(truncateTime))
}

func (t *FileTest) WriteThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Overwite a byte.
	err = t.in.Write(t.ctx, []byte("p"), 0)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(len("paco"), o.Size)

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("paco", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("paco"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) AppendThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq("taco", t.initialContents)

	// Append some data.
	err = t.in.Write(t.ctx, []byte("burrito"), int64(len("taco")))
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(len("tacoburrito"), o.Size)

	// Read the object's contents.
	contents, err := gcsutil.ReadObject(t.ctx, t.bucket, t.in.Name())

	AssertEq(nil, err)
	ExpectEq("tacoburrito", string(contents))

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(len("tacoburrito"), attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) TruncateDownwardThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(2, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(2, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) TruncateUpwardThenSync() {
	var attrs fuseops.InodeAttributes
	var err error

	AssertEq(4, len(t.initialContents))

	// Truncate upward.
	err = t.in.Truncate(t.ctx, 6)
	AssertEq(nil, err)

	t.clock.AdvanceTime(time.Second)

	// Sync.
	err = t.in.Sync(t.ctx)
	AssertEq(nil, err)

	// The generation should have advanced.
	ExpectLt(t.backingObj.Generation, t.in.SourceGeneration())

	// Stat the current object in the bucket.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(t.in.SourceGeneration(), o.Generation)
	ExpectEq(6, o.Size)

	// Check attributes.
	attrs, err = t.in.Attributes(t.ctx)
	AssertEq(nil, err)

	ExpectEq(6, attrs.Size)
	ExpectThat(attrs.Mtime, timeutil.TimeEq(o.Updated))
}

func (t *FileTest) Sync_Clobbered() {
	var err error

	// Truncate downward.
	err = t.in.Truncate(t.ctx, 2)
	AssertEq(nil, err)

	// Clobber the backing object.
	newObj, err := gcsutil.CreateObject(t.ctx, t.bucket, t.in.Name(), []byte("burrito"))
	AssertEq(nil, err)

	// Sync. The call should succeed, but nothing should change.
	err = t.in.Sync(t.ctx)

	AssertEq(nil, err)
	ExpectEq(t.backingObj.Generation, t.in.SourceGeneration())

	// The object in the bucket should not have been changed.
	statReq := &gcs.StatObjectRequest{Name: t.in.Name()}
	o, err := t.bucket.StatObject(t.ctx, statReq)

	AssertEq(nil, err)
	ExpectEq(newObj.Generation, o.Generation)
	ExpectEq(newObj.Size, o.Size)
}
<|endoftext|>