<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	. "github.com/hajimehoshi/ebiten"
	"image"
	"image/color"
	_ "image/png"
	"math"
	"testing"
)

var ebitenImageBin = ""

func openImage(path string) (*Image, image.Image, error) {
	file, err := readFile(path)
	if err != nil {
		return nil, nil, err
	}

	img, _, err := image.Decode(file)
	if err != nil {
		return nil, nil, err
	}

	eimg, err := NewImageFromImage(img, FilterNearest)
	if err != nil {
		return nil, nil, err
	}
	return eimg, img, nil
}

func diff(x, y uint8) uint8 {
	if x <= y {
		return y - x
	}
	return x - y
}

func TestImagePixels(t *testing.T) {
	eimg, img, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	if got := eimg.Bounds().Size(); got != img.Bounds().Size() {
		t.Errorf("img size: got %d; want %d", got, img.Bounds().Size())
	}

	for j := 0; j < eimg.Bounds().Size().Y; j++ {
		for i := 0; i < eimg.Bounds().Size().X; i++ {
			got := eimg.At(i, j)
			want := color.RGBAModel.Convert(img.At(i, j))
			if got != want {
				t.Errorf("img At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

func TestImageComposition(t *testing.T) {
	img2Color := color.NRGBA{0x24, 0x3f, 0x6a, 0x88}
	img3Color := color.NRGBA{0x85, 0xa3, 0x08, 0xd3}

	img1, _, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	w, h := img1.Bounds().Size().X, img1.Bounds().Size().Y

	img2, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_12_3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img2.DrawImage(img1, nil)
	img3.DrawImage(img2, nil)
	img_12_3.DrawImage(img3, nil)

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_1_23, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img3.DrawImage(img2, nil)
	img3.DrawImage(img1, nil)
	img_1_23.DrawImage(img3, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c1 := img_12_3.At(i, j).(color.RGBA)
			c2 := img_1_23.At(i, j).(color.RGBA)
			if 1 < diff(c1.R, c2.R) || 1 < diff(c1.G, c2.G) || 1 < diff(c1.B, c2.B) || 1 < diff(c1.A, c2.A) {
				t.Errorf("img_12_3.At(%d, %d) = %#v; img_1_23.At(%[1]d, %[2]d) = %#[4]v", i, j, c1, c2)
			}
			if c1.A == 0 {
				t.Fatalf("img_12_3.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
			if c2.A == 0 {
				t.Fatalf("img_1_23.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
		}
	}
}

func TestImageSelf(t *testing.T) {
	img, _, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.DrawImage(img, nil); err == nil {
		t.Fatalf("img.DrawImage(img, nil) doesn't return error; an error should be returned")
	}
}

func TestImageDotByDotInversion(t *testing.T) {
	img0, _, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	w, h := img0.Size()
	img1, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	op := &DrawImageOptions{}
	op.GeoM.Rotate(2 * math.Pi / 2)
	op.GeoM.Translate(float64(w), float64(h))
	img1.DrawImage(img0, op)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c0 := img0.At(i, j).(color.RGBA)
			c1 := img1.At(w-i-1, h-j-1).(color.RGBA)
			if c0 != c1 {
				t.Errorf("img0.At(%[1]d, %[2]d) should equal to img1.At(%[1]d, %[2]d) but not: %[3]v vs %[4]v", i, j, c0, c1)
			}
		}
	}
}

// TODO: Add more tests (e.g. DrawImage with color matrix)
<commit_msg>Add TestImageReplacePixels<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten_test

import (
	. "github.com/hajimehoshi/ebiten"
	"image"
	"image/color"
	"image/draw"
	_ "image/png"
	"math"
	"testing"
)

var ebitenImageBin = ""

func openImage(path string) (image.Image, error) {
	file, err := readFile(path)
	if err != nil {
		return nil, err
	}

	img, _, err := image.Decode(file)
	if err != nil {
		return nil, err
	}
	return img, nil
}

func openEbitenImage(path string) (*Image, image.Image, error) {
	img, err := openImage(path)
	if err != nil {
		return nil, nil, err
	}

	eimg, err := NewImageFromImage(img, FilterNearest)
	if err != nil {
		return nil, nil, err
	}
	return eimg, img, nil
}

func diff(x, y uint8) uint8 {
	if x <= y {
		return y - x
	}
	return x - y
}

func TestImagePixels(t *testing.T) {
	img0, img, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	if got := img0.Bounds().Size(); got != img.Bounds().Size() {
		t.Errorf("img size: got %d; want %d", got, img.Bounds().Size())
	}

	for j := 0; j < img0.Bounds().Size().Y; j++ {
		for i := 0; i < img0.Bounds().Size().X; i++ {
			got := img0.At(i, j)
			want := color.RGBAModel.Convert(img.At(i, j))
			if got != want {
				t.Errorf("img0 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

func TestImageComposition(t *testing.T) {
	img2Color := color.NRGBA{0x24, 0x3f, 0x6a, 0x88}
	img3Color := color.NRGBA{0x85, 0xa3, 0x08, 0xd3}

	img1, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}

	w, h := img1.Bounds().Size().X, img1.Bounds().Size().Y

	img2, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_12_3, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img2.DrawImage(img1, nil)
	img3.DrawImage(img2, nil)
	img_12_3.DrawImage(img3, nil)

	img2.Fill(img2Color)
	img3.Fill(img3Color)
	img_1_23, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	img3.DrawImage(img2, nil)
	img3.DrawImage(img1, nil)
	img_1_23.DrawImage(img3, nil)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c1 := img_12_3.At(i, j).(color.RGBA)
			c2 := img_1_23.At(i, j).(color.RGBA)
			if 1 < diff(c1.R, c2.R) || 1 < diff(c1.G, c2.G) || 1 < diff(c1.B, c2.B) || 1 < diff(c1.A, c2.A) {
				t.Errorf("img_12_3.At(%d, %d) = %#v; img_1_23.At(%[1]d, %[2]d) = %#[4]v", i, j, c1, c2)
			}
			if c1.A == 0 {
				t.Fatalf("img_12_3.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
			if c2.A == 0 {
				t.Fatalf("img_1_23.At(%d, %d).A = 0; nothing is rendered?", i, j)
			}
		}
	}
}

func TestImageSelf(t *testing.T) {
	img, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	if err := img.DrawImage(img, nil); err == nil {
		t.Fatalf("img.DrawImage(img, nil) doesn't return error; an error should be returned")
	}
}

func TestImageDotByDotInversion(t *testing.T) {
	img0, _, err := openEbitenImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	w, h := img0.Size()
	img1, err := NewImage(w, h, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}
	op := &DrawImageOptions{}
	op.GeoM.Rotate(2 * math.Pi / 2)
	op.GeoM.Translate(float64(w), float64(h))
	img1.DrawImage(img0, op)

	for j := 0; j < h; j++ {
		for i := 0; i < w; i++ {
			c0 := img0.At(i, j).(color.RGBA)
			c1 := img1.At(w-i-1, h-j-1).(color.RGBA)
			if c0 != c1 {
				t.Errorf("img0.At(%[1]d, %[2]d) should equal to img1.At(%[1]d, %[2]d) but not: %[3]v vs %[4]v", i, j, c0, c1)
			}
		}
	}
}

func TestReplacePixels(t *testing.T) {
	origImg, err := openImage("testdata/ebiten.png")
	if err != nil {
		t.Fatal(err)
		return
	}
	// Convert to RGBA
	img := image.NewRGBA(origImg.Bounds())
	draw.Draw(img, img.Bounds(), origImg, image.ZP, draw.Src)

	size := img.Bounds().Size()
	img0, err := NewImage(size.X, size.Y, FilterNearest)
	if err != nil {
		t.Fatal(err)
		return
	}

	img0.ReplacePixels(img.Pix)
	for j := 0; j < img0.Bounds().Size().Y; j++ {
		for i := 0; i < img0.Bounds().Size().X; i++ {
			got := img.At(i, j)
			want := img0.At(i, j)
			if got != want {
				t.Errorf("img0 At(%d, %d): got %#v; want %#v", i, j, got, want)
			}
		}
	}
}

// TODO: Add more tests (e.g. DrawImage with color matrix)
<|endoftext|>


<commit_before>package imageserver

import (
	"bytes"
	"image"
	"image/color"
	"image/png"
	"math/rand"
	"reflect"
	"testing"
)

func BenchmarkImageMarshalGob(b *testing.B) {
	image := &Image{
		Format: "png",
		Data:   make([]byte, 50*1024),
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := image.MarshalBinary()
		if err != nil {
			b.Fatal(err)
		}
	}
}

func BenchmarkImageMarshalBinaryExp(b *testing.B) {
	image := &Image{
		Format: "png",
		Data:   make([]byte, 50*1024),
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := image.MarshalBinary()
		if err != nil {
			b.Fatal(err)
		}
	}
}

func TestImage(t *testing.T) {
	image1 := CreateImage(500, 400)
	data, err := image1.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}

	image2, err := NewImageUnmarshalBinary(data)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(image2, image1) {
		t.Fatal("image not equals")
	}
}

func TestImageUnmarshalError(t *testing.T) {
	_, err := NewImageUnmarshalBinary(nil)
	if err == nil {
		t.Fatal("no error")
	}
}

func CreateImage(width, height int) *Image {
	baseImage := CreateBaseImage(width, height)
	buffer := new(bytes.Buffer)
	png.Encode(buffer, baseImage)
	return &Image{
		Format: "png",
		Data:   buffer.Bytes(),
	}
}

func CreateBaseImage(width, height int) *image.NRGBA {
	i := image.NewNRGBA(image.Rect(0, 0, width, height))
	for y, height := 0, i.Bounds().Dy(); y < height; y++ {
		for x, width := 0, i.Bounds().Dx(); x < width; x++ {
			i.Set(x, y, randColor())
		}
	}
	return i
}

func randColor() color.RGBA {
	return color.RGBA{
		R: randColorComponent(),
		G: randColorComponent(),
		B: randColorComponent(),
		A: randColorComponent(),
	}
}

func randColorComponent() uint8 {
	return uint8(rand.Int31n(256))
}
<commit_msg>remove image marshal benchmark<commit_after>package imageserver

import (
	"bytes"
	"image"
	"image/color"
	"image/png"
	"math/rand"
	"reflect"
	"testing"
)

func TestImage(t *testing.T) {
	image1 := CreateImage(500, 400)
	data, err := image1.MarshalBinary()
	if err != nil {
		t.Fatal(err)
	}

	image2, err := NewImageUnmarshalBinary(data)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(image2, image1) {
		t.Fatal("image not equals")
	}
}

func TestImageUnmarshalError(t *testing.T) {
	_, err := NewImageUnmarshalBinary(nil)
	if err == nil {
		t.Fatal("no error")
	}
}

func CreateImage(width, height int) *Image {
	baseImage := CreateBaseImage(width, height)
	buffer := new(bytes.Buffer)
	png.Encode(buffer, baseImage)
	return &Image{
		Format: "png",
		Data:   buffer.Bytes(),
	}
}

func CreateBaseImage(width, height int) *image.NRGBA {
	i := image.NewNRGBA(image.Rect(0, 0, width, height))
	for y, height := 0, i.Bounds().Dy(); y < height; y++ {
		for x, width := 0, i.Bounds().Dx(); x < width; x++ {
			i.Set(x, y, randColor())
		}
	}
	return i
}

func randColor() color.RGBA {
	return color.RGBA{
		R: randColorComponent(),
		G: randColorComponent(),
		B: randColorComponent(),
		A: randColorComponent(),
	}
}

func randColorComponent() uint8 {
	return uint8(rand.Int31n(256))
}
<|endoftext|>


<commit_before>package next

import (
	"testing"
	"time"
)

var (
	loc = time.Now().Location()
	t1  = time.Date(2015, 7, 1, 15, 4, 30, 0, loc)
)

func TestMinute(t *testing.T) {
	expected := time.Date(2015, 7, 1, 15, 5, 0, 0, loc)
	result := Minute(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}
func TestHour(t *testing.T) {
	expected := time.Date(2015, 7, 1, 16, 0, 0, 0, loc)
	result := Hour(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestDay(t *testing.T) {
	expected := time.Date(2015, 7, 2, 0, 0, 0, 0, loc)
	result := Day(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
	t2 := time.Date(2015, 7, 1, 11, 12, 0, 0, loc)
	expected = time.Date(2015, 7, 2, 0, 0, 0, 0, loc)
	result = Day(t2)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestWeek(t *testing.T) {
	expected := time.Date(2015, 7, 5, 0, 0, 0, 0, loc)
	result := Week(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestMonth(t *testing.T) {
	nextMonth := time.Date(2015, 8, 1, 0, 0, 0, 0, loc)
	result := Month(t1)
	if !result.Equal(nextMonth) {
		t.Errorf("unexpected time - wanted %v got %v\n", nextMonth, result)
	}
}
<commit_msg>improve Month testing to cross year boundaries<commit_after>package next

import (
	"testing"
	"time"
)

var (
	loc = time.Now().Location()
	t1  = time.Date(2015, 7, 1, 15, 4, 30, 0, loc)
)

func TestMinute(t *testing.T) {
	expected := time.Date(2015, 7, 1, 15, 5, 0, 0, loc)
	result := Minute(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}
func TestHour(t *testing.T) {
	expected := time.Date(2015, 7, 1, 16, 0, 0, 0, loc)
	result := Hour(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestDay(t *testing.T) {
	expected := time.Date(2015, 7, 2, 0, 0, 0, 0, loc)
	result := Day(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
	t2 := time.Date(2015, 7, 1, 11, 12, 0, 0, loc)
	expected = time.Date(2015, 7, 2, 0, 0, 0, 0, loc)
	result = Day(t2)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestWeek(t *testing.T) {
	expected := time.Date(2015, 7, 5, 0, 0, 0, 0, loc)
	result := Week(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}

func TestMonth(t *testing.T) {
	expected := time.Date(2015, 8, 1, 0, 0, 0, 0, loc)
	result := Month(t1)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
	starting := time.Date(2015, 12, 15, 0, 0, 0, 0, loc)
	expected = time.Date(2016, 1, 1, 0, 0, 0, 0, loc)
	result = Month(starting)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
	starting = time.Date(2015, 1, 28, 0, 0, 0, 0, loc)
	expected = time.Date(2015, 2, 1, 0, 0, 0, 0, loc)
	result = Month(starting)
	if !result.Equal(expected) {
		t.Errorf("unexpected time - wanted %v got %v\n", expected, result)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package pipoint

import (
	"fmt"
	"reflect"
	"time"
)

// Param is a value or a struct that has age and validity.  Updating a
// Param also fires an event.
type Param struct {
	Name    string
	value   interface{}
	updated time.Time
	params  *Params
	final   bool
}

// Ok return true if the value has been recently updated.
func (p *Param) Ok() bool {
	if p.final {
		return true
	}
	elapsed := time.Now().Sub(p.updated)
	return elapsed.Seconds() < 3
}

// Get returns the current value which may be nil.
func (p *Param) Get() interface{} {
	return p.value
}

// GetFloat64 returns the current value as a float64.
func (p *Param) GetFloat64() float64 {
	return p.value.(float64)
}

// GetInt returns the current value as an int.
func (p *Param) GetInt() int {
	return int(p.GetFloat64())
}

func isNumber(value interface{}) bool {
	if value == nil {
		return false
	}

	switch value.(type) {
	case float64, int:
		return true
	default:
		return false
	}
}

// Set the value, update validity, and notify listeners.
func (p *Param) Set(value interface{}) error {
	if p.value == nil {
		// OK, nothing set yet.
	} else if isNumber(p.value) && isNumber(value) {
		// Number -> number is fine.
	} else if reflect.TypeOf(value) != reflect.TypeOf(p.value) {
		return fmt.Errorf("Type of %v changed from %v to %v",
			p.Name, p.value, value)
	}

	if isNumber(value) {
		// Store all numbers as float64.
		switch value.(type) {
		case float64:
			// No change
		case int:
			value = float64(value.(int))
		default:
			panic("Unsupported numeric type.")
		}
	}

	p.value = value
	p.updated = time.Now()
	p.final = false
	p.params.updated(p)
	return nil
}

// SetFloat64 tries to update the value as a float64.
func (p *Param) SetFloat64(value float64) error {
	return p.Set(value)
}

// SetInt tries to update the value as an int.
func (p *Param) SetInt(value int) error {
	return p.Set(value)
}

// Inc tries to increment the integer value.
func (p *Param) Inc() error {
	return p.SetInt(p.GetInt() + 1)
}

// Finalise marks the param as always valid.
func (p *Param) Finalise() {
	p.final = true
}

// ValueVisitor is a callback for leaves in the parameter tree.
type ValueVisitor func(p *Param, path []string, value interface{})

func (p *Param) walk(visitor ValueVisitor, path []string, v reflect.Value) {
	switch v.Kind() {
	case reflect.Ptr, reflect.Interface:
		p.walk(visitor, path, v.Elem())
	case reflect.Struct:
		t := v.Type()
		for i := 0; i < v.NumField(); i++ {
			p.walk(visitor, append(path, t.Field(i).Name), v.Field(i))
		}
	default:
		visitor(p, path, v.Interface())
	}
}

// Walk calls visitor on all leaf values of this parameter.
func (p *Param) Walk(visitor ValueVisitor) {
	p.walk(visitor, []string{p.Name}, reflect.ValueOf(p.Get()))
}
<commit_msg>pipoint: add param.Update() which sets if changed.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package pipoint

import (
	"fmt"
	"reflect"
	"time"
)

// Param is a value or a struct that has age and validity.  Updating a
// Param also fires an event.
type Param struct {
	Name    string
	value   interface{}
	updated time.Time
	params  *Params
	final   bool
}

// Ok return true if the value has been recently updated.
func (p *Param) Ok() bool {
	if p.final {
		return true
	}
	elapsed := time.Now().Sub(p.updated)
	return elapsed.Seconds() < 3
}

// Get returns the current value which may be nil.
func (p *Param) Get() interface{} {
	return p.value
}

// GetFloat64 returns the current value as a float64.
func (p *Param) GetFloat64() float64 {
	return p.value.(float64)
}

// GetInt returns the current value as an int.
func (p *Param) GetInt() int {
	return int(p.GetFloat64())
}

func isNumber(value interface{}) bool {
	if value == nil {
		return false
	}

	switch value.(type) {
	case float64, int:
		return true
	default:
		return false
	}
}

// Set the value, update validity, and notify listeners.
func (p *Param) Set(value interface{}) error {
	if p.value == nil {
		// OK, nothing set yet.
	} else if isNumber(p.value) && isNumber(value) {
		// Number -> number is fine.
	} else if reflect.TypeOf(value) != reflect.TypeOf(p.value) {
		return fmt.Errorf("Type of %v changed from %v to %v",
			p.Name, p.value, value)
	}

	if isNumber(value) {
		// Store all numbers as float64.
		switch value.(type) {
		case float64:
			// No change
		case int:
			value = float64(value.(int))
		default:
			panic("Unsupported numeric type.")
		}
	}

	p.value = value
	p.updated = time.Now()
	p.final = false
	p.params.updated(p)
	return nil
}

// SetFloat64 tries to update the value as a float64.
func (p *Param) SetFloat64(value float64) error {
	return p.Set(value)
}

// SetInt tries to update the value as an int.
func (p *Param) SetInt(value int) error {
	return p.Set(value)
}

// SetInt tries to update the value as an int.
func (p *Param) UpdateInt(value int) (bool, error) {
	if p.GetInt() == value {
		return false, nil
	}
	return true, p.Set(value)
}

// Inc tries to increment the integer value.
func (p *Param) Inc() error {
	return p.SetInt(p.GetInt() + 1)
}

// Finalise marks the param as always valid.
func (p *Param) Finalise() {
	p.final = true
}

// ValueVisitor is a callback for leaves in the parameter tree.
type ValueVisitor func(p *Param, path []string, value interface{})

func (p *Param) walk(visitor ValueVisitor, path []string, v reflect.Value) {
	switch v.Kind() {
	case reflect.Ptr, reflect.Interface:
		p.walk(visitor, path, v.Elem())
	case reflect.Struct:
		t := v.Type()
		for i := 0; i < v.NumField(); i++ {
			p.walk(visitor, append(path, t.Field(i).Name), v.Field(i))
		}
	default:
		visitor(p, path, v.Interface())
	}
}

// Walk calls visitor on all leaf values of this parameter.
func (p *Param) Walk(visitor ValueVisitor) {
	p.walk(visitor, []string{p.Name}, reflect.ValueOf(p.Get()))
}
<|endoftext|>


<commit_before>package text

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/errors"
import "github.com/qlova/ilang/syntax/symbols"

func init() {
	Type.EmbeddedStatement = func(c *compiler.Compiler, list compiler.Type) {
		statement(c, true)
	}
}

var Statement = compiler.Statement {
	Detect: func(c *compiler.Compiler) bool {
		return statement(c, false)
	},
}

func statement(c *compiler.Compiler, embed bool) bool {
	if embed || c.GetVariable(c.Token()).Type.Equals(Type) {

			var name = c.Token()

			switch c.Scan() {

				case symbols.Equals:
					if embed {
						//Garbage collect! >:)
						c.Get()
						c.Copy()
						c.If()
							c.Flip()
							c.HeapList()
						c.Or()
							c.Drop()
						c.No()
					}

					var t = c.ScanExpression()
					if !t.Equals(Type) {
						c.RaiseError(errors.AssignmentMismatch(t, Type))
					}

					if embed {
						c.Int(0)
						c.HeapList()
						c.Set()
					} else {

						c.NameList(name)

					}

				case symbols.Plus:
					if embed {
						c.Unimplemented()
					}

					c.Expecting(symbols.Equals)

					c.PushList(name)

					var t = c.ScanExpression()
					if !t.Equals(Type) {
						c.RaiseError(errors.AssignmentMismatch(t, Type))
					}

					c.Call(&Join)

					c.NameList(name)

				default:
					c.Unexpected(c.Token())
			}

			return true
		}
		return false
}

<commit_msg>String numeric incrementation spec.<commit_after>package text

import "github.com/qlova/uct/compiler"
import "github.com/qlova/ilang/syntax/errors"
import "github.com/qlova/ilang/syntax/symbols"

import "github.com/qlova/ilang/types/number"

func init() {
	Type.EmbeddedStatement = func(c *compiler.Compiler, list compiler.Type) {
		statement(c, true)
	}
}

var Statement = compiler.Statement {
	Detect: func(c *compiler.Compiler) bool {
		return statement(c, false)
	},
}

func statement(c *compiler.Compiler, embed bool) bool {
	if embed || c.GetVariable(c.Token()).Type.Equals(Type) {

			var name = c.Token()

			switch c.Scan() {

				case symbols.Equals:
					if embed {
						//Garbage collect! >:)
						c.Get()
						c.Copy()
						c.If()
							c.Flip()
							c.HeapList()
						c.Or()
							c.Drop()
						c.No()
					}

					var t = c.ScanExpression()
					if !t.Equals(Type) {
						c.RaiseError(errors.AssignmentMismatch(t, Type))
					}

					if embed {
						c.Int(0)
						c.HeapList()
						c.Set()
					} else {

						c.NameList(name)

					}

				case symbols.Plus:
					if embed {
						c.Unimplemented()
					}

					c.Expecting(symbols.Equals)

					c.PushList(name)

					var t = c.ScanExpression()

					//string += 1 (increments the numeric value encoded in string)
					if t.Equals(number.Type) {
						c.Int(10)
						c.Call(&Atoi)
						c.Add()
						c.Int(10)
						c.Call(&Itoa)
						c.NameList(name)

						return true
					}

					if !t.Equals(Type) {
						c.RaiseError(errors.AssignmentMismatch(t, Type))
					}

					c.Call(&Join)

					c.NameList(name)

				default:
					c.Unexpected(c.Token())
			}

			return true
		}
		return false
}

<|endoftext|>


<commit_before>package tmongo

import (
	"errors"
	"fmt"
	"github.com/tolexo/aero/conf"
	"gopkg.in/mgo.v2"
	"time"
)

type MgoOption struct {
	Timeout time.Duration
}

// create the mongo connection string
func getMongoConnStr(container string) (string, string) {
	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	replicas := conf.String(container+".replicas", "")
	options := conf.String(container+".options", "")

	if port != "" {
		port = ":" + port
	}
	if replicas != "" {
		replicas = "," + replicas
	}
	if options != "" {
		options = "?" + options
	}
	auth := ""
	if username != "" || password != "" {
		auth = username + ":" + password + "@"
	}

	return fmt.Sprintf("mongodb://%s%s%s%s/%s%s", auth, host, port, replicas, db, options), db
}

// validate the container string
func validateContainer(container string) (db string, err error) {

	if conf.Exists(container) == false {
		err = errors.New("mongo configuration missing")
		return
	}
	db = container
	return
}

// create mongo connection
// TODO introduce parameter for connection additional settings like socket timeout
func GetMongoConn(container string, param ...MgoOption) (sess *mgo.Session, mdb string, err error) {
	var db string
	if db, err = validateContainer(container); err == nil {
		var conn string
		conn, mdb = getMongoConnStr(db)
		if mdb == "" {
			err = errors.New("mongo database name missing")
			return
		}
		pLen := len(param)
		if pLen == 1 {
			if param[0].Timeout != time.Duration(0) {
				sess, err = mgo.DialWithTimeout(conn, param[0].Timeout)
			} else {
				sess, err = mgo.Dial(conn)
			}
		} else if pLen > 1 {
			err = errors.New("More than one MgoOption structures not supported")
		} else {
			sess, err = mgo.Dial(conn)
		}
		if err == nil {
			sess.SetMode(mgo.Monotonic, true)
		}
	}
	return
}
<commit_msg>OptionalMgoT code tweak<commit_after>package tmongo

import (
	"errors"
	"fmt"
	"github.com/tolexo/aero/conf"
	"gopkg.in/mgo.v2"
	"time"
)

type MgoOption struct {
	Timeout time.Duration
}

// create the mongo connection string
func getMongoConnStr(container string) (conn string, db string, err error) {
	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db = conf.String(container+".db", "")
	replicas := conf.String(container+".replicas", "")
	options := conf.String(container+".options", "")

	if db == "" {
		err = errors.New("mongo database name missing")
		return
	}

	if port != "" {
		port = ":" + port
	}
	if replicas != "" {
		replicas = "," + replicas
	}
	if options != "" {
		options = "?" + options
	}
	auth := ""
	if username != "" || password != "" {
		auth = username + ":" + password + "@"
	}

	conn = fmt.Sprintf("mongodb://%s%s%s%s/%s%s", auth, host, port, replicas, db, options)
	return
}

// validate the container string
func validateContainer(container string) (db string, err error) {

	if conf.Exists(container) == false {
		err = errors.New("mongo configuration missing")
		return
	}
	db = container
	return
}

// create mongo connection
// TODO introduce parameter for connection additional settings like socket timeout
func GetMongoConn(container string, param ...MgoOption) (sess *mgo.Session, mdb string, err error) {
	var db string
	if db, err = validateContainer(container); err == nil {
		var conn string
		if conn, mdb, err = getMongoConnStr(db); err != nil {
			return
		}

		pLen := len(param)
		if pLen == 0 {
			sess, err = mgo.Dial(conn)
		} if pLen == 1 {
			if param[0].Timeout != time.Duration(0) {
				sess, err = mgo.DialWithTimeout(conn, param[0].Timeout)
			} else {
				sess, err = mgo.Dial(conn)
			}
		} else {
			err = errors.New("More than one MgoOption structures not supported")
		}

		if err == nil {
			sess.SetMode(mgo.Monotonic, true)
		}
	}
	return
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"errors"
	"io"

	"github.com/btcsuite/btclog"
	btcwallet "github.com/roasbeef/btcwallet/wallet"
)

// walletLog is a logger that is initialized with no output filters.  This
// means the package will not perform any logging by default until the caller
// requests it.
var walletLog btclog.Logger

// The default amount of logging is none.
func init() {
	DisableLog()
}

// DisableLog disables all library log output.  Logging output is disabled
// by default until either UseLogger or SetLogWriter are called.
func DisableLog() {
	walletLog = btclog.Disabled
}

// UseLogger uses a specified Logger to output package logging info.
// This should be used in preference to SetLogWriter if the caller is also
// using btclog.
func UseLogger(logger btclog.Logger) {
	walletLog = logger
	btcwallet.UseLogger(logger)
}

// SetLogWriter uses a specified io.Writer to output package logging info.
// This allows a caller to direct package logging output without needing a
// dependency on seelog.  If the caller is also using btclog, UseLogger should
// be used instead.
func SetLogWriter(w io.Writer, level string) error {
	if w == nil {
		return errors.New("nil writer")
	}

	lvl, ok := btclog.LogLevelFromString(level)
	if !ok {
		return errors.New("invalid log level")
	}

	l, err := btclog.NewLoggerFromWriter(w, lvl)
	if err != nil {
		return err
	}

	UseLogger(l)
	return nil
}

// logClosure is used to provide a closure over expensive logging operations
// so don't have to be performed when the logging level doesn't warrant it.
type logClosure func() string

// String invokes the underlying function and returns the result.
func (c logClosure) String() string {
	return c()
}

// newLogClosure returns a new closure over a function that returns a string
// which itself provides a Stringer interface so that it can be used with the
// logging system.
func newLogClosure(c func() string) logClosure {
	return logClosure(c)
}
<commit_msg>lnwallet: couple websockets and wtxmgr logging into package logging<commit_after>package lnwallet

import (
	"errors"
	"io"

	"github.com/btcsuite/btclog"
	"github.com/roasbeef/btcrpcclient"
	"github.com/roasbeef/btcwallet/chain"
	btcwallet "github.com/roasbeef/btcwallet/wallet"
	"github.com/roasbeef/btcwallet/wtxmgr"
)

// walletLog is a logger that is initialized with no output filters.  This
// means the package will not perform any logging by default until the caller
// requests it.
var walletLog btclog.Logger

// The default amount of logging is none.
func init() {
	DisableLog()
}

// DisableLog disables all library log output.  Logging output is disabled
// by default until either UseLogger or SetLogWriter are called.
func DisableLog() {
	walletLog = btclog.Disabled
}

// UseLogger uses a specified Logger to output package logging info.
// This should be used in preference to SetLogWriter if the caller is also
// using btclog.
func UseLogger(logger btclog.Logger) {
	walletLog = logger

	btcwallet.UseLogger(logger)
	wtxmgr.UseLogger(logger)
	btcrpcclient.UseLogger(logger)
	chain.UseLogger(logger)
}

// SetLogWriter uses a specified io.Writer to output package logging info.
// This allows a caller to direct package logging output without needing a
// dependency on seelog.  If the caller is also using btclog, UseLogger should
// be used instead.
func SetLogWriter(w io.Writer, level string) error {
	if w == nil {
		return errors.New("nil writer")
	}

	lvl, ok := btclog.LogLevelFromString(level)
	if !ok {
		return errors.New("invalid log level")
	}

	l, err := btclog.NewLoggerFromWriter(w, lvl)
	if err != nil {
		return err
	}

	UseLogger(l)
	return nil
}

// logClosure is used to provide a closure over expensive logging operations
// so don't have to be performed when the logging level doesn't warrant it.
type logClosure func() string

// String invokes the underlying function and returns the result.
func (c logClosure) String() string {
	return c()
}

// newLogClosure returns a new closure over a function that returns a string
// which itself provides a Stringer interface so that it can be used with the
// logging system.
func newLogClosure(c func() string) logClosure {
	return logClosure(c)
}
<|endoftext|>


<commit_before>package raft

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/hashicorp/go-hclog"
)

// CheckInteg will skip a test if integration testing is not enabled.
func CheckInteg(t *testing.T) {
	if !IsInteg() {
		t.SkipNow()
	}
}

// IsInteg returns a boolean telling you if we're in integ testing mode.
func IsInteg() bool {
	return os.Getenv("INTEG_TESTS") != ""
}

type RaftEnv struct {
	dir      string
	conf     *Config
	fsm      *MockFSM
	store    *InmemStore
	snapshot *FileSnapshotStore
	trans    *NetworkTransport
	raft     *Raft
	logger   hclog.Logger
}

// Release shuts down and cleans up any stored data, its not restartable after this
func (r *RaftEnv) Release() {
	r.Shutdown()
	os.RemoveAll(r.dir)
}

// Shutdown shuts down raft & transport, but keeps track of its data, its restartable
// after a Shutdown() by calling Start()
func (r *RaftEnv) Shutdown() {
	r.logger.Warn(fmt.Sprintf("Shutdown node at %v", r.raft.localAddr))
	f := r.raft.Shutdown()
	if err := f.Error(); err != nil {
		panic(err)
	}
	r.trans.Close()
}

// Restart will start a raft node that was previously Shutdown()
func (r *RaftEnv) Restart(t *testing.T) {
	trans, err := NewTCPTransport(string(r.raft.localAddr), nil, 2, time.Second, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	r.trans = trans
	r.logger.Info("starting node", "addr", trans.LocalAddr())
	raft, err := NewRaft(r.conf, r.fsm, r.store, r.store, r.snapshot, r.trans)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	r.raft = raft
}

func MakeRaft(t *testing.T, conf *Config, bootstrap bool) *RaftEnv {
	// Set the config
	if conf == nil {
		conf = inmemConfig(t)
	}

	dir, err := ioutil.TempDir("", "raft")
	if err != nil {
		t.Fatalf("err: %v ", err)
	}

	stable := NewInmemStore()

	snap, err := NewFileSnapshotStore(dir, 3, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	env := &RaftEnv{
		conf:     conf,
		dir:      dir,
		store:    stable,
		snapshot: snap,
		fsm:      &MockFSM{},
	}
	trans, err := NewTCPTransport("localhost:0", nil, 2, time.Second, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	env.logger = hclog.New(&hclog.LoggerOptions{
		Name: string(trans.LocalAddr()) + " :",
	})
	env.trans = trans

	if bootstrap {
		var configuration Configuration
		configuration.Servers = append(configuration.Servers, Server{
			Suffrage: Voter,
			ID:       conf.LocalID,
			Address:  trans.LocalAddr(),
		})
		err = BootstrapCluster(conf, stable, stable, snap, trans, configuration)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
	}
	env.logger.Info("starting node", "addr", trans.LocalAddr())
	conf.Logger = env.logger
	raft, err := NewRaft(conf, env.fsm, stable, stable, snap, trans)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	env.raft = raft
	return env
}

func WaitFor(env *RaftEnv, state RaftState) error {
	limit := time.Now().Add(200 * time.Millisecond)
	for env.raft.State() != state {
		if time.Now().Before(limit) {
			time.Sleep(10 * time.Millisecond)
		} else {
			return fmt.Errorf("failed to transition to state %v", state)
		}
	}
	return nil
}

func WaitForAny(state RaftState, envs []*RaftEnv) (*RaftEnv, error) {
	limit := time.Now().Add(200 * time.Millisecond)
CHECK:
	for _, env := range envs {
		if env.raft.State() == state {
			return env, nil
		}
	}
	if time.Now().Before(limit) {
		goto WAIT
	}
	return nil, fmt.Errorf("failed to find node in %v state", state)
WAIT:
	time.Sleep(10 * time.Millisecond)
	goto CHECK
}

func WaitFuture(f Future, t *testing.T) error {
	timer := time.AfterFunc(200*time.Millisecond, func() {
		panic(fmt.Errorf("timeout waiting for future %v", f))
	})
	defer timer.Stop()
	return f.Error()
}

func NoErr(err error, t *testing.T) {
	if err != nil {
		t.Fatalf("err: %v", err)
	}
}

func CheckConsistent(envs []*RaftEnv, t *testing.T) {
	limit := time.Now().Add(400 * time.Millisecond)
	first := envs[0]
	first.fsm.Lock()
	defer first.fsm.Unlock()
	var err error
CHECK:
	l1 := len(first.fsm.logs)
	for i := 1; i < len(envs); i++ {
		env := envs[i]
		env.fsm.Lock()
		l2 := len(env.fsm.logs)
		if l1 != l2 {
			err = fmt.Errorf("log length mismatch %d %d", l1, l2)
			env.fsm.Unlock()
			goto ERR
		}
		for idx, log := range first.fsm.logs {
			other := env.fsm.logs[idx]
			if bytes.Compare(log, other) != 0 {
				err = fmt.Errorf("log entry %d mismatch between %s/%s : '%s' / '%s'", idx, first.raft.localAddr, env.raft.localAddr, log, other)
				env.fsm.Unlock()
				goto ERR
			}
		}
		env.fsm.Unlock()
	}
	return
ERR:
	if time.Now().After(limit) {
		t.Fatalf("%v", err)
	}
	first.fsm.Unlock()
	time.Sleep(20 * time.Millisecond)
	first.fsm.Lock()
	goto CHECK
}

// return a log entry that's at least sz long that has the prefix 'test i '
func logBytes(i, sz int) []byte {
	var logBuffer bytes.Buffer
	fmt.Fprintf(&logBuffer, "test %d ", i)
	for logBuffer.Len() < sz {
		logBuffer.WriteByte('x')
	}
	return logBuffer.Bytes()
}

// Tests Raft by creating a cluster, growing it to 5 nodes while
// causing various stressful conditions
func TestRaft_Integ(t *testing.T) {
	CheckInteg(t)
	conf := DefaultConfig()
	conf.LocalID = ServerID("first")
	conf.HeartbeatTimeout = 50 * time.Millisecond
	conf.ElectionTimeout = 50 * time.Millisecond
	conf.LeaderLeaseTimeout = 50 * time.Millisecond
	conf.CommitTimeout = 5 * time.Millisecond
	conf.SnapshotThreshold = 100
	conf.TrailingLogs = 10

	// Create a single node
	env1 := MakeRaft(t, conf, true)
	NoErr(WaitFor(env1, Leader), t)

	totalApplied := 0
	applyAndWait := func(leader *RaftEnv, n, sz int) {
		// Do some commits
		var futures []ApplyFuture
		for i := 0; i < n; i++ {
			futures = append(futures, leader.raft.Apply(logBytes(i, sz), 0))
		}
		for _, f := range futures {
			NoErr(WaitFuture(f, t), t)
			leader.logger.Debug("applied", "index", f.Index(), "size", sz)
		}
		totalApplied += n
	}
	// Do some commits
	applyAndWait(env1, 100, 10)

	// Do a snapshot
	NoErr(WaitFuture(env1.raft.Snapshot(), t), t)

	// Join a few nodes!
	var envs []*RaftEnv
	for i := 0; i < 4; i++ {
		conf.LocalID = ServerID(fmt.Sprintf("next-batch-%d", i))
		env := MakeRaft(t, conf, false)
		addr := env.trans.LocalAddr()
		NoErr(WaitFuture(env1.raft.AddVoter(conf.LocalID, addr, 0, 0), t), t)
		envs = append(envs, env)
	}

	// Wait for a leader
	leader, err := WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Do some more commits
	applyAndWait(leader, 100, 10)

	// Snapshot the leader
	NoErr(WaitFuture(leader.raft.Snapshot(), t), t)

	CheckConsistent(append([]*RaftEnv{env1}, envs...), t)

	// shutdown a follower
	disconnected := envs[len(envs)-1]
	disconnected.Shutdown()

	// Do some more commits [make sure the resulting snapshot will be a reasonable size]
	applyAndWait(leader, 100, 10000)

	// snapshot the leader [leaders log should be compacted past the disconnected follower log now]
	NoErr(WaitFuture(leader.raft.Snapshot(), t), t)

	// Unfortunately we need to wait for the leader to start backing off RPCs to the down follower
	// such that when the follower comes back up it'll run an election before it gets an rpc from
	// the leader
	time.Sleep(time.Second * 5)

	// start the now out of date follower back up
	disconnected.Restart(t)

	// wait for it to get caught up
	timeout := time.Now().Add(time.Second * 10)
	for disconnected.raft.getLastApplied() < leader.raft.getLastApplied() {
		time.Sleep(time.Millisecond)
		if time.Now().After(timeout) {
			t.Fatalf("Gave up waiting for follower to get caught up to leader")
		}
	}

	CheckConsistent(append([]*RaftEnv{env1}, envs...), t)

	// Shoot two nodes in the head!
	rm1, rm2 := envs[0], envs[1]
	rm1.Release()
	rm2.Release()
	envs = envs[2:]
	time.Sleep(10 * time.Millisecond)

	// Wait for a leader
	leader, err = WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Do some more commits
	applyAndWait(leader, 100, 10)

	// Join a few new nodes!
	for i := 0; i < 2; i++ {
		conf.LocalID = ServerID(fmt.Sprintf("final-batch-%d", i))
		env := MakeRaft(t, conf, false)
		addr := env.trans.LocalAddr()
		NoErr(WaitFuture(env1.raft.AddVoter(conf.LocalID, addr, 0, 0), t), t)
		envs = append(envs, env)
	}

	// Wait for a leader
	leader, err = WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Remove the old nodes
	NoErr(WaitFuture(leader.raft.RemoveServer(rm1.raft.localID, 0, 0), t), t)
	NoErr(WaitFuture(leader.raft.RemoveServer(rm2.raft.localID, 0, 0), t), t)

	// Shoot the leader
	env1.Release()
	time.Sleep(3 * conf.HeartbeatTimeout)

	// Wait for a leader
	leader, err = WaitForAny(Leader, envs)
	NoErr(err, t)

	allEnvs := append([]*RaftEnv{env1}, envs...)
	CheckConsistent(allEnvs, t)

	if len(env1.fsm.logs) != totalApplied {
		t.Fatalf("should apply %d logs! %d", totalApplied, len(env1.fsm.logs))
	}

	for _, e := range envs {
		e.Release()
	}
}
<commit_msg>Make TestRaft_Integ more reliable<commit_after>package raft

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/hashicorp/go-hclog"
)

// CheckInteg will skip a test if integration testing is not enabled.
func CheckInteg(t *testing.T) {
	if !IsInteg() {
		t.SkipNow()
	}
}

// IsInteg returns a boolean telling you if we're in integ testing mode.
func IsInteg() bool {
	return os.Getenv("INTEG_TESTS") != ""
}

type RaftEnv struct {
	dir      string
	conf     *Config
	fsm      *MockFSM
	store    *InmemStore
	snapshot *FileSnapshotStore
	trans    *NetworkTransport
	raft     *Raft
	logger   hclog.Logger
}

// Release shuts down and cleans up any stored data, its not restartable after this
func (r *RaftEnv) Release() {
	r.Shutdown()
	os.RemoveAll(r.dir)
}

// Shutdown shuts down raft & transport, but keeps track of its data, its restartable
// after a Shutdown() by calling Start()
func (r *RaftEnv) Shutdown() {
	r.logger.Warn(fmt.Sprintf("Shutdown node at %v", r.raft.localAddr))
	f := r.raft.Shutdown()
	if err := f.Error(); err != nil {
		panic(err)
	}
	r.trans.Close()
}

// Restart will start a raft node that was previously Shutdown()
func (r *RaftEnv) Restart(t *testing.T) {
	trans, err := NewTCPTransport(string(r.raft.localAddr), nil, 2, time.Second, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	r.trans = trans
	r.logger.Info("starting node", "addr", trans.LocalAddr())
	raft, err := NewRaft(r.conf, r.fsm, r.store, r.store, r.snapshot, r.trans)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	r.raft = raft
}

func MakeRaft(t *testing.T, conf *Config, bootstrap bool) *RaftEnv {
	// Set the config
	if conf == nil {
		conf = inmemConfig(t)
	}

	dir, err := ioutil.TempDir("", "raft")
	if err != nil {
		t.Fatalf("err: %v ", err)
	}

	stable := NewInmemStore()

	snap, err := NewFileSnapshotStore(dir, 3, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	env := &RaftEnv{
		conf:     conf,
		dir:      dir,
		store:    stable,
		snapshot: snap,
		fsm:      &MockFSM{},
	}
	trans, err := NewTCPTransport("localhost:0", nil, 2, time.Second, nil)
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	env.logger = hclog.New(&hclog.LoggerOptions{
		Name: string(trans.LocalAddr()) + " :",
	})
	env.trans = trans

	if bootstrap {
		var configuration Configuration
		configuration.Servers = append(configuration.Servers, Server{
			Suffrage: Voter,
			ID:       conf.LocalID,
			Address:  trans.LocalAddr(),
		})
		err = BootstrapCluster(conf, stable, stable, snap, trans, configuration)
		if err != nil {
			t.Fatalf("err: %v", err)
		}
	}
	env.logger.Info("starting node", "addr", trans.LocalAddr())
	conf.Logger = env.logger
	raft, err := NewRaft(conf, env.fsm, stable, stable, snap, trans)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	env.raft = raft
	return env
}

func WaitFor(env *RaftEnv, state RaftState) error {
	limit := time.Now().Add(200 * time.Millisecond)
	for env.raft.State() != state {
		if time.Now().Before(limit) {
			time.Sleep(10 * time.Millisecond)
		} else {
			return fmt.Errorf("failed to transition to state %v", state)
		}
	}
	return nil
}

func WaitForAny(state RaftState, envs []*RaftEnv) (*RaftEnv, error) {
	limit := time.Now().Add(200 * time.Millisecond)
CHECK:
	for _, env := range envs {
		if env.raft.State() == state {
			return env, nil
		}
	}
	if time.Now().Before(limit) {
		goto WAIT
	}
	return nil, fmt.Errorf("failed to find node in %v state", state)
WAIT:
	time.Sleep(10 * time.Millisecond)
	goto CHECK
}

func WaitFuture(f Future, t *testing.T) error {
	timer := time.AfterFunc(1000*time.Millisecond, func() {
		panic(fmt.Errorf("timeout waiting for future %v", f))
	})
	defer timer.Stop()
	return f.Error()
}

func NoErr(err error, t *testing.T) {
	t.Helper()
	if err != nil {
		t.Fatalf("err: %v", err)
	}
}

func CheckConsistent(envs []*RaftEnv, t *testing.T) {
	limit := time.Now().Add(400 * time.Millisecond)
	first := envs[0]
	first.fsm.Lock()
	defer first.fsm.Unlock()
	var err error
CHECK:
	l1 := len(first.fsm.logs)
	for i := 1; i < len(envs); i++ {
		env := envs[i]
		env.fsm.Lock()
		l2 := len(env.fsm.logs)
		if l1 != l2 {
			err = fmt.Errorf("log length mismatch %d %d", l1, l2)
			env.fsm.Unlock()
			goto ERR
		}
		for idx, log := range first.fsm.logs {
			other := env.fsm.logs[idx]
			if bytes.Compare(log, other) != 0 {
				err = fmt.Errorf("log entry %d mismatch between %s/%s : '%s' / '%s'", idx, first.raft.localAddr, env.raft.localAddr, log, other)
				env.fsm.Unlock()
				goto ERR
			}
		}
		env.fsm.Unlock()
	}
	return
ERR:
	if time.Now().After(limit) {
		t.Fatalf("%v", err)
	}
	first.fsm.Unlock()
	time.Sleep(20 * time.Millisecond)
	first.fsm.Lock()
	goto CHECK
}

// return a log entry that's at least sz long that has the prefix 'test i '
func logBytes(i, sz int) []byte {
	var logBuffer bytes.Buffer
	fmt.Fprintf(&logBuffer, "test %d ", i)
	for logBuffer.Len() < sz {
		logBuffer.WriteByte('x')
	}
	return logBuffer.Bytes()
}

// Tests Raft by creating a cluster, growing it to 5 nodes while
// causing various stressful conditions
func TestRaft_Integ(t *testing.T) {
	CheckInteg(t)
	conf := DefaultConfig()
	conf.LocalID = ServerID("first")
	conf.HeartbeatTimeout = 50 * time.Millisecond
	conf.ElectionTimeout = 50 * time.Millisecond
	conf.LeaderLeaseTimeout = 50 * time.Millisecond
	conf.CommitTimeout = 5 * time.Millisecond
	conf.SnapshotThreshold = 100
	conf.TrailingLogs = 10

	// Create a single node
	env1 := MakeRaft(t, conf, true)
	NoErr(WaitFor(env1, Leader), t)

	totalApplied := 0
	applyAndWait := func(leader *RaftEnv, n, sz int) {
		// Do some commits
		var futures []ApplyFuture
		for i := 0; i < n; i++ {
			futures = append(futures, leader.raft.Apply(logBytes(i, sz), 0))
		}
		for _, f := range futures {
			NoErr(WaitFuture(f, t), t)
			leader.logger.Debug("applied", "index", f.Index(), "size", sz)
		}
		totalApplied += n
	}
	// Do some commits
	applyAndWait(env1, 100, 10)

	// Do a snapshot
	NoErr(WaitFuture(env1.raft.Snapshot(), t), t)

	// Join a few nodes!
	var envs []*RaftEnv
	for i := 0; i < 4; i++ {
		conf.LocalID = ServerID(fmt.Sprintf("next-batch-%d", i))
		env := MakeRaft(t, conf, false)
		addr := env.trans.LocalAddr()
		NoErr(WaitFuture(env1.raft.AddVoter(conf.LocalID, addr, 0, 0), t), t)
		envs = append(envs, env)
	}

	// Wait for a leader
	leader, err := WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Do some more commits
	applyAndWait(leader, 100, 10)

	// Snapshot the leader
	NoErr(WaitFuture(leader.raft.Snapshot(), t), t)

	CheckConsistent(append([]*RaftEnv{env1}, envs...), t)

	// shutdown a follower
	disconnected := envs[len(envs)-1]
	disconnected.Shutdown()

	// Do some more commits [make sure the resulting snapshot will be a reasonable size]
	applyAndWait(leader, 100, 10000)

	// snapshot the leader [leaders log should be compacted past the disconnected follower log now]
	NoErr(WaitFuture(leader.raft.Snapshot(), t), t)

	// Unfortunately we need to wait for the leader to start backing off RPCs to the down follower
	// such that when the follower comes back up it'll run an election before it gets an rpc from
	// the leader
	time.Sleep(time.Second * 5)

	// start the now out of date follower back up
	disconnected.Restart(t)

	// wait for it to get caught up
	timeout := time.Now().Add(time.Second * 10)
	for disconnected.raft.getLastApplied() < leader.raft.getLastApplied() {
		time.Sleep(time.Millisecond)
		if time.Now().After(timeout) {
			t.Fatalf("Gave up waiting for follower to get caught up to leader")
		}
	}

	CheckConsistent(append([]*RaftEnv{env1}, envs...), t)

	// Shoot two nodes in the head!
	rm1, rm2 := envs[0], envs[1]
	rm1.Release()
	rm2.Release()
	envs = envs[2:]
	time.Sleep(10 * time.Millisecond)

	// Wait for a leader
	leader, err = WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Do some more commits
	applyAndWait(leader, 100, 10)

	// Join a few new nodes!
	for i := 0; i < 2; i++ {
		conf.LocalID = ServerID(fmt.Sprintf("final-batch-%d", i))
		env := MakeRaft(t, conf, false)
		addr := env.trans.LocalAddr()
		NoErr(WaitFuture(leader.raft.AddVoter(conf.LocalID, addr, 0, 0), t), t)
		envs = append(envs, env)

		leader, err = WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
		NoErr(err, t)
	}

	// Wait for a leader
	leader, err = WaitForAny(Leader, append([]*RaftEnv{env1}, envs...))
	NoErr(err, t)

	// Remove the old nodes
	NoErr(WaitFuture(leader.raft.RemoveServer(rm1.raft.localID, 0, 0), t), t)
	NoErr(WaitFuture(leader.raft.RemoveServer(rm2.raft.localID, 0, 0), t), t)

	// Shoot the leader
	env1.Release()
	time.Sleep(3 * conf.HeartbeatTimeout)

	// Wait for a leader
	leader, err = WaitForAny(Leader, envs)
	NoErr(err, t)

	allEnvs := append([]*RaftEnv{env1}, envs...)
	CheckConsistent(allEnvs, t)

	if len(env1.fsm.logs) != totalApplied {
		t.Fatalf("should apply %d logs! %d", totalApplied, len(env1.fsm.logs))
	}

	for _, e := range envs {
		e.Release()
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2012 Mostafa Hajizadeh
// Copyright (C) 2014-2022 Steve Francia

// package fsync keeps two files or directories in sync.
//
//         err := fsync.Sync("~/dst", ".")
//
// After the above code, if err is nil, every file and directory in the current
// directory is copied to ~/dst and has the same permissions. Consequent calls
// will only copy changed or new files.
//
// SyncTo is a helper function which helps you sync a groups of files or
// directories into a single destination. For instance, calling
//
//     SyncTo("public", "build/app.js", "build/app.css", "images", "fonts")
//
// is equivalent to calling
//
//     Sync("public/app.js", "build/app.js")
//     Sync("public/app.css", "build/app.css")
//     Sync("public/images", "images")
//     Sync("public/fonts", "fonts")
//
// Actually, this is how SyncTo is implemented: consequent calls to Sync.
//
// By default, sync code ignores extra files in the destination that don’t have
// identicals in the source. Setting Delete field of a Syncer to true changes
// this behavior and deletes these extra files.

package fsync

import (
	"bytes"
	"errors"
	"io"
	iofs "io/fs"
	"os"
	"path/filepath"
	"runtime"

	"github.com/spf13/afero"
)

var (
	ErrFileOverDir = errors.New(
		"fsync: trying to overwrite a non-empty directory with a file")
)

// FileInfo contains the shared methods between os.FileInfo and fs.DirEntry.
type FileInfo interface {
	Name() string
	IsDir() bool
}

// Sync copies files and directories inside src into dst.
func Sync(dst, src string) error {
	return NewSyncer().Sync(dst, src)
}

// SyncTo syncs srcs files and directories into to directory.
func SyncTo(to string, srcs ...string) error {
	return NewSyncer().SyncTo(to, srcs...)
}

// Type Syncer provides functions for syncing files.
type Syncer struct {
	// Set this to true to delete files in the destination that don't exist
	// in the source.
	Delete bool
	// To allow certain files to remain in the destination, implement this function.
	// Return true to skip file, false to delete.
	// Note that src may be either os.FileInfo or fs.DirEntry depending on the file system.
	DeleteFilter func(f FileInfo) bool
	// By default, modification times are synced. This can be turned off by
	// setting this to true.
	NoTimes bool
	// NoChmod disables permission mode syncing.
	NoChmod bool
	// Implement this function to skip Chmod syncing for only certain files
	// or directories. Return true to skip Chmod.
	// Note that src may be either os.FileInfo or fs.DirEntry depending on the file system.
	ChmodFilter func(dst, src FileInfo) bool

	// TODO add options for not checking content for equality

	SrcFs  afero.Fs
	DestFs afero.Fs
}

// NewSyncer creates a new instance of Syncer with default options.
func NewSyncer() *Syncer {
	s := Syncer{SrcFs: new(afero.OsFs), DestFs: new(afero.OsFs)}
	s.DeleteFilter = func(f FileInfo) bool {
		return false
	}
	return &s
}

// Sync copies files and directories inside src into dst.
func (s *Syncer) Sync(dst, src string) error {
	// make sure src exists
	if _, err := s.SrcFs.Stat(src); err != nil {
		return err
	}
	// return error instead of replacing a non-empty directory with a file
	if b, err := s.checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return s.syncRecover(dst, src)
}

// SyncTo syncs srcs files or directories into to directory.
func (s *Syncer) SyncTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := filepath.Join(to, filepath.Base(src))
		if err := s.Sync(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// syncRecover handles errors and calls sync
func (s *Syncer) syncRecover(dst, src string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			err = r.(error)
		}
	}()

	s.sync(dst, src)
	return nil
}

// sync updates dst to match with src, handling both files and directories.
func (s *Syncer) sync(dst, src string) {
	// sync permissions and modification times after handling content
	defer s.syncstats(dst, src)

	// read files info
	dstat, err := s.DestFs.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		panic(err)
	}
	sstat, err := s.SrcFs.Stat(src)
	if err != nil && os.IsNotExist(err) {
		return // src was deleted before we could copy it
	}
	check(err)

	if !sstat.IsDir() {
		// src is a file
		// delete dst if its a directory
		if dstat != nil && dstat.IsDir() {
			check(s.DestFs.RemoveAll(dst))
		}
		if !s.equal(dst, src) {
			// perform copy
			df, err := s.DestFs.Create(dst)
			check(err)
			defer df.Close()
			sf, err := s.SrcFs.Open(src)
			if os.IsNotExist(err) {
				return
			}
			check(err)
			defer sf.Close()
			_, err = io.Copy(df, sf)
			if os.IsNotExist(err) {
				return
			}
			check(err)
		}
		return
	}

	// src is a directory
	// make dst if necessary
	if dstat == nil {
		// dst does not exist; create directory
		check(s.DestFs.MkdirAll(dst, 0755)) // permissions will be synced later
	} else if !dstat.IsDir() {
		// dst is a file; remove and create directory
		check(s.DestFs.Remove(dst))
		check(s.DestFs.MkdirAll(dst, 0755)) // permissions will be synced later
	}

	// make a map of filenames for quick lookup; used in deletion
	// deletion below
	m := make(map[string]bool)
	err = withDirEntry(s.SrcFs, src, func(fi FileInfo) bool {
		dst2 := filepath.Join(dst, fi.Name())
		src2 := filepath.Join(src, fi.Name())
		s.sync(dst2, src2)
		m[fi.Name()] = true

		return false
	})

	if os.IsNotExist(err) {
		return
	}
	check(err)

	// delete files from dst that does not exist in src
	if s.Delete {
		err = withDirEntry(s.DestFs, dst, func(fi FileInfo) bool {
			if !m[fi.Name()] && !s.DeleteFilter(fi) {
				check(s.DestFs.RemoveAll(filepath.Join(dst, fi.Name())))
			}
			return false
		})
		check(err)

	}
}

// syncstats makes sure dst has the same pemissions and modification time as src
func (s *Syncer) syncstats(dst, src string) {
	// get file infos; return if not exist and panic if error
	dstat, err1 := s.DestFs.Stat(dst)
	sstat, err2 := s.SrcFs.Stat(src)
	if os.IsNotExist(err1) || os.IsNotExist(err2) {
		return
	}
	check(err1)
	check(err2)

	// update dst's permission bits
	noChmod := s.NoChmod
	if !noChmod && s.ChmodFilter != nil {
		noChmod = s.ChmodFilter(dstat, sstat)
	}
	if !noChmod {
		if dstat.Mode().Perm() != sstat.Mode().Perm() {
			check(s.DestFs.Chmod(dst, sstat.Mode().Perm()))
		}
	}

	// update dst's modification time
	if !s.NoTimes {
		if !dstat.ModTime().Equal(sstat.ModTime()) {
			err := s.DestFs.Chtimes(dst, sstat.ModTime(), sstat.ModTime())
			check(err)
		}
	}
}

// equal returns true if both dst and src files are equal
func (s *Syncer) equal(dst, src string) bool {
	// get file infos
	info1, err1 := s.DestFs.Stat(dst)
	info2, err2 := s.SrcFs.Stat(src)
	if os.IsNotExist(err1) || os.IsNotExist(err2) {
		return false
	}
	check(err1)
	check(err2)

	// check sizes
	if info1.Size() != info2.Size() {
		return false
	}

	// both have the same size, check the contents
	f1, err := s.DestFs.Open(dst)
	check(err)
	defer f1.Close()
	f2, err := s.SrcFs.Open(src)
	check(err)
	defer f2.Close()
	buf1 := make([]byte, 1000)
	buf2 := make([]byte, 1000)
	for {
		// read from both
		n1, err := f1.Read(buf1)
		if err != nil && err != io.EOF {
			panic(err)
		}
		n2, err := f2.Read(buf2)
		if err != nil && err != io.EOF {
			panic(err)
		}

		// compare read bytes
		if !bytes.Equal(buf1[:n1], buf2[:n2]) {
			return false
		}

		// end of both files
		if n1 == 0 && n2 == 0 {
			break
		}
	}

	return true
}

// checkDir returns true if dst is a non-empty directory and src is a file
func (s *Syncer) checkDir(dst, src string) (b bool, err error) {
	// read file info
	dstat, err := s.DestFs.Stat(dst)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	sstat, err := s.SrcFs.Stat(src)
	if err != nil {
		return false, err
	}

	// return false is dst is not a directory or src is a directory
	if !dstat.IsDir() || sstat.IsDir() {
		return false, nil
	}

	// dst is a directory and src is a file
	// check if dst is non-empty
	// read dst directory
	var isNonEmpty bool
	err = withDirEntry(s.DestFs, dst, func(FileInfo) bool {
		isNonEmpty = true
		return true
	})

	return isNonEmpty, nil
}

func withDirEntry(fs afero.Fs, path string, fn func(FileInfo) bool) error {
	f, err := fs.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	if rdf, ok := f.(iofs.ReadDirFile); ok {
		fis, err := rdf.ReadDir(-1)
		if err != nil {
			return err
		}
		for _, fi := range fis {
			if fn(fi) {
				return nil
			}
		}
		return nil
	}

	fis, err := f.Readdir(-1)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fn(fi) {
			return nil
		}
	}

	return nil
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Don't repeat Stat in equal<commit_after>// Copyright (C) 2012 Mostafa Hajizadeh
// Copyright (C) 2014-2022 Steve Francia

// package fsync keeps two files or directories in sync.
//
//         err := fsync.Sync("~/dst", ".")
//
// After the above code, if err is nil, every file and directory in the current
// directory is copied to ~/dst and has the same permissions. Consequent calls
// will only copy changed or new files.
//
// SyncTo is a helper function which helps you sync a groups of files or
// directories into a single destination. For instance, calling
//
//     SyncTo("public", "build/app.js", "build/app.css", "images", "fonts")
//
// is equivalent to calling
//
//     Sync("public/app.js", "build/app.js")
//     Sync("public/app.css", "build/app.css")
//     Sync("public/images", "images")
//     Sync("public/fonts", "fonts")
//
// Actually, this is how SyncTo is implemented: consequent calls to Sync.
//
// By default, sync code ignores extra files in the destination that don’t have
// identicals in the source. Setting Delete field of a Syncer to true changes
// this behavior and deletes these extra files.

package fsync

import (
	"bytes"
	"errors"
	"io"
	iofs "io/fs"
	"os"
	"path/filepath"
	"runtime"

	"github.com/spf13/afero"
)

var (
	ErrFileOverDir = errors.New(
		"fsync: trying to overwrite a non-empty directory with a file")
)

// FileInfo contains the shared methods between os.FileInfo and fs.DirEntry.
type FileInfo interface {
	Name() string
	IsDir() bool
}

// Sync copies files and directories inside src into dst.
func Sync(dst, src string) error {
	return NewSyncer().Sync(dst, src)
}

// SyncTo syncs srcs files and directories into to directory.
func SyncTo(to string, srcs ...string) error {
	return NewSyncer().SyncTo(to, srcs...)
}

// Type Syncer provides functions for syncing files.
type Syncer struct {
	// Set this to true to delete files in the destination that don't exist
	// in the source.
	Delete bool
	// To allow certain files to remain in the destination, implement this function.
	// Return true to skip file, false to delete.
	// Note that src may be either os.FileInfo or fs.DirEntry depending on the file system.
	DeleteFilter func(f FileInfo) bool
	// By default, modification times are synced. This can be turned off by
	// setting this to true.
	NoTimes bool
	// NoChmod disables permission mode syncing.
	NoChmod bool
	// Implement this function to skip Chmod syncing for only certain files
	// or directories. Return true to skip Chmod.
	ChmodFilter func(dst, src os.FileInfo) bool

	// TODO add options for not checking content for equality

	SrcFs  afero.Fs
	DestFs afero.Fs
}

// NewSyncer creates a new instance of Syncer with default options.
func NewSyncer() *Syncer {
	s := Syncer{SrcFs: new(afero.OsFs), DestFs: new(afero.OsFs)}
	s.DeleteFilter = func(f FileInfo) bool {
		return false
	}
	return &s
}

// Sync copies files and directories inside src into dst.
func (s *Syncer) Sync(dst, src string) error {
	// make sure src exists
	if _, err := s.SrcFs.Stat(src); err != nil {
		return err
	}
	// return error instead of replacing a non-empty directory with a file
	if b, err := s.checkDir(dst, src); err != nil {
		return err
	} else if b {
		return ErrFileOverDir
	}

	return s.syncRecover(dst, src)
}

// SyncTo syncs srcs files or directories into to directory.
func (s *Syncer) SyncTo(to string, srcs ...string) error {
	for _, src := range srcs {
		dst := filepath.Join(to, filepath.Base(src))
		if err := s.Sync(dst, src); err != nil {
			return err
		}
	}
	return nil
}

// syncRecover handles errors and calls sync
func (s *Syncer) syncRecover(dst, src string) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if _, ok := r.(runtime.Error); ok {
				panic(r)
			}
			err = r.(error)
		}
	}()

	s.sync(dst, src)
	return nil
}

// sync updates dst to match with src, handling both files and directories.
func (s *Syncer) sync(dst, src string) {
	// sync permissions and modification times after handling content
	defer s.syncstats(dst, src)

	// read files info
	dstat, err := s.DestFs.Stat(dst)
	if err != nil && !os.IsNotExist(err) {
		panic(err)
	}
	sstat, err := s.SrcFs.Stat(src)
	if err != nil && os.IsNotExist(err) {
		return // src was deleted before we could copy it
	}
	check(err)

	if !sstat.IsDir() {
		// src is a file
		// delete dst if its a directory
		if dstat != nil && dstat.IsDir() {
			check(s.DestFs.RemoveAll(dst))
		}
		if !s.equal(dst, src, dstat, sstat) {
			// perform copy
			df, err := s.DestFs.Create(dst)
			check(err)
			defer df.Close()
			sf, err := s.SrcFs.Open(src)
			if os.IsNotExist(err) {
				return
			}
			check(err)
			defer sf.Close()
			_, err = io.Copy(df, sf)
			if os.IsNotExist(err) {
				return
			}
			check(err)
		}
		return
	}

	// src is a directory
	// make dst if necessary
	if dstat == nil {
		// dst does not exist; create directory
		check(s.DestFs.MkdirAll(dst, 0755)) // permissions will be synced later
	} else if !dstat.IsDir() {
		// dst is a file; remove and create directory
		check(s.DestFs.Remove(dst))
		check(s.DestFs.MkdirAll(dst, 0755)) // permissions will be synced later
	}

	// make a map of filenames for quick lookup; used in deletion
	// deletion below
	m := make(map[string]bool)
	err = withDirEntry(s.SrcFs, src, func(fi FileInfo) bool {
		dst2 := filepath.Join(dst, fi.Name())
		src2 := filepath.Join(src, fi.Name())
		s.sync(dst2, src2)
		m[fi.Name()] = true

		return false
	})

	if os.IsNotExist(err) {
		return
	}
	check(err)

	// delete files from dst that does not exist in src
	if s.Delete {
		err = withDirEntry(s.DestFs, dst, func(fi FileInfo) bool {
			if !m[fi.Name()] && !s.DeleteFilter(fi) {
				check(s.DestFs.RemoveAll(filepath.Join(dst, fi.Name())))
			}
			return false
		})
		check(err)

	}
}

// syncstats makes sure dst has the same pemissions and modification time as src
func (s *Syncer) syncstats(dst, src string) {
	// get file infos; return if not exist and panic if error
	dstat, err1 := s.DestFs.Stat(dst)
	sstat, err2 := s.SrcFs.Stat(src)
	if os.IsNotExist(err1) || os.IsNotExist(err2) {
		return
	}
	check(err1)
	check(err2)

	// update dst's permission bits
	noChmod := s.NoChmod
	if !noChmod && s.ChmodFilter != nil {
		noChmod = s.ChmodFilter(dstat, sstat)
	}
	if !noChmod {
		if dstat.Mode().Perm() != sstat.Mode().Perm() {
			check(s.DestFs.Chmod(dst, sstat.Mode().Perm()))
		}
	}

	// update dst's modification time
	if !s.NoTimes {
		if !dstat.ModTime().Equal(sstat.ModTime()) {
			err := s.DestFs.Chtimes(dst, sstat.ModTime(), sstat.ModTime())
			check(err)
		}
	}
}

// equal returns true if both dst and src files are equal
func (s *Syncer) equal(dst, src string, dstat, sstat os.FileInfo) bool {
	if sstat == nil || dstat == nil {
		return false
	}

	// check sizes
	if dstat.Size() != sstat.Size() {
		return false
	}

	// both have the same size, check the contents
	f1, err := s.DestFs.Open(dst)
	check(err)
	defer f1.Close()
	f2, err := s.SrcFs.Open(src)
	check(err)
	defer f2.Close()
	buf1 := make([]byte, 1000)
	buf2 := make([]byte, 1000)
	for {
		// read from both
		n1, err := f1.Read(buf1)
		if err != nil && err != io.EOF {
			panic(err)
		}
		n2, err := f2.Read(buf2)
		if err != nil && err != io.EOF {
			panic(err)
		}

		// compare read bytes
		if !bytes.Equal(buf1[:n1], buf2[:n2]) {
			return false
		}

		// end of both files
		if n1 == 0 && n2 == 0 {
			break
		}
	}

	return true
}

// checkDir returns true if dst is a non-empty directory and src is a file
func (s *Syncer) checkDir(dst, src string) (b bool, err error) {
	// read file info
	dstat, err := s.DestFs.Stat(dst)
	if os.IsNotExist(err) {
		return false, nil
	} else if err != nil {
		return false, err
	}
	sstat, err := s.SrcFs.Stat(src)
	if err != nil {
		return false, err
	}

	// return false is dst is not a directory or src is a directory
	if !dstat.IsDir() || sstat.IsDir() {
		return false, nil
	}

	// dst is a directory and src is a file
	// check if dst is non-empty
	// read dst directory
	var isNonEmpty bool
	err = withDirEntry(s.DestFs, dst, func(FileInfo) bool {
		isNonEmpty = true
		return true
	})

	return isNonEmpty, err
}

func withDirEntry(fs afero.Fs, path string, fn func(FileInfo) bool) error {
	f, err := fs.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	if rdf, ok := f.(iofs.ReadDirFile); ok {
		fis, err := rdf.ReadDir(-1)
		if err != nil {
			return err
		}
		for _, fi := range fis {
			if fn(fi) {
				return nil
			}
		}
		return nil
	}

	fis, err := f.Readdir(-1)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if fn(fi) {
			return nil
		}
	}

	return nil
}

func check(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"text/template"
	"time"
)

// tmplFuncs contains the function map for templates that will be rendered by
// jgtr.
var tmplFuncs = template.FuncMap{
	"notLast": func(slice []interface{}, i int) bool {
		return (i != len(slice)-1)
	},

	"timeParse": time.Parse,
	"timeNow":   time.Now,

	"stringSplit":  strings.Split,
	"stringFields": strings.Fields,
	"stringJoin":   strings.Join,
	"stringUpper":  strings.ToUpper,
	"stringLower":  strings.ToLower,
	"stringTitle":  strings.ToTitle,

	// sliceSort sorts a slice of empty interfaces, using the implementation
	// provided by GenericSlice.
	"sliceSort": func(s []interface{}) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		sort.Sort(GenericSlice(ret))
		return ret
	},
	// sliceSortKey sorts a slice of empty interfaces on the given map key,
	// using the implementation provided by MapSlice.
	"sliceSortKey": func(s []interface{}, k string) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		sort.Stable(MapSlice{ret, k})
		return ret
	},
	// sliceReverse reverses a slice of empty interfaces.
	"sliceReverse": func(s []interface{}) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		for i, j := 0, len(ret)-1; i < j; i, j = i+1, j-1 {
			ret[i], ret[j] = ret[j], ret[i]
		}
		return ret
	},
}

// GenericSlice implements sort.Interface on a slice of interface{}, with the
// assumption that all of those interface{} have the same underlying type, and
// that type is some kind of int, float or string. The ordering that
// GenericSlice imposes on those elements is the expected ordering (ie sorting
// by ascending magnitude for ints/floats, and lexicographically for strings).
type GenericSlice []interface{}

// Less compares two elements of a GenericSlice. If the two elements are
// comparable (ie they are both ints, uints, floats or strings), then Less will
// return the result of a less-than comparison between those two elements. If
// the two elements are not comparable (ie their types do not match, or they do
// not have a natural ordering), then Less will panic.
func (p GenericSlice) Less(i, j int) bool {
	leftV := reflect.ValueOf(p[i])
	rightV := reflect.ValueOf(p[j])
	return valueLt(leftV, rightV)
}

func (p GenericSlice) Len() int      { return len(p) }
func (p GenericSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// MapSlice implements sort.Interface on a slice of interface{}, with the
// assumption that all of those interface{} have the underlying type
// map[string]interface{}. The ordering that MapSlice imposes on those maps
// is the standard ordering of a certain key, specified by Key. All maps in
// Slice must either omit that key, or contain a value for that key, such
// that all the maps contain the same type of value for that key.
type MapSlice struct {
	Slice []interface{}
	Key   string
}

// Less compares two elements of a MapSlice. If both elements contain the key
// specified by Key, then Less will return the result of comparing the values
// associated with Key. If those values are of noncomparable types, Less will
// panic.
// If an element does not contain the key specified by Key, then it should sort
// before the other element.
// If both elements do not contain the key specified by Key, then they are
// considered to be equal in ordering, so Less will return false.
// If either element is not actually a map[string]interface{}, Less will panic.
func (p MapSlice) Less(i, j int) bool {
	// several layers of indirection in this code
	// you have to reflect.ValueOf on the []interface{}, which reveals that the
	// interface{} is a map[string]interface{}
	// then you have to MapIndex on that map, which gives you another Value,
	// whose Kind is interface (because the map contains interfaces)
	k := reflect.ValueOf(p.Key)
	leftV := reflect.ValueOf(p.Slice[i]).MapIndex(k)
	rightV := reflect.ValueOf(p.Slice[j]).MapIndex(k)

	// if the key was not in the map, the Value will be the zero Value
	// we consider this to be "less than everything"
	// ordering of these comparisons is important:
	// if they're both the zero Value, we return false, not true
	if !rightV.IsValid() {
		return false
	}
	if !leftV.IsValid() {
		return true
	}

	// if the key was in the map, then we have to indirect through the
	// interface using Elem, to get the underlying value
	return valueLt(leftV.Elem(), rightV.Elem())
}
func (p MapSlice) Len() int      { return len(p.Slice) }
func (p MapSlice) Swap(i, j int) { p.Slice[i], p.Slice[j] = p.Slice[j], p.Slice[i] }

// valueLT performs a less-than comparison on two reflect.Values. The Values
// must both be valid receivers for one of the following reflection methods:
// Value.Int, Value.Uint, Value.Float, or Value.String. This function will panic
// if the Values do not have comparable underlying values, or if their Kinds
// are not comparable in this way.
func valueLt(leftV, rightV reflect.Value) bool {
	switch leftV.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return leftV.Int() < rightV.Int()
	case reflect.Uint, reflect.Uintptr, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return leftV.Uint() < rightV.Uint()
	case reflect.Float32, reflect.Float64:
		return leftV.Float() < rightV.Float()
	case reflect.String:
		return leftV.String() < rightV.String()
	}
	panic(fmt.Errorf("Attempting to compare items of noncomparable types: %v %v", leftV.Kind(), rightV.Kind()))
}
<commit_msg>Remove some extraneous variable names in GenericSlice.Less<commit_after>package main

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"text/template"
	"time"
)

// tmplFuncs contains the function map for templates that will be rendered by
// jgtr.
var tmplFuncs = template.FuncMap{
	"notLast": func(slice []interface{}, i int) bool {
		return (i != len(slice)-1)
	},

	"timeParse": time.Parse,
	"timeNow":   time.Now,

	"stringSplit":  strings.Split,
	"stringFields": strings.Fields,
	"stringJoin":   strings.Join,
	"stringUpper":  strings.ToUpper,
	"stringLower":  strings.ToLower,
	"stringTitle":  strings.ToTitle,

	// sliceSort sorts a slice of empty interfaces, using the implementation
	// provided by GenericSlice.
	"sliceSort": func(s []interface{}) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		sort.Sort(GenericSlice(ret))
		return ret
	},
	// sliceSortKey sorts a slice of empty interfaces on the given map key,
	// using the implementation provided by MapSlice.
	"sliceSortKey": func(s []interface{}, k string) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		sort.Stable(MapSlice{ret, k})
		return ret
	},
	// sliceReverse reverses a slice of empty interfaces.
	"sliceReverse": func(s []interface{}) []interface{} {
		ret := make([]interface{}, len(s))
		copy(ret, s)

		for i, j := 0, len(ret)-1; i < j; i, j = i+1, j-1 {
			ret[i], ret[j] = ret[j], ret[i]
		}
		return ret
	},
}

// GenericSlice implements sort.Interface on a slice of interface{}, with the
// assumption that all of those interface{} have the same underlying type, and
// that type is some kind of int, float or string. The ordering that
// GenericSlice imposes on those elements is the expected ordering (ie sorting
// by ascending magnitude for ints/floats, and lexicographically for strings).
type GenericSlice []interface{}

// Less compares two elements of a GenericSlice. If the two elements are
// comparable (ie they are both ints, uints, floats or strings), then Less will
// return the result of a less-than comparison between those two elements. If
// the two elements are not comparable (ie their types do not match, or they do
// not have a natural ordering), then Less will panic.
func (p GenericSlice) Less(i, j int) bool {
	return valueLt(reflect.ValueOf(p[i]), reflect.ValueOf(p[j]))
}

func (p GenericSlice) Len() int      { return len(p) }
func (p GenericSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

// MapSlice implements sort.Interface on a slice of interface{}, with the
// assumption that all of those interface{} have the underlying type
// map[string]interface{}. The ordering that MapSlice imposes on those maps
// is the standard ordering of a certain key, specified by Key. All maps in
// Slice must either omit that key, or contain a value for that key, such
// that all the maps contain the same type of value for that key.
type MapSlice struct {
	Slice []interface{}
	Key   string
}

// Less compares two elements of a MapSlice. If both elements contain the key
// specified by Key, then Less will return the result of comparing the values
// associated with Key. If those values are of noncomparable types, Less will
// panic.
// If an element does not contain the key specified by Key, then it should sort
// before the other element.
// If both elements do not contain the key specified by Key, then they are
// considered to be equal in ordering, so Less will return false.
// If either element is not actually a map[string]interface{}, Less will panic.
func (p MapSlice) Less(i, j int) bool {
	// several layers of indirection in this code
	// you have to reflect.ValueOf on the []interface{}, which reveals that the
	// interface{} is a map[string]interface{}
	// then you have to MapIndex on that map, which gives you another Value,
	// whose Kind is interface (because the map contains interfaces)
	k := reflect.ValueOf(p.Key)
	leftV := reflect.ValueOf(p.Slice[i]).MapIndex(k)
	rightV := reflect.ValueOf(p.Slice[j]).MapIndex(k)

	// if the key was not in the map, the Value will be the zero Value
	// we consider this to be "less than everything"
	// ordering of these comparisons is important:
	// if they're both the zero Value, we return false, not true
	if !rightV.IsValid() {
		return false
	}
	if !leftV.IsValid() {
		return true
	}

	// if the key was in the map, then we have to indirect through the
	// interface using Elem, to get the underlying value
	return valueLt(leftV.Elem(), rightV.Elem())
}
func (p MapSlice) Len() int      { return len(p.Slice) }
func (p MapSlice) Swap(i, j int) { p.Slice[i], p.Slice[j] = p.Slice[j], p.Slice[i] }

// valueLT performs a less-than comparison on two reflect.Values. The Values
// must both be valid receivers for one of the following reflection methods:
// Value.Int, Value.Uint, Value.Float, or Value.String. This function will panic
// if the Values do not have comparable underlying values, or if their Kinds
// are not comparable in this way.
func valueLt(leftV, rightV reflect.Value) bool {
	switch leftV.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return leftV.Int() < rightV.Int()
	case reflect.Uint, reflect.Uintptr, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return leftV.Uint() < rightV.Uint()
	case reflect.Float32, reflect.Float64:
		return leftV.Float() < rightV.Float()
	case reflect.String:
		return leftV.String() < rightV.String()
	}
	panic(fmt.Errorf("Attempting to compare items of noncomparable types: %v %v", leftV.Kind(), rightV.Kind()))
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ipam

import (
	"database/sql"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"log"
	"strings"
)

// Endpoint represents an endpoint (a VM, a Kubernetes Pod, etc.)
// that is to get an IP address.
type Endpoint struct {
	Ip           string         `json:"ip,omitempty"`
	TenantID     string         `json:"tenant_id,omitempty"`
	SegmentID    string         `json:"segment_id,omitempty"`
	HostId       string         `json:"host_id,omitempty"`
	Name         string         `json:"name,omitempty"`
	RequestToken sql.NullString `json:"request_token" sql:"unique"`
	// Ordinal number of this Endpoint in the host/tenant combination
	NetworkID uint64 `json:"-"`
	// Calculated effective network ID of this Endpoint --
	// taking into account stride (endpoint space bits)
	// and alignment thereof. This is used in IP calculation.
	EffectiveNetworkID uint64 `json:"-"`
	// Whether it is in use (for purposes of reclaiming)
	InUse bool   `json:"-"`
	Id    uint64 `sql:"AUTO_INCREMENT",json:"-"`
}
type ipamStore struct {
	common.DbStore
}

// deleteEndpoint releases the IP(s) owned by the endpoint into assignable
// pool.
func (ipamStore *ipamStore) deleteEndpoint(ip string) (Endpoint, error) {
	tx := ipamStore.DbStore.Db.Begin()
	results := make([]Endpoint, 0)
	tx.Where(&Endpoint{Ip: ip}).Find(&results)
	if len(results) == 0 {
		tx.Rollback()
		return Endpoint{}, common.NewError404("endpoint", ip)
	}
	if len(results) > 1 {
		// This cannot happen by constraints...
		tx.Rollback()
		errMsg := fmt.Sprintf("Expected one result for ip %s, got %v", ip, results)
		log.Printf(errMsg)
		return Endpoint{}, common.NewError500(errors.New(errMsg))
	}
	tx = tx.Model(Endpoint{}).Where("ip = ?", ip).Update("in_use", false)
	err := common.MakeMultiError(tx.GetErrors())
	if err != nil {
		tx.Rollback()
		return Endpoint{}, err
	}
	tx.Commit()
	return results[0], nil
}

// addEndpoint allocates an IP address and stores it in the
// database.
func (ipamStore *ipamStore) addEndpoint(endpoint *Endpoint, upToEndpointIpInt uint64, dc common.Datacenter) error {

	var err error
	tx := ipamStore.DbStore.Db.Begin()

	if endpoint.RequestToken.Valid && endpoint.RequestToken.String != "" {
		var existingEndpoints []Endpoint
		var count int
		tx.Where("request_token = ?", endpoint.RequestToken.String).Find(&existingEndpoints).Count(&count)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 1: %v", err)
			tx.Rollback()
			return err
		}
		if count > 0 {
			// This will only be 1, because of unique constraint.
			tx.Rollback()
			log.Printf("Found existing %s: %+v", endpoint.RequestToken.String, existingEndpoints[0])
			endpoint.EffectiveNetworkID = existingEndpoints[0].EffectiveNetworkID
			endpoint.HostId = existingEndpoints[0].HostId
			endpoint.Id = existingEndpoints[0].Id
			endpoint.InUse = existingEndpoints[0].InUse
			endpoint.Name = existingEndpoints[0].Name
			endpoint.NetworkID = existingEndpoints[0].NetworkID
			endpoint.RequestToken = existingEndpoints[0].RequestToken
			endpoint.SegmentID = existingEndpoints[0].SegmentID
			endpoint.TenantID = existingEndpoints[0].TenantID
			return nil
		}
	}

	hostId := endpoint.HostId
	endpoint.InUse = true
	tenantId := endpoint.TenantID
	segId := endpoint.SegmentID
	filter := "host_id = ? AND tenant_id = ? AND segment_id = ? "

	var sel string
	// First, find the MAX network ID available for this host/segment combination.
	sel = "IFNULL(MAX(network_id),-1)+1"
	log.Printf("IpamStore: Calling SELECT %s FROM endpoints WHERE %s;", sel, fmt.Sprintf(strings.Replace(filter, "?", "%s", 3), hostId, tenantId, segId))
	row := tx.Model(Endpoint{}).Where(filter, hostId, tenantId, segId).Select(sel).Row()
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 2: %v", err)
		tx.Rollback()
		return err
	}

	netID := sql.NullInt64{}
	row.Scan(&netID)
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 3: %v", err)
		tx.Rollback()
		return err
	}

	log.Printf("IpamStore: max net ID: %v", netID)

	maxEffNetID := uint64(1<<(dc.EndpointSpaceBits+dc.EndpointBits) - 1)

	// Does this exceed max bits?
	endpoint.NetworkID = uint64(netID.Int64)
	endpoint.EffectiveNetworkID = getEffectiveNetworkID(endpoint.NetworkID, dc.EndpointSpaceBits)
	if endpoint.EffectiveNetworkID <= maxEffNetID {
		// Does not exceed max bits, all good.
		//		log.Printf("IpamStore: Effective network ID for network ID %d (stride %d): %d\n", endpoint.NetworkID, dc.EndpointSpaceBits, endpoint.EffectiveNetworkID)
		ipInt := upToEndpointIpInt | endpoint.EffectiveNetworkID
		//		log.Printf("IpamStore: %d | %d = %d", upToEndpointIpInt, endpoint.EffectiveNetworkID, ipInt)
		endpoint.Ip = common.IntToIPv4(ipInt).String()
		tx = tx.Create(endpoint)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 4: %v", err)
			tx.Rollback()
			return err
		}
		tx.Commit()
		return nil
	}

	// Out of bits, see if we can reuse an earlier allocated address...
	log.Printf("IpamStore: New effective network ID is %d, exceeds maximum %d\n", endpoint.EffectiveNetworkID, maxEffNetID)
	// See if there is a formerly allocated IP already that has been released
	// (marked "in_use")
	sel = "MIN(network_id), ip"
	log.Printf("IpamStore: Calling SELECT %s FROM endpoints WHERE %s;", sel, fmt.Sprintf(strings.Replace(filter+"AND in_use = 0", "?", "%s", 3), hostId, tenantId, segId))
	// In containerized setup, not using group by leads to failure due to
	// incompatible sql mode, thus use "GROUP BY network_id, ip" to avoid
	// this failure.
	row = tx.Model(Endpoint{}).Where(filter+"AND in_use = 0", hostId, tenantId, segId).Select(sel).Group("network_id, ip").Row()
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 5: %v", err)
		tx.Rollback()
		return err
	}
	netID = sql.NullInt64{}
	var ip string
	row.Scan(&netID, &ip)
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 6: %v", err)
		tx.Rollback()
		return err
	}
	if netID.Valid {
		endpoint.Ip = ip
		tx = tx.Model(Endpoint{}).Where("ip = ?", ip).Update("in_use", true)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 7: %v", err)
			tx.Rollback()
			return err
		}
		tx.Commit()
		return nil
	}
	tx.Rollback()
	return common.NewError("Out of IP addresses.")

}

// getEffectiveNetworkID gets effective number of an Endpoint
// on a given host (see endpoint.EffectiveNetworkID).
func getEffectiveNetworkID(EndpointNetworkID uint64, stride uint) uint64 {
	var effectiveEndpointNetworkID uint64
	// We start with 3 because we reserve 1 for gateway
	// and 2 for DHCP.
	effectiveEndpointNetworkID = 3 + (1<<stride)*EndpointNetworkID
	return effectiveEndpointNetworkID
}

// Entities implements Entities method of Service interface.
func (ipamStore *ipamStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = &Endpoint{}
	return retval
}

// CreateSchemaPostProcess implements CreateSchemaPostProcess method of
// Service interface.
func (ipamStore *ipamStore) CreateSchemaPostProcess() error {
	db := ipamStore.Db
	log.Printf("ipamStore.CreateSchemaPostProcess(), DB is %v", db)
	db.Model(&Endpoint{}).AddUniqueIndex("idx_tenant_segment_host_network_id", "tenant_id", "segment_id", "host_id", "network_id")
	db.Model(&Endpoint{}).AddUniqueIndex("idx_ip", "ip")
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>group by should not include the field being aggregated<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ipam

import (
	"database/sql"
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"log"
	"strings"
)

// Endpoint represents an endpoint (a VM, a Kubernetes Pod, etc.)
// that is to get an IP address.
type Endpoint struct {
	Ip           string         `json:"ip,omitempty"`
	TenantID     string         `json:"tenant_id,omitempty"`
	SegmentID    string         `json:"segment_id,omitempty"`
	HostId       string         `json:"host_id,omitempty"`
	Name         string         `json:"name,omitempty"`
	RequestToken sql.NullString `json:"request_token" sql:"unique"`
	// Ordinal number of this Endpoint in the host/tenant combination
	NetworkID uint64 `json:"-"`
	// Calculated effective network ID of this Endpoint --
	// taking into account stride (endpoint space bits)
	// and alignment thereof. This is used in IP calculation.
	EffectiveNetworkID uint64 `json:"-"`
	// Whether it is in use (for purposes of reclaiming)
	InUse bool   `json:"-"`
	Id    uint64 `sql:"AUTO_INCREMENT",json:"-"`
}
type ipamStore struct {
	common.DbStore
}

// deleteEndpoint releases the IP(s) owned by the endpoint into assignable
// pool.
func (ipamStore *ipamStore) deleteEndpoint(ip string) (Endpoint, error) {
	tx := ipamStore.DbStore.Db.Begin()
	results := make([]Endpoint, 0)
	tx.Where(&Endpoint{Ip: ip}).Find(&results)
	if len(results) == 0 {
		tx.Rollback()
		return Endpoint{}, common.NewError404("endpoint", ip)
	}
	if len(results) > 1 {
		// This cannot happen by constraints...
		tx.Rollback()
		errMsg := fmt.Sprintf("Expected one result for ip %s, got %v", ip, results)
		log.Printf(errMsg)
		return Endpoint{}, common.NewError500(errors.New(errMsg))
	}
	tx = tx.Model(Endpoint{}).Where("ip = ?", ip).Update("in_use", false)
	err := common.MakeMultiError(tx.GetErrors())
	if err != nil {
		tx.Rollback()
		return Endpoint{}, err
	}
	tx.Commit()
	return results[0], nil
}

// addEndpoint allocates an IP address and stores it in the
// database.
func (ipamStore *ipamStore) addEndpoint(endpoint *Endpoint, upToEndpointIpInt uint64, dc common.Datacenter) error {

	var err error
	tx := ipamStore.DbStore.Db.Begin()

	if endpoint.RequestToken.Valid && endpoint.RequestToken.String != "" {
		var existingEndpoints []Endpoint
		var count int
		tx.Where("request_token = ?", endpoint.RequestToken.String).Find(&existingEndpoints).Count(&count)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 1: %v", err)
			tx.Rollback()
			return err
		}
		if count > 0 {
			// This will only be 1, because of unique constraint.
			tx.Rollback()
			log.Printf("Found existing %s: %+v", endpoint.RequestToken.String, existingEndpoints[0])
			endpoint.EffectiveNetworkID = existingEndpoints[0].EffectiveNetworkID
			endpoint.HostId = existingEndpoints[0].HostId
			endpoint.Id = existingEndpoints[0].Id
			endpoint.InUse = existingEndpoints[0].InUse
			endpoint.Name = existingEndpoints[0].Name
			endpoint.NetworkID = existingEndpoints[0].NetworkID
			endpoint.RequestToken = existingEndpoints[0].RequestToken
			endpoint.SegmentID = existingEndpoints[0].SegmentID
			endpoint.TenantID = existingEndpoints[0].TenantID
			return nil
		}
	}

	hostId := endpoint.HostId
	endpoint.InUse = true
	tenantId := endpoint.TenantID
	segId := endpoint.SegmentID
	filter := "host_id = ? AND tenant_id = ? AND segment_id = ? "

	var sel string
	// First, find the MAX network ID available for this host/segment combination.
	sel = "IFNULL(MAX(network_id),-1)+1"
	log.Printf("IpamStore: Calling SELECT %s FROM endpoints WHERE %s;", sel, fmt.Sprintf(strings.Replace(filter, "?", "%s", 3), hostId, tenantId, segId))
	row := tx.Model(Endpoint{}).Where(filter, hostId, tenantId, segId).Select(sel).Row()
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 2: %v", err)
		tx.Rollback()
		return err
	}

	netID := sql.NullInt64{}
	row.Scan(&netID)
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 3: %v", err)
		tx.Rollback()
		return err
	}

	log.Printf("IpamStore: max net ID: %v", netID)

	maxEffNetID := uint64(1<<(dc.EndpointSpaceBits+dc.EndpointBits) - 1)

	// Does this exceed max bits?
	endpoint.NetworkID = uint64(netID.Int64)
	endpoint.EffectiveNetworkID = getEffectiveNetworkID(endpoint.NetworkID, dc.EndpointSpaceBits)
	if endpoint.EffectiveNetworkID <= maxEffNetID {
		// Does not exceed max bits, all good.
		//		log.Printf("IpamStore: Effective network ID for network ID %d (stride %d): %d\n", endpoint.NetworkID, dc.EndpointSpaceBits, endpoint.EffectiveNetworkID)
		ipInt := upToEndpointIpInt | endpoint.EffectiveNetworkID
		//		log.Printf("IpamStore: %d | %d = %d", upToEndpointIpInt, endpoint.EffectiveNetworkID, ipInt)
		endpoint.Ip = common.IntToIPv4(ipInt).String()
		tx = tx.Create(endpoint)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 4: %v", err)
			tx.Rollback()
			return err
		}
		tx.Commit()
		return nil
	}

	// Out of bits, see if we can reuse an earlier allocated address...
	log.Printf("IpamStore: New effective network ID is %d, exceeds maximum %d\n", endpoint.EffectiveNetworkID, maxEffNetID)
	// See if there is a formerly allocated IP already that has been released
	// (marked "in_use")
	sel = "MIN(network_id), ip"
	log.Printf("IpamStore: Calling SELECT %s FROM endpoints WHERE %s;", sel, fmt.Sprintf(strings.Replace(filter+"AND in_use = 0", "?", "%s", 3), hostId, tenantId, segId))
	// In containerized setup, not using group by leads to failure due to
	// incompatible sql mode, thus use "GROUP BY network_id, ip" to avoid
	// this failure.
	row = tx.Model(Endpoint{}).Where(filter+"AND in_use = 0", hostId, tenantId, segId).Select(sel).Group("ip").Row()
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 5: %v", err)
		tx.Rollback()
		return err
	}
	netID = sql.NullInt64{}
	var ip string
	row.Scan(&netID, &ip)
	err = common.GetDbErrors(tx)
	if err != nil {
		log.Printf("IPAM Errors 6: %v", err)
		tx.Rollback()
		return err
	}
	if netID.Valid {
		endpoint.Ip = ip
		tx = tx.Model(Endpoint{}).Where("ip = ?", ip).Update("in_use", true)
		err = common.GetDbErrors(tx)
		if err != nil {
			log.Printf("IPAM Errors 7: %v", err)
			tx.Rollback()
			return err
		}
		tx.Commit()
		return nil
	}
	tx.Rollback()
	return common.NewError("Out of IP addresses.")

}

// getEffectiveNetworkID gets effective number of an Endpoint
// on a given host (see endpoint.EffectiveNetworkID).
func getEffectiveNetworkID(EndpointNetworkID uint64, stride uint) uint64 {
	var effectiveEndpointNetworkID uint64
	// We start with 3 because we reserve 1 for gateway
	// and 2 for DHCP.
	effectiveEndpointNetworkID = 3 + (1<<stride)*EndpointNetworkID
	return effectiveEndpointNetworkID
}

// Entities implements Entities method of Service interface.
func (ipamStore *ipamStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = &Endpoint{}
	return retval
}

// CreateSchemaPostProcess implements CreateSchemaPostProcess method of
// Service interface.
func (ipamStore *ipamStore) CreateSchemaPostProcess() error {
	db := ipamStore.Db
	log.Printf("ipamStore.CreateSchemaPostProcess(), DB is %v", db)
	db.Model(&Endpoint{}).AddUniqueIndex("idx_tenant_segment_host_network_id", "tenant_id", "segment_id", "host_id", "network_id")
	db.Model(&Endpoint{}).AddUniqueIndex("idx_ip", "ip")
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>