<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("throttle")))
	ExpectThat(err, Error(HasSubstr("cancel")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			AssertEq(&buf[2], &p[0])
			err = errors.New("")
		}

		return
	}

	// Call
	t.reader.Read(buf)

	ExpectEq(2, callCount)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsError() {
	// Wrapped
	var callCount int
	expectedErr := errors.New("taco")

	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = 11
			err = expectedErr
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(2+11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsEOF() {
	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = 11
			err = io.EOF
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(2+11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSucceedsInFull() {
	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = len(p)
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(16, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	buf := make([]byte, 2048)
	AssertGt(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(&buf[0], &p[0])
		ExpectEq(t.throttle.Capacity(), len(p))

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}
<commit_msg>Fixed a test bug.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("Cancelled")))
	ExpectThat(err, Error(HasSubstr("throttle")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			AssertEq(&buf[2], &p[0])
			err = errors.New("")
		}

		return
	}

	// Call
	t.reader.Read(buf)

	ExpectEq(2, callCount)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsError() {
	// Wrapped
	var callCount int
	expectedErr := errors.New("taco")

	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = 11
			err = expectedErr
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(2+11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondReturnsEOF() {
	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = 11
			err = io.EOF
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(2+11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSucceedsInFull() {
	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			n = len(p)
		}

		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(16, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	buf := make([]byte, 2048)
	AssertGt(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(&buf[0], &p[0])
		ExpectEq(t.throttle.Capacity(), len(p))

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}
<|endoftext|>


<commit_before>package tui

import (
	"bytes"
	"image"
	"testing"
)

func TestMask_Full(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	var want string

	want = `
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_Inset(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(1, 1, 9, 9), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	var want string

	want = `
..........
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
..........
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_FirstCell(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(0, 0, 1, 1), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	var want string

	want = `
█.........
..........
..........
..........
..........
..........
..........
..........
..........
..........
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_LastCell(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(9, 9, 10, 10), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	var want string

	want = `
..........
..........
..........
..........
..........
..........
..........
..........
..........
.........█
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

type testCell struct {
	Rune  rune
	Style Style
}

type testSurface struct {
	cells   map[image.Point]testCell
	cursor  image.Point
	size    image.Point
	emptyCh rune
}

func newTestSurface(w, h int) *testSurface {
	return &testSurface{
		cells:   make(map[image.Point]testCell),
		size:    image.Point{w, h},
		emptyCh: '.',
	}
}

func (s *testSurface) SetCell(x, y int, ch rune, style Style) {
	s.cells[image.Point{x, y}] = testCell{
		Rune:  ch,
		Style: style,
	}
}

func (s *testSurface) SetCursor(x, y int) {
	s.cursor = image.Point{x, y}
}

func (s *testSurface) HideCursor() {
	s.cursor = image.Point{}
}

func (s *testSurface) Begin() {
	s.cells = make(map[image.Point]testCell)
}

func (s *testSurface) End() {
	// NOP
}

func (s *testSurface) Size() image.Point {
	return s.size
}

func (s *testSurface) String() string {
	var buf bytes.Buffer
	buf.WriteRune('\n')
	for j := 0; j < s.size.Y; j++ {
		for i := 0; i < s.size.X; i++ {
			if cell, ok := s.cells[image.Point{i, j}]; ok {
				buf.WriteRune(cell.Rune)
			} else {
				buf.WriteRune(s.emptyCh)
			}
		}
		buf.WriteRune('\n')
	}
	return buf.String()
}
<commit_msg>Fix linter issues<commit_after>package tui

import (
	"bytes"
	"image"
	"testing"
)

func TestMask_Full(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	want := `
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
██████████
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_Inset(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(1, 1, 9, 9), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	want := `
..........
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
.████████.
..........
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_FirstCell(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(0, 0, 1, 1), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	want := `
█.........
..........
..........
..........
..........
..........
..........
..........
..........
..........
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

func TestMask_LastCell(t *testing.T) {
	surface := newTestSurface(10, 10)

	p := Painter{surface: surface}

	p.WithMask(image.Rect(0, 0, 10, 10), func(p *Painter) {
		p.WithMask(image.Rect(9, 9, 10, 10), func(p *Painter) {
			sz := p.surface.Size()
			for x := 0; x < sz.X; x++ {
				for y := 0; y < sz.Y; y++ {
					p.DrawRune(x, y, '█')
				}
			}
		})
	})

	want := `
..........
..........
..........
..........
..........
..........
..........
..........
..........
.........█
`
	if surface.String() != want {
		t.Errorf("got = \n%s\n\nwant = \n%s", surface.String(), want)
	}
}

type testCell struct {
	Rune  rune
	Style Style
}

type testSurface struct {
	cells   map[image.Point]testCell
	cursor  image.Point
	size    image.Point
	emptyCh rune
}

func newTestSurface(w, h int) *testSurface {
	return &testSurface{
		cells:   make(map[image.Point]testCell),
		size:    image.Point{w, h},
		emptyCh: '.',
	}
}

func (s *testSurface) SetCell(x, y int, ch rune, style Style) {
	s.cells[image.Point{x, y}] = testCell{
		Rune:  ch,
		Style: style,
	}
}

func (s *testSurface) SetCursor(x, y int) {
	s.cursor = image.Point{x, y}
}

func (s *testSurface) HideCursor() {
	s.cursor = image.Point{}
}

func (s *testSurface) Begin() {
	s.cells = make(map[image.Point]testCell)
}

func (s *testSurface) End() {
	// NOP
}

func (s *testSurface) Size() image.Point {
	return s.size
}

func (s *testSurface) String() string {
	var buf bytes.Buffer
	buf.WriteRune('\n')
	for j := 0; j < s.size.Y; j++ {
		for i := 0; i < s.size.X; i++ {
			if cell, ok := s.cells[image.Point{i, j}]; ok {
				buf.WriteRune(cell.Rune)
			} else {
				buf.WriteRune(s.emptyCh)
			}
		}
		buf.WriteRune('\n')
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package godbg

import (
	"fmt"
	"os"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestProject(t *testing.T) {
	Convey("Test buffers", t, func() {

		Convey("By Default, equals to std", func() {
			So(Out(), ShouldEqual, os.Stdout)
			So(Err(), ShouldEqual, os.Stderr)
		})
		Convey("When set to buffer, no longer equals to std", func() {
			SetBuffers(nil)
			So(Out(), ShouldNotEqual, os.Stdout)
			So(Err(), ShouldNotEqual, os.Stderr)
		})
		Convey("By Default, a new pdbg instance buffer equals to std", func() {
			apdbg := NewPdbg()
			So(apdbg.Out(), ShouldEqual, os.Stdout)
			So(apdbg.Err(), ShouldEqual, os.Stderr)
		})
		Convey("By Default, a new pdbg instance set to buffer writes no longer equals to std", func() {
			apdbg := NewPdbg(SetBuffers)
			So(apdbg.Out(), ShouldNotEqual, os.Stdout)
			So(apdbg.Err(), ShouldNotEqual, os.Stderr)
		})
		Convey("Test custom buffer on global pdbg", func() {
			pdbg.bout = nil
			pdbg.sout = nil
			pdbg.berr = nil
			pdbg.serr = nil
			fmt.Fprintln(Out(), "test0 content0")
			So(OutString(), ShouldEqual, ``)
			fmt.Fprintln(Err(), "err0 content0")
			So(ErrString(), ShouldEqual, ``)
			SetBuffers(nil)
			fmt.Fprintln(Out(), "test content")
			fmt.Fprintln(Err(), "err1 cerr")
			fmt.Fprintln(Err(), "err2 cerr2")
			fmt.Fprint(Out(), "test2 content2")
			So(OutString(), ShouldEqual, `test content
test2 content2`)
			So(ErrString(), ShouldEqual, `err1 cerr
err2 cerr2
`)
		})

		Convey("Test custom buffer reset on global pdbg", func() {
			SetBuffers(nil)
			fmt.Fprint(Out(), "test content")
			So(OutString(), ShouldEqual, `test content`)
			fmt.Fprint(Err(), "err1 cerr")
			So(ErrString(), ShouldEqual, `err1 cerr`)
			ResetIOs()
			fmt.Fprint(Out(), "test2 content2")
			So(OutString(), ShouldEqual, `test2 content2`)
			fmt.Fprint(Err(), "err2 cerr2")
			So(ErrString(), ShouldEqual, `err2 cerr2`)
		})

		Convey("Test custom buffer on custom pdbg", func() {
			apdbg := NewPdbg(SetBuffers)
			fmt.Fprintln(apdbg.Out(), "test content")
			fmt.Fprintln(apdbg.Err(), "err1 cerr")
			fmt.Fprintln(apdbg.Err(), "err2 cerr2")
			fmt.Fprint(apdbg.Out(), "test2 content2")
			So(apdbg.OutString(), ShouldEqual, `test content
test2 content2`)
			So(apdbg.ErrString(), ShouldEqual, `err1 cerr
err2 cerr2
`)
		})
		Convey("Test custom buffer reset on custom pdbg", func() {
			apdbg := NewPdbg(SetBuffers)
			fmt.Fprint(apdbg.Out(), "test content")
			So(apdbg.OutString(), ShouldEqual, `test content`)
			fmt.Fprint(apdbg.Err(), "err1 cerr")
			So(apdbg.ErrString(), ShouldEqual, `err1 cerr`)
			apdbg.ResetIOs()
			fmt.Fprint(apdbg.Out(), "test2 content2")
			So(apdbg.OutString(), ShouldEqual, `test2 content2`)
			fmt.Fprint(apdbg.Err(), "err2 cerr2")
			So(apdbg.ErrString(), ShouldEqual, `err2 cerr2`)
		})
	})

	Convey("Test pdbg print functions", t, func() {
		Convey("Test pdbg print with global instance", func() {
			SetBuffers(nil)
			Pdbgf("test")
			So(ErrString(), ShouldEqual,
				`[func.011:95]
  test
`)
			ResetIOs()
			prbgtest()
			So(ErrString(), ShouldEqual,
				` [prbgtest:4] (func.011:101)
   prbgtest content
`)
		})

		Convey("Test pdbg print with custom instance", func() {
			apdbg := NewPdbg(SetBuffers)
			apdbg.Pdbgf("test2")
			So(apdbg.ErrString(), ShouldEqual,
				`[func.012:110]
  test2
`)
			apdbg.ResetIOs()
			prbgtestCustom(apdbg)
			So(apdbg.ErrString(), ShouldEqual,
				` [prbgtestCustom:8] (func.012:116)
   prbgtest content2
`)
			apdbg.ResetIOs()
			apdbg.pdbgTestInstance()
			So(apdbg.ErrString(), ShouldEqual,
				` [*Pdbg.pdbgTestInstance:12] (func.012:122)
   pdbgTestInstance content3
`)
		})
	})
}
<commit_msg>Test exclude on global method<commit_after>package godbg

import (
	"fmt"
	"os"
	"testing"
	. "github.com/smartystreets/goconvey/convey"
)

func TestProject(t *testing.T) {
	Convey("Test buffers", t, func() {

		Convey("By Default, equals to std", func() {
			So(Out(), ShouldEqual, os.Stdout)
			So(Err(), ShouldEqual, os.Stderr)
		})
		Convey("When set to buffer, no longer equals to std", func() {
			SetBuffers(nil)
			So(Out(), ShouldNotEqual, os.Stdout)
			So(Err(), ShouldNotEqual, os.Stderr)
		})
		Convey("By Default, a new pdbg instance buffer equals to std", func() {
			apdbg := NewPdbg()
			So(apdbg.Out(), ShouldEqual, os.Stdout)
			So(apdbg.Err(), ShouldEqual, os.Stderr)
		})
		Convey("By Default, a new pdbg instance set to buffer writes no longer equals to std", func() {
			apdbg := NewPdbg(SetBuffers)
			So(apdbg.Out(), ShouldNotEqual, os.Stdout)
			So(apdbg.Err(), ShouldNotEqual, os.Stderr)
		})
		Convey("Test custom buffer on global pdbg", func() {
			pdbg.bout = nil
			pdbg.sout = nil
			pdbg.berr = nil
			pdbg.serr = nil
			fmt.Fprintln(Out(), "test0 content0")
			So(OutString(), ShouldEqual, ``)
			fmt.Fprintln(Err(), "err0 content0")
			So(ErrString(), ShouldEqual, ``)
			SetBuffers(nil)
			fmt.Fprintln(Out(), "test content")
			fmt.Fprintln(Err(), "err1 cerr")
			fmt.Fprintln(Err(), "err2 cerr2")
			fmt.Fprint(Out(), "test2 content2")
			So(OutString(), ShouldEqual, `test content
test2 content2`)
			So(ErrString(), ShouldEqual, `err1 cerr
err2 cerr2
`)
		})

		Convey("Test custom buffer reset on global pdbg", func() {
			SetBuffers(nil)
			fmt.Fprint(Out(), "test content")
			So(OutString(), ShouldEqual, `test content`)
			fmt.Fprint(Err(), "err1 cerr")
			So(ErrString(), ShouldEqual, `err1 cerr`)
			ResetIOs()
			fmt.Fprint(Out(), "test2 content2")
			So(OutString(), ShouldEqual, `test2 content2`)
			fmt.Fprint(Err(), "err2 cerr2")
			So(ErrString(), ShouldEqual, `err2 cerr2`)
		})

		Convey("Test custom buffer on custom pdbg", func() {
			apdbg := NewPdbg(SetBuffers)
			fmt.Fprintln(apdbg.Out(), "test content")
			fmt.Fprintln(apdbg.Err(), "err1 cerr")
			fmt.Fprintln(apdbg.Err(), "err2 cerr2")
			fmt.Fprint(apdbg.Out(), "test2 content2")
			So(apdbg.OutString(), ShouldEqual, `test content
test2 content2`)
			So(apdbg.ErrString(), ShouldEqual, `err1 cerr
err2 cerr2
`)
		})
		Convey("Test custom buffer reset on custom pdbg", func() {
			apdbg := NewPdbg(SetBuffers)
			fmt.Fprint(apdbg.Out(), "test content")
			So(apdbg.OutString(), ShouldEqual, `test content`)
			fmt.Fprint(apdbg.Err(), "err1 cerr")
			So(apdbg.ErrString(), ShouldEqual, `err1 cerr`)
			apdbg.ResetIOs()
			fmt.Fprint(apdbg.Out(), "test2 content2")
			So(apdbg.OutString(), ShouldEqual, `test2 content2`)
			fmt.Fprint(apdbg.Err(), "err2 cerr2")
			So(apdbg.ErrString(), ShouldEqual, `err2 cerr2`)
		})
	})

	Convey("Test pdbg print functions", t, func() {
		Convey("Test pdbg print with global instance", func() {
			SetBuffers(nil)
			Pdbgf("test")
			So(ErrString(), ShouldEqual,
				`[func.011:95]
  test
`)
			ResetIOs()
			prbgtest()
			So(ErrString(), ShouldEqual,
				` [prbgtest:4] (func.011:101)
   prbgtest content
`)
		})

		Convey("Test pdbg print with custom instance", func() {
			apdbg := NewPdbg(SetBuffers)
			apdbg.Pdbgf("test2")
			So(apdbg.ErrString(), ShouldEqual,
				`[func.012:110]
  test2
`)
			apdbg.ResetIOs()
			prbgtestCustom(apdbg)
			So(apdbg.ErrString(), ShouldEqual,
				` [prbgtestCustom:8] (func.012:116)
   prbgtest content2
`)
			apdbg.ResetIOs()
			apdbg.pdbgTestInstance()
			So(apdbg.ErrString(), ShouldEqual,
				` [*Pdbg.pdbgTestInstance:12] (func.012:122)
   pdbgTestInstance content3
`)
		})
	})

	Convey("Test pdbg excludes functions", t, func() {
		Convey("Test pdbg exclude with global instance", func() {
			SetBuffers(nil)
			pdbg.SetExcludes([]string{"globalNo"})
			globalPdbgExcludeTest()
			So(ErrString(), ShouldEqual,
				` [globalPdbgExcludeTest:16] (func.014:134)
   calling no
`)
		})
	})
}
<|endoftext|>


<commit_before>package repository

import (
	"fmt"
	"io"
	"os"
	"path"

	"github.com/hoffie/larasync/helpers/atomic"
)

// FileContentStorage is the basic implementation of the ContentStorage
// implementation which stores the data into the file system.
type FileContentStorage struct {
	StoragePath string
}

// newFileContentStorage generates a file content storage with the
// given path.
func newFileContentStorage(path string) *FileContentStorage {
	return &FileContentStorage{
		StoragePath: path,
	}
}

// CreateDir ensures that the file blob storage directory exists.
func (f *FileContentStorage) CreateDir() error {
	err := os.Mkdir(f.StoragePath, defaultDirPerms)

	if err != nil && !os.IsExist(err) {
		return err
	}
	return nil
}

// storagePathFor returns the storage path for the data entry.
func (f *FileContentStorage) storagePathFor(contentID string) string {
	return path.Join(f.StoragePath, contentID)
}

// tempStoragePathFor returns the storage path which should be used to buffer
// data before it should be finalized.
func (f *FileContentStorage) tempStoragePathFor(contentID string) string {
	storagePath := f.storagePathFor(contentID)
	return fmt.Sprintf("%s.tmp", storagePath)
}

// Get returns the file handle for the given contentID.
// If there is no data stored for the Id it should return a
// os.ErrNotExists error.
func (f *FileContentStorage) Get(contentID string) (io.ReadCloser, error) {
	if f.Exists(contentID) {
		return os.Open(f.storagePathFor(contentID))
	}
	return nil, os.ErrNotExist
}

// Set sets the data of the given contentID in the blob storage.
func (f *FileContentStorage) Set(contentID string, reader io.Reader) error {
	blobStoragePath := f.storagePathFor(contentID)

	writer, err := atomic.NewStandardWriter(blobStoragePath, defaultFilePerms)
	if err != nil {
		return err
	}

	_, err = io.Copy(writer, reader)
	if err != nil {
		writer.Abort()
		writer.Close()
		return err
	}

	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

// Exists checks if the given entry is stored in the database.
func (f *FileContentStorage) Exists(contentID string) bool {
	_, err := os.Stat(f.storagePathFor(contentID))
	if err != nil {
		return !os.IsNotExist(err)
	}
	return true
}

// Delete removes the data with the given contentID from the store.
func (f *FileContentStorage) Delete(contentID string) error {
	return os.Remove(f.storagePathFor(contentID))
}
<commit_msg>repository: removed not used tmpStoragePathFor in the FileContentStorage struct implementation.<commit_after>package repository

import (
	"fmt"
	"io"
	"os"
	"path"

	"github.com/hoffie/larasync/helpers/atomic"
)

// FileContentStorage is the basic implementation of the ContentStorage
// implementation which stores the data into the file system.
type FileContentStorage struct {
	StoragePath string
}

// newFileContentStorage generates a file content storage with the
// given path.
func newFileContentStorage(path string) *FileContentStorage {
	return &FileContentStorage{
		StoragePath: path,
	}
}

// CreateDir ensures that the file blob storage directory exists.
func (f *FileContentStorage) CreateDir() error {
	err := os.Mkdir(f.StoragePath, defaultDirPerms)

	if err != nil && !os.IsExist(err) {
		return err
	}
	return nil
}

// storagePathFor returns the storage path for the data entry.
func (f *FileContentStorage) storagePathFor(contentID string) string {
	return path.Join(f.StoragePath, contentID)
}

// Get returns the file handle for the given contentID.
// If there is no data stored for the Id it should return a
// os.ErrNotExists error.
func (f *FileContentStorage) Get(contentID string) (io.ReadCloser, error) {
	if f.Exists(contentID) {
		return os.Open(f.storagePathFor(contentID))
	}
	return nil, os.ErrNotExist
}

// Set sets the data of the given contentID in the blob storage.
func (f *FileContentStorage) Set(contentID string, reader io.Reader) error {
	blobStoragePath := f.storagePathFor(contentID)

	writer, err := atomic.NewStandardWriter(blobStoragePath, defaultFilePerms)
	if err != nil {
		return err
	}

	_, err = io.Copy(writer, reader)
	if err != nil {
		writer.Abort()
		writer.Close()
		return err
	}

	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

// Exists checks if the given entry is stored in the database.
func (f *FileContentStorage) Exists(contentID string) bool {
	_, err := os.Stat(f.storagePathFor(contentID))
	if err != nil {
		return !os.IsNotExist(err)
	}
	return true
}

// Delete removes the data with the given contentID from the store.
func (f *FileContentStorage) Delete(contentID string) error {
	return os.Remove(f.storagePathFor(contentID))
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"net"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEip() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEipCreate,
		Read:   resourceAwsEipRead,
		Update: resourceAwsEipUpdate,
		Delete: resourceAwsEipDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"vpc": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"instance": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"network_interface": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"allocation_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"association_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"public_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"private_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"associate_with_private_ip": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsEipCreate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	// By default, we're not in a VPC
	domainOpt := ""
	if v := d.Get("vpc"); v != nil && v.(bool) {
		domainOpt = "vpc"
	}

	allocOpts := &ec2.AllocateAddressInput{
		Domain: aws.String(domainOpt),
	}

	log.Printf("[DEBUG] EIP create configuration: %#v", allocOpts)
	allocResp, err := ec2conn.AllocateAddress(allocOpts)
	if err != nil {
		return fmt.Errorf("Error creating EIP: %s", err)
	}

	// The domain tells us if we're in a VPC or not
	d.Set("domain", allocResp.Domain)

	// Assign the eips (unique) allocation id for use later
	// the EIP api has a conditional unique ID (really), so
	// if we're in a VPC we need to save the ID as such, otherwise
	// it defaults to using the public IP
	log.Printf("[DEBUG] EIP Allocate: %#v", allocResp)
	if d.Get("domain").(string) == "vpc" {
		d.SetId(*allocResp.AllocationId)
	} else {
		d.SetId(*allocResp.PublicIp)
	}

	log.Printf("[INFO] EIP ID: %s (domain: %v)", d.Id(), *allocResp.Domain)
	return resourceAwsEipUpdate(d, meta)
}

func resourceAwsEipRead(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	domain := resourceAwsEipDomain(d)
	id := d.Id()

	req := &ec2.DescribeAddressesInput{}

	if domain == "vpc" {
		req.AllocationIds = []*string{aws.String(id)}
	} else {
		req.PublicIps = []*string{aws.String(id)}
	}

	log.Printf(
		"[DEBUG] EIP describe configuration: %s (domain: %s)",
		req, domain)

	describeAddresses, err := ec2conn.DescribeAddresses(req)
	if err != nil {
		if ec2err, ok := err.(awserr.Error); ok && (ec2err.Code() == "InvalidAllocationID.NotFound" || ec2err.Code() == "InvalidAddress.NotFound") {
			d.SetId("")
			return nil
		}

		return fmt.Errorf("Error retrieving EIP: %s", err)
	}

	// Verify AWS returned our EIP
	if len(describeAddresses.Addresses) != 1 ||
		domain == "vpc" && *describeAddresses.Addresses[0].AllocationId != id ||
		*describeAddresses.Addresses[0].PublicIp != id {
		if err != nil {
			return fmt.Errorf("Unable to find EIP: %#v", describeAddresses.Addresses)
		}
	}

	address := describeAddresses.Addresses[0]

	d.Set("association_id", address.AssociationId)
	if address.InstanceId != nil {
		d.Set("instance", address.InstanceId)
	} else {
		d.Set("instance", "")
	}
	if address.NetworkInterfaceId != nil {
		d.Set("network_interface", address.NetworkInterfaceId)
	} else {
		d.Set("network_interface", "")
	}
	d.Set("private_ip", address.PrivateIpAddress)
	d.Set("public_ip", address.PublicIp)

	// On import (domain never set, which it must've been if we created),
	// set the 'vpc' attribute depending on if we're in a VPC.
	if address.Domain != nil {
		d.Set("vpc", *address.Domain == "vpc")
	}

	d.Set("domain", address.Domain)

	// Force ID to be an Allocation ID if we're on a VPC
	// This allows users to import the EIP based on the IP if they are in a VPC
	if *address.Domain == "vpc" && net.ParseIP(id) != nil {
		log.Printf("[DEBUG] Re-assigning EIP ID (%s) to it's Allocation ID (%s)", d.Id(), *address.AllocationId)
		d.SetId(*address.AllocationId)
	}

	return nil
}

func resourceAwsEipUpdate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	domain := resourceAwsEipDomain(d)

	// Associate to instance or interface if specified
	v_instance, ok_instance := d.GetOk("instance")
	v_interface, ok_interface := d.GetOk("network_interface")

	// If we are updating an EIP that is not newly created, and we are attached to
	// an instance or interface, detach first.
	if (d.Get("instance").(string) != "" || d.Get("association_id").(string) != "") && !d.IsNewResource() {
		if err := disassociateEip(d, meta); err != nil {
			return err
		}
	}

	if ok_instance || ok_interface {
		instanceId := v_instance.(string)
		networkInterfaceId := v_interface.(string)

		assocOpts := &ec2.AssociateAddressInput{
			InstanceId: aws.String(instanceId),
			PublicIp:   aws.String(d.Id()),
		}

		// more unique ID conditionals
		if domain == "vpc" {
			var privateIpAddress *string
			if v := d.Get("associate_with_private_ip").(string); v != "" {
				privateIpAddress = aws.String(v)
			}
			assocOpts = &ec2.AssociateAddressInput{
				NetworkInterfaceId: aws.String(networkInterfaceId),
				InstanceId:         aws.String(instanceId),
				AllocationId:       aws.String(d.Id()),
				PrivateIpAddress:   privateIpAddress,
			}
		}

		log.Printf("[DEBUG] EIP associate configuration: %s (domain: %s)", assocOpts, domain)

		err := resource.Retry(5*time.Minute, func() *resource.RetryError {
			_, err := ec2conn.AssociateAddress(assocOpts)
			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok {
					if awsErr.Code() == "InvalidAllocationID.NotFound" {
						return resource.RetryableError(awsErr)
					}
				}
				return resource.NonRetryableError(err)
			}
			return nil
		})
		if err != nil {
			// Prevent saving instance if association failed
			// e.g. missing internet gateway in VPC
			d.Set("instance", "")
			d.Set("network_interface", "")
			return fmt.Errorf("Failure associating EIP: %s", err)
		}
	}

	return resourceAwsEipRead(d, meta)
}

func resourceAwsEipDelete(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	if err := resourceAwsEipRead(d, meta); err != nil {
		return err
	}
	if d.Id() == "" {
		// This might happen from the read
		return nil
	}

	// If we are attached to an instance or interface, detach first.
	if d.Get("instance").(string) != "" || d.Get("association_id").(string) != "" {
		if err := disassociateEip(d, meta); err != nil {
			return err
		}
	}

	domain := resourceAwsEipDomain(d)
	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		var err error
		switch domain {
		case "vpc":
			log.Printf(
				"[DEBUG] EIP release (destroy) address allocation: %v",
				d.Id())
			_, err = ec2conn.ReleaseAddress(&ec2.ReleaseAddressInput{
				AllocationId: aws.String(d.Id()),
			})
		case "standard":
			log.Printf("[DEBUG] EIP release (destroy) address: %v", d.Id())
			_, err = ec2conn.ReleaseAddress(&ec2.ReleaseAddressInput{
				PublicIp: aws.String(d.Id()),
			})
		}

		if err == nil {
			return nil
		}
		if _, ok := err.(awserr.Error); !ok {
			return resource.NonRetryableError(err)
		}

		return resource.RetryableError(err)
	})
}

func resourceAwsEipDomain(d *schema.ResourceData) string {
	if v, ok := d.GetOk("domain"); ok {
		return v.(string)
	} else if strings.Contains(d.Id(), "eipalloc") {
		// We have to do this for backwards compatibility since TF 0.1
		// didn't have the "domain" computed attribute.
		return "vpc"
	}

	return "standard"
}

func disassociateEip(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn
	log.Printf("[DEBUG] Disassociating EIP: %s", d.Id())
	var err error
	switch resourceAwsEipDomain(d) {
	case "vpc":
		_, err = ec2conn.DisassociateAddress(&ec2.DisassociateAddressInput{
			AssociationId: aws.String(d.Get("association_id").(string)),
		})
	case "standard":
		_, err = ec2conn.DisassociateAddress(&ec2.DisassociateAddressInput{
			PublicIp: aws.String(d.Get("public_ip").(string)),
		})
	}

	// First check if the association ID is not found. If this
	// is the case, then it was already disassociated somehow,
	// and that is okay. The most commmon reason for this is that
	// the instance or ENI it was attached it was destroyed.
	if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidAssociationID.NotFound" {
		err = nil
	}
	return err
}
<commit_msg>r/resource_aws_eip: implement retry reading EIPs<commit_after>package aws

import (
	"fmt"
	"log"
	"net"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsEip() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsEipCreate,
		Read:   resourceAwsEipRead,
		Update: resourceAwsEipUpdate,
		Delete: resourceAwsEipDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"vpc": &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"instance": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"network_interface": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"allocation_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"association_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"public_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"private_ip": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"associate_with_private_ip": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
		},
	}
}

func resourceAwsEipCreate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	// By default, we're not in a VPC
	domainOpt := ""
	if v := d.Get("vpc"); v != nil && v.(bool) {
		domainOpt = "vpc"
	}

	allocOpts := &ec2.AllocateAddressInput{
		Domain: aws.String(domainOpt),
	}

	log.Printf("[DEBUG] EIP create configuration: %#v", allocOpts)
	allocResp, err := ec2conn.AllocateAddress(allocOpts)
	if err != nil {
		return fmt.Errorf("Error creating EIP: %s", err)
	}

	// The domain tells us if we're in a VPC or not
	d.Set("domain", allocResp.Domain)

	// Assign the eips (unique) allocation id for use later
	// the EIP api has a conditional unique ID (really), so
	// if we're in a VPC we need to save the ID as such, otherwise
	// it defaults to using the public IP
	log.Printf("[DEBUG] EIP Allocate: %#v", allocResp)
	if d.Get("domain").(string) == "vpc" {
		d.SetId(*allocResp.AllocationId)
	} else {
		d.SetId(*allocResp.PublicIp)
	}

	log.Printf("[INFO] EIP ID: %s (domain: %v)", d.Id(), *allocResp.Domain)
	return resourceAwsEipUpdate(d, meta)
}

func resourceAwsEipRead(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	domain := resourceAwsEipDomain(d)
	id := d.Id()

	req := &ec2.DescribeAddressesInput{}

	if domain == "vpc" {
		req.AllocationIds = []*string{aws.String(id)}
	} else {
		req.PublicIps = []*string{aws.String(id)}
	}

	log.Printf(
		"[DEBUG] EIP describe configuration: %s (domain: %s)",
		req, domain)

	var err error
	var describeAddresses *ec2.DescribeAddressesOutput

	if d.IsNewResource() {
		err := resource.Retry(15*time.Minute, func() *resource.RetryError {
			describeAddresses, err = ec2conn.DescribeAddresses(req)
			if err != nil {
				awsErr, ok := err.(awserr.Error)
				if ok && (awsErr.Code() == "InvalidAllocationID.NotFound" ||
					awsErr.Code() == "InvalidAddress.NotFound") {
					return resource.RetryableError(err)
				}

				return resource.NonRetryableError(err)
			}
			return nil
		})
		if err != nil {
			return fmt.Errorf("Error retrieving EIP: %s", err)
		}
	} else {
		describeAddresses, err = ec2conn.DescribeAddresses(req)
		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if ok && (awsErr.Code() == "InvalidAllocationID.NotFound" ||
				awsErr.Code() == "InvalidAddress.NotFound") {
				log.Printf("[WARN] EIP not found, removing from state: %s", req)
				return nil
			}
			return err
		}
	}

	// Verify AWS returned our EIP
	if len(describeAddresses.Addresses) != 1 ||
		domain == "vpc" && *describeAddresses.Addresses[0].AllocationId != id ||
		*describeAddresses.Addresses[0].PublicIp != id {
		if err != nil {
			return fmt.Errorf("Unable to find EIP: %#v", describeAddresses.Addresses)
		}
	}

	address := describeAddresses.Addresses[0]

	d.Set("association_id", address.AssociationId)
	if address.InstanceId != nil {
		d.Set("instance", address.InstanceId)
	} else {
		d.Set("instance", "")
	}
	if address.NetworkInterfaceId != nil {
		d.Set("network_interface", address.NetworkInterfaceId)
	} else {
		d.Set("network_interface", "")
	}
	d.Set("private_ip", address.PrivateIpAddress)
	d.Set("public_ip", address.PublicIp)

	// On import (domain never set, which it must've been if we created),
	// set the 'vpc' attribute depending on if we're in a VPC.
	if address.Domain != nil {
		d.Set("vpc", *address.Domain == "vpc")
	}

	d.Set("domain", address.Domain)

	// Force ID to be an Allocation ID if we're on a VPC
	// This allows users to import the EIP based on the IP if they are in a VPC
	if *address.Domain == "vpc" && net.ParseIP(id) != nil {
		log.Printf("[DEBUG] Re-assigning EIP ID (%s) to it's Allocation ID (%s)", d.Id(), *address.AllocationId)
		d.SetId(*address.AllocationId)
	}

	return nil
}

func resourceAwsEipUpdate(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	domain := resourceAwsEipDomain(d)

	// Associate to instance or interface if specified
	v_instance, ok_instance := d.GetOk("instance")
	v_interface, ok_interface := d.GetOk("network_interface")

	// If we are updating an EIP that is not newly created, and we are attached to
	// an instance or interface, detach first.
	if (d.Get("instance").(string) != "" || d.Get("association_id").(string) != "") && !d.IsNewResource() {
		if err := disassociateEip(d, meta); err != nil {
			return err
		}
	}

	if ok_instance || ok_interface {
		instanceId := v_instance.(string)
		networkInterfaceId := v_interface.(string)

		assocOpts := &ec2.AssociateAddressInput{
			InstanceId: aws.String(instanceId),
			PublicIp:   aws.String(d.Id()),
		}

		// more unique ID conditionals
		if domain == "vpc" {
			var privateIpAddress *string
			if v := d.Get("associate_with_private_ip").(string); v != "" {
				privateIpAddress = aws.String(v)
			}
			assocOpts = &ec2.AssociateAddressInput{
				NetworkInterfaceId: aws.String(networkInterfaceId),
				InstanceId:         aws.String(instanceId),
				AllocationId:       aws.String(d.Id()),
				PrivateIpAddress:   privateIpAddress,
			}
		}

		log.Printf("[DEBUG] EIP associate configuration: %s (domain: %s)", assocOpts, domain)

		err := resource.Retry(5*time.Minute, func() *resource.RetryError {
			_, err := ec2conn.AssociateAddress(assocOpts)
			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok {
					if awsErr.Code() == "InvalidAllocationID.NotFound" {
						return resource.RetryableError(awsErr)
					}
				}
				return resource.NonRetryableError(err)
			}
			return nil
		})
		if err != nil {
			// Prevent saving instance if association failed
			// e.g. missing internet gateway in VPC
			d.Set("instance", "")
			d.Set("network_interface", "")
			return fmt.Errorf("Failure associating EIP: %s", err)
		}
	}

	return resourceAwsEipRead(d, meta)
}

func resourceAwsEipDelete(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn

	if err := resourceAwsEipRead(d, meta); err != nil {
		return err
	}
	if d.Id() == "" {
		// This might happen from the read
		return nil
	}

	// If we are attached to an instance or interface, detach first.
	if d.Get("instance").(string) != "" || d.Get("association_id").(string) != "" {
		if err := disassociateEip(d, meta); err != nil {
			return err
		}
	}

	domain := resourceAwsEipDomain(d)
	return resource.Retry(3*time.Minute, func() *resource.RetryError {
		var err error
		switch domain {
		case "vpc":
			log.Printf(
				"[DEBUG] EIP release (destroy) address allocation: %v",
				d.Id())
			_, err = ec2conn.ReleaseAddress(&ec2.ReleaseAddressInput{
				AllocationId: aws.String(d.Id()),
			})
		case "standard":
			log.Printf("[DEBUG] EIP release (destroy) address: %v", d.Id())
			_, err = ec2conn.ReleaseAddress(&ec2.ReleaseAddressInput{
				PublicIp: aws.String(d.Id()),
			})
		}

		if err == nil {
			return nil
		}
		if _, ok := err.(awserr.Error); !ok {
			return resource.NonRetryableError(err)
		}

		return resource.RetryableError(err)
	})
}

func resourceAwsEipDomain(d *schema.ResourceData) string {
	if v, ok := d.GetOk("domain"); ok {
		return v.(string)
	} else if strings.Contains(d.Id(), "eipalloc") {
		// We have to do this for backwards compatibility since TF 0.1
		// didn't have the "domain" computed attribute.
		return "vpc"
	}

	return "standard"
}

func disassociateEip(d *schema.ResourceData, meta interface{}) error {
	ec2conn := meta.(*AWSClient).ec2conn
	log.Printf("[DEBUG] Disassociating EIP: %s", d.Id())
	var err error
	switch resourceAwsEipDomain(d) {
	case "vpc":
		_, err = ec2conn.DisassociateAddress(&ec2.DisassociateAddressInput{
			AssociationId: aws.String(d.Get("association_id").(string)),
		})
	case "standard":
		_, err = ec2conn.DisassociateAddress(&ec2.DisassociateAddressInput{
			PublicIp: aws.String(d.Get("public_ip").(string)),
		})
	}

	// First check if the association ID is not found. If this
	// is the case, then it was already disassociated somehow,
	// and that is okay. The most commmon reason for this is that
	// the instance or ENI it was attached it was destroyed.
	if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidAssociationID.NotFound" {
		err = nil
	}
	return err
}
<|endoftext|>


<commit_before>package githubprovider

import (
	"os"
	"testing"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

func TestGithubAddAndRemoveUser(t *testing.T) {
	orgClient := github.NewClient(
		oauth2.NewClient(
			oauth2.NoContext,
			oauth2.StaticTokenSource(
				&oauth2.Token{
					AccessToken: os.Getenv("GITHUB_ORG_ACCESS_TOKEN"),
				},
			),
		),
	)
	userClient := github.NewClient(
		oauth2.NewClient(
			oauth2.NoContext,
			oauth2.StaticTokenSource(
				&oauth2.Token{
					AccessToken: os.Getenv("GITHUB_USER_ACCESS_TOKEN"),
				},
			),
		),
	)

	org := "organizasyon"
	user := "mehmetalikoding"
	teamNames := []string{"team"}
	role := "member"
	active := "active"

	teamIDs, err := GetTeamIDs(orgClient, org, teamNames)

	optAddOrgMembership := &github.OrganizationAddTeamMembershipOptions{
		Role: role,
	}

	for _, teamID := range teamIDs {
		_, _, err := orgClient.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			t.Fatalf("Error while adding user to team: %+v ", err)
		}
	}

	// When you try to add a user to a team twice,
	// it doesn't return any error
	for _, teamID := range teamIDs {
		_, _, err := orgClient.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			t.Fatalf("Error while adding user to team: %+v ", err)
		}
	}

	membership := &github.Membership{
		// state should be active to add the user into organization
		State: &active,

		// Role is the required for the membership
		Role: &role,
	}

	_, _, err = userClient.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		t.Fatalf("Error while adding user to organization: %+v ", err)
	}

	// When you try to add a user to an organization twice,
	// it doesn't return any error
	_, _, err = userClient.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		t.Fatalf("Error while adding user to organization: %+v ", err)
	}

	// You can remove a member from organization
	_, err = orgClient.Organizations.RemoveMember(org, user)
	if err != nil {
		t.Fatalf("Error while removing user from organization and teams of the organization: %+v", err)
	}

}
<commit_msg>tests/github: add length test related with github team name error<commit_after>package githubprovider

import (
	"os"
	"testing"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

func TestGithubAddAndRemoveUser(t *testing.T) {
	orgClient := github.NewClient(
		oauth2.NewClient(
			oauth2.NoContext,
			oauth2.StaticTokenSource(
				&oauth2.Token{
					AccessToken: os.Getenv("GITHUB_ORG_ACCESS_TOKEN"),
				},
			),
		),
	)
	userClient := github.NewClient(
		oauth2.NewClient(
			oauth2.NoContext,
			oauth2.StaticTokenSource(
				&oauth2.Token{
					AccessToken: os.Getenv("GITHUB_USER_ACCESS_TOKEN"),
				},
			),
		),
	)

	org := "organizasyon"
	user := "mehmetalikoding"
	teamNames := []string{"team"}
	role := "member"
	active := "active"

	teamIDs, err := GetTeamIDs(orgClient, org, teamNames)

	if len(teamIDs) != len(teamNames) {
		t.Fatalf("Error team name is not found")
	}

	optAddOrgMembership := &github.OrganizationAddTeamMembershipOptions{
		Role: role,
	}

	for _, teamID := range teamIDs {
		_, _, err := orgClient.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			t.Fatalf("Error while adding user to team: %+v ", err)
		}
	}

	// When you try to add a user to a team twice,
	// it doesn't return any error
	for _, teamID := range teamIDs {
		_, _, err := orgClient.Organizations.AddTeamMembership(teamID, user, optAddOrgMembership)
		if err != nil {
			t.Fatalf("Error while adding user to team: %+v ", err)
		}
	}

	membership := &github.Membership{
		// state should be active to add the user into organization
		State: &active,

		// Role is the required for the membership
		Role: &role,
	}

	_, _, err = userClient.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		t.Fatalf("Error while adding user to organization: %+v ", err)
	}

	// When you try to add a user to an organization twice,
	// it doesn't return any error
	_, _, err = userClient.Organizations.EditOrgMembership("", org, membership)
	if err != nil {
		t.Fatalf("Error while adding user to organization: %+v ", err)
	}

	// You can remove a member from organization
	_, err = orgClient.Organizations.RemoveMember(org, user)
	if err != nil {
		t.Fatalf("Error while removing user from organization and teams of the organization: %+v", err)
	}

}
<|endoftext|>


<commit_before>//// Copyright 2015 Spring Rain Software Compnay LTD. All Rights Reserved.
//// Licensed under the MIT (MIT-LICENSE.txt) license.
package proxy

import (
	"fmt"
	"github.com/wfxiang08/cyutils/utils/atomic2"
	log "github.com/wfxiang08/cyutils/utils/rolling_log"
	topozk "github.com/wfxiang08/go-zookeeper/zk"
	"github.com/wfxiang08/go_thrift/thrift"
	"github.com/wfxiang08/thrift_rpc_base/rpc_utils"
	"os"
	"os/signal"
	"runtime/debug"
	"strings"
	"syscall"
	"time"
)

type Server interface {
	Run()
}
type ServerFactorory func(config *ServiceConfig) Server
type ProxyFactorory func(config *ProxyConfig) Server

//
// Thrift Server的参数
//
type ThriftRpcServer struct {
	ZkAddr          string
	ProductName     string
	ServiceName     string
	FrontendAddr    string
	Topo            *Topology
	Processor       thrift.TProcessor
	Verbose         bool
	lastRequestTime atomic2.Int64
	config          *ServiceConfig
}

func NewThriftRpcServer(config *ServiceConfig, processor thrift.TProcessor) *ThriftRpcServer {
	log.Printf("FrontAddr: %s\n", Magenta(config.FrontendAddr))

	return &ThriftRpcServer{
		config:       config,
		ZkAddr:       config.ZkAddr,
		ProductName:  config.ProductName,
		ServiceName:  config.Service,
		FrontendAddr: config.FrontendAddr,
		Verbose:      config.Verbose,
		Processor:    processor,
	}

}

//
// 根据前端的地址生成服务的id
// 例如: 127.0.0.1:5555 --> 127_0_0_1_5555
// "/Users/feiwang/gowork/src/github.com/wfxiang08/rpc_proxyiplocation/aa.sock"
func GetServiceIdentity(frontendAddr string) string {

	fid := strings.Replace(frontendAddr, ".", "_", -1)
	fid = strings.Replace(fid, ":", "_", -1)
	fid = strings.Replace(fid, "/", "", -1)
	if len(fid) > 20 {
		// unix domain socket
		fid = fid[len(fid)-20 : len(fid)]
	}
	return fid

}

//
// 去ZK注册当前的Service
//
func RegisterService(serviceName, frontendAddr, serviceId string, topo *Topology, evtExit chan interface{},
	workDir string, codeUrlVerion string, state *atomic2.Bool, stateChan chan bool) *ServiceEndpoint {

	// 1. 准备数据
	// 记录Service Endpoint的信息
	servicePath := topo.ProductServicePath(serviceName)

	// 确保东西存在
	if ok, _ := topo.Exist(servicePath); !ok {
		topo.CreateDir(servicePath)
	}

	// 用来从zookeeper获取事件
	evtbus := make(chan interface{})

	// 2. 将信息添加到Zk中, 并且监控Zk的状态(如果添加失败会怎么样?)
	endpoint := NewServiceEndpoint(serviceName, serviceId, frontendAddr, workDir, codeUrlVerion)

	// deployPath

	// 为了保证Add是干净的，需要先删除，保证自己才是Owner
	endpoint.DeleteServiceEndpoint(topo)

	// 如果没有状态，或状态为true, 则上线
	if state == nil || state.Get() {
		endpoint.AddServiceEndpoint(topo)
	}

	go func() {

		for true {
			// Watch的目的是监控: 当前的zk session的状态, 如果session出现异常，则重新注册
			_, err := topo.WatchNode(servicePath, evtbus)

			if err == nil {
				// 如果成功添加Watch, 则等待退出，或者ZK Event
				select {
				case <-evtExit:
					return
				case <-stateChan:
					// 如何状态变化(则重新注册)
					endpoint.DeleteServiceEndpoint(topo)
					if state == nil || state.Get() {
						endpoint.AddServiceEndpoint(topo)
					}
				case e := <-evtbus:
					event := e.(topozk.Event)
					if event.State == topozk.StateExpired ||
						event.Type == topozk.EventNotWatching {
						// Session过期了，则需要删除之前的数据，
						// 因为当前的session不是之前的数据的Owner
						endpoint.DeleteServiceEndpoint(topo)

						if state == nil || state.Get() {
							endpoint.AddServiceEndpoint(topo)
						}

					}
				}

			} else {
				// 如果添加Watch失败，则等待退出，或等待timer接触，进行下一次尝试
				timer := time.NewTimer(time.Second * time.Duration(10))
				log.ErrorErrorf(err, "Reg Failed, Wait 10 seconds...., %v", err)
				select {
				case <-evtExit:
					return
				case <-timer.C:
					// pass
				}
			}

		}
	}()
	return endpoint
}

//
// 后端如何处理一个Request
//
func (p *ThriftRpcServer) Dispatch(r *Request) error {
	transport := NewTMemoryBufferWithBuf(r.Request.Data)
	ip := thrift.NewTBinaryProtocolTransport(transport)

	slice := getSlice(0, DEFAULT_SLICE_LEN)
	transport = NewTMemoryBufferWithBuf(slice)
	op := thrift.NewTBinaryProtocolTransport(transport)
	p.Processor.Process(ip, op)

	r.Response.Data = transport.Bytes()

	_, _, seqId, _ := DecodeThriftTypIdSeqId(r.Response.Data)

	log.Debugf("SeqId: %d vs. %d, Dispatch Over", r.Request.SeqId, seqId)
	//	// 如果transport重新分配了内存，则立即归还slice
	//	if cap(r.Response.Data) != DEFAULT_SLICE_LEN {
	//		returnSlice(slice)
	//	}
	return nil
}

func (p *ThriftRpcServer) Run() {
	//	// 1. 创建到zk的连接
	// 有条件地做服务注册
	registerService := !p.config.StandAlone // 不独立运行则注册服务
	if registerService {
		p.Topo = NewTopology(p.ProductName, p.ZkAddr)
	}

	// 127.0.0.1:5555 --> 127_0_0_1:5555
	lbServiceName := GetServiceIdentity(p.FrontendAddr)

	exitSignal := make(chan os.Signal, 1)
	signal.Notify(exitSignal, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)

	// syscall.SIGKILL
	// kill -9 pid
	// kill -s SIGKILL pid 还是留给运维吧

	if len(p.config.FalconClient) > 0 {
		StartTicker(p.config.FalconClient, p.ServiceName)
	}

	// 初始状态为不上线
	var state atomic2.Bool
	state.Set(false)
	stateChan := make(chan bool)

	// 注册服务
	evtExit := make(chan interface{})

	var endpoint *ServiceEndpoint = nil
	if registerService {
		endpoint = RegisterService(p.ServiceName, p.FrontendAddr, lbServiceName,
			p.Topo, evtExit, p.config.WorkDir, p.config.CodeUrlVersion,
			&state, stateChan)
	}

	// 3. 读取"前端"的配置
	var transport thrift.TServerTransport
	var err error

	isUnixDomain := false
	// 127.0.0.1:9999(以:区分不同的类型)
	if !strings.Contains(p.FrontendAddr, ":") {
		if rpc_utils.FileExist(p.FrontendAddr) {
			os.Remove(p.FrontendAddr)
		}
		transport, err = rpc_utils.NewTServerUnixDomain(p.FrontendAddr)
		isUnixDomain = true
	} else {
		transport, err = thrift.NewTServerSocket(p.FrontendAddr)
	}

	if err != nil {
		log.ErrorErrorf(err, Red("Server Socket Create Failed: %v"), err)
		panic(fmt.Sprintf("Invalid FrontendAddr: %s", p.FrontendAddr))
	}

	err = transport.Listen()
	if err != nil {
		log.ErrorErrorf(err, Red("Server Socket Open Failed: %v"), err)
		panic(fmt.Sprintf("Server Socket Open Failed: %s", p.FrontendAddr))
	}

	ch := make(chan thrift.TTransport, 4096)
	defer close(ch)

	// 强制退出? TODO: Graceful退出
	go func() {
		<-exitSignal
		log.Info(Magenta("Receive Exit Signals...."))

		if registerService {
			evtExit <- true
		}
		if registerService {
			endpoint.DeleteServiceEndpoint(p.Topo)
		} else {
			// 如果没有注册到后端服务器，则直接不再接受新请求
			transport.Interrupt()
		}

		// 等待
		start := time.Now().Unix()
		for true {
			// 如果5s内没有接受到新的请求了，则退出
			now := time.Now().Unix()
			if now-p.lastRequestTime.Get() > 5 {
				log.Info(Red("Graceful Exit..."))
				break
			} else {
				log.Printf(Cyan("Sleeping %d seconds\n"), now-start)
				time.Sleep(time.Second)
			}
		}

		// 确定处理完毕数据之后，关闭transport
		if registerService {
			transport.Interrupt()
		}
		transport.Close()
	}()

	go func() {
		var address string
		for c := range ch {
			// 为每个Connection建立一个Session
			socket, ok := c.(rpc_utils.SocketAddr)

			if ok {
				if isUnixDomain {
					address = p.FrontendAddr
				} else {
					address = socket.Addr().String()
				}
			} else {
				address = "unknow"
			}

			// Session独立处理自己的请求
			if registerService {
				x := NewNonBlockSession(c, address, p.Verbose, &p.lastRequestTime)
				go x.Serve(p, 1000)
			} else {
				go func() {
					// 打印异常信息
					defer func() {
						c.Close()
						if e := recover(); e != nil {
							log.Errorf("panic in processor: %v: %s", e, debug.Stack())
						}
					}()

					ip := thrift.NewTBinaryProtocolTransport(thrift.NewTFramedTransport(c))
					op := thrift.NewTBinaryProtocolTransport(thrift.NewTFramedTransport(c))

					for true {
						ok, err := p.Processor.Process(ip, op)

						if err, ok := err.(thrift.TTransportException); ok && err.TypeId() == thrift.END_OF_FILE {
							return
						} else if err != nil {
							// 如果是网络问题，则直接报错
							if !strings.Contains(err.Error(), "use of closed network connection") {
								log.ErrorErrorf(err, "Error processing request, quit")
							}
							return
						}
						if err, ok := err.(thrift.TApplicationException); ok && err.TypeId() == thrift.UNKNOWN_METHOD {
							log.ErrorErrorf(err, "Error processing request, continue")
							continue
						}
						if !ok {
							log.ErrorErrorf(err, "Process Not OK, stopped")
							break
						}

					}
				}()
			}
		}
	}()

	// 准备上线服务
	state.Set(true)
	if registerService {
		stateChan <- true
	}

	log.Printf("Begin to accept requests...")
	// Accept什么时候出错，出错之后如何处理呢?
	for {
		c, err := transport.Accept()
		if err != nil {
			break
		} else {
			ch <- c
		}
	}
}
<commit_msg>重新整理rpc的异常处理<commit_after>//// Copyright 2015 Spring Rain Software Compnay LTD. All Rights Reserved.
//// Licensed under the MIT (MIT-LICENSE.txt) license.
package proxy

import (
	"fmt"
	"github.com/wfxiang08/cyutils/utils/atomic2"
	log "github.com/wfxiang08/cyutils/utils/rolling_log"
	topozk "github.com/wfxiang08/go-zookeeper/zk"
	"github.com/wfxiang08/go_thrift/thrift"
	"github.com/wfxiang08/thrift_rpc_base/rpc_utils"
	"os"
	"os/signal"
	"runtime/debug"
	"strings"
	"syscall"
	"time"
)

type Server interface {
	Run()
}
type ServerFactorory func(config *ServiceConfig) Server
type ProxyFactorory func(config *ProxyConfig) Server

//
// Thrift Server的参数
//
type ThriftRpcServer struct {
	ZkAddr          string
	ProductName     string
	ServiceName     string
	FrontendAddr    string
	Topo            *Topology
	Processor       thrift.TProcessor
	Verbose         bool
	lastRequestTime atomic2.Int64
	config          *ServiceConfig
}

func NewThriftRpcServer(config *ServiceConfig, processor thrift.TProcessor) *ThriftRpcServer {
	log.Printf("FrontAddr: %s\n", Magenta(config.FrontendAddr))

	return &ThriftRpcServer{
		config:       config,
		ZkAddr:       config.ZkAddr,
		ProductName:  config.ProductName,
		ServiceName:  config.Service,
		FrontendAddr: config.FrontendAddr,
		Verbose:      config.Verbose,
		Processor:    processor,
	}

}

//
// 根据前端的地址生成服务的id
// 例如: 127.0.0.1:5555 --> 127_0_0_1_5555
// "/Users/feiwang/gowork/src/github.com/wfxiang08/rpc_proxyiplocation/aa.sock"
func GetServiceIdentity(frontendAddr string) string {

	fid := strings.Replace(frontendAddr, ".", "_", -1)
	fid = strings.Replace(fid, ":", "_", -1)
	fid = strings.Replace(fid, "/", "", -1)
	if len(fid) > 20 {
		// unix domain socket
		fid = fid[len(fid)-20 : len(fid)]
	}
	return fid

}

//
// 去ZK注册当前的Service
//
func RegisterService(serviceName, frontendAddr, serviceId string, topo *Topology, evtExit chan interface{},
	workDir string, codeUrlVerion string, state *atomic2.Bool, stateChan chan bool) *ServiceEndpoint {

	// 1. 准备数据
	// 记录Service Endpoint的信息
	servicePath := topo.ProductServicePath(serviceName)

	// 确保东西存在
	if ok, _ := topo.Exist(servicePath); !ok {
		topo.CreateDir(servicePath)
	}

	// 用来从zookeeper获取事件
	evtbus := make(chan interface{})

	// 2. 将信息添加到Zk中, 并且监控Zk的状态(如果添加失败会怎么样?)
	endpoint := NewServiceEndpoint(serviceName, serviceId, frontendAddr, workDir, codeUrlVerion)

	// deployPath

	// 为了保证Add是干净的，需要先删除，保证自己才是Owner
	endpoint.DeleteServiceEndpoint(topo)

	// 如果没有状态，或状态为true, 则上线
	if state == nil || state.Get() {
		endpoint.AddServiceEndpoint(topo)
	}

	go func() {

		for true {
			// Watch的目的是监控: 当前的zk session的状态, 如果session出现异常，则重新注册
			_, err := topo.WatchNode(servicePath, evtbus)

			if err == nil {
				// 如果成功添加Watch, 则等待退出，或者ZK Event
				select {
				case <-evtExit:
					return
				case <-stateChan:
					// 如何状态变化(则重新注册)
					endpoint.DeleteServiceEndpoint(topo)
					if state == nil || state.Get() {
						endpoint.AddServiceEndpoint(topo)
					}
				case e := <-evtbus:
					event := e.(topozk.Event)
					if event.State == topozk.StateExpired ||
						event.Type == topozk.EventNotWatching {
						// Session过期了，则需要删除之前的数据，
						// 因为当前的session不是之前的数据的Owner
						endpoint.DeleteServiceEndpoint(topo)

						if state == nil || state.Get() {
							endpoint.AddServiceEndpoint(topo)
						}

					}
				}

			} else {
				// 如果添加Watch失败，则等待退出，或等待timer接触，进行下一次尝试
				timer := time.NewTimer(time.Second * time.Duration(10))
				log.ErrorErrorf(err, "Reg Failed, Wait 10 seconds...., %v", err)
				select {
				case <-evtExit:
					return
				case <-timer.C:
					// pass
				}
			}

		}
	}()
	return endpoint
}

//
// 后端如何处理一个Request
//
func (p *ThriftRpcServer) Dispatch(r *Request) error {
	transport := NewTMemoryBufferWithBuf(r.Request.Data)
	ip := thrift.NewTBinaryProtocolTransport(transport)

	slice := getSlice(0, DEFAULT_SLICE_LEN)
	transport = NewTMemoryBufferWithBuf(slice)
	op := thrift.NewTBinaryProtocolTransport(transport)
	p.Processor.Process(ip, op)

	r.Response.Data = transport.Bytes()

	_, _, seqId, _ := DecodeThriftTypIdSeqId(r.Response.Data)

	log.Debugf("SeqId: %d vs. %d, Dispatch Over", r.Request.SeqId, seqId)
	//	// 如果transport重新分配了内存，则立即归还slice
	//	if cap(r.Response.Data) != DEFAULT_SLICE_LEN {
	//		returnSlice(slice)
	//	}
	return nil
}

func (p *ThriftRpcServer) Run() {
	//	// 1. 创建到zk的连接
	// 有条件地做服务注册
	registerService := !p.config.StandAlone // 不独立运行则注册服务
	if registerService {
		p.Topo = NewTopology(p.ProductName, p.ZkAddr)
	}

	// 127.0.0.1:5555 --> 127_0_0_1:5555
	lbServiceName := GetServiceIdentity(p.FrontendAddr)

	exitSignal := make(chan os.Signal, 1)
	signal.Notify(exitSignal, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)

	// syscall.SIGKILL
	// kill -9 pid
	// kill -s SIGKILL pid 还是留给运维吧

	if len(p.config.FalconClient) > 0 {
		StartTicker(p.config.FalconClient, p.ServiceName)
	}

	// 初始状态为不上线
	var state atomic2.Bool
	state.Set(false)
	stateChan := make(chan bool)

	// 注册服务
	evtExit := make(chan interface{})

	var endpoint *ServiceEndpoint = nil
	if registerService {
		endpoint = RegisterService(p.ServiceName, p.FrontendAddr, lbServiceName,
			p.Topo, evtExit, p.config.WorkDir, p.config.CodeUrlVersion,
			&state, stateChan)
	}

	// 3. 读取"前端"的配置
	var transport thrift.TServerTransport
	var err error

	isUnixDomain := false
	// 127.0.0.1:9999(以:区分不同的类型)
	if !strings.Contains(p.FrontendAddr, ":") {
		if rpc_utils.FileExist(p.FrontendAddr) {
			os.Remove(p.FrontendAddr)
		}
		transport, err = rpc_utils.NewTServerUnixDomain(p.FrontendAddr)
		isUnixDomain = true
	} else {
		transport, err = thrift.NewTServerSocket(p.FrontendAddr)
	}

	if err != nil {
		log.ErrorErrorf(err, Red("Server Socket Create Failed: %v"), err)
		panic(fmt.Sprintf("Invalid FrontendAddr: %s", p.FrontendAddr))
	}

	err = transport.Listen()
	if err != nil {
		log.ErrorErrorf(err, Red("Server Socket Open Failed: %v"), err)
		panic(fmt.Sprintf("Server Socket Open Failed: %s", p.FrontendAddr))
	}

	ch := make(chan thrift.TTransport, 4096)
	defer close(ch)

	// 强制退出? TODO: Graceful退出
	go func() {
		<-exitSignal
		log.Info(Magenta("Receive Exit Signals...."))

		if registerService {
			evtExit <- true
		}
		if registerService {
			endpoint.DeleteServiceEndpoint(p.Topo)
		} else {
			// 如果没有注册到后端服务器，则直接不再接受新请求
			transport.Interrupt()
		}

		// 等待
		start := time.Now().Unix()
		for true {
			// 如果5s内没有接受到新的请求了，则退出
			now := time.Now().Unix()
			if now-p.lastRequestTime.Get() > 5 {
				log.Info(Red("Graceful Exit..."))
				break
			} else {
				log.Printf(Cyan("Sleeping %d seconds\n"), now-start)
				time.Sleep(time.Second)
			}
		}

		// 确定处理完毕数据之后，关闭transport
		if registerService {
			transport.Interrupt()
		}
		transport.Close()
	}()

	go func() {
		var address string
		for c := range ch {
			// 为每个Connection建立一个Session
			socket, ok := c.(rpc_utils.SocketAddr)

			if ok {
				if isUnixDomain {
					address = p.FrontendAddr
				} else {
					address = socket.Addr().String()
				}
			} else {
				address = "unknow"
			}

			// Session独立处理自己的请求
			if registerService {
				x := NewNonBlockSession(c, address, p.Verbose, &p.lastRequestTime)
				go x.Serve(p, 1000)
			} else {
				go func() {
					// 打印异常信息
					defer func() {
						c.Close()
						if e := recover(); e != nil {
							log.Errorf("panic in processor: %v: %s", e, debug.Stack())
						}
					}()

					ip := thrift.NewTBinaryProtocolTransport(thrift.NewTFramedTransport(c))
					op := thrift.NewTBinaryProtocolTransport(thrift.NewTFramedTransport(c))

					for true {
						ok, err := p.Processor.Process(ip, op)

						if err != nil {
							// 如果链路出现异常（必须结束)
							if err, ok := err.(thrift.TTransportException); ok && (err.TypeId() == thrift.END_OF_FILE ||
								strings.Contains(err.Error(), "use of closed network connection")) {
								return
							} else if err != nil {
								// 其他链路问题，直接报错，退出
								log.ErrorErrorf(err, "Error processing request, quit")
								return
							}

							// 如果是方法未知，则直接报错，然后跳过
							if err, ok := err.(thrift.TApplicationException); ok && err.TypeId() == thrift.UNKNOWN_METHOD {
								log.ErrorErrorf(err, "Error processing request, continue")
								continue
							}

							// INTERNAL_ERROR --> ok: true 可以继续
							// PROTOCOL_ERROR --> ok: false
						}

						// 其他情况下，ok == false，意味io过程中存在读写错误，connection上存在脏数据
						// 用户自定义的异常，必须继承自: *services.RpcException, 否则整个流程就容易出现问题
						if !ok {
							log.Printf("Process Not OK, stopped")
							break
						}

					}
				}()
			}
		}
	}()

	// 准备上线服务
	state.Set(true)
	if registerService {
		stateChan <- true
	}

	log.Printf("Begin to accept requests...")
	// Accept什么时候出错，出错之后如何处理呢?
	for {
		c, err := transport.Accept()
		if err != nil {
			break
		} else {
			ch <- c
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"regexp"

	"crypto/sha1"
	"fmt"
	"io"

	"log"

	"github.com/dtop/go.demo.iam/iam/wrappers"
	"github.com/dtop/go.ginject"
)

type (

	// UserLogin is the user login handler
	UserLogin struct {
		UserID    string
		SessionID string `form:"sess" binding:"required"`
		Email     string `form:"eml" binding:"required"`
		Password  string `form:"pwd" binding:"required"`

		Db     *wrappers.MySQL `inject:"db"`
		errors []Err
	}

	Err struct {
		Field   string `json:"field"`
		Message string `json:"message"`
	}
)

// NewUserLogin returns a brand new user login model
func NewUserLogin(deps ginject.Injector) *UserLogin {

	ulog := &UserLogin{errors: make([]Err, 0)}
	deps.Apply(ulog)

	return ulog
}

// CheckLogin checks if eml/pwd is valid and assigns the userID if so
func (ul *UserLogin) CheckLogin() []Err {

	if ul.SessionID == "" {
		ul.addError("SessionID was missing, please start over", "none")
	}

	if ul.Email == "" || !isEmail(ul.Email) {
		ul.addError("Please enter a valid email address", "eml")
	}

	if len(ul.Password) < 6 {
		ul.addError("Please enter a valid password", "pwd")
	}

	if len(ul.errors) > 0 {
		return ul.errors
	}

	result, err := ul.Db.Query(func(db *sql.DB) (*sql.Rows, error) {

		query := "SELECT id, password FROM user_data WHERE email LIKE ?"
		return db.Query(query, ul.Email)
	})

	if err != nil {
		ul.addError("server error occured, please try later", "none")
		return ul.errors
	}

	if !result.Next() {
		log.Println("unknown record")
		ul.addError("could not find this username/password combination", "none")
		return ul.errors
	}

	var (
		ident string
		pass  string
	)

	if err := result.Scan(&ident, &pass); err != nil {
		ul.addError("server error occured, please try later", "none")
		return ul.errors
	}

	if hashPassword(ul.Password) != pass {
		log.Println(hashPassword(ul.Password), " does not match ", pass)
		ul.addError("could not find this username/password combination", "none")
		return ul.errors
	}

	ul.UserID = ident
	return ul.errors
}

func (ul *UserLogin) addError(text, field string) {

	ul.errors = append(ul.errors, Err{Field: field, Message: text})
}

// Adopted from StackOverflow =)
func isEmail(mail string) bool {

	pttrn := "^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$"
	rxEmail := regexp.MustCompile(pttrn)
	return rxEmail.MatchString(mail)
}

func hashPassword(pwd string) string {

	h := sha1.New()
	io.WriteString(h, pwd)
	return fmt.Sprintf("%x", h.Sum(nil))
}
<commit_msg>comment<commit_after>package models

import (
	"database/sql"
	"regexp"

	"crypto/sha1"
	"fmt"
	"io"

	"log"

	"github.com/dtop/go.demo.iam/iam/wrappers"
	"github.com/dtop/go.ginject"
)

type (

	// UserLogin is the user login handler
	UserLogin struct {
		UserID    string
		SessionID string `form:"sess" binding:"required"`
		Email     string `form:"eml" binding:"required"`
		Password  string `form:"pwd" binding:"required"`

		Db     *wrappers.MySQL `inject:"db"`
		errors []Err
	}

	// Err is the error struct
	Err struct {
		Field   string `json:"field"`
		Message string `json:"message"`
	}
)

// NewUserLogin returns a brand new user login model
func NewUserLogin(deps ginject.Injector) *UserLogin {

	ulog := &UserLogin{errors: make([]Err, 0)}
	deps.Apply(ulog)

	return ulog
}

// CheckLogin checks if eml/pwd is valid and assigns the userID if so
func (ul *UserLogin) CheckLogin() []Err {

	if ul.SessionID == "" {
		ul.addError("SessionID was missing, please start over", "none")
	}

	if ul.Email == "" || !isEmail(ul.Email) {
		ul.addError("Please enter a valid email address", "eml")
	}

	if len(ul.Password) < 6 {
		ul.addError("Please enter a valid password", "pwd")
	}

	if len(ul.errors) > 0 {
		return ul.errors
	}

	result, err := ul.Db.Query(func(db *sql.DB) (*sql.Rows, error) {

		query := "SELECT id, password FROM user_data WHERE email LIKE ?"
		return db.Query(query, ul.Email)
	})

	if err != nil {
		ul.addError("server error occured, please try later", "none")
		return ul.errors
	}

	if !result.Next() {
		log.Println("unknown record")
		ul.addError("could not find this username/password combination", "none")
		return ul.errors
	}

	var (
		ident string
		pass  string
	)

	if err := result.Scan(&ident, &pass); err != nil {
		ul.addError("server error occured, please try later", "none")
		return ul.errors
	}

	if hashPassword(ul.Password) != pass {
		log.Println(hashPassword(ul.Password), " does not match ", pass)
		ul.addError("could not find this username/password combination", "none")
		return ul.errors
	}

	ul.UserID = ident
	return ul.errors
}

func (ul *UserLogin) addError(text, field string) {

	ul.errors = append(ul.errors, Err{Field: field, Message: text})
}

// Adopted from StackOverflow =)
func isEmail(mail string) bool {

	pttrn := "^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$"
	rxEmail := regexp.MustCompile(pttrn)
	return rxEmail.MatchString(mail)
}

func hashPassword(pwd string) string {

	h := sha1.New()
	io.WriteString(h, pwd)
	return fmt.Sprintf("%x", h.Sum(nil))
}
<|endoftext|>


<commit_before>package meta_cache

import (
	"context"
	"fmt"
	"os"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
)

// need to have logic similar to FilerStoreWrapper
// e.g. fill fileId field for chunks

type MetaCache struct {
	localStore filer.VirtualFilerStore
	sync.RWMutex
	visitedBoundary *bounded_tree.BoundedTree
	uidGidMapper    *UidGidMapper
	invalidateFunc  func(util.FullPath)
}

func NewMetaCache(dbFolder string, baseDir util.FullPath, uidGidMapper *UidGidMapper, invalidateFunc func(util.FullPath)) *MetaCache {
	return &MetaCache{
		localStore:      openMetaStore(dbFolder),
		visitedBoundary: bounded_tree.NewBoundedTree(baseDir),
		uidGidMapper:    uidGidMapper,
		invalidateFunc: func(fullpath util.FullPath) {
			invalidateFunc(fullpath)
		},
	}
}

func openMetaStore(dbFolder string) filer.VirtualFilerStore {

	os.RemoveAll(dbFolder)
	os.MkdirAll(dbFolder, 0755)

	store := &leveldb.LevelDBStore{}
	config := &cacheConfig{
		dir: dbFolder,
	}

	if err := store.Initialize(config, ""); err != nil {
		glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
	}

	return filer.NewFilerStoreWrapper(store)

}

func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	return mc.doInsertEntry(ctx, entry)
}

func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
	return mc.localStore.InsertEntry(ctx, entry)
}

func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()

	oldDir, _ := oldPath.DirAndName()
	if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
		if oldPath != "" {
			if newEntry != nil && oldPath == newEntry.FullPath {
				// skip the unnecessary deletion
				// leave the update to the following InsertEntry operation
			} else {
				glog.V(3).Infof("DeleteEntry %s/%s", oldPath, oldPath.Name())
				if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil {
					return err
				}
			}
		}
	} else {
		// println("unknown old directory:", oldDir)
	}

	if newEntry != nil {
		newDir, _ := newEntry.DirAndName()
		if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
			glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name())
			if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil {
				return err
			}
		}
	}
	return nil
}

func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	return mc.localStore.UpdateEntry(ctx, entry)
}

func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) {
	mc.RLock()
	defer mc.RUnlock()
	entry, err = mc.localStore.FindEntry(ctx, fp)
	if err != nil {
		return nil, err
	}
	mc.mapIdFromFilerToLocal(entry)
	return
}

func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
	mc.Lock()
	defer mc.Unlock()
	return mc.localStore.DeleteEntry(ctx, fp)
}

func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error {
	mc.RLock()
	defer mc.RUnlock()

	if !mc.visitedBoundary.HasVisited(dirPath) {
		return fmt.Errorf("unsynchronized dir: %v", dirPath)
	}

	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
		mc.mapIdFromFilerToLocal(entry)
		return eachEntryFunc(entry)
	})
	if err != nil {
		return err
	}
	return err
}

func (mc *MetaCache) Shutdown() {
	mc.Lock()
	defer mc.Unlock()
	mc.localStore.Shutdown()
}

func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) {
	entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid)
}
<commit_msg>fix logs<commit_after>package meta_cache

import (
	"context"
	"fmt"
	"os"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/filer/leveldb"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/bounded_tree"
)

// need to have logic similar to FilerStoreWrapper
// e.g. fill fileId field for chunks

type MetaCache struct {
	localStore filer.VirtualFilerStore
	sync.RWMutex
	visitedBoundary *bounded_tree.BoundedTree
	uidGidMapper    *UidGidMapper
	invalidateFunc  func(util.FullPath)
}

func NewMetaCache(dbFolder string, baseDir util.FullPath, uidGidMapper *UidGidMapper, invalidateFunc func(util.FullPath)) *MetaCache {
	return &MetaCache{
		localStore:      openMetaStore(dbFolder),
		visitedBoundary: bounded_tree.NewBoundedTree(baseDir),
		uidGidMapper:    uidGidMapper,
		invalidateFunc: func(fullpath util.FullPath) {
			invalidateFunc(fullpath)
		},
	}
}

func openMetaStore(dbFolder string) filer.VirtualFilerStore {

	os.RemoveAll(dbFolder)
	os.MkdirAll(dbFolder, 0755)

	store := &leveldb.LevelDBStore{}
	config := &cacheConfig{
		dir: dbFolder,
	}

	if err := store.Initialize(config, ""); err != nil {
		glog.Fatalf("Failed to initialize metadata cache store for %s: %+v", store.GetName(), err)
	}

	return filer.NewFilerStoreWrapper(store)

}

func (mc *MetaCache) InsertEntry(ctx context.Context, entry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	return mc.doInsertEntry(ctx, entry)
}

func (mc *MetaCache) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
	return mc.localStore.InsertEntry(ctx, entry)
}

func (mc *MetaCache) AtomicUpdateEntryFromFiler(ctx context.Context, oldPath util.FullPath, newEntry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()

	oldDir, _ := oldPath.DirAndName()
	if mc.visitedBoundary.HasVisited(util.FullPath(oldDir)) {
		if oldPath != "" {
			if newEntry != nil && oldPath == newEntry.FullPath {
				// skip the unnecessary deletion
				// leave the update to the following InsertEntry operation
			} else {
				glog.V(3).Infof("DeleteEntry %s", oldPath)
				if err := mc.localStore.DeleteEntry(ctx, oldPath); err != nil {
					return err
				}
			}
		}
	} else {
		// println("unknown old directory:", oldDir)
	}

	if newEntry != nil {
		newDir, _ := newEntry.DirAndName()
		if mc.visitedBoundary.HasVisited(util.FullPath(newDir)) {
			glog.V(3).Infof("InsertEntry %s/%s", newDir, newEntry.Name())
			if err := mc.localStore.InsertEntry(ctx, newEntry); err != nil {
				return err
			}
		}
	}
	return nil
}

func (mc *MetaCache) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
	mc.Lock()
	defer mc.Unlock()
	return mc.localStore.UpdateEntry(ctx, entry)
}

func (mc *MetaCache) FindEntry(ctx context.Context, fp util.FullPath) (entry *filer.Entry, err error) {
	mc.RLock()
	defer mc.RUnlock()
	entry, err = mc.localStore.FindEntry(ctx, fp)
	if err != nil {
		return nil, err
	}
	mc.mapIdFromFilerToLocal(entry)
	return
}

func (mc *MetaCache) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
	mc.Lock()
	defer mc.Unlock()
	return mc.localStore.DeleteEntry(ctx, fp)
}

func (mc *MetaCache) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) error {
	mc.RLock()
	defer mc.RUnlock()

	if !mc.visitedBoundary.HasVisited(dirPath) {
		return fmt.Errorf("unsynchronized dir: %v", dirPath)
	}

	_, err := mc.localStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *filer.Entry) bool {
		mc.mapIdFromFilerToLocal(entry)
		return eachEntryFunc(entry)
	})
	if err != nil {
		return err
	}
	return err
}

func (mc *MetaCache) Shutdown() {
	mc.Lock()
	defer mc.Unlock()
	mc.localStore.Shutdown()
}

func (mc *MetaCache) mapIdFromFilerToLocal(entry *filer.Entry) {
	entry.Attr.Uid, entry.Attr.Gid = mc.uidGidMapper.FilerToLocal(entry.Attr.Uid, entry.Attr.Gid)
}
<|endoftext|>