<commit_before>package core

import (
	"sync"
	"sync/atomic"

	"github.com/coel-lang/coel/src/lib/debug"
)

type thunkState int32

const (
	normal thunkState = iota
	app
	spinLock
)

// Thunk you all!
type Thunk struct {
	result    Value
	function  *Thunk
	args      Arguments
	state     thunkState
	blackHole sync.WaitGroup
	info      debug.Info
}

// Normal creates a thunk of a normal value as its result.
func Normal(v Value) *Thunk {
	if t, ok := v.(*Thunk); ok {
		return t
	}

	return &Thunk{result: v, state: normal}
}

// App creates a thunk applying a function to arguments.
func App(f *Thunk, args Arguments) *Thunk {
	return AppWithInfo(f, args, debug.NewGoInfo(1))
}

// AppWithInfo is the same as App except that it stores debug information
// in the thunk.
func AppWithInfo(f *Thunk, args Arguments, i debug.Info) *Thunk {
	t := &Thunk{
		function: f,
		args:     args,
		state:    app,
		info:     i,
	}
	t.blackHole.Add(1)
	return t
}

// PApp is not PPap.
func PApp(f *Thunk, ps ...*Thunk) *Thunk {
	return AppWithInfo(f, NewPositionalArguments(ps...), debug.NewGoInfo(1))
}

// evalAny evaluates a thunk and returns a pure or impure (effect) value.
func (t *Thunk) evalAny() Value {
	if t.lock(normal) {
		for {
			v := t.swapFunction(nil).Eval()

			if t.chainError(v) {
				break
			}

			f, ok := v.(callable)

			if !ok {
				t.result = NotCallableError(v).Eval()
				break
			}

			t.result = f.call(t.swapArguments(Arguments{}))

			if t.chainError(t.result) {
				break
			}

			child, ok := t.result.(*Thunk)

			if !ok {
				break
			}

			if ok := child.delegateEval(t); !ok {
				t.result = child.evalAny()
				t.chainError(t.result)
				break
			}
		}

		assertValueIsNormal("Thunk.result", t.result)

		t.finalize()
	} else {
		t.blackHole.Wait()
	}

	assertValueIsNormal("Thunk.result", t.result)

	return t.result
}

func (t *Thunk) lock(s thunkState) bool {
	for {
		switch t.loadState() {
		case normal:
			return false
		case app:
			if t.compareAndSwapState(app, s) {
				return true
			}
		}
	}
}

func (t *Thunk) delegateEval(parent *Thunk) bool {
	if t.lock(spinLock) {
		parent.function = t.swapFunction(identity)
		parent.args = t.swapArguments(Arguments{[]*Thunk{parent}, nil, nil, nil})
		parent.info = t.info
		t.storeState(app)
		return true
	}

	return false
}

func (t *Thunk) swapFunction(new *Thunk) *Thunk {
	old := t.function
	t.function = new
	return old
}

func (t *Thunk) swapArguments(new Arguments) Arguments {
	old := t.args
	t.args = new
	return old
}

func (t *Thunk) finalize() {
	t.function = nil
	t.args = Arguments{}
	t.storeState(normal)
	t.blackHole.Done()
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapInt32((*int32)(&t.state), int32(old), int32(new))
}

func (t *Thunk) loadState() thunkState {
	return thunkState(atomic.LoadInt32((*int32)(&t.state)))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreInt32((*int32)(&t.state), int32(new))
}

func (t *Thunk) chainError(v Value) bool {
	if e, ok := v.(ErrorType); ok {
		t.result = e.Chain(t.info)
		return true
	}

	return false
}

func assertValueIsNormal(s string, v Value) {
	if _, ok := v.(*Thunk); ok {
		panic(s + " is *Thunk")
	}
}

// Eval evaluates a pure value.
func (t *Thunk) Eval() Value {
	if _, ok := t.evalAny().(effectType); ok {
		return impureFunctionError().Eval().(ErrorType).Chain(t.info)
	}

	return t.result
}

// EvalEffect evaluates an effect expression.
func (t *Thunk) EvalEffect() Value {
	v := t.evalAny()
	e, ok := v.(effectType)

	if !ok {
		return NotEffectError(v).Eval().(ErrorType).Chain(t.info)
	}

	return e.value.Eval()
}
<commit_msg>Remove chainError method<commit_after>package core

import (
	"sync"
	"sync/atomic"

	"github.com/coel-lang/coel/src/lib/debug"
)

type thunkState int32

const (
	normal thunkState = iota
	app
	spinLock
)

// Thunk you all!
type Thunk struct {
	result    Value
	function  *Thunk
	args      Arguments
	state     thunkState
	blackHole sync.WaitGroup
	info      debug.Info
}

// Normal creates a thunk of a normal value as its result.
func Normal(v Value) *Thunk {
	if t, ok := v.(*Thunk); ok {
		return t
	}

	return &Thunk{result: v, state: normal}
}

// App creates a thunk applying a function to arguments.
func App(f *Thunk, args Arguments) *Thunk {
	return AppWithInfo(f, args, debug.NewGoInfo(1))
}

// AppWithInfo is the same as App except that it stores debug information
// in the thunk.
func AppWithInfo(f *Thunk, args Arguments, i debug.Info) *Thunk {
	t := &Thunk{
		function: f,
		args:     args,
		state:    app,
		info:     i,
	}
	t.blackHole.Add(1)
	return t
}

// PApp is not PPap.
func PApp(f *Thunk, ps ...*Thunk) *Thunk {
	return AppWithInfo(f, NewPositionalArguments(ps...), debug.NewGoInfo(1))
}

// evalAny evaluates a thunk and returns a pure or impure (effect) value.
func (t *Thunk) evalAny() Value {
	if t.lock(normal) {
		for {
			v := t.swapFunction(nil).Eval()

			if _, ok := v.(ErrorType); ok {
				t.result = v
				break
			}

			f, ok := v.(callable)

			if !ok {
				t.result = NotCallableError(v).Eval()
				break
			}

			t.result = f.call(t.swapArguments(Arguments{}))

			if _, ok := t.result.(ErrorType); ok {
				break
			}

			child, ok := t.result.(*Thunk)

			if !ok {
				break
			}

			if ok := child.delegateEval(t); !ok {
				t.result = child.evalAny()
				break
			}
		}

		assertValueIsNormal("Thunk.result", t.result)

		if e, ok := t.result.(ErrorType); ok {
			t.result = e.Chain(t.info)
		}

		t.finalize()
	} else {
		t.blackHole.Wait()
	}

	assertValueIsNormal("Thunk.result", t.result)

	return t.result
}

func (t *Thunk) lock(s thunkState) bool {
	for {
		switch t.loadState() {
		case normal:
			return false
		case app:
			if t.compareAndSwapState(app, s) {
				return true
			}
		}
	}
}

func (t *Thunk) delegateEval(parent *Thunk) bool {
	if t.lock(spinLock) {
		parent.function = t.swapFunction(identity)
		parent.args = t.swapArguments(Arguments{[]*Thunk{parent}, nil, nil, nil})
		parent.info = t.info
		t.storeState(app)
		return true
	}

	return false
}

func (t *Thunk) swapFunction(new *Thunk) *Thunk {
	old := t.function
	t.function = new
	return old
}

func (t *Thunk) swapArguments(new Arguments) Arguments {
	old := t.args
	t.args = new
	return old
}

func (t *Thunk) finalize() {
	t.function = nil
	t.args = Arguments{}
	t.storeState(normal)
	t.blackHole.Done()
}

func (t *Thunk) compareAndSwapState(old, new thunkState) bool {
	return atomic.CompareAndSwapInt32((*int32)(&t.state), int32(old), int32(new))
}

func (t *Thunk) loadState() thunkState {
	return thunkState(atomic.LoadInt32((*int32)(&t.state)))
}

func (t *Thunk) storeState(new thunkState) {
	atomic.StoreInt32((*int32)(&t.state), int32(new))
}

func assertValueIsNormal(s string, v Value) {
	if _, ok := v.(*Thunk); ok {
		panic(s + " is *Thunk")
	}
}

// Eval evaluates a pure value.
func (t *Thunk) Eval() Value {
	if _, ok := t.evalAny().(effectType); ok {
		return impureFunctionError().Eval().(ErrorType).Chain(t.info)
	}

	return t.result
}

// EvalEffect evaluates an effect expression.
func (t *Thunk) EvalEffect() Value {
	v := t.evalAny()
	e, ok := v.(effectType)

	if !ok {
		return NotEffectError(v).Eval().(ErrorType).Chain(t.info)
	}

	return e.value.Eval()
}
<|endoftext|>


<commit_before>/*
Package lmdbsync provides advanced synchronization for LMDB environments at the
cost of performance.  The package provides a drop-in replacement for *lmdb.Env
that can be used in situations where the database may be resized or where the
flag lmdb.NoLock is used.

Bypassing an Env's methods to access the underlying lmdb.Env is not safe.  The
severity of such usage depends such behavior should be strictly avoided as it
may produce undefined behavior from the LMDB C library.

Resizing the environment

The Env type synchronizes all calls to Env.SetMapSize so that it may, with some
caveats, be safely called in the presence of concurrent transactions after an
environment has been opened.  All running transactions must complete before the
method will be called on the underlying lmdb.Env.

If an open transaction depends on a change in map size then the Env will
deadlock and block all future transactions.  When using a Handler to
automatically resize the map this implies the restriction that transactions
must terminate independently of the creation and termination of other
transactions to avoid deadlock

In the simplest example, a view transaction that attempts an update on the
underlying Env will deadlock the environment if the map is full and a Handler
attempts to resize the map so the update may be retried.

	env.View(func(txn *lmdb.Txn) (err error) {
		v, err := txn.Get(db, key)
		if err != nil {
			return err
		}
		err = env.Update(func(txn *lmdb.Txn) (err error) { // deadlock on lmdb.MapFull!
			txn.Put(dbi, key, append(v, b...))
		})
		return err
	}

The update should instead be prepared inside the view and then executed
following its termination.  This removes the implicit dependence of the view on
calls to Env.SetMapSize().

	var v []byte
	env.View(func(txn *lmdb.Txn) (err error) {
		// RawRead isn't used because the value will be used outside the
		// transaction.
		v, err = txn.Get(db, key)
		if err != nil {
			return err
		}
		return nil
	}
	if err != nil {
		// ...
	}
	err = env.Update(func(txn *lmdb.Txn) (err error) { // no deadlock, even if env is resized!
		txn.Put(dbi, key, append(v, b...))
	})

The developers of LMDB officially recommend against applications changing the
memory map size for an open database.  It requires careful synchronization by
all processes accessing the database file.  And, a large memory map will not
affect disk usage on operating systems that support sparse files (e.g. Linux,
not OS X).

See mdb_env_set_mapsize.

Multi-processing (MapResized)

Using the Handler interface provided by the package MapResizedHandler can be
used to automatically resize an enviornment when a lmdb.MapResized error is
encountered.  Usage of the MapResizedHandler puts important caveats on how one
can safely work with transactions.  See the function documentation for more
detailed information.

When other processes may change an environment's map size it is extremely
important to ensure that transactions terminate independent of all other
transactions.  The MapResized error may be returned at the beginning of any
transaction.

See mdb_txn_begin and MDB_MAP_RESIZED.

MapFull

Similar to the MapResizedHandler the MapFullHandler will automatically resize
the map and retry transactions when a MapFull error is encountered.  Usage of
the MapFullHandler puts important caveats on how one can safely work with
transactions.  See the function documentation for more detailed information.

The caveats on transactions are lessened if lmdb.MapFull is the only error
being handled (when multi-processing is not a concern).  The only requirement
then is that view transactions not depend on the termination of updates
transactions.

See mdb_env_set_mapsize and MDB_MAP_FULL.

NoLock

When the lmdb.NoLock flag is set on an environment Env handles all transaction
synchronization using Go structures and is an experimental feature.  It is
unclear what benefits this provides.

Usage of lmdb.NoLock requires that update transactions acquire an exclusive
lock on the environment.  In such cases it is required that view transactions
execute independently of update transactions, a requirement more strict than
that from handling MapFull.

See mdb_env_open and MDB_NOLOCK.
*/
package lmdbsync

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/bmatsuo/lmdb-go/lmdb"
)

type envBagKey int

// BagEnv returns the Env corresponding to a Bag in the HandleTxnErr method of
// a Handler.
func BagEnv(b Bag) *Env {
	env, _ := b.Value(envBagKey(0)).(*Env)
	return env
}

func bagWithEnv(b Bag, env *Env) Bag {
	return BagWith(b, envBagKey(0), env)
}

// Env wraps an *lmdb.Env, receiving all the same methods and proxying some to
// provide transaction management.  Transactions run by an Env handle
// lmdb.MapResized error transparently through additional synchronization.
// Additionally, Env is safe to use on environments setting the lmdb.NoLock
// flag.  When in NoLock mode write transactions block all read transactions
// from running (in addition to blocking other write transactions like a normal
// lmdb.Env would).
//
// Env proxies several methods to provide synchronization required for safe
// operation in some scenarios.  It is important not byprass proxies and call
// the methods directly on the underlying lmdb.Env or synchronization may be
// interfered with.  Calling proxied methods directly on the lmdb.Env may
// result in poor transaction performance or unspecified behavior in from the C
// library.
type Env struct {
	*lmdb.Env
	Handlers HandlerChain
	bag      Bag
	noLock   bool
	txnlock  sync.RWMutex
}

// NewEnv returns an newly allocated Env that wraps env.  If env is nil then
// lmdb.NewEnv() will be called to allocate an lmdb.Env.
func NewEnv(env *lmdb.Env, h ...Handler) (*Env, error) {
	var err error
	if env == nil {
		env, err = lmdb.NewEnv()
		if err != nil {
			return nil, err
		}
	}

	flags, err := env.Flags()
	if err != nil {
		return nil, err
	}
	noLock := flags&lmdb.NoLock != 0

	chain := append(HandlerChain(nil), h...)

	_env := &Env{
		Env:      env,
		Handlers: chain,
		noLock:   noLock,
		bag:      Background(),
	}
	return _env, nil
}

// Open is a proxy for r.Env.Open() that detects the lmdb.NoLock flag to
// properly manage transaction synchronization.
func (r *Env) Open(path string, flags uint, mode os.FileMode) error {
	err := r.Env.Open(path, flags, mode)
	if err != nil {
		// no update to flags occurred
		return err
	}

	if flags&lmdb.NoLock != 0 {
		r.noLock = true
	}

	return nil
}

// SetMapSize is a proxy for r.Env.SetMapSize() that blocks while concurrent
// transactions are in progress.
func (r *Env) SetMapSize(size int64) error {
	return r.setMapSize(size, 0)
}

func (r *Env) setMapSize(size int64, delay time.Duration) error {
	r.txnlock.Lock()
	if delay > 0 {
		// wait before adopting a map size set from another process. hold on to
		// the transaction lock so that other transactions don't attempt to
		// begin while waiting.
		time.Sleep(delay)
	}
	err := r.Env.SetMapSize(size)
	r.txnlock.Unlock()
	return err
}

// BeginTxn overrides the r.Env.BeginTxn and always returns an error.  An
// unmanaged transaction.
func (r *Env) BeginTxn(parent *lmdb.Txn, flags uint) (*lmdb.Txn, error) {
	return nil, fmt.Errorf("lmdbsync: unmanaged transactions are not supported")
}

// RunTxn is a proxy for r.Env.RunTxn().
//
// If lmdb.NoLock is set on r.Env then RunTxn will block while other updates
// are in progress, regardless of flags.
func (r *Env) RunTxn(flags uint, op lmdb.TxnOp) (err error) {
	readonly := flags&lmdb.Readonly != 0
	return r.runHandler(readonly, func() error { return r.Env.RunTxn(flags, op) }, r.Handlers)
}

// View is a proxy for r.Env.View().
//
// If lmdb.NoLock is set on r.Env then View will block until any running update
// completes.
func (r *Env) View(op lmdb.TxnOp) error {
	return r.runHandler(true, func() error { return r.Env.View(op) }, r.Handlers)
}

// Update is a proxy for r.Env.Update().
//
// If lmdb.NoLock is set on r.Env then Update blocks until all other
// transactions have terminated and blocks all other transactions from running
// while in progress (including readonly transactions).
func (r *Env) Update(op lmdb.TxnOp) error {
	return r.runHandler(false, func() error { return r.Env.Update(op) }, r.Handlers)
}

// UpdateLocked is a proxy for r.Env.UpdateLocked().
//
// If lmdb.NoLock is set on r.Env then UpdateLocked blocks until all other
// transactions have terminated and blocks all other transactions from running
// while in progress (including readonly transactions).
func (r *Env) UpdateLocked(op lmdb.TxnOp) error {
	return r.runHandler(false, func() error { return r.Env.UpdateLocked(op) }, r.Handlers)
}

// WithHandler returns a TxnRunner than handles transaction errors r.Handlers
// chained with h.
func (r *Env) WithHandler(h Handler) TxnRunner {
	return &handlerRunner{
		env: r,
		h:   r.Handlers.Append(h),
	}
}

func (r *Env) runHandler(readonly bool, fn func() error, h Handler) error {
	b := bagWithEnv(r.bag, r)
	for {
		err := r.run(readonly, fn)
		b, err = h.HandleTxnErr(b, err)
		if err != ErrTxnRetry {
			return err
		}
	}
}
func (r *Env) run(readonly bool, fn func() error) error {
	var err error
	if r.noLock && !readonly {
		r.txnlock.Lock()
		err = fn()
		r.txnlock.Unlock()
	} else {
		r.txnlock.RLock()
		err = fn()
		r.txnlock.RUnlock()
	}
	return err
}
<commit_msg>lmdbsync: fix go doc section header<commit_after>/*
Package lmdbsync provides advanced synchronization for LMDB environments at the
cost of performance.  The package provides a drop-in replacement for *lmdb.Env
that can be used in situations where the database may be resized or where the
flag lmdb.NoLock is used.

Bypassing an Env's methods to access the underlying lmdb.Env is not safe.  The
severity of such usage depends such behavior should be strictly avoided as it
may produce undefined behavior from the LMDB C library.

Resizing the environment

The Env type synchronizes all calls to Env.SetMapSize so that it may, with some
caveats, be safely called in the presence of concurrent transactions after an
environment has been opened.  All running transactions must complete before the
method will be called on the underlying lmdb.Env.

If an open transaction depends on a change in map size then the Env will
deadlock and block all future transactions.  When using a Handler to
automatically resize the map this implies the restriction that transactions
must terminate independently of the creation and termination of other
transactions to avoid deadlock

In the simplest example, a view transaction that attempts an update on the
underlying Env will deadlock the environment if the map is full and a Handler
attempts to resize the map so the update may be retried.

	env.View(func(txn *lmdb.Txn) (err error) {
		v, err := txn.Get(db, key)
		if err != nil {
			return err
		}
		err = env.Update(func(txn *lmdb.Txn) (err error) { // deadlock on lmdb.MapFull!
			txn.Put(dbi, key, append(v, b...))
		})
		return err
	}

The update should instead be prepared inside the view and then executed
following its termination.  This removes the implicit dependence of the view on
calls to Env.SetMapSize().

	var v []byte
	env.View(func(txn *lmdb.Txn) (err error) {
		// RawRead isn't used because the value will be used outside the
		// transaction.
		v, err = txn.Get(db, key)
		if err != nil {
			return err
		}
		return nil
	}
	if err != nil {
		// ...
	}
	err = env.Update(func(txn *lmdb.Txn) (err error) { // no deadlock, even if env is resized!
		txn.Put(dbi, key, append(v, b...))
	})

The developers of LMDB officially recommend against applications changing the
memory map size for an open database.  It requires careful synchronization by
all processes accessing the database file.  And, a large memory map will not
affect disk usage on operating systems that support sparse files (e.g. Linux,
not OS X).

See mdb_env_set_mapsize.

MapResized

When multiple processes access and resize an environment it is not uncommon to
encounter a MapResized error which requires synchronized action before
continuing normal operations.

Using the Handler interface provided by the package MapResizedHandler can be
used to automatically resize an enviornment when a lmdb.MapResized error is
encountered.  However, usage of the MapResizedHandler puts important caveats on
how one can safely work with transactions.  See the function documentation for
more detailed information.

When other processes may change an environment's map size it is extremely
important to ensure that transactions terminate independent of all other
transactions.  The MapResized error may be returned at the beginning of any
transaction.

See mdb_txn_begin and MDB_MAP_RESIZED.

MapFull

Similar to the MapResizedHandler the MapFullHandler will automatically resize
the map and retry transactions when a MapFull error is encountered.  Usage of
the MapFullHandler puts important caveats on how one can safely work with
transactions.  See the function documentation for more detailed information.

The caveats on transactions are lessened if lmdb.MapFull is the only error
being handled (when multi-processing is not a concern).  The only requirement
then is that view transactions not depend on the termination of updates
transactions.

See mdb_env_set_mapsize and MDB_MAP_FULL.

NoLock

When the lmdb.NoLock flag is set on an environment Env handles all transaction
synchronization using Go structures and is an experimental feature.  It is
unclear what benefits this provides.

Usage of lmdb.NoLock requires that update transactions acquire an exclusive
lock on the environment.  In such cases it is required that view transactions
execute independently of update transactions, a requirement more strict than
that from handling MapFull.

See mdb_env_open and MDB_NOLOCK.
*/
package lmdbsync

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/bmatsuo/lmdb-go/lmdb"
)

type envBagKey int

// BagEnv returns the Env corresponding to a Bag in the HandleTxnErr method of
// a Handler.
func BagEnv(b Bag) *Env {
	env, _ := b.Value(envBagKey(0)).(*Env)
	return env
}

func bagWithEnv(b Bag, env *Env) Bag {
	return BagWith(b, envBagKey(0), env)
}

// Env wraps an *lmdb.Env, receiving all the same methods and proxying some to
// provide transaction management.  Transactions run by an Env handle
// lmdb.MapResized error transparently through additional synchronization.
// Additionally, Env is safe to use on environments setting the lmdb.NoLock
// flag.  When in NoLock mode write transactions block all read transactions
// from running (in addition to blocking other write transactions like a normal
// lmdb.Env would).
//
// Env proxies several methods to provide synchronization required for safe
// operation in some scenarios.  It is important not byprass proxies and call
// the methods directly on the underlying lmdb.Env or synchronization may be
// interfered with.  Calling proxied methods directly on the lmdb.Env may
// result in poor transaction performance or unspecified behavior in from the C
// library.
type Env struct {
	*lmdb.Env
	Handlers HandlerChain
	bag      Bag
	noLock   bool
	txnlock  sync.RWMutex
}

// NewEnv returns an newly allocated Env that wraps env.  If env is nil then
// lmdb.NewEnv() will be called to allocate an lmdb.Env.
func NewEnv(env *lmdb.Env, h ...Handler) (*Env, error) {
	var err error
	if env == nil {
		env, err = lmdb.NewEnv()
		if err != nil {
			return nil, err
		}
	}

	flags, err := env.Flags()
	if err != nil {
		return nil, err
	}
	noLock := flags&lmdb.NoLock != 0

	chain := append(HandlerChain(nil), h...)

	_env := &Env{
		Env:      env,
		Handlers: chain,
		noLock:   noLock,
		bag:      Background(),
	}
	return _env, nil
}

// Open is a proxy for r.Env.Open() that detects the lmdb.NoLock flag to
// properly manage transaction synchronization.
func (r *Env) Open(path string, flags uint, mode os.FileMode) error {
	err := r.Env.Open(path, flags, mode)
	if err != nil {
		// no update to flags occurred
		return err
	}

	if flags&lmdb.NoLock != 0 {
		r.noLock = true
	}

	return nil
}

// SetMapSize is a proxy for r.Env.SetMapSize() that blocks while concurrent
// transactions are in progress.
func (r *Env) SetMapSize(size int64) error {
	return r.setMapSize(size, 0)
}

func (r *Env) setMapSize(size int64, delay time.Duration) error {
	r.txnlock.Lock()
	if delay > 0 {
		// wait before adopting a map size set from another process. hold on to
		// the transaction lock so that other transactions don't attempt to
		// begin while waiting.
		time.Sleep(delay)
	}
	err := r.Env.SetMapSize(size)
	r.txnlock.Unlock()
	return err
}

// BeginTxn overrides the r.Env.BeginTxn and always returns an error.  An
// unmanaged transaction.
func (r *Env) BeginTxn(parent *lmdb.Txn, flags uint) (*lmdb.Txn, error) {
	return nil, fmt.Errorf("lmdbsync: unmanaged transactions are not supported")
}

// RunTxn is a proxy for r.Env.RunTxn().
//
// If lmdb.NoLock is set on r.Env then RunTxn will block while other updates
// are in progress, regardless of flags.
func (r *Env) RunTxn(flags uint, op lmdb.TxnOp) (err error) {
	readonly := flags&lmdb.Readonly != 0
	return r.runHandler(readonly, func() error { return r.Env.RunTxn(flags, op) }, r.Handlers)
}

// View is a proxy for r.Env.View().
//
// If lmdb.NoLock is set on r.Env then View will block until any running update
// completes.
func (r *Env) View(op lmdb.TxnOp) error {
	return r.runHandler(true, func() error { return r.Env.View(op) }, r.Handlers)
}

// Update is a proxy for r.Env.Update().
//
// If lmdb.NoLock is set on r.Env then Update blocks until all other
// transactions have terminated and blocks all other transactions from running
// while in progress (including readonly transactions).
func (r *Env) Update(op lmdb.TxnOp) error {
	return r.runHandler(false, func() error { return r.Env.Update(op) }, r.Handlers)
}

// UpdateLocked is a proxy for r.Env.UpdateLocked().
//
// If lmdb.NoLock is set on r.Env then UpdateLocked blocks until all other
// transactions have terminated and blocks all other transactions from running
// while in progress (including readonly transactions).
func (r *Env) UpdateLocked(op lmdb.TxnOp) error {
	return r.runHandler(false, func() error { return r.Env.UpdateLocked(op) }, r.Handlers)
}

// WithHandler returns a TxnRunner than handles transaction errors r.Handlers
// chained with h.
func (r *Env) WithHandler(h Handler) TxnRunner {
	return &handlerRunner{
		env: r,
		h:   r.Handlers.Append(h),
	}
}

func (r *Env) runHandler(readonly bool, fn func() error, h Handler) error {
	b := bagWithEnv(r.bag, r)
	for {
		err := r.run(readonly, fn)
		b, err = h.HandleTxnErr(b, err)
		if err != ErrTxnRetry {
			return err
		}
	}
}
func (r *Env) run(readonly bool, fn func() error) error {
	var err error
	if r.noLock && !readonly {
		r.txnlock.Lock()
		err = fn()
		r.txnlock.Unlock()
	} else {
		r.txnlock.RLock()
		err = fn()
		r.txnlock.RUnlock()
	}
	return err
}
<|endoftext|>


<commit_before>// Package websession facilitates using http.Handlers with web-session.
//
// Overview:
//
// - HandlerUiView implements new*Session for you; you just need to supply
//   an http.Handler and an implementation of GetViewInfo.
// - GetSessionData allows http.Handlers to access session information.
// - SessionData contains the information passed to the new*Session methods.
package websession
<commit_msg>Tweak readme.go for websession.<commit_after>// Package websession facilitates using http.Handlers with web-session.
//
// HandlerUiView is the most central thing exposed by this package.
package websession
<|endoftext|>


<commit_before>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// ErrNoSystemd error is returned when the system is detected to
// have no support for systemd.
var ErrNoSystemd = errors.New("No systemd support found")

// Service type is a resource which manages services on a
// GNU/Linux system running with systemd.
//
// Example:
//   svc = resource.service.new("nginx")
//   svc.state = "running"
//   svc.enable = true
type Service struct {
	Base

	// If true then enable the service during boot-time
	Enable bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(name string) (Resource, error) {
	if !util.IsRunningSystemd() {
		return nil, ErrNoSystemd
	}

	s := &Service{
		Base: Base{
			Name:          name,
			Type:          "service",
			State:         "running",
			Require:       make([]string, 0),
			PresentStates: []string{"present", "running"},
			AbsentStates:  []string{"absent", "stopped"},
			Concurrent:    true,
		},
		Enable: true,
		unit:   fmt.Sprintf("%s.service", name),
	}

	return s, nil
}

// unitProperty retrieves the requested property for the service unit
func (s *Service) unitProperty(name string) (*dbus.Property, error) {
	conn, err := dbus.New()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	property, err := conn.GetUnitProperty(s.unit, name)

	return property, err
}

// unitIsEnabled checks if the unit is enabled or disabled
func (s *Service) unitIsEnabled() (bool, error) {
	unitState, err := s.unitProperty("UnitFileState")
	if err != nil {
		return false, err
	}

	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		return true, nil
	case "disabled", "masked", "masked-runtime":
		return false, nil
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("enabling service\n")

	units := []string{s.unit}
	_, changes, err := conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Log("%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("disabling service\n")

	units := []string{s.unit}
	changes, err := conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Log("%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// setUnitState enables or disables the unit
func (s *Service) setUnitState() error {
	enabled, err := s.unitIsEnabled()
	if err != nil {
		return err
	}

	if s.Enable && !enabled {
		if err := s.enableUnit(); err != nil {
			return err
		}
	} else {
		if err := s.disableUnit(); err != nil {
			return err
		}
	}

	return s.daemonReload()
}

// daemonReload instructs systemd to reload it's configuration
func (s *Service) daemonReload() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	return conn.Reload()
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	state := State{
		Current:  "unknown",
		Want:     s.State,
		Outdated: false,
	}

	// Check if the unit is started/stopped
	activeState, err := s.unitProperty("ActiveState")
	if err != nil {
		return state, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		state.Current = "running"
	case "inactive", "failed", "deactivating":
		state.Current = "stopped"
	}

	enabled, err := s.unitIsEnabled()
	if err != nil {
		return state, err
	}

	if s.Enable != enabled {
		state.Outdated = true
	}

	return state, nil
}

// Create starts the service unit
func (s *Service) Create() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("starting service\n")

	ch := make(chan string)
	jobID, err := conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Log("systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Delete stops the service unit
func (s *Service) Delete() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("stopping service\n")

	ch := make(chan string)
	jobID, err := conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Log("systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Update updates the service unit state
func (s *Service) Update() error {
	return s.setUnitState()
}

func init() {
	item := RegistryItem{
		Type:      "service",
		Provider:  NewService,
		Namespace: DefaultNamespace,
	}

	Register(item)
}
<commit_msg>resource: remove redundant logic from Service resource<commit_after>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// ErrNoSystemd error is returned when the system is detected to
// have no support for systemd.
var ErrNoSystemd = errors.New("No systemd support found")

// Service type is a resource which manages services on a
// GNU/Linux system running with systemd.
//
// Example:
//   svc = resource.service.new("nginx")
//   svc.state = "running"
//   svc.enable = true
type Service struct {
	Base

	// If true then enable the service during boot-time
	Enable bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(name string) (Resource, error) {
	if !util.IsRunningSystemd() {
		return nil, ErrNoSystemd
	}

	s := &Service{
		Base: Base{
			Name:          name,
			Type:          "service",
			State:         "running",
			Require:       make([]string, 0),
			PresentStates: []string{"present", "running"},
			AbsentStates:  []string{"absent", "stopped"},
			Concurrent:    true,
		},
		Enable: true,
		unit:   fmt.Sprintf("%s.service", name),
	}

	return s, nil
}

// unitProperty retrieves the requested property for the service unit
func (s *Service) unitProperty(name string) (*dbus.Property, error) {
	conn, err := dbus.New()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	property, err := conn.GetUnitProperty(s.unit, name)

	return property, err
}

// unitIsEnabled checks if the unit is enabled or disabled
func (s *Service) unitIsEnabled() (bool, error) {
	unitState, err := s.unitProperty("UnitFileState")
	if err != nil {
		return false, err
	}

	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		return true, nil
	case "disabled", "masked", "masked-runtime":
		return false, nil
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("enabling service\n")

	units := []string{s.unit}
	_, changes, err := conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Log("%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("disabling service\n")

	units := []string{s.unit}
	changes, err := conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Log("%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// setUnitState enables or disables the unit
func (s *Service) setUnitState() error {
	enabled, err := s.unitIsEnabled()
	if err != nil {
		return err
	}

	var action func() error
	if s.Enable && !enabled {
		action = s.enableUnit
	} else {
		action = s.disableUnit
	}

	if err := action(); err != nil {
		return err
	}

	return s.daemonReload()
}

// daemonReload instructs systemd to reload it's configuration
func (s *Service) daemonReload() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	return conn.Reload()
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	state := State{
		Current:  "unknown",
		Want:     s.State,
		Outdated: false,
	}

	// Check if the unit is started/stopped
	activeState, err := s.unitProperty("ActiveState")
	if err != nil {
		return state, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		state.Current = "running"
	case "inactive", "failed", "deactivating":
		state.Current = "stopped"
	}

	enabled, err := s.unitIsEnabled()
	if err != nil {
		return state, err
	}

	if s.Enable != enabled {
		state.Outdated = true
	}

	return state, nil
}

// Create starts the service unit
func (s *Service) Create() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("starting service\n")

	ch := make(chan string)
	jobID, err := conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Log("systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Delete stops the service unit
func (s *Service) Delete() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Log("stopping service\n")

	ch := make(chan string)
	jobID, err := conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Log("systemd job id %d result: %s\n", jobID, result)

	return s.setUnitState()
}

// Update updates the service unit state
func (s *Service) Update() error {
	return s.setUnitState()
}

func init() {
	item := RegistryItem{
		Type:      "service",
		Provider:  NewService,
		Namespace: DefaultNamespace,
	}

	Register(item)
}
<|endoftext|>


<commit_before>// package extensions implements various helper addons for Colly
package extensions
<commit_msg>[fix] package comment should be of the form "Package extensions ..."<commit_after>// Package extensions implements various helper addons for Colly
package extensions
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * io/serializer.go                                       *
 *                                                        *
 * hprose seriaizer for Go.                               *
 *                                                        *
 * LastModified: Aug 18, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package io

// Serializer is a interface for serializing build-in type
type Serializer interface {
	Serialize(writer *Writer, v interface{}) error
}

type refSerializer struct {
	value Serializer
}

func (s refSerializer) Serialize(writer *Writer, v interface{}) error {
	if ok, err := writer.WriteRef(v); ok || err != nil {
		return err
	}
	return s.value.Serialize(writer, v)
}

type nilSerializer struct{}

func (*nilSerializer) Serialize(writer *Writer, v interface{}) (err error) {
	return writer.WriteNil()
}

type boolSerializer struct{}

func (*boolSerializer) Serialize(writer *Writer, v interface{}) (err error) {
	return writer.WriteBool(v.(bool))
}

type intSerializer struct{}

func (*intSerializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt(int64(v.(int)))
}

type int8Serializer struct{}

func (*int8Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(int32(v.(int8)))
}

type int16Serializer struct{}

func (*int16Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(int32(v.(int16)))
}

type int32Serializer struct{}

func (*int32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(v.(int32))
}

type int64Serializer struct{}

func (*int64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt(v.(int64))
}

type uintSerializer struct{}

func (*uintSerializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint)))
}

type uint8Serializer struct{}

func (*uint8Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint8)))
}

type uint16Serializer struct{}

func (*uint16Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint16)))
}

type uint32Serializer struct{}

func (*uint32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint32)))
}

type uint64Serializer struct{}

func (*uint64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(v.(uint64))
}

type float32Serializer struct{}

func (*float32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteFloat(float64(v.(float32)), 32)
}

type float64Serializer struct{}

func (*float64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteFloat(v.(float64), 64)
}
<commit_msg>Improved refSerializer<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * io/serializer.go                                       *
 *                                                        *
 * hprose seriaizer for Go.                               *
 *                                                        *
 * LastModified: Aug 18, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package io

// Serializer is a interface for serializing build-in type
type Serializer interface {
	Serialize(writer *Writer, v interface{}) error
}

type refSerializer struct {
	value Serializer
}

func (s *refSerializer) Serialize(writer *Writer, v interface{}) error {
	if ok, err := writer.WriteRef(v); ok || err != nil {
		return err
	}
	return s.value.Serialize(writer, v)
}

type nilSerializer struct{}

func (*nilSerializer) Serialize(writer *Writer, v interface{}) (err error) {
	return writer.WriteNil()
}

type boolSerializer struct{}

func (*boolSerializer) Serialize(writer *Writer, v interface{}) (err error) {
	return writer.WriteBool(v.(bool))
}

type intSerializer struct{}

func (*intSerializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt(int64(v.(int)))
}

type int8Serializer struct{}

func (*int8Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(int32(v.(int8)))
}

type int16Serializer struct{}

func (*int16Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(int32(v.(int16)))
}

type int32Serializer struct{}

func (*int32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt32(v.(int32))
}

type int64Serializer struct{}

func (*int64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteInt(v.(int64))
}

type uintSerializer struct{}

func (*uintSerializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint)))
}

type uint8Serializer struct{}

func (*uint8Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint8)))
}

type uint16Serializer struct{}

func (*uint16Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint16)))
}

type uint32Serializer struct{}

func (*uint32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(uint64(v.(uint32)))
}

type uint64Serializer struct{}

func (*uint64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteUint(v.(uint64))
}

type float32Serializer struct{}

func (*float32Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteFloat(float64(v.(float32)), 32)
}

type float64Serializer struct{}

func (*float64Serializer) Serialize(writer *Writer, v interface{}) error {
	return writer.WriteFloat(v.(float64), 64)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016 Charith Ellawala
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gcgrpcpool

import (
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"

	"github.com/charithe/gcgrpcpool/gcgrpc"
	"github.com/golang/groupcache"
	"github.com/golang/groupcache/consistenthash"
	pb "github.com/golang/groupcache/groupcachepb"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const defaultReplicas = 50

type GRPCPool struct {
	self        string
	opts        GRPCPoolOptions
	mu          sync.Mutex
	peers       *consistenthash.Map
	grpcGetters map[string]*grpcGetter
}

type GRPCPoolOptions struct {
	Replicas        int
	HashFn          consistenthash.Hash
	PeerDialOptions []grpc.DialOption
}

func NewGRPCPool(self string, server *grpc.Server) *GRPCPool {
	return NewGRPCPoolOptions(self, server, nil)
}

var grpcPoolCreated bool

func NewGRPCPoolOptions(self string, server *grpc.Server, opts *GRPCPoolOptions) *GRPCPool {
	if grpcPoolCreated {
		panic("NewGRPCPool must be called only once")
	}

	grpcPoolCreated = true

	pool := &GRPCPool{
		self:        self,
		grpcGetters: make(map[string]*grpcGetter),
	}

	if opts != nil {
		pool.opts = *opts
	}

	if pool.opts.Replicas == 0 {
		pool.opts.Replicas = defaultReplicas
	}

	if pool.opts.PeerDialOptions == nil {
		pool.opts.PeerDialOptions = []grpc.DialOption{grpc.WithInsecure()}
	}

	pool.peers = consistenthash.New(pool.opts.Replicas, pool.opts.HashFn)
	groupcache.RegisterPeerPicker(func() groupcache.PeerPicker { return pool })
	gcgrpc.RegisterPeerServer(server, pool)
	return pool
}

func (gp *GRPCPool) Set(peers ...string) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	gp.peers = consistenthash.New(gp.opts.Replicas, gp.opts.HashFn)
	tempGetters := make(map[string]*grpcGetter, len(peers))
	for _, peer := range peers {
		if getter, exists := gp.grpcGetters[peer]; exists == true {
			tempGetters[peer] = getter
			delete(gp.grpcGetters, peer)
		} else {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.Warnf("Failed to open connection to [%s] : %v", peer, err)
			} else {
				tempGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}

	for p, g := range gp.grpcGetters {
		g.close()
		delete(gp.grpcGetters, p)
	}

	gp.grpcGetters = tempGetters
}

func (gp *GRPCPool) PickPeer(key string) (groupcache.ProtoGetter, bool) {
	gp.mu.Lock()
	defer gp.mu.Unlock()

	if gp.peers.IsEmpty() {
		return nil, false
	}

	if peer := gp.peers.Get(key); peer != gp.self {
		return gp.grpcGetters[peer], true
	}
	return nil, false
}

func (gp *GRPCPool) Retrieve(ctx context.Context, req *gcgrpc.RetrieveRequest) (*gcgrpc.RetrieveResponse, error) {
	group := groupcache.GetGroup(req.Group)
	if group == nil {
		return nil, fmt.Errorf("Unable to find group [%s]", req.Group)
	}
	group.Stats.ServerRequests.Add(1)
	var value []byte
	err := group.Get(ctx, req.Key, groupcache.AllocatingByteSliceSink(&value))
	if err != nil {
		return nil, fmt.Errorf("Failed to retrieve [%s]: %v", req, err)
	}

	return &gcgrpc.RetrieveResponse{Value: value}, nil
}

func (gp *GRPCPool) AddPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		if _, exists := gp.grpcGetters[peer]; exists != true {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.Warnf("Failed to open connection to [%s]: %v", peer, err)
			} else {
				log.Infof("Adding peer [%s]", peer)
				gp.grpcGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}
	return &gcgrpc.Ack{}, nil

}
func (gp *GRPCPool) RemovePeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	return &gcgrpc.Ack{}, nil
}
func (gp *GRPCPool) SetPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	return &gcgrpc.Ack{}, nil
}

type grpcGetter struct {
	address string
	conn    *grpc.ClientConn
}

func newGRPCGetter(address string, dialOpts ...grpc.DialOption) (*grpcGetter, error) {
	conn, err := grpc.Dial(address, dialOpts...)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to [%s]: %v", address, err)
	}
	return &grpcGetter{address: address, conn: conn}, nil
}

func (g *grpcGetter) Get(ctx groupcache.Context, in *pb.GetRequest, out *pb.GetResponse) error {
	client := gcgrpc.NewPeerClient(g.conn)
	resp, err := client.Retrieve(context.Background(), &gcgrpc.RetrieveRequest{Group: *in.Group, Key: *in.Key})
	if err != nil {
		return fmt.Errorf("Failed to GET [%s]: %v", in, err)
	}

	out.Value = resp.Value
	return nil
}

func (g *grpcGetter) close() {
	if g.conn != nil {
		g.conn.Close()
	}
}
<commit_msg>Add RPC methods to add/remove/set peers<commit_after>/*
 * Copyright 2016 Charith Ellawala
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gcgrpcpool

import (
	"fmt"
	"sync"

	log "github.com/Sirupsen/logrus"

	"github.com/charithe/gcgrpcpool/gcgrpc"
	"github.com/golang/groupcache"
	"github.com/golang/groupcache/consistenthash"
	pb "github.com/golang/groupcache/groupcachepb"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

const defaultReplicas = 50

type GRPCPool struct {
	self        string
	opts        GRPCPoolOptions
	mu          sync.Mutex
	peers       *consistenthash.Map
	grpcGetters map[string]*grpcGetter
}

type GRPCPoolOptions struct {
	Replicas        int
	HashFn          consistenthash.Hash
	PeerDialOptions []grpc.DialOption
}

func NewGRPCPool(self string, server *grpc.Server) *GRPCPool {
	return NewGRPCPoolOptions(self, server, nil)
}

var grpcPoolCreated bool

func NewGRPCPoolOptions(self string, server *grpc.Server, opts *GRPCPoolOptions) *GRPCPool {
	if grpcPoolCreated {
		panic("NewGRPCPool must be called only once")
	}

	grpcPoolCreated = true

	pool := &GRPCPool{
		self:        self,
		grpcGetters: make(map[string]*grpcGetter),
	}

	if opts != nil {
		pool.opts = *opts
	}

	if pool.opts.Replicas == 0 {
		pool.opts.Replicas = defaultReplicas
	}

	if pool.opts.PeerDialOptions == nil {
		pool.opts.PeerDialOptions = []grpc.DialOption{grpc.WithInsecure()}
	}

	pool.peers = consistenthash.New(pool.opts.Replicas, pool.opts.HashFn)
	groupcache.RegisterPeerPicker(func() groupcache.PeerPicker { return pool })
	gcgrpc.RegisterPeerServer(server, pool)
	return pool
}

func (gp *GRPCPool) Set(peers ...string) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	gp.peers = consistenthash.New(gp.opts.Replicas, gp.opts.HashFn)
	tempGetters := make(map[string]*grpcGetter, len(peers))
	for _, peer := range peers {
		if getter, exists := gp.grpcGetters[peer]; exists == true {
			tempGetters[peer] = getter
			delete(gp.grpcGetters, peer)
		} else {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				tempGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}

	for p, g := range gp.grpcGetters {
		g.close()
		delete(gp.grpcGetters, p)
	}

	gp.grpcGetters = tempGetters
}

func (gp *GRPCPool) PickPeer(key string) (groupcache.ProtoGetter, bool) {
	gp.mu.Lock()
	defer gp.mu.Unlock()

	if gp.peers.IsEmpty() {
		return nil, false
	}

	if peer := gp.peers.Get(key); peer != gp.self {
		return gp.grpcGetters[peer], true
	}
	return nil, false
}

func (gp *GRPCPool) Retrieve(ctx context.Context, req *gcgrpc.RetrieveRequest) (*gcgrpc.RetrieveResponse, error) {
	group := groupcache.GetGroup(req.Group)
	if group == nil {
		log.Warnf("Unable to find group [%s]", req.Group)
		return nil, fmt.Errorf("Unable to find group [%s]", req.Group)
	}
	group.Stats.ServerRequests.Add(1)
	var value []byte
	err := group.Get(ctx, req.Key, groupcache.AllocatingByteSliceSink(&value))
	if err != nil {
		log.WithError(err).Warnf("Failed to retrieve [%s]", req)
		return nil, fmt.Errorf("Failed to retrieve [%s]: %v", req, err)
	}

	return &gcgrpc.RetrieveResponse{Value: value}, nil
}

func (gp *GRPCPool) AddPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		if _, exists := gp.grpcGetters[peer]; exists != true {
			getter, err := newGRPCGetter(peer, gp.opts.PeerDialOptions...)
			if err != nil {
				log.WithError(err).Warnf("Failed to open connection to [%s]", peer)
			} else {
				log.Infof("Adding peer [%s]", peer)
				gp.grpcGetters[peer] = getter
				gp.peers.Add(peer)
			}
		}
	}
	return &gcgrpc.Ack{}, nil

}

func (gp *GRPCPool) RemovePeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.mu.Lock()
	defer gp.mu.Unlock()
	for _, peer := range peers.PeerAddr {
		log.Infof("Removing peer [%s]", peer)
		delete(gp.grpcGetters, peer)
	}
	return &gcgrpc.Ack{}, nil
}

func (gp *GRPCPool) SetPeers(ctx context.Context, peers *gcgrpc.Peers) (*gcgrpc.Ack, error) {
	gp.Set(peers.PeerAddr...)
	return &gcgrpc.Ack{}, nil
}

type grpcGetter struct {
	address string
	conn    *grpc.ClientConn
}

func newGRPCGetter(address string, dialOpts ...grpc.DialOption) (*grpcGetter, error) {
	conn, err := grpc.Dial(address, dialOpts...)
	if err != nil {
		return nil, fmt.Errorf("Failed to connect to [%s]: %v", address, err)
	}
	return &grpcGetter{address: address, conn: conn}, nil
}

func (g *grpcGetter) Get(ctx groupcache.Context, in *pb.GetRequest, out *pb.GetResponse) error {
	client := gcgrpc.NewPeerClient(g.conn)
	resp, err := client.Retrieve(context.Background(), &gcgrpc.RetrieveRequest{Group: *in.Group, Key: *in.Key})
	if err != nil {
		return fmt.Errorf("Failed to GET [%s]: %v", in, err)
	}

	out.Value = resp.Value
	return nil
}

func (g *grpcGetter) close() {
	if g.conn != nil {
		g.conn.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcs

import (
	"errors"
	"io"
	"net/http"

	"golang.org/x/net/context"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
)

// Bucket represents a GCS bucket, pre-bound with a bucket name and necessary
// authorization information.
//
// Each method that may block accepts a context object that is used for
// deadlines and cancellation. Users need not package authorization information
// into the context object (using cloud.WithContext or similar).
type Bucket interface {
	Name() string

	// List the objects in the bucket that meet the criteria defined by the
	// query, returning a result object that contains the results and potentially
	// a cursor for retrieving the next portion of the larger set of results.
	ListObjects(ctx context.Context, query *storage.Query) (*storage.Objects, error)

	// Create a reader for the contents of the object with the given name. The
	// caller must arrange for the reader to be closed when it is no longer
	// needed.
	NewReader(ctx context.Context, objectName string) (io.ReadCloser, error)

	// Return an ObjectWriter that can be used to create or overwrite an object
	// with the given name.
	NewWriter(ctx context.Context, objectName string) (ObjectWriter, error)
}

type bucket struct {
	projID string
	client *http.Client
	name   string
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(ctx context.Context, query *storage.Query) (*storage.Objects, error) {
	authContext := cloud.WithContext(ctx, b.projID, b.client)
	return storage.ListObjects(authContext, b.name, query)
}

func (b *bucket) NewReader(ctx context.Context, objectName string) (io.ReadCloser, error) {
	authContext := cloud.WithContext(ctx, b.projID, b.client)
	return storage.NewReader(authContext, b.name, objectName)
}

func (b *bucket) NewWriter(ctx context.Context, objectName string) (ObjectWriter, error) {
	return nil, errors.New("TODO(jacobsa): Implement NewWriter.")
}
<commit_msg>NewWriter should accept full attributes, not just the object name.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcs

import (
	"errors"
	"io"
	"net/http"

	"golang.org/x/net/context"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
)

// Bucket represents a GCS bucket, pre-bound with a bucket name and necessary
// authorization information.
//
// Each method that may block accepts a context object that is used for
// deadlines and cancellation. Users need not package authorization information
// into the context object (using cloud.WithContext or similar).
type Bucket interface {
	Name() string

	// List the objects in the bucket that meet the criteria defined by the
	// query, returning a result object that contains the results and potentially
	// a cursor for retrieving the next portion of the larger set of results.
	ListObjects(ctx context.Context, query *storage.Query) (*storage.Objects, error)

	// Create a reader for the contents of the object with the given name. The
	// caller must arrange for the reader to be closed when it is no longer
	// needed.
	NewReader(ctx context.Context, objectName string) (io.ReadCloser, error)

	// Return an ObjectWriter that can be used to create or overwrite an object
	// with the given attributes. attrs.Name must be specified.
	NewWriter(ctx context.Context, attrs *storage.ObjectAttrs) (ObjectWriter, error)
}

type bucket struct {
	projID string
	client *http.Client
	name   string
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) ListObjects(ctx context.Context, query *storage.Query) (*storage.Objects, error) {
	authContext := cloud.WithContext(ctx, b.projID, b.client)
	return storage.ListObjects(authContext, b.name, query)
}

func (b *bucket) NewReader(ctx context.Context, objectName string) (io.ReadCloser, error) {
	authContext := cloud.WithContext(ctx, b.projID, b.client)
	return storage.NewReader(authContext, b.name, objectName)
}

func (b *bucket) NewWriter(ctx context.Context, attrs *storage.ObjectAttrs) (ObjectWriter, error) {
	return nil, errors.New("TODO(jacobsa): Implement NewWriter.")
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package charset

import (
	"bytes"
	"encoding/xml"
	"io/ioutil"
	"runtime"
	"strings"
	"testing"

	"golang.org/x/text/transform"
)

func transformString(t transform.Transformer, s string) (string, error) {
	r := transform.NewReader(strings.NewReader(s), t)
	b, err := ioutil.ReadAll(r)
	return string(b), err
}

type testCase struct {
	utf8, other, otherEncoding string
}

// testCases for encoding and decoding.
var testCases = []testCase{
	{"Résumé", "Résumé", "utf8"},
	{"Résumé", "R\xe9sum\xe9", "latin1"},
	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
	{"Hello, world", "Hello, world", "ASCII"},
	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
	{"Résumé", "R\x8esum\x8e", "macintosh"},
	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
	{"Kağan", "Ka\xf0an", "windows-1254"},
	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
	{"latviešu", "latvie\xf0u", "windows-1257"},
	{"Việt", "Vi\xea\xf2t", "windows-1258"},
	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
	{"㧯", "\x82\x31\x89\x38", "gb18030"},
	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
	{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
	{"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
	{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
	{"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"},
}

func TestDecode(t *testing.T) {
	testCases := append(testCases, []testCase{
		// Replace multi-byte maximum subpart of ill-formed subsequence with
		// single replacement character (WhatWG requirement).
		{"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
	}...)
	for _, tc := range testCases {
		e, _ := Lookup(tc.otherEncoding)
		if e == nil {
			t.Errorf("%s: not found", tc.otherEncoding)
			continue
		}
		s, err := transformString(e.NewDecoder(), tc.other)
		if err != nil {
			t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
			continue
		}
		if s != tc.utf8 {
			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
		}
	}
}

func TestEncode(t *testing.T) {
	testCases := append(testCases, []testCase{
		// Use Go-style replacement.
		{"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
		// U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
		{"Gdańsk", "Gdańsk", "ISO-8859-11"},
		{"\ufffd", "�", "ISO-8859-11"},
		{"a\xe1\x80b", "a��b", "ISO-8859-11"},
	}...)
	for _, tc := range testCases {
		e, _ := Lookup(tc.otherEncoding)
		if e == nil {
			t.Errorf("%s: not found", tc.otherEncoding)
			continue
		}
		s, err := transformString(e.NewEncoder(), tc.utf8)
		if err != nil {
			t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
			continue
		}
		if s != tc.other {
			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
		}
	}
}

var sniffTestCases = []struct {
	filename, declared, want string
}{
	{"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"UTF-16LE-BOM.html", "", "utf-16le"},
	{"UTF-16BE-BOM.html", "", "utf-16be"},
	{"meta-content-attribute.html", "text/html", "iso-8859-15"},
	{"meta-charset-attribute.html", "text/html", "iso-8859-15"},
	{"No-encoding-declaration.html", "text/html", "utf-8"},
	{"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
	{"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
	{"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
}

func TestSniff(t *testing.T) {
	switch runtime.GOOS {
	case "nacl": // platforms that don't permit direct file system access
		t.Skipf("not supported on %q", runtime.GOOS)
	}

	for _, tc := range sniffTestCases {
		content, err := ioutil.ReadFile("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("%s: error reading file: %v", tc.filename, err)
			continue
		}

		_, name, _ := DetermineEncoding(content, tc.declared)
		if name != tc.want {
			t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
			continue
		}
	}
}

func TestReader(t *testing.T) {
	switch runtime.GOOS {
	case "nacl": // platforms that don't permit direct file system access
		t.Skipf("not supported on %q", runtime.GOOS)
	}

	for _, tc := range sniffTestCases {
		content, err := ioutil.ReadFile("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("%s: error reading file: %v", tc.filename, err)
			continue
		}

		r, err := NewReader(bytes.NewReader(content), tc.declared)
		if err != nil {
			t.Errorf("%s: error creating reader: %v", tc.filename, err)
			continue
		}

		got, err := ioutil.ReadAll(r)
		if err != nil {
			t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
			continue
		}

		e, _ := Lookup(tc.want)
		want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
		if err != nil {
			t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
			continue
		}

		if !bytes.Equal(got, want) {
			t.Errorf("%s: got %q, want %q", tc.filename, got, want)
			continue
		}
	}
}

var metaTestCases = []struct {
	meta, want string
}{
	{"", ""},
	{"text/html", ""},
	{"text/html; charset utf-8", ""},
	{"text/html; charset=latin-2", "latin-2"},
	{"text/html; charset; charset = utf-8", "utf-8"},
	{`charset="big5"`, "big5"},
	{"charset='shift_jis'", "shift_jis"},
}

func TestFromMeta(t *testing.T) {
	for _, tc := range metaTestCases {
		got := fromMetaElement(tc.meta)
		if got != tc.want {
			t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
		}
	}
}

func TestXML(t *testing.T) {
	const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><Word>r\xe9sum\xe9</Word>"

	d := xml.NewDecoder(strings.NewReader(s))
	d.CharsetReader = NewReaderLabel

	var a struct {
		Word string
	}
	err := d.Decode(&a)
	if err != nil {
		t.Fatalf("Decode: %v", err)
	}

	want := "résumé"
	if a.Word != want {
		t.Errorf("got %q, want %q", a.Word, want)
	}
}
<commit_msg>html/charset: replace EUC-KR test<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package charset

import (
	"bytes"
	"encoding/xml"
	"io/ioutil"
	"runtime"
	"strings"
	"testing"

	"golang.org/x/text/transform"
)

func transformString(t transform.Transformer, s string) (string, error) {
	r := transform.NewReader(strings.NewReader(s), t)
	b, err := ioutil.ReadAll(r)
	return string(b), err
}

type testCase struct {
	utf8, other, otherEncoding string
}

// testCases for encoding and decoding.
var testCases = []testCase{
	{"Résumé", "Résumé", "utf8"},
	{"Résumé", "R\xe9sum\xe9", "latin1"},
	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
	{"Hello, world", "Hello, world", "ASCII"},
	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
	{"Résumé", "R\x8esum\x8e", "macintosh"},
	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
	{"Kağan", "Ka\xf0an", "windows-1254"},
	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
	{"latviešu", "latvie\xf0u", "windows-1257"},
	{"Việt", "Vi\xea\xf2t", "windows-1258"},
	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
	{"㧯", "\x82\x31\x89\x38", "gb18030"},
	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
	{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
	{"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
	{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
	{"다음과 같은 조건을 따라야 합니다: 저작자표시", "\xb4\xd9\xc0\xbd\xb0\xfa \xb0\xb0\xc0\xba \xc1\xb6\xb0\xc7\xc0\xbb \xb5\xfb\xb6\xf3\xbe\xdf \xc7մϴ\xd9: \xc0\xfa\xc0\xdb\xc0\xdaǥ\xbd\xc3", "EUC-KR"},
}

func TestDecode(t *testing.T) {
	testCases := append(testCases, []testCase{
		// Replace multi-byte maximum subpart of ill-formed subsequence with
		// single replacement character (WhatWG requirement).
		{"Rés\ufffdumé", "Rés\xe1\x80umé", "utf8"},
	}...)
	for _, tc := range testCases {
		e, _ := Lookup(tc.otherEncoding)
		if e == nil {
			t.Errorf("%s: not found", tc.otherEncoding)
			continue
		}
		s, err := transformString(e.NewDecoder(), tc.other)
		if err != nil {
			t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
			continue
		}
		if s != tc.utf8 {
			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
		}
	}
}

func TestEncode(t *testing.T) {
	testCases := append(testCases, []testCase{
		// Use Go-style replacement.
		{"Rés\xe1\x80umé", "Rés\ufffd\ufffdumé", "utf8"},
		// U+0144 LATIN SMALL LETTER N WITH ACUTE not supported by encoding.
		{"Gdańsk", "Gdańsk", "ISO-8859-11"},
		{"\ufffd", "�", "ISO-8859-11"},
		{"a\xe1\x80b", "a��b", "ISO-8859-11"},
	}...)
	for _, tc := range testCases {
		e, _ := Lookup(tc.otherEncoding)
		if e == nil {
			t.Errorf("%s: not found", tc.otherEncoding)
			continue
		}
		s, err := transformString(e.NewEncoder(), tc.utf8)
		if err != nil {
			t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
			continue
		}
		if s != tc.other {
			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
		}
	}
}

var sniffTestCases = []struct {
	filename, declared, want string
}{
	{"HTTP-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"UTF-16LE-BOM.html", "", "utf-16le"},
	{"UTF-16BE-BOM.html", "", "utf-16be"},
	{"meta-content-attribute.html", "text/html", "iso-8859-15"},
	{"meta-charset-attribute.html", "text/html", "iso-8859-15"},
	{"No-encoding-declaration.html", "text/html", "utf-8"},
	{"HTTP-vs-UTF-8-BOM.html", "text/html; charset=iso-8859-15", "utf-8"},
	{"HTTP-vs-meta-content.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"HTTP-vs-meta-charset.html", "text/html; charset=iso-8859-15", "iso-8859-15"},
	{"UTF-8-BOM-vs-meta-content.html", "text/html", "utf-8"},
	{"UTF-8-BOM-vs-meta-charset.html", "text/html", "utf-8"},
}

func TestSniff(t *testing.T) {
	switch runtime.GOOS {
	case "nacl": // platforms that don't permit direct file system access
		t.Skipf("not supported on %q", runtime.GOOS)
	}

	for _, tc := range sniffTestCases {
		content, err := ioutil.ReadFile("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("%s: error reading file: %v", tc.filename, err)
			continue
		}

		_, name, _ := DetermineEncoding(content, tc.declared)
		if name != tc.want {
			t.Errorf("%s: got %q, want %q", tc.filename, name, tc.want)
			continue
		}
	}
}

func TestReader(t *testing.T) {
	switch runtime.GOOS {
	case "nacl": // platforms that don't permit direct file system access
		t.Skipf("not supported on %q", runtime.GOOS)
	}

	for _, tc := range sniffTestCases {
		content, err := ioutil.ReadFile("testdata/" + tc.filename)
		if err != nil {
			t.Errorf("%s: error reading file: %v", tc.filename, err)
			continue
		}

		r, err := NewReader(bytes.NewReader(content), tc.declared)
		if err != nil {
			t.Errorf("%s: error creating reader: %v", tc.filename, err)
			continue
		}

		got, err := ioutil.ReadAll(r)
		if err != nil {
			t.Errorf("%s: error reading from charset.NewReader: %v", tc.filename, err)
			continue
		}

		e, _ := Lookup(tc.want)
		want, err := ioutil.ReadAll(transform.NewReader(bytes.NewReader(content), e.NewDecoder()))
		if err != nil {
			t.Errorf("%s: error decoding with hard-coded charset name: %v", tc.filename, err)
			continue
		}

		if !bytes.Equal(got, want) {
			t.Errorf("%s: got %q, want %q", tc.filename, got, want)
			continue
		}
	}
}

var metaTestCases = []struct {
	meta, want string
}{
	{"", ""},
	{"text/html", ""},
	{"text/html; charset utf-8", ""},
	{"text/html; charset=latin-2", "latin-2"},
	{"text/html; charset; charset = utf-8", "utf-8"},
	{`charset="big5"`, "big5"},
	{"charset='shift_jis'", "shift_jis"},
}

func TestFromMeta(t *testing.T) {
	for _, tc := range metaTestCases {
		got := fromMetaElement(tc.meta)
		if got != tc.want {
			t.Errorf("%q: got %q, want %q", tc.meta, got, tc.want)
		}
	}
}

func TestXML(t *testing.T) {
	const s = "<?xml version=\"1.0\" encoding=\"windows-1252\"?><Word>r\xe9sum\xe9</Word>"

	d := xml.NewDecoder(strings.NewReader(s))
	d.CharsetReader = NewReaderLabel

	var a struct {
		Word string
	}
	err := d.Decode(&a)
	if err != nil {
		t.Fatalf("Decode: %v", err)
	}

	want := "résumé"
	if a.Word != want {
		t.Errorf("got %q, want %q", a.Word, want)
	}
}
<|endoftext|>


<commit_before>package gen

import (
	"io"
	"strconv"
)

const (
	structArraySizeVar = "ssz"
	structMapSizeVar   = "isz"
	mapSizeVar         = "msz"
	sliceSizeVar       = "xsz"
	arraySizeVar       = "asz"
)

func decode(w io.Writer) *decodeGen {
	return &decodeGen{
		p:        printer{w: w},
		hasfield: false,
	}
}

type decodeGen struct {
	passes
	p        printer
	hasfield bool
}

func (d *decodeGen) Method() Method { return Decode }

func (d *decodeGen) needsField() {
	if d.hasfield {
		return
	}
	d.p.print("\nvar field []byte; _ = field")
	d.hasfield = true
}

func (d *decodeGen) Execute(p Elem) error {
	p = d.applyall(p)
	if p == nil {
		return nil
	}
	d.hasfield = false
	if !d.p.ok() {
		return d.p.err
	}

	if !IsPrintable(p) {
		return nil
	}

	d.p.comment("DecodeMsg implements msgp.Decodable")

	d.p.printf("\nfunc (%s %s) DecodeMsg(dc *msgp.Reader) (err error) {", p.Varname(), methodReceiver(p))
	next(d, p)
	d.p.nakedReturn()
	unsetReceiver(p)
	return d.p.err
}

func (d *decodeGen) gStruct(s *Struct) {
	if !d.p.ok() {
		return
	}
	if s.AsTuple {
		d.structAsTuple(s)
	} else {
		d.structAsMap(s)
	}
	return
}

func (d *decodeGen) assignAndCheck(name string, typ string) {
	if !d.p.ok() {
		return
	}
	d.p.printf("\n%s, err = dc.Read%s()", name, typ)
	d.p.print(errcheck)
}

func (d *decodeGen) structAsTuple(s *Struct) {
	nfields := len(s.Fields)

	d.p.declare(structArraySizeVar, u32)
	d.assignAndCheck(structArraySizeVar, arrayHeader)
	d.p.arrayCheck(strconv.Itoa(nfields), structArraySizeVar)
	for i := range s.Fields {
		if !d.p.ok() {
			return
		}
		next(d, s.Fields[i].FieldElem)
	}
}

func (d *decodeGen) structAsMap(s *Struct) {
	d.needsField()
	d.p.declare(structMapSizeVar, u32)
	d.assignAndCheck(structMapSizeVar, mapHeader)

	d.p.print("\nfor isz > 0 {\nisz--")
	d.assignAndCheck("field", mapKey)
	d.p.print("\nswitch msgp.UnsafeString(field) {")
	for i := range s.Fields {
		d.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag)
		next(d, s.Fields[i].FieldElem)
		if !d.p.ok() {
			return
		}
	}
	d.p.print("\ndefault:\nerr = dc.Skip()")
	d.p.print(errcheck)
	d.p.closeblock() // close switch
	d.p.closeblock() // close for loop
}

func (d *decodeGen) gBase(b *BaseElem) {
	if !d.p.ok() {
		return
	}

	// open block for 'tmp'
	if b.Convert {
		d.p.printf("\n{ var tmp %s", b.BaseType())
	}

	vname := b.Varname()  // e.g. "z.FieldOne"
	bname := b.BaseName() // e.g. "Float64"

	// handle special cases
	// for object type.
	switch b.Value {
	case Bytes:
		if b.Convert {
			d.p.printf("\ntmp, err = dc.ReadBytes([]byte(%s))", vname)
		} else {
			d.p.printf("\n%s, err = dc.ReadBytes(%s)", vname, vname)
		}
	case IDENT:
		d.p.printf("\nerr = %s.DecodeMsg(dc)", vname)
	case Ext:
		d.p.printf("\nerr = dc.ReadExtension(%s)", vname)
	default:
		if b.Convert {
			d.p.printf("\ntmp, err = dc.Read%s()", bname)
		} else {
			d.p.printf("\n%s, err = dc.Read%s()", vname, bname)
		}
	}

	// close block for 'tmp'
	if b.Convert {
		d.p.printf("\n%s = %s(tmp)\n}", vname, b.FromBase())
	}

	d.p.print(errcheck)
}

func (d *decodeGen) gMap(m *Map) {
	if !d.p.ok() {
		return
	}

	// resize or allocate map
	d.p.declare(mapSizeVar, u32)
	d.assignAndCheck(mapSizeVar, mapHeader)
	d.p.resizeMap(mapSizeVar, m)

	// for element in map, read string/value
	// pair and assign
	d.p.print("\nfor msz > 0 {\nmsz--")
	d.p.declare(m.Keyidx, "string")
	d.p.declare(m.Validx, m.Value.TypeName())
	d.assignAndCheck(m.Keyidx, stringTyp)
	next(d, m.Value)
	d.p.mapAssign(m)
	d.p.closeblock()
}

func (d *decodeGen) gSlice(s *Slice) {
	if !d.p.ok() {
		return
	}
	d.p.declare(sliceSizeVar, u32)
	d.assignAndCheck(sliceSizeVar, arrayHeader)
	d.p.resizeSlice(sliceSizeVar, s)
	d.p.rangeBlock(s.Index, s.Varname(), d, s.Els)
}

func (d *decodeGen) gArray(a *Array) {
	if !d.p.ok() {
		return
	}

	// special case if we have [const]byte
	if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) {
		d.p.printf("\nerr = dc.ReadExactBytes(%s[:])", a.Varname())
		d.p.print(errcheck)
		return
	}

	d.p.declare(arraySizeVar, u32)
	d.assignAndCheck(arraySizeVar, arrayHeader)
	d.p.arrayCheck(a.Size, arraySizeVar)

	d.p.rangeBlock(a.Index, a.Varname(), d, a.Els)
}

func (d *decodeGen) gPtr(p *Ptr) {
	if !d.p.ok() {
		return
	}
	d.p.print("\nif dc.IsNil() {")
	d.p.print("\nerr = dc.ReadNil()")
	d.p.print(errcheck)
	d.p.printf("\n%s = nil\n} else {", p.Varname())
	d.p.initPtr(p)
	next(d, p.Value)
	d.p.closeblock()
}
<commit_msg>Fix variable redeclaration in nested tuples<commit_after>package gen

import (
	"io"
	"strconv"
)

const (
	structArraySizeVar = "ssz"
	structMapSizeVar   = "isz"
	mapSizeVar         = "msz"
	sliceSizeVar       = "xsz"
	arraySizeVar       = "asz"
)

func decode(w io.Writer) *decodeGen {
	return &decodeGen{
		p:        printer{w: w},
		hasfield: false,
	}
}

type decodeGen struct {
	passes
	p        printer
	hasfield bool
}

func (d *decodeGen) Method() Method { return Decode }

func (d *decodeGen) needsField() {
	if d.hasfield {
		return
	}
	d.p.print("\nvar field []byte; _ = field")
	d.hasfield = true
}

func (d *decodeGen) Execute(p Elem) error {
	p = d.applyall(p)
	if p == nil {
		return nil
	}
	d.hasfield = false
	if !d.p.ok() {
		return d.p.err
	}

	if !IsPrintable(p) {
		return nil
	}

	d.p.comment("DecodeMsg implements msgp.Decodable")

	d.p.printf("\nfunc (%s %s) DecodeMsg(dc *msgp.Reader) (err error) {", p.Varname(), methodReceiver(p))
	next(d, p)
	d.p.nakedReturn()
	unsetReceiver(p)
	return d.p.err
}

func (d *decodeGen) gStruct(s *Struct) {
	if !d.p.ok() {
		return
	}
	if s.AsTuple {
		d.structAsTuple(s)
	} else {
		d.structAsMap(s)
	}
	return
}

func (d *decodeGen) assignAndCheck(name string, typ string) {
	if !d.p.ok() {
		return
	}
	d.p.printf("\n%s, err = dc.Read%s()", name, typ)
	d.p.print(errcheck)
}

func (d *decodeGen) structAsTuple(s *Struct) {
	nfields := len(s.Fields)

	d.p.print("\n{")
	d.p.declare(structArraySizeVar, u32)
	d.assignAndCheck(structArraySizeVar, arrayHeader)
	d.p.arrayCheck(strconv.Itoa(nfields), structArraySizeVar)
	d.p.print("\n}")
	for i := range s.Fields {
		if !d.p.ok() {
			return
		}
		next(d, s.Fields[i].FieldElem)
	}
}

func (d *decodeGen) structAsMap(s *Struct) {
	d.needsField()
	d.p.declare(structMapSizeVar, u32)
	d.assignAndCheck(structMapSizeVar, mapHeader)

	d.p.print("\nfor isz > 0 {\nisz--")
	d.assignAndCheck("field", mapKey)
	d.p.print("\nswitch msgp.UnsafeString(field) {")
	for i := range s.Fields {
		d.p.printf("\ncase \"%s\":", s.Fields[i].FieldTag)
		next(d, s.Fields[i].FieldElem)
		if !d.p.ok() {
			return
		}
	}
	d.p.print("\ndefault:\nerr = dc.Skip()")
	d.p.print(errcheck)
	d.p.closeblock() // close switch
	d.p.closeblock() // close for loop
}

func (d *decodeGen) gBase(b *BaseElem) {
	if !d.p.ok() {
		return
	}

	// open block for 'tmp'
	if b.Convert {
		d.p.printf("\n{ var tmp %s", b.BaseType())
	}

	vname := b.Varname()  // e.g. "z.FieldOne"
	bname := b.BaseName() // e.g. "Float64"

	// handle special cases
	// for object type.
	switch b.Value {
	case Bytes:
		if b.Convert {
			d.p.printf("\ntmp, err = dc.ReadBytes([]byte(%s))", vname)
		} else {
			d.p.printf("\n%s, err = dc.ReadBytes(%s)", vname, vname)
		}
	case IDENT:
		d.p.printf("\nerr = %s.DecodeMsg(dc)", vname)
	case Ext:
		d.p.printf("\nerr = dc.ReadExtension(%s)", vname)
	default:
		if b.Convert {
			d.p.printf("\ntmp, err = dc.Read%s()", bname)
		} else {
			d.p.printf("\n%s, err = dc.Read%s()", vname, bname)
		}
	}

	// close block for 'tmp'
	if b.Convert {
		d.p.printf("\n%s = %s(tmp)\n}", vname, b.FromBase())
	}

	d.p.print(errcheck)
}

func (d *decodeGen) gMap(m *Map) {
	if !d.p.ok() {
		return
	}

	// resize or allocate map
	d.p.declare(mapSizeVar, u32)
	d.assignAndCheck(mapSizeVar, mapHeader)
	d.p.resizeMap(mapSizeVar, m)

	// for element in map, read string/value
	// pair and assign
	d.p.print("\nfor msz > 0 {\nmsz--")
	d.p.declare(m.Keyidx, "string")
	d.p.declare(m.Validx, m.Value.TypeName())
	d.assignAndCheck(m.Keyidx, stringTyp)
	next(d, m.Value)
	d.p.mapAssign(m)
	d.p.closeblock()
}

func (d *decodeGen) gSlice(s *Slice) {
	if !d.p.ok() {
		return
	}
	d.p.declare(sliceSizeVar, u32)
	d.assignAndCheck(sliceSizeVar, arrayHeader)
	d.p.resizeSlice(sliceSizeVar, s)
	d.p.rangeBlock(s.Index, s.Varname(), d, s.Els)
}

func (d *decodeGen) gArray(a *Array) {
	if !d.p.ok() {
		return
	}

	// special case if we have [const]byte
	if be, ok := a.Els.(*BaseElem); ok && (be.Value == Byte || be.Value == Uint8) {
		d.p.printf("\nerr = dc.ReadExactBytes(%s[:])", a.Varname())
		d.p.print(errcheck)
		return
	}

	d.p.declare(arraySizeVar, u32)
	d.assignAndCheck(arraySizeVar, arrayHeader)
	d.p.arrayCheck(a.Size, arraySizeVar)

	d.p.rangeBlock(a.Index, a.Varname(), d, a.Els)
}

func (d *decodeGen) gPtr(p *Ptr) {
	if !d.p.ok() {
		return
	}
	d.p.print("\nif dc.IsNil() {")
	d.p.print("\nerr = dc.ReadNil()")
	d.p.print(errcheck)
	d.p.printf("\n%s = nil\n} else {", p.Varname())
	d.p.initPtr(p)
	next(d, p.Value)
	d.p.closeblock()
}
<|endoftext|>