<commit_before>package core

import (
	"fmt"
	"io"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

// SharedState is a state which nodes in a topology can access. It can be a
// machine learning model, a data structure for aggregation (like a histgram),
// a configuration information for specific Boxes, and so on.
//
// SharedState doesn't have methods to read it's internal data because internal
// data representation heavily depends on each SharedState implementation. The
// easiest way to use SharedState from a component is to obtain the actual
// data type via the type assertion. See examples to learn more about how to
// use it.
//
// If a SharedState also implements Writer interface, it can be updated via
// SharedStateSink. Write method in it writes a tuple to the state. How tuples
// are processed depends on each SharedState. For example, a machine learning
// model might use a tuple as a training data, and another state could compute
// the average of a specific field. Write may return fatal or temporary errors
// as Box.Process does. See the documentation of Box.Process for details.
//
// Write method might be called after Terminate method is called. When it
// occurs, Write should return an error. Also, Write and Terminate can be
// called concurrently.
type SharedState interface {
	// Terminate finalizes the state. The state can no longer be used after
	// this method is called. This method doesn't have to be idempotent.
	//
	// Write or other methods the actual instance has might be called after
	// Terminate method is called. When it occurs, they should return an error.
	// Also, Terminate and them can be called concurrently.
	Terminate(ctx *Context) error
}

// SavableSharedState is a SharedState which can be persisted through Save
// method. Providing forward/backward compatibility of the saved file format
// is the responsibility of the author of the state.
//
// Because the best way of implementing Load method depends on each SharedState,
// it doesn't always have to be provided with Save method.
type SavableSharedState interface {
	SharedState

	// Save writes data of the state to a given writer. Save receives parameters
	// which are used to customize the behavior of the method. Parameters are
	// defined by each component and there's no common definition.
	//
	// Save and other methods can be called concurrently.
	Save(ctx *Context, w io.Writer, params data.Map) error
}

// LoadableSharedState is a SharedState which can be persisted through Save
// and Load method.
type LoadableSharedState interface {
	SavableSharedState

	// Load overwrites the state with save data. Parameters don't have to be
	// same as Save's parameters. They can even be completely different.
	// There MUST NOT be a required parameter. Values of required parameters
	// should be saved with the state itself.
	//
	// Load and other methods including Save can be called concurrently.
	Load(ctx *Context, r io.Reader, params data.Map) error
}

// TODO: Add MixiableSharedState interface

// SharedStateRegistry manages SharedState with names assigned to each state.
type SharedStateRegistry interface {
	// Add adds a state to the registry. It fails if the registry already has
	// a state having the same name. Add also calls SharedState.Init. If it
	// fails Add returns an error and doesn't register the SharedState. The
	// caller doesn't have to call Terminate on failure.
	//
	// Don't add the same instance of SharedState more than once to registries.
	// Otherwise, Init and Terminate methods of the state will be called
	// multiple times.
	Add(name, typeName string, s SharedState) error

	// Get returns a SharedState having the name in the registry. It returns
	// NotExistError if the registry doesn't have the state.
	Get(name string) (SharedState, error)

	// Type returns a type of a SharedState. It returns NotExistError if the
	// registry doesn't have the state.
	Type(name string) (string, error)

	// Replace replaces the previous SharedState instance with a new instance.
	// The previous instance is returned on success if any. The previous state
	// will not be terminated by the registry and the caller must call
	// Terminate. The type name must be same as the previous state's type name.
	//
	// The given SharedState is terminated when it cannot be replaced.
	Replace(name, typeName string, s SharedState) (SharedState, error)

	// List returns a map containing all SharedState the registry has.
	// The map returned from this method can safely be modified.
	List() (map[string]SharedState, error)

	// Remove removes a SharedState the registry has. It automatically
	// terminates the state. If SharedState.Terminate failed, Remove returns an
	// error. However, even if it returns an error, the state is removed from
	// the registry.
	//
	// Remove also returns the removed SharedState if the registry has it. When
	// SharedState.Terminate fails, Remove returns both the removed SharedState
	// and an error. If the registry doesn't have a SharedState having the name,
	// it returns a nil SharedState and NotExistError.
	Remove(name string) (SharedState, error)
}

type defaultSharedStateInfo struct {
	state    SharedState
	typeName string
}

type defaultSharedStateRegistry struct {
	ctx    *Context
	m      sync.RWMutex
	states map[string]*defaultSharedStateInfo
}

// NewDefaultSharedStateRegistry create a default registry of SharedStates.
func NewDefaultSharedStateRegistry(ctx *Context) SharedStateRegistry {
	return &defaultSharedStateRegistry{
		ctx:    ctx,
		states: map[string]*defaultSharedStateInfo{},
	}
}

func (r *defaultSharedStateRegistry) Add(name, typeName string, s SharedState) error {
	err := func() error {
		r.m.Lock()
		defer r.m.Unlock()
		if _, ok := r.states[name]; ok {
			return fmt.Errorf("the registry already has a state '%v'", name)
		}
		r.states[name] = &defaultSharedStateInfo{
			state:    s,
			typeName: typeName,
		}
		return nil
	}()
	if err != nil {
		if err := r.closeSharedState(s); err != nil {
			r.ctx.ErrLog(err).WithField("state_name", name).
				Errorf("Cannot terminate a state which couldn't be added to the registry due to name duplication")
		}
		return err // This is the original error
	}
	return nil
}

func (r *defaultSharedStateRegistry) closeSharedState(s SharedState) (err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
			} else {
				err = fmt.Errorf("SharedState.Terminate panicked: %v", e)
			}
		}
	}()
	return s.Terminate(r.ctx)
}

func (r *defaultSharedStateRegistry) Get(name string) (SharedState, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	if s, ok := r.states[name]; ok {
		return s.state, nil
	}
	return nil, NotExistError(fmt.Errorf("state '%v' was not found", name))
}

func (r *defaultSharedStateRegistry) Type(name string) (string, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	if s, ok := r.states[name]; ok {
		return s.typeName, nil
	}
	return "", NotExistError(fmt.Errorf("state '%v' was not found", name))
}

func (r *defaultSharedStateRegistry) Replace(name, typeName string, s SharedState) (SharedState, error) {
	r.m.Lock()
	defer r.m.Unlock()
	prev, ok := r.states[name]
	if ok {
		if prev.typeName != typeName {
			if err := r.closeSharedState(s); err != nil {
				r.ctx.ErrLog(err).WithField("state_name", name).
					WithField("state_type", typeName).WithField("prev_state_type", prev.typeName).
					Errorf("Cannot terminate a state which couldn't be replaced due to a type mismatch")
			}
			return nil, fmt.Errorf("state '%v' has a different type from the previous state's type", name)
		}
	}
	r.states[name] = &defaultSharedStateInfo{
		state:    s,
		typeName: typeName,
	}
	if prev == nil {
		return nil, nil
	}
	return prev.state, nil
}

func (r *defaultSharedStateRegistry) List() (map[string]SharedState, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	m := make(map[string]SharedState, len(r.states))
	for n, s := range r.states {
		m[n] = s.state
	}
	return m, nil
}

func (r *defaultSharedStateRegistry) Remove(name string) (SharedState, error) {
	s := func() SharedState {
		r.m.Lock()
		defer r.m.Unlock()
		if s, ok := r.states[name]; ok {
			delete(r.states, name)
			return s.state
		}
		return nil
	}()
	if s == nil {
		return nil, NotExistError(fmt.Errorf("state '%v' was not found", name))
	}

	if err := s.Terminate(r.ctx); err != nil {
		return s, err
	}
	return s, nil
}

// sharedStateSink represents a shared state. sharedStateSink refers to a shared state by name.
type sharedStateSink struct {
	name string
}

// NewSharedStateSink creates a sink that writes to SharedState.
func NewSharedStateSink(ctx *Context, name string) (Sink, error) {
	// Get SharedState by name
	state, err := ctx.SharedStates.Get(name)
	if err != nil {
		return nil, err
	}

	// It fails if the shared state cannot be written
	_, ok := state.(Writer)
	if !ok {
		return nil, fmt.Errorf("'%v' state cannot be written", name)
	}

	s := &sharedStateSink{
		name: name,
	}
	return s, nil
}

func (s *sharedStateSink) Write(ctx *Context, t *Tuple) error {
	state, err := ctx.SharedStates.Get(s.name)
	if err != nil {
		return err
	}

	// It fails if the shared state cannot be written
	writer, ok := state.(Writer)
	if !ok {
		return fmt.Errorf("'%v' state cannot be written", s.name)
	}

	return writer.Write(ctx, t)
}

func (s *sharedStateSink) Close(ctx *Context) error {
	// SharedState must not be terminated when this sink is closed because
	// the state is still being used by other components.
	return nil
}
<commit_msg>Add TODO comment for optimization.<commit_after>package core

import (
	"fmt"
	"io"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

// SharedState is a state which nodes in a topology can access. It can be a
// machine learning model, a data structure for aggregation (like a histgram),
// a configuration information for specific Boxes, and so on.
//
// SharedState doesn't have methods to read it's internal data because internal
// data representation heavily depends on each SharedState implementation. The
// easiest way to use SharedState from a component is to obtain the actual
// data type via the type assertion. See examples to learn more about how to
// use it.
//
// If a SharedState also implements Writer interface, it can be updated via
// SharedStateSink. Write method in it writes a tuple to the state. How tuples
// are processed depends on each SharedState. For example, a machine learning
// model might use a tuple as a training data, and another state could compute
// the average of a specific field. Write may return fatal or temporary errors
// as Box.Process does. See the documentation of Box.Process for details.
//
// Write method might be called after Terminate method is called. When it
// occurs, Write should return an error. Also, Write and Terminate can be
// called concurrently.
type SharedState interface {
	// Terminate finalizes the state. The state can no longer be used after
	// this method is called. This method doesn't have to be idempotent.
	//
	// Write or other methods the actual instance has might be called after
	// Terminate method is called. When it occurs, they should return an error.
	// Also, Terminate and them can be called concurrently.
	Terminate(ctx *Context) error
}

// SavableSharedState is a SharedState which can be persisted through Save
// method. Providing forward/backward compatibility of the saved file format
// is the responsibility of the author of the state.
//
// Because the best way of implementing Load method depends on each SharedState,
// it doesn't always have to be provided with Save method.
type SavableSharedState interface {
	SharedState

	// Save writes data of the state to a given writer. Save receives parameters
	// which are used to customize the behavior of the method. Parameters are
	// defined by each component and there's no common definition.
	//
	// Save and other methods can be called concurrently.
	Save(ctx *Context, w io.Writer, params data.Map) error
}

// LoadableSharedState is a SharedState which can be persisted through Save
// and Load method.
type LoadableSharedState interface {
	SavableSharedState

	// Load overwrites the state with save data. Parameters don't have to be
	// same as Save's parameters. They can even be completely different.
	// There MUST NOT be a required parameter. Values of required parameters
	// should be saved with the state itself.
	//
	// Load and other methods including Save can be called concurrently.
	Load(ctx *Context, r io.Reader, params data.Map) error
}

// TODO: Add MixiableSharedState interface

// SharedStateRegistry manages SharedState with names assigned to each state.
type SharedStateRegistry interface {
	// Add adds a state to the registry. It fails if the registry already has
	// a state having the same name. Add also calls SharedState.Init. If it
	// fails Add returns an error and doesn't register the SharedState. The
	// caller doesn't have to call Terminate on failure.
	//
	// Don't add the same instance of SharedState more than once to registries.
	// Otherwise, Init and Terminate methods of the state will be called
	// multiple times.
	Add(name, typeName string, s SharedState) error

	// Get returns a SharedState having the name in the registry. It returns
	// NotExistError if the registry doesn't have the state.
	Get(name string) (SharedState, error)

	// Type returns a type of a SharedState. It returns NotExistError if the
	// registry doesn't have the state.
	Type(name string) (string, error)

	// Replace replaces the previous SharedState instance with a new instance.
	// The previous instance is returned on success if any. The previous state
	// will not be terminated by the registry and the caller must call
	// Terminate. The type name must be same as the previous state's type name.
	//
	// The given SharedState is terminated when it cannot be replaced.
	Replace(name, typeName string, s SharedState) (SharedState, error)

	// List returns a map containing all SharedState the registry has.
	// The map returned from this method can safely be modified.
	List() (map[string]SharedState, error)

	// Remove removes a SharedState the registry has. It automatically
	// terminates the state. If SharedState.Terminate failed, Remove returns an
	// error. However, even if it returns an error, the state is removed from
	// the registry.
	//
	// Remove also returns the removed SharedState if the registry has it. When
	// SharedState.Terminate fails, Remove returns both the removed SharedState
	// and an error. If the registry doesn't have a SharedState having the name,
	// it returns a nil SharedState and NotExistError.
	Remove(name string) (SharedState, error)
}

type defaultSharedStateInfo struct {
	state    SharedState
	typeName string
}

type defaultSharedStateRegistry struct {
	ctx    *Context
	m      sync.RWMutex
	states map[string]*defaultSharedStateInfo
}

// NewDefaultSharedStateRegistry create a default registry of SharedStates.
func NewDefaultSharedStateRegistry(ctx *Context) SharedStateRegistry {
	return &defaultSharedStateRegistry{
		ctx:    ctx,
		states: map[string]*defaultSharedStateInfo{},
	}
}

func (r *defaultSharedStateRegistry) Add(name, typeName string, s SharedState) error {
	err := func() error {
		r.m.Lock()
		defer r.m.Unlock()
		if _, ok := r.states[name]; ok {
			return fmt.Errorf("the registry already has a state '%v'", name)
		}
		r.states[name] = &defaultSharedStateInfo{
			state:    s,
			typeName: typeName,
		}
		return nil
	}()
	if err != nil {
		if err := r.closeSharedState(s); err != nil {
			r.ctx.ErrLog(err).WithField("state_name", name).
				Errorf("Cannot terminate a state which couldn't be added to the registry due to name duplication")
		}
		return err // This is the original error
	}
	return nil
}

func (r *defaultSharedStateRegistry) closeSharedState(s SharedState) (err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
			} else {
				err = fmt.Errorf("SharedState.Terminate panicked: %v", e)
			}
		}
	}()
	return s.Terminate(r.ctx)
}

func (r *defaultSharedStateRegistry) Get(name string) (SharedState, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	if s, ok := r.states[name]; ok {
		return s.state, nil
	}
	return nil, NotExistError(fmt.Errorf("state '%v' was not found", name))
}

func (r *defaultSharedStateRegistry) Type(name string) (string, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	if s, ok := r.states[name]; ok {
		return s.typeName, nil
	}
	return "", NotExistError(fmt.Errorf("state '%v' was not found", name))
}

func (r *defaultSharedStateRegistry) Replace(name, typeName string, s SharedState) (SharedState, error) {
	r.m.Lock()
	defer r.m.Unlock()
	prev, ok := r.states[name]
	if ok {
		if prev.typeName != typeName {
			if err := r.closeSharedState(s); err != nil {
				r.ctx.ErrLog(err).WithField("state_name", name).
					WithField("state_type", typeName).WithField("prev_state_type", prev.typeName).
					Errorf("Cannot terminate a state which couldn't be replaced due to a type mismatch")
			}
			return nil, fmt.Errorf("state '%v' has a different type from the previous state's type", name)
		}
	}
	r.states[name] = &defaultSharedStateInfo{
		state:    s,
		typeName: typeName,
	}
	if prev == nil {
		return nil, nil
	}
	return prev.state, nil
}

func (r *defaultSharedStateRegistry) List() (map[string]SharedState, error) {
	r.m.RLock()
	defer r.m.RUnlock()
	m := make(map[string]SharedState, len(r.states))
	for n, s := range r.states {
		m[n] = s.state
	}
	return m, nil
}

func (r *defaultSharedStateRegistry) Remove(name string) (SharedState, error) {
	s := func() SharedState {
		r.m.Lock()
		defer r.m.Unlock()
		if s, ok := r.states[name]; ok {
			delete(r.states, name)
			return s.state
		}
		return nil
	}()
	if s == nil {
		return nil, NotExistError(fmt.Errorf("state '%v' was not found", name))
	}

	if err := s.Terminate(r.ctx); err != nil {
		return s, err
	}
	return s, nil
}

// sharedStateSink represents a shared state. sharedStateSink refers to a shared state by name.
type sharedStateSink struct {
	name string
}

// NewSharedStateSink creates a sink that writes to SharedState.
func NewSharedStateSink(ctx *Context, name string) (Sink, error) {
	// Get SharedState by name
	state, err := ctx.SharedStates.Get(name)
	if err != nil {
		return nil, err
	}

	// It fails if the shared state cannot be written
	_, ok := state.(Writer)
	if !ok {
		return nil, fmt.Errorf("'%v' state cannot be written", name)
	}

	// TODO: check whether the state is a LoadableSharedState for optimization.
	// When the state is a LoadableSharedState, we can omit state loading in Write() method.

	s := &sharedStateSink{
		name: name,
	}
	return s, nil
}

func (s *sharedStateSink) Write(ctx *Context, t *Tuple) error {
	state, err := ctx.SharedStates.Get(s.name)
	if err != nil {
		return err
	}

	// It fails if the shared state cannot be written
	writer, ok := state.(Writer)
	if !ok {
		return fmt.Errorf("'%v' state cannot be written", s.name)
	}

	return writer.Write(ctx, t)
}

func (s *sharedStateSink) Close(ctx *Context) error {
	// SharedState must not be terminated when this sink is closed because
	// the state is still being used by other components.
	return nil
}
<|endoftext|>


<commit_before>package wallet

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// AgeDelay indicates how long the wallet will wait before allowing the
	// user to double-spend a transaction under standard circumstances. The
	// rationale is that most transactions are meant to be submitted to the
	// blockchain immediately, and ones that take more than AgeDelay blocks
	// have probably failed in some way.
	AgeDelay = 80

	// TransactionFee is yet another deprecated-on-arrival constant that says
	// how large the transaction fees should be. This should really be a
	// function supplied by the transaction pool.
	TransactionFee = 10
)

// A Wallet uses the state and transaction pool to track the unconfirmed
// balance of a user. All of the keys are stored in 'saveDir'/wallet.dat.
//
// One feature of the wallet is preventing accidental double spends. The wallet
// will block an output from being spent if it has been spent in the last
// 'AgeDelay' blocks. This is managed by tracking a global age for the wallet
// and then an age for each output, set to the age of the wallet that the
// output was most recently spent. If the wallet is 'AgeDelay' blocks older
// than an output, then the output can be spent again.
//
// A second feature of the wallet is the transaction builder, which is a series
// of functions that can be used to build independent transactions for use with
// untrusted parties. The transactions can be cobbled together piece by piece
// and then signed. When using the transaction builder, the wallet will always
// have exact outputs (by creating another transaction first if needed) and
// thus the transaction does not need to be spent for the transaction builder
// to be able to use any refunds.
type Wallet struct {
	state            modules.ConsensusSet
	tpool            modules.TransactionPool
	unconfirmedDiffs []modules.SiacoinOutputDiff

	// Location of the wallet directory, for saving and loading keys.
	saveDir string

	// A key contains all the information necessary to spend a particular
	// address, as well as all the known outputs that use the address.
	//
	// age is a tool to determine whether or not an output can be spent. When
	// an output is spent by the wallet, the age of the output is marked equal
	// to the age of the wallet. It will not be spent again until the age is
	// `AgeDelay` less than the wallet. The wallet ages by 1 every block. The
	// wallet can also be manually aged, which is a convenient and efficient
	// way of resetting spent outputs. Transactions are not intended to be
	// broadcast for a while can be given an age that is much greater than the
	// wallet.
	//
	// Timelocked keys is a list of addresses found in `keys` that can't be
	// spent until a certain height. The wallet will use `timelockedKeys` to
	// mark keys as unspendable until the timelock has lifted.
	//
	// Visible keys will be displayed to the user.
	consensusHeight  types.BlockHeight
	age              int
	keys             map[types.UnlockHash]*key
	timelockedKeys   map[types.BlockHeight][]types.UnlockHash
	visibleAddresses map[types.UnlockHash]struct{}
	siafundAddresses map[types.UnlockHash]struct{}
	siafundOutputs   map[types.SiafundOutputID]types.SiafundOutput

	// transactions is a list of transactions that are currently being built by
	// the wallet. Each transaction has a unique id, which is enforced by the
	// transactionCounter.
	transactionCounter int
	transactions       map[string]*openTransaction

	subscribers []chan struct{}

	mu *sync.RWMutex
}

// New creates a new wallet, loading any known addresses from the input file
// name and then using the file to save in the future.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, saveDir string) (w *Wallet, err error) {
	if cs == nil {
		err = errors.New("wallet cannot use a nil state")
		return
	}
	if tpool == nil {
		err = errors.New("wallet cannot use a nil transaction pool")
		return
	}

	w = &Wallet{
		state: cs,
		tpool: tpool,

		saveDir: saveDir,

		age:              AgeDelay + 100,
		keys:             make(map[types.UnlockHash]*key),
		timelockedKeys:   make(map[types.BlockHeight][]types.UnlockHash),
		visibleAddresses: make(map[types.UnlockHash]struct{}),
		siafundAddresses: make(map[types.UnlockHash]struct{}),
		siafundOutputs:   make(map[types.SiafundOutputID]types.SiafundOutput),

		transactions: make(map[string]*openTransaction),

		mu: sync.New(modules.SafeMutexDelay, 1),
	}

	// Create the wallet folder.
	err = os.MkdirAll(saveDir, 0700)
	if err != nil {
		return
	}

	// Try to load a previously saved wallet file. If it doesn't exist, assume
	// that we're creating a new wallet file.
	// TODO: log warning if no file found?
	err = w.load()
	if os.IsNotExist(err) {
		err = nil
		// No wallet file exists... make a visible address for the user.
		_, _, err = w.coinAddress(true)
		if err != nil {
			return nil, err
		}
	}
	if err != nil {
		err = fmt.Errorf("couldn't load wallet file %s: %v", saveDir, err)
		// TODO: try to recover from wallet.backup?
		return
	}

	w.tpool.TransactionPoolSubscribe(w)

	return
}

func (w *Wallet) Close() error {
	id := w.mu.RLock()
	defer w.mu.RUnlock(id)
	return w.save()
}

// SpendCoins creates a transaction sending 'amount' to 'dest'. The transaction
// is submitted to the transaction pool and is also returned.
func (w *Wallet) SpendCoins(amount types.Currency, dest types.UnlockHash) (t types.Transaction, err error) {
	tPoolFee := types.NewCurrency64(10).Mul(types.SiacoinPrecision)
	amount = amount.Add(tPoolFee)

	// Create and send the transaction.
	output := types.SiacoinOutput{
		Value:      amount,
		UnlockHash: dest,
	}
	id, err := w.RegisterTransaction(t)
	if err != nil {
		return
	}
	_, err = w.FundTransaction(id, amount)
	if err != nil {
		return
	}
	_, _, err = w.AddSiacoinOutput(id, output)
	_, _, err = w.AddMinerFee(id, tPoolFee)
	if err != nil {
		return
	}
	_, _, err = w.AddOutput(id, output)
	if err != nil {
		return
	}
	t, err = w.SignTransaction(id, true)
	if err != nil {
		return
	}
	err = w.tpool.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<commit_msg>Fixed misnamed function in merge conflict<commit_after>package wallet

import (
	"errors"
	"fmt"
	"os"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/sync"
	"github.com/NebulousLabs/Sia/types"
)

const (
	// AgeDelay indicates how long the wallet will wait before allowing the
	// user to double-spend a transaction under standard circumstances. The
	// rationale is that most transactions are meant to be submitted to the
	// blockchain immediately, and ones that take more than AgeDelay blocks
	// have probably failed in some way.
	AgeDelay = 80

	// TransactionFee is yet another deprecated-on-arrival constant that says
	// how large the transaction fees should be. This should really be a
	// function supplied by the transaction pool.
	TransactionFee = 10
)

// A Wallet uses the state and transaction pool to track the unconfirmed
// balance of a user. All of the keys are stored in 'saveDir'/wallet.dat.
//
// One feature of the wallet is preventing accidental double spends. The wallet
// will block an output from being spent if it has been spent in the last
// 'AgeDelay' blocks. This is managed by tracking a global age for the wallet
// and then an age for each output, set to the age of the wallet that the
// output was most recently spent. If the wallet is 'AgeDelay' blocks older
// than an output, then the output can be spent again.
//
// A second feature of the wallet is the transaction builder, which is a series
// of functions that can be used to build independent transactions for use with
// untrusted parties. The transactions can be cobbled together piece by piece
// and then signed. When using the transaction builder, the wallet will always
// have exact outputs (by creating another transaction first if needed) and
// thus the transaction does not need to be spent for the transaction builder
// to be able to use any refunds.
type Wallet struct {
	state            modules.ConsensusSet
	tpool            modules.TransactionPool
	unconfirmedDiffs []modules.SiacoinOutputDiff

	// Location of the wallet directory, for saving and loading keys.
	saveDir string

	// A key contains all the information necessary to spend a particular
	// address, as well as all the known outputs that use the address.
	//
	// age is a tool to determine whether or not an output can be spent. When
	// an output is spent by the wallet, the age of the output is marked equal
	// to the age of the wallet. It will not be spent again until the age is
	// `AgeDelay` less than the wallet. The wallet ages by 1 every block. The
	// wallet can also be manually aged, which is a convenient and efficient
	// way of resetting spent outputs. Transactions are not intended to be
	// broadcast for a while can be given an age that is much greater than the
	// wallet.
	//
	// Timelocked keys is a list of addresses found in `keys` that can't be
	// spent until a certain height. The wallet will use `timelockedKeys` to
	// mark keys as unspendable until the timelock has lifted.
	//
	// Visible keys will be displayed to the user.
	consensusHeight  types.BlockHeight
	age              int
	keys             map[types.UnlockHash]*key
	timelockedKeys   map[types.BlockHeight][]types.UnlockHash
	visibleAddresses map[types.UnlockHash]struct{}
	siafundAddresses map[types.UnlockHash]struct{}
	siafundOutputs   map[types.SiafundOutputID]types.SiafundOutput

	// transactions is a list of transactions that are currently being built by
	// the wallet. Each transaction has a unique id, which is enforced by the
	// transactionCounter.
	transactionCounter int
	transactions       map[string]*openTransaction

	subscribers []chan struct{}

	mu *sync.RWMutex
}

// New creates a new wallet, loading any known addresses from the input file
// name and then using the file to save in the future.
func New(cs modules.ConsensusSet, tpool modules.TransactionPool, saveDir string) (w *Wallet, err error) {
	if cs == nil {
		err = errors.New("wallet cannot use a nil state")
		return
	}
	if tpool == nil {
		err = errors.New("wallet cannot use a nil transaction pool")
		return
	}

	w = &Wallet{
		state: cs,
		tpool: tpool,

		saveDir: saveDir,

		age:              AgeDelay + 100,
		keys:             make(map[types.UnlockHash]*key),
		timelockedKeys:   make(map[types.BlockHeight][]types.UnlockHash),
		visibleAddresses: make(map[types.UnlockHash]struct{}),
		siafundAddresses: make(map[types.UnlockHash]struct{}),
		siafundOutputs:   make(map[types.SiafundOutputID]types.SiafundOutput),

		transactions: make(map[string]*openTransaction),

		mu: sync.New(modules.SafeMutexDelay, 1),
	}

	// Create the wallet folder.
	err = os.MkdirAll(saveDir, 0700)
	if err != nil {
		return
	}

	// Try to load a previously saved wallet file. If it doesn't exist, assume
	// that we're creating a new wallet file.
	// TODO: log warning if no file found?
	err = w.load()
	if os.IsNotExist(err) {
		err = nil
		// No wallet file exists... make a visible address for the user.
		_, _, err = w.coinAddress(true)
		if err != nil {
			return nil, err
		}
	}
	if err != nil {
		err = fmt.Errorf("couldn't load wallet file %s: %v", saveDir, err)
		// TODO: try to recover from wallet.backup?
		return
	}

	w.tpool.TransactionPoolSubscribe(w)

	return
}

func (w *Wallet) Close() error {
	id := w.mu.RLock()
	defer w.mu.RUnlock(id)
	return w.save()
}

// SpendCoins creates a transaction sending 'amount' to 'dest'. The transaction
// is submitted to the transaction pool and is also returned.
func (w *Wallet) SpendCoins(amount types.Currency, dest types.UnlockHash) (t types.Transaction, err error) {
	tPoolFee := types.NewCurrency64(10).Mul(types.SiacoinPrecision)
	amount = amount.Add(tPoolFee)

	// Create and send the transaction.
	output := types.SiacoinOutput{
		Value:      amount,
		UnlockHash: dest,
	}
	id, err := w.RegisterTransaction(t)
	if err != nil {
		return
	}
	_, err = w.FundTransaction(id, amount)
	if err != nil {
		return
	}
	_, _, err = w.AddSiacoinOutput(id, output)
	if err != nil {
		return
	}
	_, _, err = w.AddMinerFee(id, tPoolFee)
	if err != nil {
		return
	}
	t, err = w.SignTransaction(id, true)
	if err != nil {
		return
	}
	err = w.tpool.AcceptTransaction(t)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package httpexpect

import (
	"github.com/moul/http2curl"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

// CompactPrinter implements Printer. It prints requests in compact form.
type CompactPrinter struct {
	logger Logger
}

// NewCompactPrinter returns a new CompactPrinter given a logger.
func NewCompactPrinter(logger Logger) CompactPrinter {
	return CompactPrinter{logger}
}

// Request implements Printer.Request.
func (p CompactPrinter) Request(req *http.Request) {
	if req != nil {
		p.logger.Logf("%s %s", req.Method, req.URL)
	}
}

// Response implements Printer.Response.
func (CompactPrinter) Response(*http.Response, time.Duration) {
}

// DebugPrinter implements Printer. Uses net/http/httputil to dump
// both requests and responses.
type DebugPrinter struct {
	logger Logger
	body   bool
}

// NewDebugPrinter returns a new DebugPrinter given a logger and body
// flag. If body is true, request and response body is also printed.
func NewDebugPrinter(logger Logger, body bool) DebugPrinter {
	return DebugPrinter{logger, body}
}

// Request implements Printer.Request.
func (p DebugPrinter) Request(req *http.Request) {
	if req == nil {
		return
	}

	dump, err := httputil.DumpRequestOut(req, p.body)
	if err != nil {
		panic(err)
	}
	p.logger.Logf("%s", dump)
}

// Response implements Printer.Response.
func (p DebugPrinter) Response(resp *http.Response, duration time.Duration) {
	if resp == nil {
		return
	}

	dump, err := httputil.DumpResponse(resp, p.body)
	if err != nil {
		panic(err)
	}

	text := strings.Replace(string(dump), "\r\n", "\n", -1)
	lines := strings.SplitN(text, "\n", 2)

	p.logger.Logf("%s %s\n%s", lines[0], duration, lines[1])
}

// CurlPrinter implements Printer. Uses http2curl to dump requests as
// curl commands.
type CurlPrinter struct {
	logger Logger
}

// NewCurlPrinter returns a new CurlPrinter given a logger.
func NewCurlPrinter(logger Logger) CurlPrinter {
	return CurlPrinter{logger}
}

// Request implements Printer.Request.
func (p CurlPrinter) Request(req *http.Request) {
	if req != nil {
		cmd, err := http2curl.GetCurlCommand(req)
		if err != nil {
			panic(err)
		}
		p.logger.Logf("%s", cmd.String())
	}
}

// Response implements Printer.Response.
func (CurlPrinter) Response(*http.Response, time.Duration) {
}
<commit_msg>Fix panic in DebugPrinter<commit_after>package httpexpect

import (
	"github.com/moul/http2curl"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

// CompactPrinter implements Printer. It prints requests in compact form.
type CompactPrinter struct {
	logger Logger
}

// NewCompactPrinter returns a new CompactPrinter given a logger.
func NewCompactPrinter(logger Logger) CompactPrinter {
	return CompactPrinter{logger}
}

// Request implements Printer.Request.
func (p CompactPrinter) Request(req *http.Request) {
	if req != nil {
		p.logger.Logf("%s %s", req.Method, req.URL)
	}
}

// Response implements Printer.Response.
func (CompactPrinter) Response(*http.Response, time.Duration) {
}

// DebugPrinter implements Printer. Uses net/http/httputil to dump
// both requests and responses.
type DebugPrinter struct {
	logger Logger
	body   bool
}

// NewDebugPrinter returns a new DebugPrinter given a logger and body
// flag. If body is true, request and response body is also printed.
func NewDebugPrinter(logger Logger, body bool) DebugPrinter {
	return DebugPrinter{logger, body}
}

// Request implements Printer.Request.
func (p DebugPrinter) Request(req *http.Request) {
	if req == nil {
		return
	}

	dump, err := httputil.DumpRequest(req, p.body)
	if err != nil {
		panic(err)
	}
	p.logger.Logf("%s", dump)
}

// Response implements Printer.Response.
func (p DebugPrinter) Response(resp *http.Response, duration time.Duration) {
	if resp == nil {
		return
	}

	dump, err := httputil.DumpResponse(resp, p.body)
	if err != nil {
		panic(err)
	}

	text := strings.Replace(string(dump), "\r\n", "\n", -1)
	lines := strings.SplitN(text, "\n", 2)

	p.logger.Logf("%s %s\n%s", lines[0], duration, lines[1])
}

// CurlPrinter implements Printer. Uses http2curl to dump requests as
// curl commands.
type CurlPrinter struct {
	logger Logger
}

// NewCurlPrinter returns a new CurlPrinter given a logger.
func NewCurlPrinter(logger Logger) CurlPrinter {
	return CurlPrinter{logger}
}

// Request implements Printer.Request.
func (p CurlPrinter) Request(req *http.Request) {
	if req != nil {
		cmd, err := http2curl.GetCurlCommand(req)
		if err != nil {
			panic(err)
		}
		p.logger.Logf("%s", cmd.String())
	}
}

// Response implements Printer.Response.
func (CurlPrinter) Response(*http.Response, time.Duration) {
}
<|endoftext|>


<commit_before>package main

import (
	"fullerite/config"
	"fullerite/handler"
	"fullerite/metric"
)

func startHandlers(c config.Config) (handlers []handler.Handler) {
	log.Info("Starting handlers...")
	for name, config := range c.Handlers {
		handlers = append(handlers, startHandler(name, c, config))
	}
	return handlers
}

func startHandler(name string, globalConfig config.Config, instanceConfig map[string]interface{}) handler.Handler {
	log.Debug("Starting handler ", name)
	handlerInst := handler.New(name)

	// apply any global configs
	handlerInst.SetInterval(config.GetAsInt(globalConfig.Interval, handler.DefaultInterval))
	handlerInst.SetPrefix(globalConfig.Prefix)
	handlerInst.SetDefaultDimensions(globalConfig.DefaultDimensions)

	// now apply the handler level configs
	handlerInst.Configure(instanceConfig)

	go handlerInst.Run()
	return handlerInst
}

func writeToHandlers(handlers []handler.Handler, metric metric.Metric) {
	for _, handler := range handlers {
		handler.Channel() <- metric
	}
}
<commit_msg>log handler start as Info<commit_after>package main

import (
	"fullerite/config"
	"fullerite/handler"
	"fullerite/metric"
)

func startHandlers(c config.Config) (handlers []handler.Handler) {
	log.Info("Starting handlers...")
	for name, config := range c.Handlers {
		handlers = append(handlers, startHandler(name, c, config))
	}
	return handlers
}

func startHandler(name string, globalConfig config.Config, instanceConfig map[string]interface{}) handler.Handler {
	log.Info("Starting handler ", name)
	handlerInst := handler.New(name)

	// apply any global configs
	handlerInst.SetInterval(config.GetAsInt(globalConfig.Interval, handler.DefaultInterval))
	handlerInst.SetPrefix(globalConfig.Prefix)
	handlerInst.SetDefaultDimensions(globalConfig.DefaultDimensions)

	// now apply the handler level configs
	handlerInst.Configure(instanceConfig)

	go handlerInst.Run()
	return handlerInst
}

func writeToHandlers(handlers []handler.Handler, metric metric.Metric) {
	for _, handler := range handlers {
		handler.Channel() <- metric
	}
}
<|endoftext|>


<commit_before>package gorm

import (
	"fmt"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"
)

func (s *DB) clone() *DB {
	db := DB{db: s.db, parent: s.parent, logMode: s.logMode, data: s.data, Error: s.Error}

	if s.search == nil {
		db.search = &search{}
	} else {
		db.search = s.search.clone()
	}

	db.search.db = &db
	return &db
}

func (s *DB) new() *DB {
	db := DB{db: s.db, parent: s.parent, logMode: s.logMode, data: s.data, Error: s.Error, search: &search{}}
	db.search.db = &db
	return &db
}

func (s *DB) do(data interface{}) *Do {
	s.data = data
	do := Do{db: s}
	do.setModel(data)
	return &do
}

func (s *DB) fileWithLineNum() string {
	for i := 5; i < 15; i++ {
		_, file, line, ok := runtime.Caller(i)
		if ok && (!regexp.MustCompile(`jinzhu/gorm/.*.go`).MatchString(file) || regexp.MustCompile(`jinzhu/gorm/.*test.go`).MatchString(file)) {
			return fmt.Sprintf("%v:%v", strings.TrimPrefix(file, os.Getenv("GOPATH")+"src/"), line)
		}
	}
	return ""
}

func (s *DB) err(err error) error {
	if err != nil {
		s.Error = err
		if s.logMode == 0 {
			if err != RecordNotFound {
				go fmt.Println(s.fileWithLineNum(), err)
			}
		} else {
			s.warn(err)
		}
	}
	return err
}

func (s *DB) hasError() bool {
	return s.Error != nil
}

func (s *DB) print(level string, v ...interface{}) {
	if s.logMode == 2 || level == "debug" {
		if _, ok := s.parent.logger.(Logger); !ok {
			fmt.Println("logger haven't been set, using os.Stdout")
			s.parent.logger = default_logger
		}
		args := []interface{}{level}
		s.parent.logger.(Logger).Print(append(args, v...)...)
	}
}

func (s *DB) warn(v ...interface{}) {
	go s.print("warn", v...)
}

func (s *DB) info(v ...interface{}) {
	go s.print("info", v...)
}

func (s *DB) slog(sql string, t time.Time, vars ...interface{}) {
	go s.print("sql", time.Now().Sub(t), sql, vars)
}

func (s *DB) debug(v ...interface{}) {
	go s.print("debug", v...)
}
<commit_msg>ignore error `sql: Scan error on column index...`<commit_after>package gorm

import (
	"fmt"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"
)

func (s *DB) clone() *DB {
	db := DB{db: s.db, parent: s.parent, logMode: s.logMode, data: s.data, Error: s.Error}

	if s.search == nil {
		db.search = &search{}
	} else {
		db.search = s.search.clone()
	}

	db.search.db = &db
	return &db
}

func (s *DB) new() *DB {
	db := DB{db: s.db, parent: s.parent, logMode: s.logMode, data: s.data, Error: s.Error, search: &search{}}
	db.search.db = &db
	return &db
}

func (s *DB) do(data interface{}) *Do {
	s.data = data
	do := Do{db: s}
	do.setModel(data)
	return &do
}

func (s *DB) fileWithLineNum() string {
	for i := 5; i < 15; i++ {
		_, file, line, ok := runtime.Caller(i)
		if ok && (!regexp.MustCompile(`jinzhu/gorm/.*.go`).MatchString(file) || regexp.MustCompile(`jinzhu/gorm/.*test.go`).MatchString(file)) {
			return fmt.Sprintf("%v:%v", strings.TrimPrefix(file, os.Getenv("GOPATH")+"src/"), line)
		}
	}
	return ""
}

func (s *DB) err(err error) error {
	if err == nil {
		return nil
	}

	if s.logMode == 0 {
		if err != RecordNotFound {
			go fmt.Println(s.fileWithLineNum(), err)
			error_str := err.Error()
			if regexp.MustCompile(`^sql: Scan error on column index`).MatchString(error_str) {
				return nil
			}
		}
	} else {
		s.warn(err)
	}

	s.Error = err
	return err
}

func (s *DB) hasError() bool {
	return s.Error != nil
}

func (s *DB) print(level string, v ...interface{}) {
	if s.logMode == 2 || level == "debug" {
		if _, ok := s.parent.logger.(Logger); !ok {
			fmt.Println("logger haven't been set, using os.Stdout")
			s.parent.logger = default_logger
		}
		args := []interface{}{level}
		s.parent.logger.(Logger).Print(append(args, v...)...)
	}
}

func (s *DB) warn(v ...interface{}) {
	go s.print("warn", v...)
}

func (s *DB) info(v ...interface{}) {
	go s.print("info", v...)
}

func (s *DB) slog(sql string, t time.Time, vars ...interface{}) {
	go s.print("sql", time.Now().Sub(t), sql, vars)
}

func (s *DB) debug(v ...interface{}) {
	go s.print("debug", v...)
}
<|endoftext|>


<commit_before>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFormatters map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFormatters: make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFormatters()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFormatters does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFormatters() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFormatters[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			phName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(phName)
			case "p":
				replacement = t.Param(phName)
			case "t":
				replacement = t.Tag(phName)
			default:
				Fail("Replace failed for placeholder ", phName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				substPtn := regexp.MustCompile("s\\/([^\\/]+)\\/([^\\/]*)\\/")
				trimEndPtn := regexp.MustCompile("%(.*)")

				for _, restPart := range restParts {
					if substPtn.MatchString(restPart) {
						mbits := substPtn.FindStringSubmatch(restPart)
						search := mbits[1]
						replace := mbits[2]
						replacement = strings.Replace(replacement, search, replace, 1)
					}
					if trimEndPtn.MatchString(restPart) {
						mbits := trimEndPtn.FindStringSubmatch(restPart)
						end := mbits[1]
						if end == replacement[len(replacement)-len(end):] {
							replacement = replacement[:len(replacement)-len(end)]
						}
					}
				}
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	p.PathFormatters[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("%s: CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d)\n", p.Name(), p.CoresPerTask, cap(p.workflow.concurrentTasks))
	}

	tasks := []*Task{}
	for t := range p.createTasks() {
		// Collect tasks so we can later wait for their done-signal before sending outputs
		tasks = append(tasks, t)

		// Sending FIFOs for the task
		for oname, oip := range t.OutIPs {
			if oip.doStream {
				if oip.FifoFileExists() {
					Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
				}
				oip.CreateFifo()
				p.Out(oname).Send(oip)
			}
		}

		// Execute task in separate go-routine
		go t.Execute()
	}

	// Wait for tasks to finish, in the order they were started (thus maintaining
	// order of IPs), and then sending output IPs
	for _, t := range tasks {
		<-t.Done
		for oname, oip := range t.OutIPs {
			if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
				p.Out(oname).Send(oip)
			}
			// Remove any FIFO file
			if oip.doStream && oip.FifoFileExists() {
				os.Remove(oip.FifoPath())
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}
		tags := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFormatters, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}
<commit_msg>Init ports also on InParamPort and SetOutFunc<commit_after>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFormatters map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFormatters: make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFormatters()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFormatters does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFormatters() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFormatters[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	if _, ok := p.inParamPorts[portName]; !ok {
		p.InitInParamPort(p, portName)
	}
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			phName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(phName)
			case "p":
				replacement = t.Param(phName)
			case "t":
				replacement = t.Tag(phName)
			default:
				Fail("Replace failed for placeholder ", phName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				substPtn := regexp.MustCompile("s\\/([^\\/]+)\\/([^\\/]*)\\/")
				trimEndPtn := regexp.MustCompile("%(.*)")

				for _, restPart := range restParts {
					if substPtn.MatchString(restPart) {
						mbits := substPtn.FindStringSubmatch(restPart)
						search := mbits[1]
						replace := mbits[2]
						replacement = strings.Replace(replacement, search, replace, 1)
					}
					if trimEndPtn.MatchString(restPart) {
						mbits := trimEndPtn.FindStringSubmatch(restPart)
						end := mbits[1]
						if end == replacement[len(replacement)-len(end):] {
							replacement = replacement[:len(replacement)-len(end)]
						}
					}
				}
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.PathFormatters[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("%s: CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d)\n", p.Name(), p.CoresPerTask, cap(p.workflow.concurrentTasks))
	}

	tasks := []*Task{}
	for t := range p.createTasks() {
		// Collect tasks so we can later wait for their done-signal before sending outputs
		tasks = append(tasks, t)

		// Sending FIFOs for the task
		for oname, oip := range t.OutIPs {
			if oip.doStream {
				if oip.FifoFileExists() {
					Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
				}
				oip.CreateFifo()
				p.Out(oname).Send(oip)
			}
		}

		// Execute task in separate go-routine
		go t.Execute()
	}

	// Wait for tasks to finish, in the order they were started (thus maintaining
	// order of IPs), and then sending output IPs
	for _, t := range tasks {
		<-t.Done
		for oname, oip := range t.OutIPs {
			if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
				p.Out(oname).Send(oip)
			}
			// Remove any FIFO file
			if oip.doStream && oip.FifoFileExists() {
				os.Remove(oip.FifoPath())
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}
		tags := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFormatters, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	pubsub "google.golang.org/api/pubsub/v1"
	storage "google.golang.org/api/storage/v1"

	log "github.com/Sirupsen/logrus"
)

type (
	ProcessConfig struct {
		Command  *CommandConfig  `json:"command,omitempty"`
		Job      *JobConfig      `json:"job,omitempty"`
		Progress *ProgressConfig `json:"progress,omitempty"`
		Log      *LogConfig      `json:"log,omitempty"`
	}
)

func (c *ProcessConfig) setup(args []string) error {
	if c.Command == nil {
		c.Command = &CommandConfig{}
	}
	c.Command.Template = args
	err := c.Log.setup()
	if err != nil {
		return err
	}
	return nil
}

func LoadProcessConfig(path string) (*ProcessConfig, error) {
	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	funcMap := template.FuncMap{"env": os.Getenv}
	t, err := template.New("config").Funcs(funcMap).Parse(string(raw))
	if err != nil {
		return nil, err
	}

	env := map[string]string{}
	for _, s := range os.Environ() {
		parts := strings.SplitN(s, "=", 2)
		env[parts[0]] = parts[1]
	}

	buf := new(bytes.Buffer)
	t.Execute(buf, env)

	var res ProcessConfig
	err = json.Unmarshal(buf.Bytes(), &res)
	if err != nil {
		return nil, err
	}
	return &res, nil
}

type (
	Process struct {
		config       *ProcessConfig
		subscription *JobSubscription
		notification *ProgressNotification
		storage      *CloudStorage
	}
)

func (p *Process) setup(ctx context.Context) error {
	// https://github.com/google/google-api-go-client#application-default-credentials-example
	client, err := google.DefaultClient(ctx, pubsub.PubsubScope, storage.DevstorageReadWriteScope)

	if err != nil {
		log.Fatalln("Failed to create DefaultClient")
		return err
	}

	// Create a storageService
	storageService, err := storage.New(client)
	if err != nil {
		logAttrs := log.Fields{"client": client, "error": err}
		log.WithFields(logAttrs).Fatalln("Failed to create storage.Service")
		return err
	}
	p.storage = &CloudStorage{storageService.Objects}

	// Creates a pubsubService
	pubsubService, err := pubsub.New(client)
	if err != nil {
		logAttrs := log.Fields{"client": client, "error": err}
		log.WithFields(logAttrs).Fatalln("Failed to create pubsub.Service")
		return err
	}

	p.subscription = &JobSubscription{
		config: p.config.Job,
		puller: &pubsubPuller{pubsubService.Projects.Subscriptions},
	}
	p.notification = &ProgressNotification{
		config:    p.config.Progress,
		publisher: &pubsubPublisher{pubsubService.Projects.Topics},
	}
	return nil
}

func (p *Process) run() error {
	logAttrs :=
		log.Fields{
			"VERSION": VERSION,
			"config": map[string]interface{}{
				"command":  p.config.Command,
				"job":      p.config.Job,
				"progress": p.config.Progress,
				"log":      p.config.Log,
			},
		}
	log.WithFields(logAttrs).Infoln("Start listening")
	err := p.subscription.listen(func(msg *JobMessage) error {
		job := &Job{
			config:       p.config.Command,
			message:      msg,
			notification: p.notification,
			storage:      p.storage,
		}
		err := job.run()
		if err != nil {
			logAttrs := log.Fields{"error": err, "msg": msg}
			log.WithFields(logAttrs).Fatalln("Kpbg Error")
			return err
		}
		return nil
	})
	return err
}
<commit_msg>:+1: Fix typo in log<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"strings"
	"text/template"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"

	pubsub "google.golang.org/api/pubsub/v1"
	storage "google.golang.org/api/storage/v1"

	log "github.com/Sirupsen/logrus"
)

type (
	ProcessConfig struct {
		Command  *CommandConfig  `json:"command,omitempty"`
		Job      *JobConfig      `json:"job,omitempty"`
		Progress *ProgressConfig `json:"progress,omitempty"`
		Log      *LogConfig      `json:"log,omitempty"`
	}
)

func (c *ProcessConfig) setup(args []string) error {
	if c.Command == nil {
		c.Command = &CommandConfig{}
	}
	c.Command.Template = args
	err := c.Log.setup()
	if err != nil {
		return err
	}
	return nil
}

func LoadProcessConfig(path string) (*ProcessConfig, error) {
	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	funcMap := template.FuncMap{"env": os.Getenv}
	t, err := template.New("config").Funcs(funcMap).Parse(string(raw))
	if err != nil {
		return nil, err
	}

	env := map[string]string{}
	for _, s := range os.Environ() {
		parts := strings.SplitN(s, "=", 2)
		env[parts[0]] = parts[1]
	}

	buf := new(bytes.Buffer)
	t.Execute(buf, env)

	var res ProcessConfig
	err = json.Unmarshal(buf.Bytes(), &res)
	if err != nil {
		return nil, err
	}
	return &res, nil
}

type (
	Process struct {
		config       *ProcessConfig
		subscription *JobSubscription
		notification *ProgressNotification
		storage      *CloudStorage
	}
)

func (p *Process) setup(ctx context.Context) error {
	// https://github.com/google/google-api-go-client#application-default-credentials-example
	client, err := google.DefaultClient(ctx, pubsub.PubsubScope, storage.DevstorageReadWriteScope)

	if err != nil {
		log.Fatalln("Failed to create DefaultClient")
		return err
	}

	// Create a storageService
	storageService, err := storage.New(client)
	if err != nil {
		logAttrs := log.Fields{"client": client, "error": err}
		log.WithFields(logAttrs).Fatalln("Failed to create storage.Service")
		return err
	}
	p.storage = &CloudStorage{storageService.Objects}

	// Creates a pubsubService
	pubsubService, err := pubsub.New(client)
	if err != nil {
		logAttrs := log.Fields{"client": client, "error": err}
		log.WithFields(logAttrs).Fatalln("Failed to create pubsub.Service")
		return err
	}

	p.subscription = &JobSubscription{
		config: p.config.Job,
		puller: &pubsubPuller{pubsubService.Projects.Subscriptions},
	}
	p.notification = &ProgressNotification{
		config:    p.config.Progress,
		publisher: &pubsubPublisher{pubsubService.Projects.Topics},
	}
	return nil
}

func (p *Process) run() error {
	logAttrs :=
		log.Fields{
			"VERSION": VERSION,
			"config": map[string]interface{}{
				"command":  p.config.Command,
				"job":      p.config.Job,
				"progress": p.config.Progress,
				"log":      p.config.Log,
			},
		}
	log.WithFields(logAttrs).Infoln("Start listening")
	err := p.subscription.listen(func(msg *JobMessage) error {
		job := &Job{
			config:       p.config.Command,
			message:      msg,
			notification: p.notification,
			storage:      p.storage,
		}
		err := job.run()
		if err != nil {
			logAttrs := log.Fields{"error": err, "msg": msg}
			log.WithFields(logAttrs).Fatalln("Job Error")
			return err
		}
		return nil
	})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	"golang.org/x/tools/cover"
)

var ErrNoBlock = errors.New("no block")

type Profile struct {
	*cover.Profile
	newBlocks []*ProfileBlock
}

func (p *Profile) getBlock(otherBlock cover.ProfileBlock) (*ProfileBlock, error) {
	for _, block := range p.newBlocks {
		if block.sameRange(otherBlock) {
			return block, nil
		}
	}
	return nil, ErrNoBlock
}

func (p *Profile) MergeBlocks() {
	for _, otherBlock := range p.Blocks {
		myBlock, err := p.getBlock(otherBlock)
		if err == nil {
			myBlock.ImportCount(otherBlock)
			continue
		}
		if err == ErrNoBlock {
			p.newBlocks = append(p.newBlocks, &ProfileBlock{&otherBlock})
			continue
		}
		if err != nil {
			panic(err)
		}
	}
	p.newBlocksToBlocks()
}

func (p *Profile) newBlocksToBlocks() {
	p.Blocks = make([]cover.ProfileBlock, len(p.newBlocks))
	for i, b := range p.newBlocks {
		p.Blocks[i] = *b.ProfileBlock
	}
}

func (p *Profile) Format() string {
	res := ""
	for _, block := range p.Blocks {
		res += fmt.Sprintf("%s:%d.%d,%d.%d %d %d\n",
			p.FileName, block.StartLine, block.StartCol,
			block.EndLine, block.EndCol,
			block.NumStmt, block.Count)
	}
	return res
}
<commit_msg>profile: fix loop variable reference problem<commit_after>package main

import (
	"errors"
	"fmt"

	"golang.org/x/tools/cover"
)

var ErrNoBlock = errors.New("no block")

type Profile struct {
	*cover.Profile
	newBlocks []*ProfileBlock
}

func (p *Profile) getBlock(otherBlock cover.ProfileBlock) (*ProfileBlock, error) {
	for _, block := range p.newBlocks {
		if block.sameRange(otherBlock) {
			return block, nil
		}
	}
	return nil, ErrNoBlock
}

func (p *Profile) MergeBlocks() {
	for i, otherBlock := range p.Blocks {
		myBlock, err := p.getBlock(otherBlock)
		fmt.Println("loop")
		if err == nil {
			myBlock.ImportCount(otherBlock)
			continue
		}
		if err == ErrNoBlock {
			fmt.Println("new block")
			p.newBlocks = append(p.newBlocks, &ProfileBlock{&p.Blocks[i]})
			continue
		}
		if err != nil {
			panic(err)
		}
	}
	p.newBlocksToBlocks()
}

func (p *Profile) newBlocksToBlocks() {
	p.Blocks = make([]cover.ProfileBlock, len(p.newBlocks))
	for i, b := range p.newBlocks {
		p.Blocks[i] = *b.ProfileBlock
	}
}

func (p *Profile) Format() string {
	res := ""
	for _, block := range p.Blocks {
		res += fmt.Sprintf("%s:%d.%d,%d.%d %d %d\n",
			p.FileName, block.StartLine, block.StartCol,
			block.EndLine, block.EndCol,
			block.NumStmt, block.Count)
	}
	return res
}
<|endoftext|>


<commit_before>package algoholic

// A trie is a data structure which stores words in a tree by letter, when the letters form a
// valid string, the node is marked terminal and a value can be stored.

type Trie struct {
	Char     rune
	Parent   *Trie
	Children map[rune]*Trie
	Terminal bool
	Value    interface{}
}

// A value indicating that the node is the root node, thus it doesn't denote a character
// itself.
const RootTrieChar = rune(0)

// Create a new trie with the specified character and parent.
func NewTrie(parent *Trie, chr rune) *Trie {
	return &Trie{chr, parent, make(map[rune]*Trie), false, nil}
}

func NewRootTrie() *Trie {
	return NewTrie(nil, RootTrieChar)
}

// Create a new trie with strings mapped to specified values.
func NewTrieFromMap(strMap map[string]interface{}) *Trie {
	ret := NewRootTrie()

	for str, val := range strMap {
		ret.Insert(str, val)
	}

	return ret
}

// Create a new trie with strings whose values we don't care about.
func NewTrieFromStrings(strs []string) *Trie {
	ret := NewRootTrie()

	for _, str := range strs {
		ret.Insert(str, nil)
	}

	return ret
}

// Find the specified string and return its trie node.
// O(m) worst-case where m is the length of the string searched for.
// Note this returns non-terminal nodes.
func (trie *Trie) FindTrie(str string) *Trie {
	if len(str) == 0 {
		return trie
	}

	if next := trie.Children[rune(str[0])]; next != nil {
		return next.FindTrie(str[1:])
	}

	return nil
}

// Find the specified string and return its value.
// O(m) worst-case where m is the length of the string searched for.
func (trie *Trie) Find(str string) (val interface{}, has bool) {
	ret := trie.FindTrie(str)

	if ret == nil || !ret.Terminal {
		// Not found.
		return
	}

	has = true
	val = ret.Value

	return
}

// Find all valid strings that consist of suffixes of the input prefix.
// O(m) worst-case where m is the length of the longest returned string.
func (trie *Trie) FindSuffixes(prefix string) []string {
	trie = trie.FindTrie(prefix)

	if trie == nil {
		return nil
	}

	var ret []string

	for suffix := range trie.Walk() {
		ret = append(ret, prefix+suffix[1:])
	}

	return ret
}

// Insert string, value pair into the specified trie.
// O(m) worst-case where m is the length of the inserted string.
func (trie *Trie) Insert(str string, val interface{}) {
	var (
		i   int
		chr rune
	)

	// Search through existing nodes.
	for i, chr = range str {
		if next, has := trie.Children[chr]; has {
			trie = next
		} else {
			break
		}
	}

	// Insert nodes as necessary.
	for _, chr = range str[i:] {
		next := NewTrie(trie, chr)
		trie.Children[chr] = next
		trie = next
	}
	trie.Terminal = true
	trie.Value = val
}

// Recursively walk through all children of the input trie, adding string, value pairs to
// trieMap as the walk is performed. Trie is traversed in pre-order.
// O(n) where n is the number of nodes in the input trie.
func (trie *Trie) doWalk(trieMap map[string]interface{}, prev []rune) {
	// TODO: Use something other than a hash for map to allow alphabetical output ordering.

	if trie.Char != RootTrie {
		prev = append(prev, trie.Char)
	}

	if trie.Terminal {
		str := string(prev)
		trieMap[str] = trie.Value
	}

	for _, child := range trie.Children {
		child.doWalk(trieMap, prev)
	}
}

// Recursively walk through all children of the input trie, returning a map of string, value
// pairs.
// O(n) where n is the number of nodes in the input trie.
func (trie *Trie) Walk() map[string]interface{} {
	ret := make(map[string]interface{})
	trie.doWalk(ret, nil)
	return ret
}
<commit_msg>Make Walk() behave as a walk function is expected to.<commit_after>package algoholic

// A trie is a data structure which stores words in a tree by letter, when the letters form a
// valid string, the node is marked terminal and a value can be stored.

type Trie struct {
	Char     rune
	Parent   *Trie
	Children map[rune]*Trie
	Terminal bool
	Value    interface{}
}

// A value indicating that the node is the root node, thus it doesn't denote a character
// itself.
const RootTrieChar = rune(0)

// Create a new trie with the specified character and parent.
func NewTrie(parent *Trie, chr rune) *Trie {
	return &Trie{chr, parent, make(map[rune]*Trie), false, nil}
}

func NewRootTrie() *Trie {
	return NewTrie(nil, RootTrieChar)
}

// Create a new trie with strings mapped to specified values.
func NewTrieFromMap(strMap map[string]interface{}) *Trie {
	ret := NewRootTrie()

	for str, val := range strMap {
		ret.Insert(str, val)
	}

	return ret
}

// Create a new trie with strings whose values we don't care about.
func NewTrieFromStrings(strs []string) *Trie {
	ret := NewRootTrie()

	for _, str := range strs {
		ret.Insert(str, nil)
	}

	return ret
}

// Find the specified string and return its trie node.
// O(m) worst-case where m is the length of the string searched for.
// Note this returns non-terminal nodes.
func (trie *Trie) FindTrie(str string) *Trie {
	if len(str) == 0 {
		return trie
	}

	if next := trie.Children[rune(str[0])]; next != nil {
		return next.FindTrie(str[1:])
	}

	return nil
}

// Find the specified string and return its value.
// O(m) worst-case where m is the length of the string searched for.
func (trie *Trie) Find(str string) (val interface{}, has bool) {
	ret := trie.FindTrie(str)

	if ret == nil || !ret.Terminal {
		// Not found.
		return
	}

	has = true
	val = ret.Value

	return
}

// Find all valid strings that consist of suffixes of the input prefix.
// O(m) worst-case where m is the length of the longest returned string.
func (trie *Trie) FindSuffixes(prefix string) []string {
	trie = trie.FindTrie(prefix)

	if trie == nil {
		return nil
	}

	var ret []string

	for suffix := range trie.Walk() {
		ret = append(ret, prefix+suffix[1:])
	}

	return ret
}

// Insert string, value pair into the specified trie.
// O(m) worst-case where m is the length of the inserted string.
func (trie *Trie) Insert(str string, val interface{}) {
	var (
		i   int
		chr rune
	)

	// Search through existing nodes.
	for i, chr = range str {
		if next, has := trie.Children[chr]; has {
			trie = next
		} else {
			break
		}
	}

	// Insert nodes as necessary.
	for _, chr = range str[i:] {
		next := NewTrie(trie, chr)
		trie.Children[chr] = next
		trie = next
	}
	trie.Terminal = true
	trie.Value = val
}

// Recursively walk through all children of the input trie, adding string, value pairs to
// trieMap as the walk is performed. Trie is traversed in pre-order.
// O(n) where n is the number of nodes in the input trie.
func (trie *Trie) doWalk(trieMap map[string]interface{}, prev []rune) {
	// TODO: Use something other than a hash for map to allow alphabetical output ordering.

	if trie.Char != RootTrie {
		prev = append(prev, trie.Char)
	}

	if trie.Terminal {
		str := string(prev)
		trieMap[str] = trie.Value
	}

	for _, child := range trie.Children {
		child.doWalk(trieMap, prev)
	}
}

// Recursively walk through all children of the input trie in preorder executing the specified
// function on each trie node.
//O(n) where n is the number of nodes in the input trie.
func (trie *Trie) Walk(fn func(*Trie)) {
	fn(trie)
	for _, child := range trie.Children {
		child.Walk(fn)
	}
}
<|endoftext|>


<commit_before>package circular

import (
	"strings"

	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/h"
	"github.com/alecthomas/chroma/lexers/internal"
)

// PHTML lexer is PHP in HTML.
var PHTML = internal.Register(DelegatingLexer(h.HTML, MustNewLazyLexer(
	&Config{
		Name:            "PHTML",
		Aliases:         []string{"phtml"},
		Filenames:       []string{"*.phtml"},
		MimeTypes:       []string{"application/x-php", "application/x-httpd-php", "application/x-httpd-php3", "application/x-httpd-php4", "application/x-httpd-php5"},
		DotAll:          true,
		CaseInsensitive: true,
		EnsureNL:        true,
	},
	phtmlRules,
).SetAnalyser(func(text string) float32 {
	if strings.Contains(text, "<?php") {
		return 0.5
	}
	return 0.0
})))

func phtmlRules() Rules {
	return Rules{
		"root": {
			{`<\?(php)?`, CommentPreproc, Push("php")},
			{`[^<]+`, Other, nil},
			{`<`, Other, nil},
		},
	}.Merge(phpCommonRules())
}
<commit_msg>File name matches and mime matches should be moved to phtml from php lexer (#477)<commit_after>package circular

import (
	"strings"

	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/h"
	"github.com/alecthomas/chroma/lexers/internal"
)

// PHTML lexer is PHP in HTML.
var PHTML = internal.Register(DelegatingLexer(h.HTML, MustNewLazyLexer(
	&Config{
		Name:            "PHTML",
		Aliases:         []string{"phtml"},
		Filenames:       []string{"*.phtml", "*.php", "*.php[345]", "*.inc"},
		MimeTypes:       []string{"application/x-php", "application/x-httpd-php", "application/x-httpd-php3", "application/x-httpd-php4", "application/x-httpd-php5", "text/x-php"},
		DotAll:          true,
		CaseInsensitive: true,
		EnsureNL:        true,
		Priority:        2,
	},
	phtmlRules,
).SetAnalyser(func(text string) float32 {
	if strings.Contains(text, "<?php") {
		return 0.5
	}
	return 0.0
})))

func phtmlRules() Rules {
	return Rules{
		"root": {
			{`<\?(php)?`, CommentPreproc, Push("php")},
			{`[^<]+`, Other, nil},
			{`<`, Other, nil},
		},
	}.Merge(phpCommonRules())
}
<|endoftext|>


<commit_before>package mysqld_cmd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/config"
)

type MysqldCmd interface {
	RecoverSeqno() (string, error)
}

type mysqldCmd struct {
	logger       lager.Logger
	mysqldconfig config.Config
}

func NewMysqldCmd(logger lager.Logger, mysqldconfig config.Config) MysqldCmd {
	return &mysqldCmd{
		logger:       logger,
		mysqldconfig: mysqldconfig,
	}
}

/*
* Why?
*
* Galera does not provide an elegant way to determine seqno if the DB is not
* running.
* The mysqld --wsrep-recover cmd prints the seqno to stderr (lines starts with `WSREP: Recovered position:`)
* This command writes its stderr to a log file specified by the `--log-error`
* flag
 */
func (m *mysqldCmd) RecoverSeqno() (string, error) {

	errorLogFile := path.Join(os.TempDir(), "galera-healthcheck-mysqld-log.err")
	os.RemoveAll(errorLogFile) //ensure log is empty

	cmd := exec.Command(m.mysqldconfig.MysqldPath,
		fmt.Sprintf("--defaults-file=%s", m.mysqldconfig.MyCnfPath),
		"--wsrep-recover",
		fmt.Sprintf("--log-error=%s", errorLogFile))

	stdout, cmdErr := cmd.CombinedOutput()
	stderr, readingLogErr := ioutil.ReadFile(errorLogFile)
	if readingLogErr != nil {
		stderr = []byte("failed to read stderr")
	}

	if cmdErr != nil {
		m.logger.Error("Error running mysqld recovery", cmdErr, lager.Data{
			"stdout": stdout,
			"stderr": stderr,
		})
		return "", cmdErr
	} else {
		m.logger.Debug(string(stdout))
	}

	seqNoRegex := `WSREP: Recovered position:.*:(\d+)`
	re := regexp.MustCompile(seqNoRegex)
	sequenceNumberLogLine := re.FindStringSubmatch(string(stderr))

	if len(sequenceNumberLogLine) < 2 {
		// First match is the whole string, second match is the seq no
		err := errors.New(fmt.Sprintf("Couldn't find regex: %s Log Line: %s", seqNoRegex, sequenceNumberLogLine))
		m.logger.Error("Failed to parse seqno from logs", err)
		return "", err
	}

	sequenceNumber := sequenceNumberLogLine[1]
	return sequenceNumber, nil
}
<commit_msg>Fix regex to accept -1 as "valid" sequence number<commit_after>package mysqld_cmd

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"

	"code.cloudfoundry.org/lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/config"
)

type MysqldCmd interface {
	RecoverSeqno() (string, error)
}

type mysqldCmd struct {
	logger       lager.Logger
	mysqldconfig config.Config
}

func NewMysqldCmd(logger lager.Logger, mysqldconfig config.Config) MysqldCmd {
	return &mysqldCmd{
		logger:       logger,
		mysqldconfig: mysqldconfig,
	}
}

/*
* Why?
*
* Galera does not provide an elegant way to determine seqno if the DB is not
* running.
* The mysqld --wsrep-recover cmd prints the seqno to stderr (lines starts with `WSREP: Recovered position:`)
* This command writes its stderr to a log file specified by the `--log-error`
* flag
 */
func (m *mysqldCmd) RecoverSeqno() (string, error) {

	errorLogFile := path.Join(os.TempDir(), "galera-healthcheck-mysqld-log.err")
	os.RemoveAll(errorLogFile) //ensure log is empty

	cmd := exec.Command(m.mysqldconfig.MysqldPath,
		fmt.Sprintf("--defaults-file=%s", m.mysqldconfig.MyCnfPath),
		"--wsrep-recover",
		fmt.Sprintf("--log-error=%s", errorLogFile))

	stdout, cmdErr := cmd.CombinedOutput()
	stderr, readingLogErr := ioutil.ReadFile(errorLogFile)
	if readingLogErr != nil {
		stderr = []byte("failed to read stderr")
	}

	if cmdErr != nil {
		m.logger.Error("Error running mysqld recovery", cmdErr, lager.Data{
			"stdout": stdout,
			"stderr": stderr,
		})
		return "", cmdErr
	} else {
		m.logger.Debug(string(stdout))
	}

	seqNoRegex := `WSREP: Recovered position:.*:(-?\d+)`
	re := regexp.MustCompile(seqNoRegex)
	sequenceNumberLogLine := re.FindStringSubmatch(string(stderr))

	if len(sequenceNumberLogLine) < 2 {
		// First match is the whole string, second match is the seq no
		err := errors.New(fmt.Sprintf("Couldn't find regex: %s Log Line: %s", seqNoRegex, sequenceNumberLogLine))
		m.logger.Error("Failed to parse seqno from logs", err)
		return "", err
	}

	sequenceNumber := sequenceNumberLogLine[1]
	return sequenceNumber, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"crypto/sha256"
	"encoding/hex"
)

var maxSize = flag.Int64("maxSize", 20971520, "the maximum size in bytes a user is allowed to upload")
var addr = flag.String("addr", "localhost:8000", "host:port format IP address to listen on")
var subpath = flag.String("subpath", "/", "configure a subdirectory, for use with a reverse proxy (example: ./9000server -subpath=/image9000/)")
var logrequests = flag.Bool("logrequests", false, "print all HTTP requests to stdout")

var acceptedfmt = map[string]string{
	"image/jpeg":       "jpg",
	"image/png":        "png",
	"image/gif":        "gif",
	"video/webm":       "webm",
	"video/x-matroska": "mkv",
	"video/ogg":        "ogv",
	"application/ogg":  "ogg",
	"audio/ogg":        "ogg",
	"audio/mp3":        "mp3",
}

func CreateFileId(bt []byte, ext string) string {
	bytes := sha256.Sum256(bt)
	sha := hex.EncodeToString(bytes[:])
	return sha[:14] + "." + ext
}

func Log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if *logrequests {
			fmt.Printf("%s (%s) %s %s\n", r.RemoteAddr, r.UserAgent(), r.Method, r.URL)
		}
		handler.ServeHTTP(w, r)
	})
}

func UploadHandler(rw http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		r.ParseMultipartForm(*maxSize)
		file, _, _ := r.FormFile("file")
		if r.ContentLength > *maxSize {
			http.Error(rw, "File Too big!", http.StatusRequestEntityTooLarge)
			return
		}

		ImageReader := bufio.NewReader(file)
		ImageBuffer := make([]byte, r.ContentLength)
		_, err := ImageReader.Read(ImageBuffer)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		ImageType := http.DetectContentType(ImageBuffer)

		if acceptedfmt[ImageType] != "" {
			id := CreateFileId(ImageBuffer, acceptedfmt[ImageType])
			if _, err := os.Stat("web/img/" + id); os.IsNotExist(err) {
				err = ioutil.WriteFile("web/img/"+id, ImageBuffer, 0666)
				if err != nil {
					panic(err)
				}
				fmt.Println("redirecting to " + *subpath + "img/" + id)
				http.Redirect(rw, r, *subpath+"img/"+id+"."+acceptedfmt[ImageType], 301)
			} else {
				http.Redirect(rw, r, "https://www.youtube.com/watch?v=dQw4w9WgXcQ", 301)
			}
		} else {
			http.Error(rw, fmt.Sprintf("File type (%s) not supported.", ImageType), http.StatusBadRequest)
		}
	default:
		http.Error(rw, "Bad Request", http.StatusBadRequest)
	}
}

func main() {
	flag.Parse()

	if _, err := os.Stat("web/img"); os.IsNotExist(err) {
		os.Mkdir("web/img", 0666)
	}

	http.Handle("/", http.FileServer(http.Dir("web")))

	http.HandleFunc("/upload", UploadHandler)
	http.ListenAndServe(*addr, Log(http.DefaultServeMux))
}
<commit_msg>remove old extension concat<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"crypto/sha256"
	"encoding/hex"
)

var maxSize = flag.Int64("maxSize", 20971520, "the maximum size in bytes a user is allowed to upload")
var addr = flag.String("addr", "localhost:8000", "host:port format IP address to listen on")
var subpath = flag.String("subpath", "/", "configure a subdirectory, for use with a reverse proxy (example: ./9000server -subpath=/image9000/)")
var logrequests = flag.Bool("logrequests", false, "print all HTTP requests to stdout")

var acceptedfmt = map[string]string{
	"image/jpeg":       "jpg",
	"image/png":        "png",
	"image/gif":        "gif",
	"video/webm":       "webm",
	"video/x-matroska": "mkv",
	"video/ogg":        "ogv",
	"application/ogg":  "ogg",
	"audio/ogg":        "ogg",
	"audio/mp3":        "mp3",
}

func CreateFileId(bt []byte, ext string) string {
	bytes := sha256.Sum256(bt)
	sha := hex.EncodeToString(bytes[:])
	return sha[:14] + "." + ext
}

func Log(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if *logrequests {
			fmt.Printf("%s (%s) %s %s\n", r.RemoteAddr, r.UserAgent(), r.Method, r.URL)
		}
		handler.ServeHTTP(w, r)
	})
}

func UploadHandler(rw http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "POST":
		r.ParseMultipartForm(*maxSize)
		file, _, _ := r.FormFile("file")
		if r.ContentLength > *maxSize {
			http.Error(rw, "File Too big!", http.StatusRequestEntityTooLarge)
			return
		}

		ImageReader := bufio.NewReader(file)
		ImageBuffer := make([]byte, r.ContentLength)
		_, err := ImageReader.Read(ImageBuffer)
		if err != nil {
			http.Error(rw, err.Error(), http.StatusInternalServerError)
			return
		}

		ImageType := http.DetectContentType(ImageBuffer)

		if acceptedfmt[ImageType] != "" {
			id := CreateFileId(ImageBuffer, acceptedfmt[ImageType])
			if _, err := os.Stat("web/img/" + id); os.IsNotExist(err) {
				err = ioutil.WriteFile("web/img/"+id, ImageBuffer, 0666)
				if err != nil {
					panic(err)
				}
				http.Redirect(rw, r, *subpath+"img/"+id, 301)
			} else {
				http.Redirect(rw, r, "https://www.youtube.com/watch?v=dQw4w9WgXcQ", 301)
			}
		} else {
			http.Error(rw, fmt.Sprintf("File type (%s) not supported.", ImageType), http.StatusBadRequest)
		}
	default:
		http.Error(rw, "Bad Request", http.StatusBadRequest)
	}
}

func main() {
	flag.Parse()

	if _, err := os.Stat("web/img"); os.IsNotExist(err) {
		os.Mkdir("web/img", 0666)
	}

	http.Handle("/", http.FileServer(http.Dir("web")))

	http.HandleFunc("/upload", UploadHandler)
	http.ListenAndServe(*addr, Log(http.DefaultServeMux))
}
<|endoftext|>