<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"errors"
	"fmt"
	"math"
	"runtime"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/iam"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/api/support/bundler"
	pb "google.golang.org/genproto/googleapis/pubsub/v1"
)

const (
	// The maximum number of messages that can be in a single publish request, as
	// determined by the PubSub service.
	MaxPublishRequestCount = 1000

	// The maximum size of a single publish request in bytes, as determined by the PubSub service.
	MaxPublishRequestBytes = 1e7
)

// ErrOversizedMessage indicates that a message's size exceeds MaxPublishRequestBytes.
var ErrOversizedMessage = bundler.ErrOversizedItem

// Topic is a reference to a PubSub topic.
type Topic struct {
	s service
	// The fully qualified identifier for the topic, in the format "projects/<projid>/topics/<name>"
	name string

	// Settings for publishing messages. All changes must be made before the
	// first call to Publish. The default is DefaultPublishSettings.
	PublishSettings PublishSettings

	mu      sync.RWMutex
	stopped bool
	bundler *bundler.Bundler

	wg sync.WaitGroup

	// Channel for message bundles to be published. Close to indicate that Stop was called.
	bundlec chan []*bundledMessage

	flowController *flowController
}

// PublishSettings control the bundling of published messages.
type PublishSettings struct {

	// Publish a non-empty batch after this delay has passed.
	DelayThreshold time.Duration

	// Publish a batch when it has this many messages. The maximum is
	// MaxPublishRequestCount.
	CountThreshold int

	// Publish a batch when its size in bytes reaches this value.
	ByteThreshold int

	// MaxOutstandingMessages is the maximum number of messages for which the
	// publish can be outstanding. If MaxOutstandingMessages is 0, it will be
	// treated as if it were DefaultPublishSettings.MaxOutstandingMessages. If the
	// value is negative, then there will be no limit on the number of messages
	// that can be outstanding.
	MaxOutstandingMessages int

	// MaxOutstandingBytes is the maximum size of messages for which the publish
	// can be outstanding. If MaxOutstandingBytes is 0, it will be treated as if it
	// werer DefaultPublishSettings.MaxOutstandingBytes. If the value is negative,
	// then there will be no limit on the number of bytes that can be outstanding.
	MaxOutstandingBytes int

	// The number of goroutines that invoke the Publish RPC concurrently.
	// Defaults to a multiple of GOMAXPROCS.
	NumGoroutines int
}

// DefaultPublishSettings holds the default values for topics' BatchSettings.
var DefaultPublishSettings = PublishSettings{
	DelayThreshold:         1 * time.Millisecond,
	CountThreshold:         100,
	ByteThreshold:          1e6,
	MaxOutstandingMessages: -1,  // Default to unlimited number of messages
	MaxOutstandingBytes:    1e9, // 1G
}

// CreateTopic creates a new topic.
// The specified topic ID must start with a letter, and contain only letters
// ([A-Za-z]), numbers ([0-9]), dashes (-), underscores (_), periods (.),
// tildes (~), plus (+) or percent signs (%). It must be between 3 and 255
// characters in length, and must not start with "goog".
// If the topic already exists an error will be returned.
func (c *Client) CreateTopic(ctx context.Context, id string) (*Topic, error) {
	t := c.Topic(id)
	err := c.s.createTopic(ctx, t.name)
	return t, err
}

// Topic creates a reference to a topic.
//
// If a Topic's Publish method is called, it has background goroutines
// associated with it. Clean them up by calling Topic.Stop.
//
// Avoid creating many Topic instances if you use them to publish.
func (c *Client) Topic(id string) *Topic {
	return newTopic(c.s, fmt.Sprintf("projects/%s/topics/%s", c.projectID, id))
}

func newTopic(s service, name string) *Topic {
	// bundlec is unbuffered. A buffer would occupy memory not
	// accounted for by the bundler, so BufferedByteLimit would be a lie:
	// the actual memory consumed would be higher.
	return &Topic{
		s:               s,
		name:            name,
		PublishSettings: DefaultPublishSettings,
		bundlec:         make(chan []*bundledMessage),
	}
}

// Topics returns an iterator which returns all of the topics for the client's project.
func (c *Client) Topics(ctx context.Context) *TopicIterator {
	return &TopicIterator{
		s:    c.s,
		next: c.s.listProjectTopics(ctx, c.fullyQualifiedProjectName()),
	}
}

// TopicIterator is an iterator that returns a series of topics.
type TopicIterator struct {
	s    service
	next nextStringFunc
}

// Next returns the next topic. If there are no more topics, iterator.Done will be returned.
func (tps *TopicIterator) Next() (*Topic, error) {
	topicName, err := tps.next()
	if err != nil {
		return nil, err
	}
	return newTopic(tps.s, topicName), nil
}

// ID returns the unique idenfier of the topic within its project.
func (t *Topic) ID() string {
	slash := strings.LastIndex(t.name, "/")
	if slash == -1 {
		// name is not a fully-qualified name.
		panic("bad topic name")
	}
	return t.name[slash+1:]
}

// String returns the printable globally unique name for the topic.
func (t *Topic) String() string {
	return t.name
}

// Delete deletes the topic.
func (t *Topic) Delete(ctx context.Context) error {
	return t.s.deleteTopic(ctx, t.name)
}

// Exists reports whether the topic exists on the server.
func (t *Topic) Exists(ctx context.Context) (bool, error) {
	if t.name == "_deleted-topic_" {
		return false, nil
	}

	return t.s.topicExists(ctx, t.name)
}

func (t *Topic) IAM() *iam.Handle {
	return t.s.iamHandle(t.name)
}

// Subscriptions returns an iterator which returns the subscriptions for this topic.
func (t *Topic) Subscriptions(ctx context.Context) *SubscriptionIterator {
	// NOTE: zero or more Subscriptions that are ultimately returned by this
	// Subscriptions iterator may belong to a different project to t.
	return &SubscriptionIterator{
		s:    t.s,
		next: t.s.listTopicSubscriptions(ctx, t.name),
	}
}

var errTopicStopped = errors.New("pubsub: Stop has been called for this topic")

// Publish publishes msg to the topic asynchronously. Messages are batched and
// sent according to the topic's BatchSettings.
//
// Publish returns a non-nil PublishResult which will be ready when the
// message has been sent (or has failed to be sent) to the server.
//
// Publish creates goroutines for batching and sending messages. These goroutines
// need to be stopped by calling t.Stop(). Once stopped, future calls to Publish
// or TryPublish will immediately return a PublishResult with an error.
//
// Publish blocks until the number of messages and memory consumed by batching
// fall below MaxOutstandingMessages and MaxOutstandingBytes, respectively, or
// until ctx is Done. The ctx argument is used only for this purpose; it is
// unrelated to the context used by the background goroutines which call the
// Publish RPC.
func (t *Topic) Publish(ctx context.Context, msg *Message) *PublishResult {
	return t.publish(ctx, msg, true)
}

// TryPublish publishes msg to the topic asynchronously. Messages are batched
// and sent according to the topic's BatchSettings.
//
// If the number of messages or memory consumed by batching are above
// MaxOutstandingMessages or MaxOutstandingBytes, respectively, then TryPublish
// immediately returns nil. Otherwise, TryPublish returns a non-nil PublishResult
// which will be ready when the message has been sent (or has failed to be sent)
// to the server.
//
// TryPublish creates goroutines for batching and sending messages. These goroutines
// need to be stopped by calling t.Stop(). Once stopped, future calls to Publish
// or TryPublish will immediately return a PublishResult with an error.
func (t *Topic) TryPublish(ctx context.Context, msg *Message) *PublishResult {
	return t.publish(ctx, msg, false)
}

func (t *Topic) publish(ctx context.Context, msg *Message, waitForFC bool) *PublishResult {
	// TODO(jba): if this turns out to take significant time, try to approximate it.
	// Or, convert the messages to protos in Publish, instead of in the service.
	msg.size = proto.Size(&pb.PubsubMessage{
		Data:       msg.Data,
		Attributes: msg.Attributes,
	})
	r := &PublishResult{ready: make(chan struct{})}
	t.initBundlerAndFlowController()
	t.mu.RLock()
	defer t.mu.RUnlock()
	// TODO(aboulhosn) [from bcmills] consider changing the semantics of bundler to perform this logic so we don't have to do it here
	if t.stopped {
		r.err = errTopicStopped
		close(r.ready)
		return r
	}
	if waitForFC {
		if err := t.flowController.acquire(ctx, msg.size); err != nil {
			r.err = err
			close(r.ready)
			return r
		}
	} else if !t.flowController.tryAcquire(msg.size) {
		return nil
	}
	// TODO(jba) [from bcmills] consider using a shared channel per bundle (requires Bundler API changes; would reduce allocations)
	// The call to AddWait will never block because the bundler's BufferedByteLimit is set to MaxInt64. The topic's flowController
	// is what ensures we have capacity to publish more messages.
	err := t.bundler.AddWait(ctx, &bundledMessage{msg, r}, msg.size)
	if err != nil {
		r.err = err
		close(r.ready)
	}
	return r
}

// Send all remaining published messages and stop goroutines created for handling
// publishing. Returns once all outstanding messages have been sent or have
// failed to be sent.
func (t *Topic) Stop() {
	t.mu.Lock()
	noop := t.stopped || t.bundler == nil
	t.stopped = true
	t.mu.Unlock()
	if noop {
		return
	}
	t.bundler.Flush()
	// At this point, all pending bundles have been published and the bundler's
	// goroutines have exited, so it is OK for this goroutine to close bundlec.
	close(t.bundlec)
	t.wg.Wait()
}

// A PublishResult holds the result from a call to Publish.
type PublishResult struct {
	ready    chan struct{}
	serverID string
	err      error
}

// Ready returns a channel that is closed when the result is ready.
// When the Ready channel is closed, Get is guaranteed not to block.
func (r *PublishResult) Ready() <-chan struct{} { return r.ready }

// Get returns the server-generated message ID and/or error result of a Publish call.
// Get blocks until the Publish call completes or the context is done.
func (r *PublishResult) Get(ctx context.Context) (serverID string, err error) {
	// If the result is already ready, return it even if the context is done.
	select {
	case <-r.Ready():
		return r.serverID, r.err
	default:
	}
	select {
	case <-ctx.Done():
		return "", ctx.Err()
	case <-r.Ready():
		return r.serverID, r.err
	}
}

type bundledMessage struct {
	msg *Message
	res *PublishResult
}

func (t *Topic) initBundlerAndFlowController() {
	t.mu.RLock()
	noop := t.stopped || t.bundler != nil
	t.mu.RUnlock()
	if noop {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	// Must re-check, since we released the lock.
	if t.stopped || t.bundler != nil {
		return
	}

	// TODO(jba): use a context detached from the one passed to NewClient.
	ctx := context.TODO()
	// Unless overridden, run several goroutines per CPU to call the Publish RPC.
	n := t.PublishSettings.NumGoroutines
	if n <= 0 {
		n = 25 * runtime.GOMAXPROCS(0)
	}
	t.wg.Add(n)
	for i := 0; i < n; i++ {
		go func() {
			defer t.wg.Done()
			for b := range t.bundlec {
				t.publishMessageBundle(ctx, b)
			}
		}()
	}
	t.bundler = bundler.NewBundler(&bundledMessage{}, func(items interface{}) {
		t.bundlec <- items.([]*bundledMessage)

	})
	t.bundler.DelayThreshold = t.PublishSettings.DelayThreshold
	t.bundler.BundleCountThreshold = t.PublishSettings.CountThreshold
	if t.bundler.BundleCountThreshold > MaxPublishRequestCount {
		t.bundler.BundleCountThreshold = MaxPublishRequestCount
	}
	t.bundler.BundleByteThreshold = t.PublishSettings.ByteThreshold
	t.bundler.BufferedByteLimit = math.MaxInt64
	t.bundler.BundleByteLimit = MaxPublishRequestBytes

	maxCount := t.PublishSettings.MaxOutstandingMessages
	if maxCount == 0 {
		maxCount = DefaultPublishSettings.MaxOutstandingMessages
	}
	maxBytes := t.PublishSettings.MaxOutstandingBytes
	if maxBytes == 0 {
		maxBytes = DefaultPublishSettings.MaxOutstandingBytes
	}

	t.flowController = newFlowController(maxCount, maxBytes)
}

func (t *Topic) publishMessageBundle(ctx context.Context, bms []*bundledMessage) {
	msgs := make([]*Message, len(bms))
	for i, bm := range bms {
		msgs[i], bm.msg = bm.msg, nil // release bm.msg for GC
	}
	ids, err := t.s.publishMessages(ctx, t.name, msgs)
	for i, bm := range bms {
		if err != nil {
			bm.res.err = err
		} else {
			bm.res.serverID = ids[i]
		}
		t.flowController.release(msgs[i].size)
		close(bm.res.ready)
	}
}
<commit_msg>pubsub: localize setting of PublishResult.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"errors"
	"fmt"
	"math"
	"runtime"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/iam"
	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/api/support/bundler"
	pb "google.golang.org/genproto/googleapis/pubsub/v1"
)

const (
	// The maximum number of messages that can be in a single publish request, as
	// determined by the PubSub service.
	MaxPublishRequestCount = 1000

	// The maximum size of a single publish request in bytes, as determined by the PubSub service.
	MaxPublishRequestBytes = 1e7
)

// ErrOversizedMessage indicates that a message's size exceeds MaxPublishRequestBytes.
var ErrOversizedMessage = bundler.ErrOversizedItem

// Topic is a reference to a PubSub topic.
type Topic struct {
	s service
	// The fully qualified identifier for the topic, in the format "projects/<projid>/topics/<name>"
	name string

	// Settings for publishing messages. All changes must be made before the
	// first call to Publish. The default is DefaultPublishSettings.
	PublishSettings PublishSettings

	mu      sync.RWMutex
	stopped bool
	bundler *bundler.Bundler

	wg sync.WaitGroup

	// Channel for message bundles to be published. Close to indicate that Stop was called.
	bundlec chan []*bundledMessage

	flowController *flowController
}

// PublishSettings control the bundling of published messages.
type PublishSettings struct {

	// Publish a non-empty batch after this delay has passed.
	DelayThreshold time.Duration

	// Publish a batch when it has this many messages. The maximum is
	// MaxPublishRequestCount.
	CountThreshold int

	// Publish a batch when its size in bytes reaches this value.
	ByteThreshold int

	// MaxOutstandingMessages is the maximum number of messages for which the
	// publish can be outstanding. If MaxOutstandingMessages is 0, it will be
	// treated as if it were DefaultPublishSettings.MaxOutstandingMessages. If the
	// value is negative, then there will be no limit on the number of messages
	// that can be outstanding.
	MaxOutstandingMessages int

	// MaxOutstandingBytes is the maximum size of messages for which the publish
	// can be outstanding. If MaxOutstandingBytes is 0, it will be treated as if it
	// werer DefaultPublishSettings.MaxOutstandingBytes. If the value is negative,
	// then there will be no limit on the number of bytes that can be outstanding.
	MaxOutstandingBytes int

	// The number of goroutines that invoke the Publish RPC concurrently.
	// Defaults to a multiple of GOMAXPROCS.
	NumGoroutines int
}

// DefaultPublishSettings holds the default values for topics' BatchSettings.
var DefaultPublishSettings = PublishSettings{
	DelayThreshold:         1 * time.Millisecond,
	CountThreshold:         100,
	ByteThreshold:          1e6,
	MaxOutstandingMessages: -1,  // Default to unlimited number of messages
	MaxOutstandingBytes:    1e9, // 1G
}

// CreateTopic creates a new topic.
// The specified topic ID must start with a letter, and contain only letters
// ([A-Za-z]), numbers ([0-9]), dashes (-), underscores (_), periods (.),
// tildes (~), plus (+) or percent signs (%). It must be between 3 and 255
// characters in length, and must not start with "goog".
// If the topic already exists an error will be returned.
func (c *Client) CreateTopic(ctx context.Context, id string) (*Topic, error) {
	t := c.Topic(id)
	err := c.s.createTopic(ctx, t.name)
	return t, err
}

// Topic creates a reference to a topic.
//
// If a Topic's Publish method is called, it has background goroutines
// associated with it. Clean them up by calling Topic.Stop.
//
// Avoid creating many Topic instances if you use them to publish.
func (c *Client) Topic(id string) *Topic {
	return newTopic(c.s, fmt.Sprintf("projects/%s/topics/%s", c.projectID, id))
}

func newTopic(s service, name string) *Topic {
	// bundlec is unbuffered. A buffer would occupy memory not
	// accounted for by the bundler, so BufferedByteLimit would be a lie:
	// the actual memory consumed would be higher.
	return &Topic{
		s:               s,
		name:            name,
		PublishSettings: DefaultPublishSettings,
		bundlec:         make(chan []*bundledMessage),
	}
}

// Topics returns an iterator which returns all of the topics for the client's project.
func (c *Client) Topics(ctx context.Context) *TopicIterator {
	return &TopicIterator{
		s:    c.s,
		next: c.s.listProjectTopics(ctx, c.fullyQualifiedProjectName()),
	}
}

// TopicIterator is an iterator that returns a series of topics.
type TopicIterator struct {
	s    service
	next nextStringFunc
}

// Next returns the next topic. If there are no more topics, iterator.Done will be returned.
func (tps *TopicIterator) Next() (*Topic, error) {
	topicName, err := tps.next()
	if err != nil {
		return nil, err
	}
	return newTopic(tps.s, topicName), nil
}

// ID returns the unique idenfier of the topic within its project.
func (t *Topic) ID() string {
	slash := strings.LastIndex(t.name, "/")
	if slash == -1 {
		// name is not a fully-qualified name.
		panic("bad topic name")
	}
	return t.name[slash+1:]
}

// String returns the printable globally unique name for the topic.
func (t *Topic) String() string {
	return t.name
}

// Delete deletes the topic.
func (t *Topic) Delete(ctx context.Context) error {
	return t.s.deleteTopic(ctx, t.name)
}

// Exists reports whether the topic exists on the server.
func (t *Topic) Exists(ctx context.Context) (bool, error) {
	if t.name == "_deleted-topic_" {
		return false, nil
	}

	return t.s.topicExists(ctx, t.name)
}

func (t *Topic) IAM() *iam.Handle {
	return t.s.iamHandle(t.name)
}

// Subscriptions returns an iterator which returns the subscriptions for this topic.
func (t *Topic) Subscriptions(ctx context.Context) *SubscriptionIterator {
	// NOTE: zero or more Subscriptions that are ultimately returned by this
	// Subscriptions iterator may belong to a different project to t.
	return &SubscriptionIterator{
		s:    t.s,
		next: t.s.listTopicSubscriptions(ctx, t.name),
	}
}

var errTopicStopped = errors.New("pubsub: Stop has been called for this topic")

// Publish publishes msg to the topic asynchronously. Messages are batched and
// sent according to the topic's BatchSettings.
//
// Publish returns a non-nil PublishResult which will be ready when the
// message has been sent (or has failed to be sent) to the server.
//
// Publish creates goroutines for batching and sending messages. These goroutines
// need to be stopped by calling t.Stop(). Once stopped, future calls to Publish
// or TryPublish will immediately return a PublishResult with an error.
//
// Publish blocks until the number of messages and memory consumed by batching
// fall below MaxOutstandingMessages and MaxOutstandingBytes, respectively, or
// until ctx is Done. The ctx argument is used only for this purpose; it is
// unrelated to the context used by the background goroutines which call the
// Publish RPC.
func (t *Topic) Publish(ctx context.Context, msg *Message) *PublishResult {
	return t.publish(ctx, msg, true)
}

// TryPublish publishes msg to the topic asynchronously. Messages are batched
// and sent according to the topic's BatchSettings.
//
// If the number of messages or memory consumed by batching are above
// MaxOutstandingMessages or MaxOutstandingBytes, respectively, then TryPublish
// immediately returns nil. Otherwise, TryPublish returns a non-nil PublishResult
// which will be ready when the message has been sent (or has failed to be sent)
// to the server.
//
// TryPublish creates goroutines for batching and sending messages. These goroutines
// need to be stopped by calling t.Stop(). Once stopped, future calls to Publish
// or TryPublish will immediately return a PublishResult with an error.
func (t *Topic) TryPublish(ctx context.Context, msg *Message) *PublishResult {
	return t.publish(ctx, msg, false)
}

func (t *Topic) publish(ctx context.Context, msg *Message, waitForFC bool) *PublishResult {
	// TODO(jba): if this turns out to take significant time, try to approximate it.
	// Or, convert the messages to protos in Publish, instead of in the service.
	msg.size = proto.Size(&pb.PubsubMessage{
		Data:       msg.Data,
		Attributes: msg.Attributes,
	})
	r := &PublishResult{ready: make(chan struct{})}
	t.initBundlerAndFlowController()
	t.mu.RLock()
	defer t.mu.RUnlock()
	// TODO(aboulhosn) [from bcmills] consider changing the semantics of bundler to perform this logic so we don't have to do it here
	if t.stopped {
		r.set("", errTopicStopped)
		return r
	}
	if waitForFC {
		if err := t.flowController.acquire(ctx, msg.size); err != nil {
			r.set("", err)
			return r
		}
	} else if !t.flowController.tryAcquire(msg.size) {
		return nil
	}
	// TODO(jba) [from bcmills] consider using a shared channel per bundle (requires Bundler API changes; would reduce allocations)
	// The call to AddWait will never block because the bundler's BufferedByteLimit is set to MaxInt64. The topic's flowController
	// is what ensures we have capacity to publish more messages.
	err := t.bundler.AddWait(ctx, &bundledMessage{msg, r}, msg.size)
	if err != nil {
		r.set("", err)
	}
	return r
}

// Send all remaining published messages and stop goroutines created for handling
// publishing. Returns once all outstanding messages have been sent or have
// failed to be sent.
func (t *Topic) Stop() {
	t.mu.Lock()
	noop := t.stopped || t.bundler == nil
	t.stopped = true
	t.mu.Unlock()
	if noop {
		return
	}
	t.bundler.Flush()
	// At this point, all pending bundles have been published and the bundler's
	// goroutines have exited, so it is OK for this goroutine to close bundlec.
	close(t.bundlec)
	t.wg.Wait()
}

// A PublishResult holds the result from a call to Publish.
type PublishResult struct {
	ready    chan struct{}
	serverID string
	err      error
}

// Ready returns a channel that is closed when the result is ready.
// When the Ready channel is closed, Get is guaranteed not to block.
func (r *PublishResult) Ready() <-chan struct{} { return r.ready }

// Get returns the server-generated message ID and/or error result of a Publish call.
// Get blocks until the Publish call completes or the context is done.
func (r *PublishResult) Get(ctx context.Context) (serverID string, err error) {
	// If the result is already ready, return it even if the context is done.
	select {
	case <-r.Ready():
		return r.serverID, r.err
	default:
	}
	select {
	case <-ctx.Done():
		return "", ctx.Err()
	case <-r.Ready():
		return r.serverID, r.err
	}
}

func (r *PublishResult) set(sid string, err error) {
	r.serverID = sid
	r.err = err
	close(r.ready)
}

type bundledMessage struct {
	msg *Message
	res *PublishResult
}

func (t *Topic) initBundlerAndFlowController() {
	t.mu.RLock()
	noop := t.stopped || t.bundler != nil
	t.mu.RUnlock()
	if noop {
		return
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	// Must re-check, since we released the lock.
	if t.stopped || t.bundler != nil {
		return
	}

	// TODO(jba): use a context detached from the one passed to NewClient.
	ctx := context.TODO()
	// Unless overridden, run several goroutines per CPU to call the Publish RPC.
	n := t.PublishSettings.NumGoroutines
	if n <= 0 {
		n = 25 * runtime.GOMAXPROCS(0)
	}
	t.wg.Add(n)
	for i := 0; i < n; i++ {
		go func() {
			defer t.wg.Done()
			for b := range t.bundlec {
				t.publishMessageBundle(ctx, b)
			}
		}()
	}
	t.bundler = bundler.NewBundler(&bundledMessage{}, func(items interface{}) {
		t.bundlec <- items.([]*bundledMessage)

	})
	t.bundler.DelayThreshold = t.PublishSettings.DelayThreshold
	t.bundler.BundleCountThreshold = t.PublishSettings.CountThreshold
	if t.bundler.BundleCountThreshold > MaxPublishRequestCount {
		t.bundler.BundleCountThreshold = MaxPublishRequestCount
	}
	t.bundler.BundleByteThreshold = t.PublishSettings.ByteThreshold
	t.bundler.BufferedByteLimit = math.MaxInt64
	t.bundler.BundleByteLimit = MaxPublishRequestBytes

	maxCount := t.PublishSettings.MaxOutstandingMessages
	if maxCount == 0 {
		maxCount = DefaultPublishSettings.MaxOutstandingMessages
	}
	maxBytes := t.PublishSettings.MaxOutstandingBytes
	if maxBytes == 0 {
		maxBytes = DefaultPublishSettings.MaxOutstandingBytes
	}

	t.flowController = newFlowController(maxCount, maxBytes)
}

func (t *Topic) publishMessageBundle(ctx context.Context, bms []*bundledMessage) {
	msgs := make([]*Message, len(bms))
	for i, bm := range bms {
		msgs[i], bm.msg = bm.msg, nil // release bm.msg for GC
	}
	ids, err := t.s.publishMessages(ctx, t.name, msgs)
	for i, bm := range bms {
		if err != nil {
			bm.res.set("", err)
		} else {
			bm.res.set(ids[i], nil)
		}
		t.flowController.release(msgs[i].size)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdPullRequest = &Command{
	Run:   pullRequest,
	Usage: "pull-request [-f] [TITLE|-i ISSUE] [-b BASE] [-h HEAD]",
	Short: "Open a pull request on GitHub",
	Long: `Opens a pull request on GitHub for the project that the "origin" remote
points to. The default head of the pull request is the current branch.
Both base and head of the pull request can be explicitly given in one of
the following formats: "branch", "owner:branch", "owner/repo:branch".
This command will abort operation if it detects that the current topic
branch has local commits that are not yet pushed to its upstream branch
on the remote. To skip this check, use -f.

If TITLE is omitted, a text editor will open in which title and body of
the pull request can be entered in the same manner as git commit message.

If instead of normal TITLE an issue number is given with -i, the pull
request will be attached to an existing GitHub issue. Alternatively, instead
of title you can paste a full URL to an issue on GitHub.
`,
}

var flagPullRequestBase, flagPullRequestHead string

func init() {
	// TODO: delay calculation of owner and current branch until being used
	cmdPullRequest.Flag.StringVar(&flagPullRequestBase, "b", git.Owner()+":master", "BASE")
	cmdPullRequest.Flag.StringVar(&flagPullRequestHead, "h", git.Owner()+":"+git.CurrentBranch(), "HEAD")
}

func pullRequest(cmd *Command, args []string) {
	messageFile := filepath.Join(git.Dir(), "PULLREQ_EDITMSG")

	writePullRequestChanges(messageFile, flagPullRequestBase, flagPullRequestHead)

	editCmd := buildEditCommand(messageFile)
	err := execCmd(editCmd)
	if err != nil {
		log.Fatal(err)
	}

	title, body, err := readTitleAndBodyFromFile(messageFile)
	if err != nil {
		log.Fatal(err)
	}
	if len(title) == 0 {
		log.Fatal("Aborting due to empty pull request title")
	}

	params := PullRequestParams{title, body, flagPullRequestBase, flagPullRequestHead}
	err = gh.CreatePullRequest(git.Owner(), git.Repo(), params)
	if err != nil {
		log.Fatal(err)
	}
}

func writePullRequestChanges(messageFile, base, head string) {
	message := `
# Requesting a pull to %s from %s
#
# Write a message for this pull reuqest. The first block
# of the text is the title and the rest is description.
#
# Changes:
#
%s
`
	startRegexp := regexp.MustCompilePOSIX("^")
	endRegexp := regexp.MustCompilePOSIX(" +$")

	commitLogs := git.CommitLogs("master", "pull_request")
	commitLogs = strings.TrimSpace(commitLogs)
	commitLogs = startRegexp.ReplaceAllString(commitLogs, "# ")
	commitLogs = endRegexp.ReplaceAllString(commitLogs, "")

	message = fmt.Sprintf(message, base, head, commitLogs)
	err := ioutil.WriteFile(messageFile, []byte(message), 0644)
	if err != nil {
		log.Fatal(err)
	}
}

func buildEditCommand(messageFile string) []string {
	editCmd := make([]string, 0)
	gitEditor := git.Editor()
	editCmd = append(editCmd, gitEditor)
	r := regexp.MustCompile("^[mg]?vim$")
	if r.MatchString(gitEditor) {
		editCmd = append(editCmd, "-c")
		editCmd = append(editCmd, "set ft=gitcommit")
	}
	editCmd = append(editCmd, messageFile)

	return editCmd
}

func execCmd(command []string) error {
	cmd := exec.Command(command[0], command[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	return err
}

func readTitleAndBodyFromFile(messageFile string) (title, body string, err error) {
	f, err := os.Open(messageFile)
	defer f.Close()
	if err != nil {
		return "", "", err
	}

	reader := bufio.NewReader(f)
	return readTitleAndBody(reader)
}

func readTitleAndBody(reader *bufio.Reader) (title, body string, err error) {
	r := regexp.MustCompile("\\S")
	var titleParts, bodyParts []string

	line, err := readln(reader)
	for err == nil {
		if strings.HasPrefix(line, "#") {
			break
		}
		if len(bodyParts) == 0 && r.MatchString(line) {
			titleParts = append(titleParts, line)
		} else {
			bodyParts = append(bodyParts, line)
		}
		line, err = readln(reader)
	}

	title = strings.Join(titleParts, " ")
	title = strings.TrimSpace(title)

	body = strings.Join(bodyParts, "\n")
	body = strings.TrimSpace(body)

	return title, body, nil
}

func readln(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool  = true
		err      error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}
<commit_msg>Use pass-in base and head<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdPullRequest = &Command{
	Run:   pullRequest,
	Usage: "pull-request [-f] [TITLE|-i ISSUE] [-b BASE] [-h HEAD]",
	Short: "Open a pull request on GitHub",
	Long: `Opens a pull request on GitHub for the project that the "origin" remote
points to. The default head of the pull request is the current branch.
Both base and head of the pull request can be explicitly given in one of
the following formats: "branch", "owner:branch", "owner/repo:branch".
This command will abort operation if it detects that the current topic
branch has local commits that are not yet pushed to its upstream branch
on the remote. To skip this check, use -f.

If TITLE is omitted, a text editor will open in which title and body of
the pull request can be entered in the same manner as git commit message.

If instead of normal TITLE an issue number is given with -i, the pull
request will be attached to an existing GitHub issue. Alternatively, instead
of title you can paste a full URL to an issue on GitHub.
`,
}

var flagPullRequestBase, flagPullRequestHead string

func init() {
	// TODO: delay calculation of owner and current branch until being used
	cmdPullRequest.Flag.StringVar(&flagPullRequestBase, "b", git.Owner()+":master", "BASE")
	cmdPullRequest.Flag.StringVar(&flagPullRequestHead, "h", git.Owner()+":"+git.CurrentBranch(), "HEAD")
}

func pullRequest(cmd *Command, args []string) {
	messageFile := filepath.Join(git.Dir(), "PULLREQ_EDITMSG")

	err := writePullRequestChanges(messageFile, flagPullRequestBase, flagPullRequestHead)
	if err != nil {
		log.Fatal(err)
	}

	editCmd := buildEditCommand(messageFile)
	err = execCmd(editCmd)
	if err != nil {
		log.Fatal(err)
	}

	title, body, err := readTitleAndBodyFromFile(messageFile)
	if err != nil {
		log.Fatal(err)
	}
	if len(title) == 0 {
		log.Fatal("Aborting due to empty pull request title")
	}

	params := PullRequestParams{title, body, flagPullRequestBase, flagPullRequestHead}
	err = gh.CreatePullRequest(git.Owner(), git.Repo(), params)
	if err != nil {
		log.Fatal(err)
	}
}

func writePullRequestChanges(messageFile, base, head string) error {
	message := `
# Requesting a pull to %s from %s
#
# Write a message for this pull reuqest. The first block
# of the text is the title and the rest is description.
#
# Changes:
#
%s
`
	startRegexp := regexp.MustCompilePOSIX("^")
	endRegexp := regexp.MustCompilePOSIX(" +$")

	commitLogs := git.CommitLogs(getLocalBranch(base), getLocalBranch(head))
	commitLogs = strings.TrimSpace(commitLogs)
	commitLogs = startRegexp.ReplaceAllString(commitLogs, "# ")
	commitLogs = endRegexp.ReplaceAllString(commitLogs, "")

	message = fmt.Sprintf(message, base, head, commitLogs)

	return ioutil.WriteFile(messageFile, []byte(message), 0644)
}

func getLocalBranch(branchName string) string {
	result := strings.Split(branchName, ":")
	return result[len(result)-1]
}

func buildEditCommand(messageFile string) []string {
	editCmd := make([]string, 0)
	gitEditor := git.Editor()
	editCmd = append(editCmd, gitEditor)
	r := regexp.MustCompile("^[mg]?vim$")
	if r.MatchString(gitEditor) {
		editCmd = append(editCmd, "-c")
		editCmd = append(editCmd, "set ft=gitcommit")
	}
	editCmd = append(editCmd, messageFile)

	return editCmd
}

func execCmd(command []string) error {
	cmd := exec.Command(command[0], command[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err := cmd.Run()
	return err
}

func readTitleAndBodyFromFile(messageFile string) (title, body string, err error) {
	f, err := os.Open(messageFile)
	defer f.Close()
	if err != nil {
		return "", "", err
	}

	reader := bufio.NewReader(f)
	return readTitleAndBody(reader)
}

func readTitleAndBody(reader *bufio.Reader) (title, body string, err error) {
	r := regexp.MustCompile("\\S")
	var titleParts, bodyParts []string

	line, err := readln(reader)
	for err == nil {
		if strings.HasPrefix(line, "#") {
			break
		}
		if len(bodyParts) == 0 && r.MatchString(line) {
			titleParts = append(titleParts, line)
		} else {
			bodyParts = append(bodyParts, line)
		}
		line, err = readln(reader)
	}

	title = strings.Join(titleParts, " ")
	title = strings.TrimSpace(title)

	body = strings.Join(bodyParts, "\n")
	body = strings.TrimSpace(body)

	return title, body, nil
}

func readln(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool  = true
		err      error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}
<|endoftext|>


<commit_before>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"fmt"
	"path"

	"github.com/robertkrimen/otto"
)

func null() module {
	return func(ctx *Orbit) (val otto.Value, err error) {
		return otto.UndefinedValue(), nil
	}
}

func (ctx *Orbit) load(name string, path string) (val otto.Value, err error) {

	// Check loaded modules
	if module, ok := ctx.modules[name]; ok {
		return module, nil
	}

	// Check global modules
	if module, ok := modules[name]; ok {
		return module(ctx)
	}

	ctx.modules[name], err = find(name, path)(ctx)

	return ctx.modules[name], err

}

func find(name string, fold string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		if len(name) == 0 {
			return otto.UndefinedValue(), fmt.Errorf("No module name specified")
		}

		var files []string

		name = path.Clean(name)

		if path.IsAbs(name) == true {
			if path.Ext(name) != "" {
				files = append(files, name)
			}
			if path.Ext(name) == "" {
				files = append(files, name+".js")
				files = append(files, path.Join(name, "index.js"))
			}
		}

		if path.IsAbs(name) == false {
			if path.Ext(name) != "" {
				files = append(files, path.Join(fold, name))
			}
			if path.Ext(name) == "" {
				files = append(files, path.Join(fold, name)+".js")
				files = append(files, path.Join(fold, name, "index.js"))
			}
		}

		code, file, err := finder(ctx, files)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		return exec(code, file)(ctx)

	}
}

func main(code interface{}, full string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		fold, file := path.Split(full)

		script := fmt.Sprintf("%s\n%s\n%s", "(function(module) { var require = module.require; var exports = module.exports;", code, "})")

		module, _ := ctx.Object(`({})`)

		module.Set("id", full)
		module.Set("loaded", true)
		module.Set("filename", full)

		module.Set("exports", map[string]interface{}{})
		ctx.Set("__dirname", fold)
		ctx.Set("__filename", file)

		module.Set("require", func(call otto.FunctionCall) otto.Value {
			arg := call.Argument(0).String()
			val, err := ctx.load(arg, dir)
			if err != nil {
				ctx.Call("new Error", nil, err.Error())
			}
			return val
		})

		ret, err := ctx.Call(script, nil, module)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		exp, err := module.Get("exports")
		if err != nil {
			return otto.UndefinedValue(), err
		}

		if exp.IsFunction() {
			val, err = module.Call("exports")
			return
		}

		if exp.IsDefined() {
			val = exp
			return
		}

		if ret.IsDefined() {
			val = ret
			return
		}

		return

	}

}

func exec(code interface{}, full string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		fold, file := path.Split(full)

		script := fmt.Sprintf("%s\n%s\n%s", "(function(module) { var require = module.require; var exports = module.exports;", code, "})")

		module, _ := ctx.Object(`({})`)

		module.Set("id", full)
		module.Set("loaded", true)
		module.Set("filename", full)

		ctx.Set("__dirname", fold)
		ctx.Set("__filename", file)

		module.Set("require", func(call otto.FunctionCall) otto.Value {
			arg := call.Argument(0).String()
			val, err := ctx.load(arg, dir)
			if err != nil {
				ctx.Call("new Error", nil, err.Error())
			}
			return val
		})

		ret, err := ctx.Call(data, export, module)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		if ret.IsDefined() {
			val = ret
		}

		if ret.IsUndefined() {
			val, err = module.Get("exports")
		}

		return

	}

}
<commit_msg>Make function consistent with other functions on the context<commit_after>// Copyright © 2016 Abcum Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package orbit

import (
	"fmt"
	"path"

	"github.com/robertkrimen/otto"
)

func null() module {
	return func(ctx *Orbit) (val otto.Value, err error) {
		return otto.UndefinedValue(), nil
	}
}

func load(name string, fold string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		// Check loaded modules
		if module, ok := ctx.modules[name]; ok {
			return module, nil
		}

		// Check global modules
		if module, ok := modules[name]; ok {
			return module(ctx)
		}

		ctx.modules[name], err = find(name, fold)(ctx)

		return ctx.modules[name], err

	}
}

func find(name string, fold string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		if len(name) == 0 {
			return otto.UndefinedValue(), fmt.Errorf("No module name specified")
		}

		var files []string

		name = path.Clean(name)

		if path.IsAbs(name) == true {
			if path.Ext(name) != "" {
				files = append(files, name)
			}
			if path.Ext(name) == "" {
				files = append(files, name+".js")
				files = append(files, path.Join(name, "index.js"))
			}
		}

		if path.IsAbs(name) == false {
			if path.Ext(name) != "" {
				files = append(files, path.Join(fold, name))
			}
			if path.Ext(name) == "" {
				files = append(files, path.Join(fold, name)+".js")
				files = append(files, path.Join(fold, name, "index.js"))
			}
		}

		code, file, err := finder(ctx, files)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		return exec(code, file)(ctx)

	}
}

func main(code interface{}, full string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		fold, file := path.Split(full)

		script := fmt.Sprintf("%s\n%s\n%s", "(function(module) { var require = module.require; var exports = module.exports;", code, "})")

		module, _ := ctx.Object(`({})`)

		module.Set("id", full)
		module.Set("loaded", true)
		module.Set("filename", full)

		module.Set("exports", map[string]interface{}{})
		ctx.Set("__dirname", fold)
		ctx.Set("__filename", file)

		module.Set("require", func(call otto.FunctionCall) otto.Value {
			arg := call.Argument(0).String()
			val, err := load(arg, fold)(ctx)
			if err != nil {
				ctx.Call("new Error", nil, err.Error())
			}
			return val
		})

		ret, err := ctx.Call(script, nil, module)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		exp, err := module.Get("exports")
		if err != nil {
			return otto.UndefinedValue(), err
		}

		if exp.IsFunction() {
			val, err = module.Call("exports")
			return
		}

		if exp.IsDefined() {
			val = exp
			return
		}

		if ret.IsDefined() {
			val = ret
			return
		}

		return

	}

}

func exec(code interface{}, full string) module {
	return func(ctx *Orbit) (val otto.Value, err error) {

		fold, file := path.Split(full)

		script := fmt.Sprintf("%s\n%s\n%s", "(function(module) { var require = module.require; var exports = module.exports;", code, "})")

		module, _ := ctx.Object(`({})`)

		module.Set("id", full)
		module.Set("loaded", true)
		module.Set("filename", full)

		ctx.Set("__dirname", fold)
		ctx.Set("__filename", file)

		module.Set("require", func(call otto.FunctionCall) otto.Value {
			arg := call.Argument(0).String()
			val, err := load(arg, fold)(ctx)
			if err != nil {
				ctx.Call("new Error", nil, err.Error())
			}
			return val
		})

		ret, err := ctx.Call(data, export, module)
		if err != nil {
			return otto.UndefinedValue(), err
		}

		if ret.IsDefined() {
			val = ret
		}

		if ret.IsUndefined() {
			val, err = module.Get("exports")
		}

		return

	}

}
<|endoftext|>


<commit_before>package kiwi

import (
	"errors"
)

// Process holds general information about the process,
// as well as embedding the struct ProcPlatAttribs which contains platform-specific data
// such as Windows process handles, linux `/proc` file handles, etc.
type Process struct {
	// Embedded struct for platform specific fields
	ProcPlatAttribs

	// Platform independent process details
	PID uint64
}

// ReadInt8 reads an int8.
func (p *Process) ReadInt8(addr uintptr) (int8, error) {
	var v int8
	e := p.read(addr, &v)
	return v, e
}

// ReadInt16 reads an int16.
func (p *Process) ReadInt16(addr uintptr) (int16, error) {
	var v int16
	e := p.read(addr, &v)
	return v, e
}

// ReadInt32 reads an int32.
func (p *Process) ReadInt32(addr uintptr) (int32, error) {
	var v int32
	e := p.read(addr, &v)
	return v, e
}

// ReadInt64 reads an int64
func (p *Process) ReadInt64(addr uintptr) (int64, error) {
	var v int64
	e := p.read(addr, &v)
	return v, e
}

// ReadUint8 reads an uint8.
func (p *Process) ReadUint8(addr uintptr) (uint8, error) {
	var v uint8
	e := p.read(addr, &v)
	return v, e
}

// ReadUint16 reads an uint16.
func (p *Process) ReadUint16(addr uintptr) (uint16, error) {
	var v uint16
	e := p.read(addr, &v)
	return v, e
}

// ReadUint32 reads an uint32.
func (p *Process) ReadUint32(addr uintptr) (uint32, error) {
	var v uint32
	e := p.read(addr, &v)
	return v, e
}

// ReadUint64 reads an uint64.
func (p *Process) ReadUint64(addr uintptr) (uint64, error) {
	var v uint64
	e := p.read(addr, &v)
	return v, e
}

// ReadFloat32 reads a float32.
func (p *Process) ReadFloat32(addr uintptr) (float32, error) {
	var v float32
	e := p.read(addr, &v)
	return v, e
}

// ReadFloat64 reads a float64
func (p *Process) ReadFloat64(addr uintptr) (float64, error) {
	var v float64
	e := p.read(addr, &v)
	return v, e
}

// ReadUint32Ptr reads a uint32 pointer chain with offsets.
func (p *Process) ReadUint32Ptr(addr uintptr, offsets ...uintptr) (uint32, error) {
	curPtr, err := p.ReadUint32(addr)
	if err != nil {
		return 0, errors.New("Error while trying to read from ptr base.")
	}

	for _, offset := range offsets {
		curPtr, err = p.ReadUint32(uintptr(curPtr) + offset)
		if err != nil {
			return 0, errors.New("Error while trying to read from offset.")
		}
	}

	return curPtr, nil
}

// ReadBytes reads a slice of bytes.
func (p *Process) ReadBytes(addr uintptr, size int) ([]byte, error) {
	v := make([]byte, size)
	e := p.read(addr, &v)
	return v, e
}

// WriteInt8 writes an int8.
func (p *Process) WriteInt8(addr uintptr, v int8) error {
	return p.write(addr, &v)
}

// WriteInt16 writes an int16.
func (p *Process) WriteInt16(addr uintptr, v int16) error {
	return p.write(addr, &v)
}

// WriteInt32 writes an int32.
func (p *Process) WriteInt32(addr uintptr, v int32) error {
	return p.write(addr, &v)
}

// WriteInt64 writes an int64.
func (p *Process) WriteInt64(addr uintptr, v int64) error {
	return p.write(addr, &v)
}

// WriteUint8 writes an uint8.
func (p *Process) WriteUint8(addr uintptr, v uint8) error {
	return p.write(addr, &v)
}

// WriteUint16 writes an uint16.
func (p *Process) WriteUint16(addr uintptr, v uint16) error {
	return p.write(addr, &v)
}

// WriteUint32 writes an uint32.
func (p *Process) WriteUint32(addr uintptr, v uint32) error {
	return p.write(addr, &v)
}

// WriteUint64 writes an uint64.
func (p *Process) WriteUint64(addr uintptr, v uint64) error {
	return p.write(addr, &v)
}

// WriteFloat32 writes a float32.
func (p *Process) WriteFloat32(addr uintptr, v float32) error {
	return p.write(addr, &v)
}

// WriteFloat64 writes a float64.
func (p *Process) WriteFloat64(addr uintptr, v float64) error {
	return p.write(addr, &v)
}

// WriteBytes writes a slice of bytes.
func (p *Process) WriteBytes(addr uintptr, v []byte) error {
	return p.write(addr, &v)
}
<commit_msg>Wrap errors<commit_after>package kiwi

import (
	"fmt"
)

// Process holds general information about the process,
// as well as embedding the struct ProcPlatAttribs which contains platform-specific data
// such as Windows process handles, linux `/proc` file handles, etc.
type Process struct {
	// Embedded struct for platform specific fields
	ProcPlatAttribs

	// Platform independent process details
	PID uint64
}

// ReadInt8 reads an int8.
func (p *Process) ReadInt8(addr uintptr) (int8, error) {
	var v int8
	e := p.read(addr, &v)
	return v, e
}

// ReadInt16 reads an int16.
func (p *Process) ReadInt16(addr uintptr) (int16, error) {
	var v int16
	e := p.read(addr, &v)
	return v, e
}

// ReadInt32 reads an int32.
func (p *Process) ReadInt32(addr uintptr) (int32, error) {
	var v int32
	e := p.read(addr, &v)
	return v, e
}

// ReadInt64 reads an int64
func (p *Process) ReadInt64(addr uintptr) (int64, error) {
	var v int64
	e := p.read(addr, &v)
	return v, e
}

// ReadUint8 reads an uint8.
func (p *Process) ReadUint8(addr uintptr) (uint8, error) {
	var v uint8
	e := p.read(addr, &v)
	return v, e
}

// ReadUint16 reads an uint16.
func (p *Process) ReadUint16(addr uintptr) (uint16, error) {
	var v uint16
	e := p.read(addr, &v)
	return v, e
}

// ReadUint32 reads an uint32.
func (p *Process) ReadUint32(addr uintptr) (uint32, error) {
	var v uint32
	e := p.read(addr, &v)
	return v, e
}

// ReadUint64 reads an uint64.
func (p *Process) ReadUint64(addr uintptr) (uint64, error) {
	var v uint64
	e := p.read(addr, &v)
	return v, e
}

// ReadFloat32 reads a float32.
func (p *Process) ReadFloat32(addr uintptr) (float32, error) {
	var v float32
	e := p.read(addr, &v)
	return v, e
}

// ReadFloat64 reads a float64
func (p *Process) ReadFloat64(addr uintptr) (float64, error) {
	var v float64
	e := p.read(addr, &v)
	return v, e
}

// ReadUint32Ptr reads a uint32 pointer chain with offsets.
func (p *Process) ReadUint32Ptr(addr uintptr, offsets ...uintptr) (uint32, error) {
	curPtr, err := p.ReadUint32(addr)
	if err != nil {
		return 0, fmt.Errorf("ReadUint32 0x%X: %w", addr, err)
	}

	for _, offset := range offsets {
		curPtr, err = p.ReadUint32(uintptr(curPtr) + offset)
		if err != nil {
			return 0, fmt.Errorf("ReadUint32 0x%X: %w", addr, err)
		}
	}

	return curPtr, nil
}

// ReadBytes reads a slice of bytes.
func (p *Process) ReadBytes(addr uintptr, size int) ([]byte, error) {
	v := make([]byte, size)
	e := p.read(addr, &v)
	return v, e
}

// WriteInt8 writes an int8.
func (p *Process) WriteInt8(addr uintptr, v int8) error {
	return p.write(addr, &v)
}

// WriteInt16 writes an int16.
func (p *Process) WriteInt16(addr uintptr, v int16) error {
	return p.write(addr, &v)
}

// WriteInt32 writes an int32.
func (p *Process) WriteInt32(addr uintptr, v int32) error {
	return p.write(addr, &v)
}

// WriteInt64 writes an int64.
func (p *Process) WriteInt64(addr uintptr, v int64) error {
	return p.write(addr, &v)
}

// WriteUint8 writes an uint8.
func (p *Process) WriteUint8(addr uintptr, v uint8) error {
	return p.write(addr, &v)
}

// WriteUint16 writes an uint16.
func (p *Process) WriteUint16(addr uintptr, v uint16) error {
	return p.write(addr, &v)
}

// WriteUint32 writes an uint32.
func (p *Process) WriteUint32(addr uintptr, v uint32) error {
	return p.write(addr, &v)
}

// WriteUint64 writes an uint64.
func (p *Process) WriteUint64(addr uintptr, v uint64) error {
	return p.write(addr, &v)
}

// WriteFloat32 writes a float32.
func (p *Process) WriteFloat32(addr uintptr, v float32) error {
	return p.write(addr, &v)
}

// WriteFloat64 writes a float64.
func (p *Process) WriteFloat64(addr uintptr, v float64) error {
	return p.write(addr, &v)
}

// WriteBytes writes a slice of bytes.
func (p *Process) WriteBytes(addr uintptr, v []byte) error {
	return p.write(addr, &v)
}
<|endoftext|>


<commit_before>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFuncs      map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFuncs:      make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFuncs()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			// If the |-separated part starts with a dot, treat it as a
			// configuration for file extenion to use
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			// If the |-separated part starts with "join:"
			// then treat the character following that as the character to use
			// when joining multiple files received on a sub-stream
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFuncs does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFuncs() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFuncs[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.Process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedFileIPSliceMapKeys(kv map[string][]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	if _, ok := p.inParamPorts[portName]; !ok {
		p.InitInParamPort(p, portName)
	}
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			portName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(portName)
			case "o":
				if _, ok := t.Process.PathFuncs[portName]; !ok {
					Fail("No such pathfunc for out-port " + portName + " in task " + t.Name)
				}
				replacement = t.Process.PathFuncs[portName](t)
			case "p":
				replacement = t.Param(portName)
			case "t":
				replacement = t.Tag(portName)
			default:
				Fail("Replace failed for placeholder ", portName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				replacement = applyPathModifiers(replacement, restParts)
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.PathFuncs[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("%s: CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d)\n", p.Name(), p.CoresPerTask, cap(p.workflow.concurrentTasks))
	}

	// Using a slice to store unprocessed tasks allows us to receive tasks as
	// they are produced and to maintain the correct order of IPs. This select
	// allows us to process completed tasks as they become available. Waiting
	// for all Tasks to be spawned before processing any can cause deadlock
	// under certain workflow architectures when there are more than getBufsize()
	// Tasks per process, see #81.
	startedTasks := taskQueue{}

	var nextTask *Task
	tasks := p.createTasks()
	for tasks != nil || len(startedTasks) > 0 {
		select {
		case t, ok := <-tasks:
			if !ok {
				tasks = nil
			} else {
				// Sending FIFOs for the task
				for oname, oip := range t.OutIPs {
					if oip.doStream {
						if oip.FifoFileExists() {
							Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
						}
						oip.CreateFifo()
						p.Out(oname).Send(oip)
					}
				}

				// Execute task in separate go-routine
				go t.Execute()

				startedTasks = append(startedTasks, t)
			}
		case <-startedTasks.NextTaskDone():
			nextTask, startedTasks = startedTasks[0], startedTasks[1:]
			for oname, oip := range nextTask.OutIPs {
				if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
					p.Out(oname).Send(oip)
				}
				// Remove any FIFO file
				if oip.doStream && oip.FifoFileExists() {
					os.Remove(oip.FifoPath())
				}
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Tags need to be per Task, otherwise they are overwritten by future IPs
			tags := map[string]string{}
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFuncs, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}

type taskQueue []*Task

// NextTaskDone allows us to wait for the next task to be done if it's
// available. Otherwise, nil is returned since nil channels always block.
func (tq taskQueue) NextTaskDone() chan int {
	if len(tq) > 0 {
		return tq[0].Done
	}
	return nil
}
<commit_msg>Fix #66: Warn on duplicate port-names across in/out/param ports<commit_after>package scipipe

import (
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
)

// Process is the central component in SciPipe after Workflow. Processes are
// long-running "services" that schedules and executes Tasks based on the IPs
// and parameters received on its in-ports and parameter ports
type Process struct {
	BaseProcess
	CommandPattern string
	PathFuncs      map[string]func(*Task) string
	CustomExecute  func(*Task)
	CoresPerTask   int
	Prepend        string
	Spawn          bool
	PortInfo       map[string]*PortInfo
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewProc returns a new Process, and initializes its ports based on the
// command pattern.
func NewProc(workflow *Workflow, name string, cmd string) *Process {
	p := &Process{
		BaseProcess: NewBaseProcess(
			workflow,
			name,
		),
		CommandPattern: cmd,
		PathFuncs:      make(map[string]func(*Task) string),
		Spawn:          true,
		CoresPerTask:   1,
		PortInfo:       map[string]*PortInfo{},
	}
	workflow.AddProc(p)
	p.initPortsFromCmdPattern(cmd, nil)
	p.initDefaultPathFuncs()
	return p
}

// PortInfo is a container for various information about process ports
type PortInfo struct {
	portType  string
	extension string
	doStream  bool
	join      bool
	joinSep   string
}

// initPortsFromCmdPattern is a helper function for NewProc, that sets up in-
// and out-ports based on the shell command pattern used to create the Process.
// Ports are set up in this way:
// `{i:PORTNAME}` specifies an in-port
// `{o:PORTNAME}` specifies an out-port
// `{os:PORTNAME}` specifies an out-port that streams via a FIFO file
// `{p:PORTNAME}` a "parameter (in-)port", which means a port where parameters can be "streamed"
func (p *Process) initPortsFromCmdPattern(cmd string, params map[string]string) {
	// Find in/out port names and params and set up ports
	r := getShellCommandPlaceHolderRegex()
	ms := r.FindAllStringSubmatch(cmd, -1)

	seenPorts := map[string]string{}

	for _, m := range ms {
		portType := m[1]
		portRest := m[2]
		splitParts := strings.Split(portRest, "|")
		portName := splitParts[0]

		if _, ok := seenPorts[portName]; ok {
			Failf("Duplicate port-name (%s) in process (%s). Port names must be unique across all the in-, out- and parameter ports for each process.\n", portName, p.Name())
		}
		seenPorts[portName] = portName

		p.PortInfo[portName] = &PortInfo{portType: portType}

		for _, part := range splitParts[1:] {
			// If the |-separated part starts with a dot, treat it as a
			// configuration for file extenion to use
			fileExtPtn := regexp.MustCompile("\\.([a-z0-9\\.\\-\\_]+)")
			if fileExtPtn.MatchString(part) {
				m := fileExtPtn.FindStringSubmatch(part)
				p.PortInfo[portName].extension = m[1]
			}
			// If the |-separated part starts with "join:"
			// then treat the character following that as the character to use
			// when joining multiple files received on a sub-stream
			joinPtn := regexp.MustCompile("join:([^{}|]+)")
			if joinPtn.MatchString(part) {
				m := joinPtn.FindStringSubmatch(part)
				p.PortInfo[portName].join = true
				p.PortInfo[portName].joinSep = m[1]
			}
		}
	}

	for portName, pInfo := range p.PortInfo {
		if pInfo.portType == "o" || pInfo.portType == "os" {
			p.InitOutPort(p, portName)
			if pInfo.portType == "os" {
				p.PortInfo[portName].doStream = true
			}
		} else if pInfo.portType == "i" {
			p.InitInPort(p, portName)
		} else if pInfo.portType == "p" {
			if params == nil {
				p.InitInParamPort(p, portName)
			} else if _, ok := params[portName]; !ok {
				p.InitInParamPort(p, portName)
			}
		}
	}
}

// initDefaultPathFuncs does exactly what it name says: Initializes default
// path formatters for processes, that is used if no explicit path is set, using
// the proc.SetPath[...] methods
func (p *Process) initDefaultPathFuncs() {
	for outName := range p.OutPorts() {
		outName := outName
		p.PathFuncs[outName] = func(t *Task) string {
			pathPcs := []string{}
			for _, ipName := range sortedFileIPMapKeys(t.InIPs) {
				pathPcs = append(pathPcs, filepath.Base(t.InIP(ipName).Path()))
			}
			procName := sanitizePathFragment(t.Process.Name())
			pathPcs = append(pathPcs, procName)
			for _, paramName := range sortedStringMapKeys(t.Params) {
				pathPcs = append(pathPcs, paramName+"_"+t.Param(paramName))
			}
			for _, tagName := range sortedStringMapKeys(t.Tags) {
				pathPcs = append(pathPcs, tagName+"_"+t.Tag(tagName))
			}
			pathPcs = append(pathPcs, outName)
			fileExt := p.PortInfo[outName].extension
			if fileExt != "" {
				pathPcs = append(pathPcs, fileExt)
			}
			return strings.Join(pathPcs, ".")
		}
	}
}

func sortedFileIPMapKeys(kv map[string]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedStringMapKeys(kv map[string]string) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

func sortedFileIPSliceMapKeys(kv map[string][]*FileIP) []string {
	keys := []string{}
	for k := range kv {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// ------------------------------------------------------------------------
// Main API methods: Port accessor methods
// ------------------------------------------------------------------------

// In is a short-form for InPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) In(portName string) *InPort {
	if portName == "" && len(p.InPorts()) == 1 {
		for _, inPort := range p.InPorts() {
			return inPort // Return the (only) in-port available
		}
	}
	return p.InPort(portName)
}

// Out is a short-form for OutPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) Out(portName string) *OutPort {
	if portName == "" && len(p.OutPorts()) == 1 {
		for _, outPort := range p.OutPorts() {
			return outPort // Return the (only) out-port available
		}
	}
	return p.OutPort(portName)
}

// InParam is a short-form for InParamPort() (of BaseProcess), which works only on Process
// processes
func (p *Process) InParam(portName string) *InParamPort {
	if _, ok := p.inParamPorts[portName]; !ok {
		p.InitInParamPort(p, portName)
	}
	return p.InParamPort(portName)
}

// OutParam is a short-form for OutParamPort() (of BaseProcess), which works only on
// Process processes
func (p *Process) OutParam(portName string) *OutParamPort {
	return p.OutParamPort(portName)
}

// ------------------------------------------------------------------------
// Main API methods: Configure path formatting
// ------------------------------------------------------------------------

// SetOut initializes a port (if it does not already exist), and takes a
// configuration for its outputs paths via a pattern similar to the command
// pattern used to create new processes, with placeholder tags. Available
// placeholder tags to use are:
// {i:inport_name}
// {p:param_name}
// {t:tag_name}
// An example might be: {i:foo}.replace_with_{p:replacement}.txt
// ... given that the process contains an in-port named 'foo', and a parameter
// named 'replacement'.
// If an out-port with the specified name does not exist, it will be created.
// This allows to create out-ports for filenames that are created without explicitly
// stating a filename on the commandline, such as when only submitting a prefix.
func (p *Process) SetOut(outPortName string, pathPattern string) {
	p.SetOutFunc(outPortName, func(t *Task) string {
		path := pathPattern // Avoiding reusing the same variable in multiple instances of this func

		r := getShellCommandPlaceHolderRegex()
		matches := r.FindAllStringSubmatch(path, -1)
		for _, match := range matches {
			var replacement string

			placeHolder := match[0]
			phType := match[1]
			restMatch := match[2]

			parts := strings.Split(restMatch, "|")
			portName := parts[0]
			restParts := parts[1:]

			switch phType {
			case "i":
				replacement = t.InPath(portName)
			case "o":
				if _, ok := t.Process.PathFuncs[portName]; !ok {
					Fail("No such pathfunc for out-port " + portName + " in task " + t.Name)
				}
				replacement = t.Process.PathFuncs[portName](t)
			case "p":
				replacement = t.Param(portName)
			case "t":
				replacement = t.Tag(portName)
			default:
				Fail("Replace failed for placeholder ", portName, " for path patterh '", path, "'")
			}

			if len(restParts) > 0 {
				replacement = applyPathModifiers(replacement, restParts)
			}

			// Replace placeholder with concrete value
			path = strings.Replace(path, placeHolder, replacement, -1)
		}
		return path
	})
}

// SetOutFunc takes a function which produces a file path based on data
// available in *Task, such as concrete file paths and parameter values,
func (p *Process) SetOutFunc(outPortName string, pathFmtFunc func(task *Task) (path string)) {
	if _, ok := p.outPorts[outPortName]; !ok {
		p.InitOutPort(p, outPortName)
	}
	p.PathFuncs[outPortName] = pathFmtFunc
}

// ------------------------------------------------------------------------
// Run method
// ------------------------------------------------------------------------

// Run runs the process by instantiating and executing Tasks for all inputs
// and parameter values on its in-ports. in the case when there are no inputs
// or parameter values on the in-ports, it will run just once before it
// terminates. note that the actual execution of shell commands are done inside
// Task.Execute, not here.
func (p *Process) Run() {
	defer p.CloseOutPorts()
	// Check that CoresPerTask is a sane number
	if p.CoresPerTask > cap(p.workflow.concurrentTasks) {
		Failf("CoresPerTask (%d) can't be greater than maxConcurrentTasks of workflow (%d) in process (%s)\n", p.CoresPerTask, cap(p.workflow.concurrentTasks), p.Name())
	}

	// Using a slice to store unprocessed tasks allows us to receive tasks as
	// they are produced and to maintain the correct order of IPs. This select
	// allows us to process completed tasks as they become available. Waiting
	// for all Tasks to be spawned before processing any can cause deadlock
	// under certain workflow architectures when there are more than getBufsize()
	// Tasks per process, see #81.
	startedTasks := taskQueue{}

	var nextTask *Task
	tasks := p.createTasks()
	for tasks != nil || len(startedTasks) > 0 {
		select {
		case t, ok := <-tasks:
			if !ok {
				tasks = nil
			} else {
				// Sending FIFOs for the task
				for oname, oip := range t.OutIPs {
					if oip.doStream {
						if oip.FifoFileExists() {
							Fail("Fifo file exists, so exiting (clean up fifo files before restarting the workflow): ", oip.FifoPath())
						}
						oip.CreateFifo()
						p.Out(oname).Send(oip)
					}
				}

				// Execute task in separate go-routine
				go t.Execute()

				startedTasks = append(startedTasks, t)
			}
		case <-startedTasks.NextTaskDone():
			nextTask, startedTasks = startedTasks[0], startedTasks[1:]
			for oname, oip := range nextTask.OutIPs {
				if !oip.doStream { // Streaming (FIFO) outputs have been sent earlier
					p.Out(oname).Send(oip)
				}
				// Remove any FIFO file
				if oip.doStream && oip.FifoFileExists() {
					os.Remove(oip.FifoPath())
				}
			}
		}
	}
}

// createTasks is a helper method for Run that creates tasks based on incoming
// IPs on in-ports, and feeds them to the Run method on the returned channel ch
func (p *Process) createTasks() (ch chan *Task) {
	ch = make(chan *Task)
	go func() {
		defer close(ch)

		inIPs := map[string]*FileIP{}
		params := map[string]string{}

		inPortsOpen := true
		paramPortsOpen := true
		for {
			// Tags need to be per Task, otherwise they are overwritten by future IPs
			tags := map[string]string{}
			// Only read on in-ports if we have any
			if len(p.inPorts) > 0 {
				inIPs, inPortsOpen = p.receiveOnInPorts()
				// If in-port is closed, that means we got the last params on last iteration, so break
				if !inPortsOpen {
					break
				}
			}
			// Only read on param in-ports if we have any
			if len(p.inParamPorts) > 0 {
				params, paramPortsOpen = p.receiveOnInParamPorts()
				// If param-port is closed, that means we got the last params on last iteration, so break
				if !paramPortsOpen {
					break
				}
			}

			for iname, ip := range inIPs {
				for k, v := range ip.Tags() {
					tags[iname+"."+k] = v
				}
			}

			// Create task and send on the channel we are about to return
			ch <- NewTask(p.workflow, p, p.Name(), p.CommandPattern, inIPs, p.PathFuncs, p.PortInfo, params, tags, p.Prepend, p.CustomExecute, p.CoresPerTask)

			// If we have no in-ports nor param in-ports, we should break after the first iteration
			if len(p.inPorts) == 0 && len(p.inParamPorts) == 0 {
				break
			}
		}
	}()
	return ch
}

type taskQueue []*Task

// NextTaskDone allows us to wait for the next task to be done if it's
// available. Otherwise, nil is returned since nil channels always block.
func (tq taskQueue) NextTaskDone() chan int {
	if len(tq) > 0 {
		return tq[0].Done
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

const (
	// The indices of the values in /proc/<pid>/stat
	statPid = iota
	statComm
	statState
	statPpid
	statPgrp
	statSession
	statTtyNr
	statTpgid
	statFlags
	statMinflt
	statCminflt
	statMajflt
	statCmajflt
	statUtime
	statStime
	statCutime
	statCstime
	statPriority
	statNice
	statNumThreads
	statItrealvalue
	statStartTime
	statVsize
	statRss
	statRsslim
	statStartCode
	statEndCode
	statStartStack
	statKstKesp
	statKstKeip
	statSignal
	statBlocked
	statSigIgnore
	statSigCatch
	statWchan
	statNswap
	statCnswap
	statExitSignal
	statProcessor
	statRtPriority
	statPolicy
	statDelayActBlkioTicks
	statGuestTime
	statCguestTime
)

// Process represents an operating system process.
type Process struct {
	PID     int
	User    *user.User
	Name    string // foo
	Command string // /usr/bin/foo --args

	// Alive is a flag used by ProcessMonitor to determine if it should remove
	// this process.
	Alive bool

	// Data from /proc/<pid>/stat
	Pgrp  int
	Utime uint64
	Stime uint64

	UtimeDiff uint64
	StimeDiff uint64
}

// NewProcess returns a new Process if a process is currently running on
// the system with the passed in PID.
func NewProcess(pid int) *Process {
	p := &Process{
		PID: pid,
	}

	if err := p.Update(); err != nil {
		return nil
	}

	if err := p.parseCmdlineFile(); err != nil {
		return nil
	}

	return p
}

// Update updates the Process from various files in /proc/<pid>. It returns an
// error if the process was unable to be updated (probably because the process
// is no longer running).
func (p *Process) Update() error {
	if err := p.statProcDir(); err != nil {
		return err
	}

	if err := p.parseStatFile(); err != nil {
		return err
	}

	return nil
}

// IsKernelThread returns whether or not Process is a kernel thread.
func (p *Process) IsKernelThread() bool {
	return p.Pgrp == 0
}

// statProcDir updates p with any information it needs from statting /proc/<pid>.
func (p *Process) statProcDir() error {
	path := filepath.Join("/proc", strconv.Itoa(p.PID))

	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return err
	}

	user, err := UserByUID(strconv.FormatUint(uint64(stat.Uid), 10))
	if err != nil {
		return err
	}
	p.User = user

	return nil
}

// parseStatFile updates p with any information it needs from /proc/<pid>/stat.
func (p *Process) parseStatFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.PID), "stat")

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	line := string(data)
	values := strings.Split(line, " ")

	p.Pgrp, err = strconv.Atoi(values[statPgrp])
	if err != nil {
		panic(err)
	}

	lastUtime := p.Utime
	p.Utime, err = strconv.ParseUint(values[statUtime], 10, 64)
	if err != nil {
		panic(err)
	}
	p.UtimeDiff = p.Utime - lastUtime

	lastStime := p.Stime
	p.Stime, err = strconv.ParseUint(values[statStime], 10, 64)
	if err != nil {
		panic(err)
	}
	p.StimeDiff = p.Stime - lastStime

	return nil
}

// parseCmdlineFile sets p's Command via /proc/<pid>/cmdline.
func (p *Process) parseCmdlineFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.PID), "cmdline")

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	cmdline := string(data)
	p.Command = strings.TrimSpace(strings.Replace(cmdline, "\x00", " ", -1))
	p.Name = commandToName(p.Command)
	return nil
}

// commandToName takes a string in a format like "/usr/bin/foo --arguments"
// and returns its base name without arguments, "foo".
func commandToName(cmdline string) string {
	command := strings.Split(cmdline, " ")[0]
	if strings.HasSuffix(command, ":") {
		// For processes that set their name in a format like
		// "postgres: writer process" the value is returned as is.
		return cmdline
	}
	return path.Base(command)
}

// ByPID sorts by PID.
type ByPID []*Process

func (p ByPID) Len() int      { return len(p) }
func (p ByPID) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByPID) Less(i, j int) bool {
	return p[i].PID < p[j].PID
}

// ByUser sorts by the username of the processes user.
type ByUser []*Process

func (p ByUser) Len() int      { return len(p) }
func (p ByUser) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByUser) Less(i, j int) bool {
	return p[i].User.Username < p[j].User.Username
}

// ByCPU sorts by the amount of CPU time used since the last update.
type ByCPU []*Process

func (p ByCPU) Len() int      { return len(p) }
func (p ByCPU) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByCPU) Less(i, j int) bool {
	p1, p2 := p[i], p[j]
	p1Total := p1.UtimeDiff + p1.StimeDiff
	p2Total := p2.UtimeDiff + p2.StimeDiff
	if p1Total == p2Total {
		return p1.PID < p2.PID
	}
	return p1Total > p2Total
}

// ByTime sorts by the amount of CPU time used total.
type ByTime []*Process

func (p ByTime) Len() int      { return len(p) }
func (p ByTime) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByTime) Less(i, j int) bool {
	p1, p2 := p[i], p[j]
	p1Total := p1.Utime + p1.Stime
	p2Total := p2.Utime + p2.Stime
	if p1Total == p2Total {
		return p1.PID < p2.PID
	}
	return p1Total > p2Total
}
<commit_msg>Change filepath to path<commit_after>package main

import (
	"io/ioutil"
	"os"
	"os/user"
	"path"
	"strconv"
	"strings"
	"syscall"
)

const (
	// The indices of the values in /proc/<pid>/stat
	statPid = iota
	statComm
	statState
	statPpid
	statPgrp
	statSession
	statTtyNr
	statTpgid
	statFlags
	statMinflt
	statCminflt
	statMajflt
	statCmajflt
	statUtime
	statStime
	statCutime
	statCstime
	statPriority
	statNice
	statNumThreads
	statItrealvalue
	statStartTime
	statVsize
	statRss
	statRsslim
	statStartCode
	statEndCode
	statStartStack
	statKstKesp
	statKstKeip
	statSignal
	statBlocked
	statSigIgnore
	statSigCatch
	statWchan
	statNswap
	statCnswap
	statExitSignal
	statProcessor
	statRtPriority
	statPolicy
	statDelayActBlkioTicks
	statGuestTime
	statCguestTime
)

// Process represents an operating system process.
type Process struct {
	PID     int
	User    *user.User
	Name    string // foo
	Command string // /usr/bin/foo --args

	// Alive is a flag used by ProcessMonitor to determine if it should remove
	// this process.
	Alive bool

	// Data from /proc/<pid>/stat
	Pgrp  int
	Utime uint64
	Stime uint64

	UtimeDiff uint64
	StimeDiff uint64
}

// NewProcess returns a new Process if a process is currently running on
// the system with the passed in PID.
func NewProcess(pid int) *Process {
	p := &Process{
		PID: pid,
	}

	if err := p.Update(); err != nil {
		return nil
	}

	if err := p.parseCmdlineFile(); err != nil {
		return nil
	}

	return p
}

// Update updates the Process from various files in /proc/<pid>. It returns an
// error if the process was unable to be updated (probably because the process
// is no longer running).
func (p *Process) Update() error {
	if err := p.statProcDir(); err != nil {
		return err
	}

	if err := p.parseStatFile(); err != nil {
		return err
	}

	return nil
}

// IsKernelThread returns whether or not Process is a kernel thread.
func (p *Process) IsKernelThread() bool {
	return p.Pgrp == 0
}

// statProcDir updates p with any information it needs from statting /proc/<pid>.
func (p *Process) statProcDir() error {
	path := path.Join("/proc", strconv.Itoa(p.PID))

	var stat syscall.Stat_t
	if err := syscall.Stat(path, &stat); err != nil {
		return err
	}

	user, err := UserByUID(strconv.FormatUint(uint64(stat.Uid), 10))
	if err != nil {
		return err
	}
	p.User = user

	return nil
}

// parseStatFile updates p with any information it needs from /proc/<pid>/stat.
func (p *Process) parseStatFile() error {
	path := path.Join("/proc", strconv.Itoa(p.PID), "stat")

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	line := string(data)
	values := strings.Split(line, " ")

	p.Pgrp, err = strconv.Atoi(values[statPgrp])
	if err != nil {
		panic(err)
	}

	lastUtime := p.Utime
	p.Utime, err = strconv.ParseUint(values[statUtime], 10, 64)
	if err != nil {
		panic(err)
	}
	p.UtimeDiff = p.Utime - lastUtime

	lastStime := p.Stime
	p.Stime, err = strconv.ParseUint(values[statStime], 10, 64)
	if err != nil {
		panic(err)
	}
	p.StimeDiff = p.Stime - lastStime

	return nil
}

// parseCmdlineFile sets p's Command via /proc/<pid>/cmdline.
func (p *Process) parseCmdlineFile() error {
	path := path.Join("/proc", strconv.Itoa(p.PID), "cmdline")

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	cmdline := string(data)
	p.Command = strings.TrimSpace(strings.Replace(cmdline, "\x00", " ", -1))
	p.Name = commandToName(p.Command)
	return nil
}

// commandToName takes a string in a format like "/usr/bin/foo --arguments"
// and returns its base name without arguments, "foo".
func commandToName(cmdline string) string {
	command := strings.Split(cmdline, " ")[0]
	if strings.HasSuffix(command, ":") {
		// For processes that set their name in a format like
		// "postgres: writer process" the value is returned as is.
		return cmdline
	}
	return path.Base(command)
}

// ByPID sorts by PID.
type ByPID []*Process

func (p ByPID) Len() int      { return len(p) }
func (p ByPID) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByPID) Less(i, j int) bool {
	return p[i].PID < p[j].PID
}

// ByUser sorts by the username of the processes user.
type ByUser []*Process

func (p ByUser) Len() int      { return len(p) }
func (p ByUser) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByUser) Less(i, j int) bool {
	return p[i].User.Username < p[j].User.Username
}

// ByCPU sorts by the amount of CPU time used since the last update.
type ByCPU []*Process

func (p ByCPU) Len() int      { return len(p) }
func (p ByCPU) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByCPU) Less(i, j int) bool {
	p1, p2 := p[i], p[j]
	p1Total := p1.UtimeDiff + p1.StimeDiff
	p2Total := p2.UtimeDiff + p2.StimeDiff
	if p1Total == p2Total {
		return p1.PID < p2.PID
	}
	return p1Total > p2Total
}

// ByTime sorts by the amount of CPU time used total.
type ByTime []*Process

func (p ByTime) Len() int      { return len(p) }
func (p ByTime) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ByTime) Less(i, j int) bool {
	p1, p2 := p[i], p[j]
	p1Total := p1.Utime + p1.Stime
	p2Total := p2.Utime + p2.Stime
	if p1Total == p2Total {
		return p1.PID < p2.PID
	}
	return p1Total > p2Total
}
<|endoftext|>