<commit_before>package sqs

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/RichardKnop/machinery/v1/brokers/iface"
	"github.com/RichardKnop/machinery/v1/common"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"

	awssqs "github.com/aws/aws-sdk-go/service/sqs"
)

const (
	maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
)

// Broker represents a AWS SQS broker
// There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html
type Broker struct {
	common.Broker
	processingWG      sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal
	receivingWG       sync.WaitGroup
	stopReceivingChan chan int
	sess              *session.Session
	service           sqsiface.SQSAPI
	queueUrl          *string
}

// New creates new Broker instance
func New(cnf *config.Config) iface.Broker {
	b := &Broker{Broker: common.NewBroker(cnf)}
	if cnf.SQS != nil && cnf.SQS.Client != nil {
		// Use provided *SQS client
		b.service = cnf.SQS.Client
	} else {
		// Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials.
		// See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
		// Also, env AWS_REGION is also required
		b.sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
		b.service = awssqs.New(b.sess)
	}

	return b
}

// GetPendingTasks returns a slice of task.Signatures waiting in the queue
func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) {
	return nil, errors.New("Not implemented")
}

// StartConsuming enters a loop and waits for incoming messages
func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) {
	b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor)
	qURL := b.getQueueURL(taskProcessor)
	//save it so that it can be used later when attempting to delete task
	b.queueUrl = qURL

	deliveries := make(chan *awssqs.ReceiveMessageOutput)

	b.stopReceivingChan = make(chan int)
	b.receivingWG.Add(1)

	go func() {
		defer b.receivingWG.Done()

		log.INFO.Printf("[*] Waiting for messages on queue: %s. To exit press CTRL+C\n", *qURL)

		for {
			select {
			// A way to stop this goroutine from b.StopConsuming
			case <-b.stopReceivingChan:
				return
			default:
				output, err := b.receiveMessage(qURL)
				if err != nil {
					log.ERROR.Printf("Queue consume error: %s", err)
					continue
				}
				if len(output.Messages) == 0 {
					continue
				}

				deliveries <- output
			}

			whetherContinue, err := b.continueReceivingMessages(qURL, deliveries)
			if err != nil {
				log.ERROR.Printf("Error when receiving messages. Error: %v", err)
			}
			if whetherContinue == false {
				return
			}
		}
	}()

	if err := b.consume(deliveries, concurrency, taskProcessor); err != nil {
		return b.GetRetry(), err
	}

	return b.GetRetry(), nil
}

// StopConsuming quits the loop
func (b *Broker) StopConsuming() {
	b.Broker.StopConsuming()

	b.stopReceiving()

	// Waiting for any tasks being processed to finish
	b.processingWG.Wait()

	// Waiting for the receiving goroutine to have stopped
	b.receivingWG.Wait()
}

// Publish places a new message on the default queue
func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error {
	msg, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON marshal error: %s", err)
	}

	// Check that signature.RoutingKey is set, if not switch to DefaultQueue
	b.AdjustRoutingKey(signature)

	MsgInput := &awssqs.SendMessageInput{
		MessageBody: aws.String(string(msg)),
		QueueUrl:    aws.String(b.GetConfig().Broker + "/" + signature.RoutingKey),
	}

	// if this is a fifo queue, there needs to be some additional parameters.
	if strings.HasSuffix(signature.RoutingKey, ".fifo") {
		// Use Machinery's signature Task UUID as SQS Message Group ID.
		MsgDedupID := signature.UUID
		MsgInput.MessageDeduplicationId = aws.String(MsgDedupID)

		// Do not Use Machinery's signature Group UUID as SQS Message Group ID, instead use BrokerMessageGroupId
		MsgGroupID := signature.BrokerMessageGroupId
		if MsgGroupID == "" {
			return fmt.Errorf("please specify BrokerMessageGroupId attribute for task Signature when submitting a task to FIFO queue")
		}
		MsgInput.MessageGroupId = aws.String(MsgGroupID)
	}

	// Check the ETA signature field, if it is set and it is in the future,
	// and is not a fifo queue, set a delay in seconds for the task.
	if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") {
		now := time.Now().UTC()
		delay := signature.ETA.Sub(now)
		if delay > 0 {
			if delay > maxAWSSQSDelay {
				return errors.New("Max AWS SQS delay exceeded")
			}
			MsgInput.DelaySeconds = aws.Int64(int64(delay.Seconds()))
		}
	}

	result, err := b.service.SendMessageWithContext(ctx, MsgInput)

	if err != nil {
		log.ERROR.Printf("Error when sending a message: %v", err)
		return err

	}
	log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId)
	return nil

}

// consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal
func (b *Broker) consume(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor) error {
	pool := make(chan struct{}, concurrency)

	// initialize worker pool with maxWorkers workers
	go func() {
		b.initializePool(pool, concurrency)
	}()

	errorsChan := make(chan error)

	for {
		whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan)
		if err != nil {
			return err
		}
		if whetherContinue == false {
			return nil
		}
	}
}

// consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS
func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error {
	if len(delivery.Messages) == 0 {
		log.ERROR.Printf("received an empty message, the delivery was %v", delivery)
		return errors.New("received empty message, the delivery is " + delivery.GoString())
	}

	sig := new(tasks.Signature)
	decoder := json.NewDecoder(strings.NewReader(*delivery.Messages[0].Body))
	decoder.UseNumber()
	if err := decoder.Decode(sig); err != nil {
		log.ERROR.Printf("unmarshal error. the delivery is %v", delivery)
		return err
	}

	// If the task is not registered return an error
	// and leave the message in the queue
	if !b.IsTaskRegistered(sig.Name) {
		return fmt.Errorf("task %s is not registered", sig.Name)
	}

	err := taskProcessor.Process(sig)
	if err != nil {
		return err
	}
	// Delete message after successfully consuming and processing the message
	if err = b.deleteOne(delivery); err != nil {
		log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err)
	}
	return err
}

// deleteOne is a method delete a delivery from AWS SQS
func (b *Broker) deleteOne(delivery *awssqs.ReceiveMessageOutput) error {
	qURL := b.defaultQueueURL()
	_, err := b.service.DeleteMessage(&awssqs.DeleteMessageInput{
		QueueUrl:      qURL,
		ReceiptHandle: delivery.Messages[0].ReceiptHandle,
	})

	if err != nil {
		return err
	}
	return nil
}

// defaultQueueURL is a method returns the default queue url
func (b *Broker) defaultQueueURL() *string {
	if b.queueUrl != nil {
		return b.queueUrl
	} else {
		return aws.String(b.GetConfig().Broker + "/" + b.GetConfig().DefaultQueue)
	}

}

// receiveMessage is a method receives a message from specified queue url
func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, error) {
	var waitTimeSeconds int
	var visibilityTimeout *int
	if b.GetConfig().SQS != nil {
		waitTimeSeconds = b.GetConfig().SQS.WaitTimeSeconds
		visibilityTimeout = b.GetConfig().SQS.VisibilityTimeout
	} else {
		waitTimeSeconds = 0
	}
	input := &awssqs.ReceiveMessageInput{
		AttributeNames: []*string{
			aws.String(awssqs.MessageSystemAttributeNameSentTimestamp),
		},
		MessageAttributeNames: []*string{
			aws.String(awssqs.QueueAttributeNameAll),
		},
		QueueUrl:            qURL,
		MaxNumberOfMessages: aws.Int64(1),
		WaitTimeSeconds:     aws.Int64(int64(waitTimeSeconds)),
	}
	if visibilityTimeout != nil {
		input.VisibilityTimeout = aws.Int64(int64(*visibilityTimeout))
	}
	result, err := b.service.ReceiveMessage(input)
	if err != nil {
		return nil, err
	}
	return result, err
}

// initializePool is a method which initializes concurrency pool
func (b *Broker) initializePool(pool chan struct{}, concurrency int) {
	for i := 0; i < concurrency; i++ {
		pool <- struct{}{}
	}
}

// consumeDeliveries is a method consuming deliveries from deliveries channel
func (b *Broker) consumeDeliveries(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) {
	select {
	case err := <-errorsChan:
		return false, err
	case d := <-deliveries:
		if concurrency > 0 {
			// get worker from pool (blocks until one is available)
			<-pool
		}

		b.processingWG.Add(1)

		// Consume the task inside a goroutine so multiple tasks
		// can be processed concurrently
		go func() {

			if err := b.consumeOne(d, taskProcessor); err != nil {
				errorsChan <- err
			}

			b.processingWG.Done()

			if concurrency > 0 {
				// give worker back to pool
				pool <- struct{}{}
			}
		}()
	case <-b.GetStopChan():
		return false, nil
	}
	return true, nil
}

// continueReceivingMessages is a method returns a continue signal
func (b *Broker) continueReceivingMessages(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) {
	select {
	// A way to stop this goroutine from b.StopConsuming
	case <-b.stopReceivingChan:
		return false, nil
	default:
		output, err := b.receiveMessage(qURL)
		if err != nil {
			return true, err
		}
		if len(output.Messages) == 0 {
			return true, nil
		}
		go func() { deliveries <- output }()
	}
	return true, nil
}

// stopReceiving is a method sending a signal to stopReceivingChan
func (b *Broker) stopReceiving() {
	// Stop the receiving goroutine
	b.stopReceivingChan <- 1
}

// getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it
// otherwise using default queueName from config
func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string {
	queueName := b.GetConfig().DefaultQueue
	if taskProcessor.CustomQueue() != "" {
		queueName = taskProcessor.CustomQueue()
	}

	return aws.String(b.GetConfig().Broker + "/" + queueName)
}
<commit_msg>Fix issue with SQS broker, where it recieves more messages than it can consume at a given time<commit_after>package sqs

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/RichardKnop/machinery/v1/brokers/iface"
	"github.com/RichardKnop/machinery/v1/common"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"

	awssqs "github.com/aws/aws-sdk-go/service/sqs"
)

const (
	maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
)

// Broker represents a AWS SQS broker
// There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html
type Broker struct {
	common.Broker
	processingWG      sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal
	receivingWG       sync.WaitGroup
	stopReceivingChan chan int
	sess              *session.Session
	service           sqsiface.SQSAPI
	queueUrl          *string
}

// New creates new Broker instance
func New(cnf *config.Config) iface.Broker {
	b := &Broker{Broker: common.NewBroker(cnf)}
	if cnf.SQS != nil && cnf.SQS.Client != nil {
		// Use provided *SQS client
		b.service = cnf.SQS.Client
	} else {
		// Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials.
		// See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
		// Also, env AWS_REGION is also required
		b.sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
		b.service = awssqs.New(b.sess)
	}

	return b
}

// GetPendingTasks returns a slice of task.Signatures waiting in the queue
func (b *Broker) GetPendingTasks(queue string) ([]*tasks.Signature, error) {
	return nil, errors.New("Not implemented")
}

// StartConsuming enters a loop and waits for incoming messages
func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) {
	b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor)
	qURL := b.getQueueURL(taskProcessor)
	//save it so that it can be used later when attempting to delete task
	b.queueUrl = qURL

	deliveries := make(chan *awssqs.ReceiveMessageOutput, concurrency)
	pool := make(chan struct{}, concurrency)

	// initialize worker pool with maxWorkers workers
	for i := 0; i < concurrency; i++ {
		pool <- struct{}{}
	}
	b.stopReceivingChan = make(chan int)
	b.receivingWG.Add(1)

	go func() {
		defer b.receivingWG.Done()

		log.INFO.Printf("[*] Waiting for messages on queue: %s. To exit press CTRL+C\n", *qURL)

		for {
			select {
			// A way to stop this goroutine from b.StopConsuming
			case <-b.stopReceivingChan:
				close(deliveries)
				return
			case <-pool:
				output, err := b.receiveMessage(qURL)
				if err == nil && len(output.Messages) > 0 {
					deliveries <- output

				} else {
					//return back to pool right away
					pool <- struct{}{}
					if err != nil {
						log.ERROR.Printf("Queue consume error: %s", err)
					}

				}
			}

		}
	}()

	if err := b.consume(deliveries, concurrency, taskProcessor, pool); err != nil {
		return b.GetRetry(), err
	}

	return b.GetRetry(), nil
}

// StopConsuming quits the loop
func (b *Broker) StopConsuming() {
	b.Broker.StopConsuming()

	b.stopReceiving()

	// Waiting for any tasks being processed to finish
	b.processingWG.Wait()

	// Waiting for the receiving goroutine to have stopped
	b.receivingWG.Wait()
}

// Publish places a new message on the default queue
func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error {
	msg, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON marshal error: %s", err)
	}

	// Check that signature.RoutingKey is set, if not switch to DefaultQueue
	b.AdjustRoutingKey(signature)

	MsgInput := &awssqs.SendMessageInput{
		MessageBody: aws.String(string(msg)),
		QueueUrl:    aws.String(b.GetConfig().Broker + "/" + signature.RoutingKey),
	}

	// if this is a fifo queue, there needs to be some additional parameters.
	if strings.HasSuffix(signature.RoutingKey, ".fifo") {
		// Use Machinery's signature Task UUID as SQS Message Group ID.
		MsgDedupID := signature.UUID
		MsgInput.MessageDeduplicationId = aws.String(MsgDedupID)

		// Do not Use Machinery's signature Group UUID as SQS Message Group ID, instead use BrokerMessageGroupId
		MsgGroupID := signature.BrokerMessageGroupId
		if MsgGroupID == "" {
			return fmt.Errorf("please specify BrokerMessageGroupId attribute for task Signature when submitting a task to FIFO queue")
		}
		MsgInput.MessageGroupId = aws.String(MsgGroupID)
	}

	// Check the ETA signature field, if it is set and it is in the future,
	// and is not a fifo queue, set a delay in seconds for the task.
	if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") {
		now := time.Now().UTC()
		delay := signature.ETA.Sub(now)
		if delay > 0 {
			if delay > maxAWSSQSDelay {
				return errors.New("Max AWS SQS delay exceeded")
			}
			MsgInput.DelaySeconds = aws.Int64(int64(delay.Seconds()))
		}
	}

	result, err := b.service.SendMessageWithContext(ctx, MsgInput)

	if err != nil {
		log.ERROR.Printf("Error when sending a message: %v", err)
		return err

	}
	log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId)
	return nil

}

// consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal
func (b *Broker) consume(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error {

	errorsChan := make(chan error)

	for {
		whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan)
		if err != nil {
			return err
		}
		if whetherContinue == false {
			return nil
		}
	}
}

// consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS
func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error {
	if len(delivery.Messages) == 0 {
		log.ERROR.Printf("received an empty message, the delivery was %v", delivery)
		return errors.New("received empty message, the delivery is " + delivery.GoString())
	}

	sig := new(tasks.Signature)
	decoder := json.NewDecoder(strings.NewReader(*delivery.Messages[0].Body))
	decoder.UseNumber()
	if err := decoder.Decode(sig); err != nil {
		log.ERROR.Printf("unmarshal error. the delivery is %v", delivery)
		return err
	}

	// If the task is not registered return an error
	// and leave the message in the queue
	if !b.IsTaskRegistered(sig.Name) {
		return fmt.Errorf("task %s is not registered", sig.Name)
	}

	err := taskProcessor.Process(sig)
	if err != nil {
		return err
	}
	// Delete message after successfully consuming and processing the message
	if err = b.deleteOne(delivery); err != nil {
		log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err)
	}
	return err
}

// deleteOne is a method delete a delivery from AWS SQS
func (b *Broker) deleteOne(delivery *awssqs.ReceiveMessageOutput) error {
	qURL := b.defaultQueueURL()
	_, err := b.service.DeleteMessage(&awssqs.DeleteMessageInput{
		QueueUrl:      qURL,
		ReceiptHandle: delivery.Messages[0].ReceiptHandle,
	})

	if err != nil {
		return err
	}
	return nil
}

// defaultQueueURL is a method returns the default queue url
func (b *Broker) defaultQueueURL() *string {
	if b.queueUrl != nil {
		return b.queueUrl
	} else {
		return aws.String(b.GetConfig().Broker + "/" + b.GetConfig().DefaultQueue)
	}

}

// receiveMessage is a method receives a message from specified queue url
func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, error) {
	var waitTimeSeconds int
	var visibilityTimeout *int
	if b.GetConfig().SQS != nil {
		waitTimeSeconds = b.GetConfig().SQS.WaitTimeSeconds
		visibilityTimeout = b.GetConfig().SQS.VisibilityTimeout
	} else {
		waitTimeSeconds = 0
	}
	input := &awssqs.ReceiveMessageInput{
		AttributeNames: []*string{
			aws.String(awssqs.MessageSystemAttributeNameSentTimestamp),
		},
		MessageAttributeNames: []*string{
			aws.String(awssqs.QueueAttributeNameAll),
		},
		QueueUrl:            qURL,
		MaxNumberOfMessages: aws.Int64(1),
		WaitTimeSeconds:     aws.Int64(int64(waitTimeSeconds)),
	}
	if visibilityTimeout != nil {
		input.VisibilityTimeout = aws.Int64(int64(*visibilityTimeout))
	}
	result, err := b.service.ReceiveMessage(input)
	if err != nil {
		return nil, err
	}
	return result, err
}

// initializePool is a method which initializes concurrency pool
func (b *Broker) initializePool(pool chan struct{}, concurrency int) {
	for i := 0; i < concurrency; i++ {
		pool <- struct{}{}
	}
}

// consumeDeliveries is a method consuming deliveries from deliveries channel
func (b *Broker) consumeDeliveries(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) {
	select {
	case err := <-errorsChan:
		return false, err
	case d := <-deliveries:

		b.processingWG.Add(1)

		// Consume the task inside a goroutine so multiple tasks
		// can be processed concurrently
		go func() {

			if err := b.consumeOne(d, taskProcessor); err != nil {
				errorsChan <- err
			}

			b.processingWG.Done()

			if concurrency > 0 {
				// give worker back to pool
				pool <- struct{}{}
			}
		}()
	case <-b.GetStopChan():
		return false, nil
	}
	return true, nil
}

// continueReceivingMessages is a method returns a continue signal
func (b *Broker) continueReceivingMessages(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) {
	select {
	// A way to stop this goroutine from b.StopConsuming
	case <-b.stopReceivingChan:
		return false, nil
	default:
		output, err := b.receiveMessage(qURL)
		if err != nil {
			return true, err
		}
		if len(output.Messages) == 0 {
			return true, nil
		}
		go func() { deliveries <- output }()
	}
	return true, nil
}

// stopReceiving is a method sending a signal to stopReceivingChan
func (b *Broker) stopReceiving() {
	// Stop the receiving goroutine
	b.stopReceivingChan <- 1
}

// getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it
// otherwise using default queueName from config
func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string {
	queueName := b.GetConfig().DefaultQueue
	if taskProcessor.CustomQueue() != "" {
		queueName = taskProcessor.CustomQueue()
	}

	return aws.String(b.GetConfig().Broker + "/" + queueName)
}
<|endoftext|>


<commit_before>package aetest

import (
	"bufio"
	"context"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"time"

	"google.golang.org/appengine/v2"
	"google.golang.org/appengine/v2/internal"
)

// Instance represents a running instance of the development API Server.
type Instance interface {
	// Close kills the child api_server.py process, releasing its resources.
	io.Closer
	// NewRequest returns an *http.Request associated with this instance.
	NewRequest(method, urlStr string, body io.Reader) (*http.Request, error)
}

// Options is used to specify options when creating an Instance.
type Options struct {
	// AppID specifies the App ID to use during tests.
	// By default, "testapp".
	AppID string
	// StronglyConsistentDatastore is whether the local datastore should be
	// strongly consistent. This will diverge from production behaviour.
	StronglyConsistentDatastore bool
	// SupportDatastoreEmulator is whether use Cloud Datastore Emulator or
	// use old SQLite based Datastore backend or use default settings.
	SupportDatastoreEmulator *bool
	// SuppressDevAppServerLog is whether the dev_appserver running in tests
	// should output logs.
	SuppressDevAppServerLog bool
	// StartupTimeout is a duration to wait for instance startup.
	// By default, 15 seconds.
	StartupTimeout time.Duration
}

// NewContext starts an instance of the development API server, and returns
// a context that will route all API calls to that server, as well as a
// closure that must be called when the Context is no longer required.
func NewContext() (context.Context, func(), error) {
	inst, err := NewInstance(nil)
	if err != nil {
		return nil, nil, err
	}
	req, err := inst.NewRequest("GET", "/", nil)
	if err != nil {
		inst.Close()
		return nil, nil, err
	}
	ctx := appengine.NewContext(req)
	return ctx, func() {
		inst.Close()
	}, nil
}

// PrepareDevAppserver is a hook which, if set, will be called before the
// dev_appserver.py is started, each time it is started. If aetest.NewContext
// is invoked from the goapp test tool, this hook is unnecessary.
var PrepareDevAppserver func() error

// NewInstance launches a running instance of api_server.py which can be used
// for multiple test Contexts that delegate all App Engine API calls to that
// instance.
// If opts is nil the default values are used.
func NewInstance(opts *Options) (Instance, error) {
	i := &instance{
		opts:           opts,
		appID:          "testapp",
		startupTimeout: 15 * time.Second,
	}
	if opts != nil {
		if opts.AppID != "" {
			i.appID = opts.AppID
		}
		if opts.StartupTimeout > 0 {
			i.startupTimeout = opts.StartupTimeout
		}
	}
	if err := i.startChild(); err != nil {
		return nil, err
	}
	return i, nil
}

func newSessionID() string {
	var buf [16]byte
	io.ReadFull(rand.Reader, buf[:])
	return fmt.Sprintf("%x", buf[:])
}

// instance implements the Instance interface.
type instance struct {
	opts           *Options
	child          *exec.Cmd
	apiURL         *url.URL // base URL of API HTTP server
	adminURL       string   // base URL of admin HTTP server
	appDir         string
	appID          string
	startupTimeout time.Duration
}

// NewRequest returns an *http.Request associated with this instance.
func (i *instance) NewRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}

	// Associate this request.
	return internal.RegisterTestRequest(req, i.apiURL, "dev~"+i.appID), nil
}

// Close kills the child api_server.py process, releasing its resources.
func (i *instance) Close() (err error) {
	child := i.child
	if child == nil {
		return nil
	}
	defer func() {
		i.child = nil
		err1 := os.RemoveAll(i.appDir)
		if err == nil {
			err = err1
		}
	}()

	if p := child.Process; p != nil {
		errc := make(chan error, 1)
		go func() {
			errc <- child.Wait()
		}()

		// Call the quit handler on the admin server.
		res, err := http.Get(i.adminURL + "/quit")
		if err != nil {
			p.Kill()
			return fmt.Errorf("unable to call /quit handler: %v", err)
		}
		res.Body.Close()
		select {
		case <-time.After(15 * time.Second):
			p.Kill()
			return errors.New("timeout killing child process")
		case err = <-errc:
			// Do nothing.
		}
	}
	return
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func findPython() (path string, err error) {
	for _, name := range []string{"python2.7", "python"} {
		path, err = exec.LookPath(name)
		if err == nil {
			return
		}
	}
	return
}

func findDevAppserver() (string, error) {
	if p := os.Getenv("APPENGINE_DEV_APPSERVER"); p != "" {
		if fileExists(p) {
			return p, nil
		}
		return "", fmt.Errorf("invalid APPENGINE_DEV_APPSERVER environment variable; path %q doesn't exist", p)
	}
	return exec.LookPath("dev_appserver.py")
}

var apiServerAddrRE = regexp.MustCompile(`Starting API server at: (\S+)`)
var adminServerAddrRE = regexp.MustCompile(`Starting admin server at: (\S+)`)

func (i *instance) startChild() (err error) {
	if PrepareDevAppserver != nil {
		if err := PrepareDevAppserver(); err != nil {
			return err
		}
	}
	executable := os.Getenv("APPENGINE_DEV_APPSERVER_BINARY")
	var appserverArgs []string
	if len(executable) == 0 {
		executable, err = findPython()
		if err != nil {
			return fmt.Errorf("Could not find python interpreter: %v", err)
		}
		devAppserver, err := findDevAppserver()
		if err != nil {
			return fmt.Errorf("Could not find dev_appserver.py: %v", err)
		}
		appserverArgs = append(appserverArgs, devAppserver)
	}

	i.appDir, err = ioutil.TempDir("", "appengine-aetest")
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			os.RemoveAll(i.appDir)
		}
	}()
	err = os.Mkdir(filepath.Join(i.appDir, "app"), 0755)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(i.appDir, "app", "app.yaml"), []byte(i.appYAML()), 0644)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(i.appDir, "app", "stubapp.go"), []byte(appSource), 0644)
	if err != nil {
		return err
	}

	appserverArgs = append(appserverArgs,
		"--port=0",
		"--api_port=0",
		"--admin_port=0",
		"--automatic_restart=false",
		"--skip_sdk_update_check=true",
		"--clear_datastore=true",
		"--clear_search_indexes=true",
		"--datastore_path", filepath.Join(i.appDir, "datastore"),
	)
	if i.opts != nil && i.opts.StronglyConsistentDatastore {
		appserverArgs = append(appserverArgs, "--datastore_consistency_policy=consistent")
	}
	if i.opts != nil && i.opts.SupportDatastoreEmulator != nil {
		appserverArgs = append(appserverArgs, fmt.Sprintf("--support_datastore_emulator=%t", *i.opts.SupportDatastoreEmulator))
	}
	appserverArgs = append(appserverArgs, filepath.Join(i.appDir, "app"))

	i.child = exec.Command(executable, appserverArgs...)

	i.child.Stdout = os.Stdout
	var stderr io.Reader
	stderr, err = i.child.StderrPipe()
	if err != nil {
		return err
	}

	if err = i.child.Start(); err != nil {
		return err
	}

	// Read stderr until we have read the URLs of the API server and admin interface.
	errc := make(chan error, 1)
	go func() {
		s := bufio.NewScanner(stderr)
		for s.Scan() {
			// Pass stderr along as we go so the user can see it.
			if !(i.opts != nil && i.opts.SuppressDevAppServerLog) {
				fmt.Fprintln(os.Stderr, s.Text())
			}
			if match := apiServerAddrRE.FindStringSubmatch(s.Text()); match != nil {
				u, err := url.Parse(match[1])
				if err != nil {
					errc <- fmt.Errorf("failed to parse API URL %q: %v", match[1], err)
					return
				}
				i.apiURL = u
			}
			if match := adminServerAddrRE.FindStringSubmatch(s.Text()); match != nil {
				i.adminURL = match[1]
			}
			if i.adminURL != "" && i.apiURL != nil {
				// Pass along stderr to the user after we're done with it.
				if !(i.opts != nil && i.opts.SuppressDevAppServerLog) {
					go io.Copy(os.Stderr, stderr)
				}
				break
			}
		}
		errc <- s.Err()
	}()

	select {
	case <-time.After(i.startupTimeout):
		if p := i.child.Process; p != nil {
			p.Kill()
		}
		return errors.New("timeout starting child process")
	case err := <-errc:
		if err != nil {
			return fmt.Errorf("error reading child process stderr: %v", err)
		}
	}
	if i.adminURL == "" {
		return errors.New("unable to find admin server URL")
	}
	if i.apiURL == nil {
		return errors.New("unable to find API server URL")
	}
	return nil
}

func (i *instance) appYAML() string {
	return fmt.Sprintf(appYAMLTemplate, i.appID)
}

const appYAMLTemplate = `
application: %s
version: 1
runtime: go111

handlers:
- url: /.*
  script: _go_app
`

const appSource = `
package main
import "google.golang.org/appengine/v2"
func main() { appengine.Main() }
`
<commit_msg>re-apply PR #232 to v2 (#289)<commit_after>package aetest

import (
	"bufio"
	"context"
	"crypto/rand"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"time"

	"google.golang.org/appengine/v2"
	"google.golang.org/appengine/v2/internal"
)

// Instance represents a running instance of the development API Server.
type Instance interface {
	// Close kills the child api_server.py process, releasing its resources.
	io.Closer
	// NewRequest returns an *http.Request associated with this instance.
	NewRequest(method, urlStr string, body io.Reader) (*http.Request, error)
}

// Options is used to specify options when creating an Instance.
type Options struct {
	// AppID specifies the App ID to use during tests.
	// By default, "testapp".
	AppID string
	// StronglyConsistentDatastore is whether the local datastore should be
	// strongly consistent. This will diverge from production behaviour.
	StronglyConsistentDatastore bool
	// SupportDatastoreEmulator is whether use Cloud Datastore Emulator or
	// use old SQLite based Datastore backend or use default settings.
	SupportDatastoreEmulator *bool
	// SuppressDevAppServerLog is whether the dev_appserver running in tests
	// should output logs.
	SuppressDevAppServerLog bool
	// StartupTimeout is a duration to wait for instance startup.
	// By default, 15 seconds.
	StartupTimeout time.Duration
}

// NewContext starts an instance of the development API server, and returns
// a context that will route all API calls to that server, as well as a
// closure that must be called when the Context is no longer required.
func NewContext() (context.Context, func(), error) {
	inst, err := NewInstance(nil)
	if err != nil {
		return nil, nil, err
	}
	req, err := inst.NewRequest("GET", "/", nil)
	if err != nil {
		inst.Close()
		return nil, nil, err
	}
	ctx := appengine.NewContext(req)
	return ctx, func() {
		inst.Close()
	}, nil
}

// PrepareDevAppserver is a hook which, if set, will be called before the
// dev_appserver.py is started, each time it is started. If aetest.NewContext
// is invoked from the goapp test tool, this hook is unnecessary.
var PrepareDevAppserver func() error

// NewInstance launches a running instance of api_server.py which can be used
// for multiple test Contexts that delegate all App Engine API calls to that
// instance.
// If opts is nil the default values are used.
func NewInstance(opts *Options) (Instance, error) {
	i := &instance{
		opts:           opts,
		appID:          "testapp",
		startupTimeout: 15 * time.Second,
	}
	if opts != nil {
		if opts.AppID != "" {
			i.appID = opts.AppID
		}
		if opts.StartupTimeout > 0 {
			i.startupTimeout = opts.StartupTimeout
		}
	}
	if err := i.startChild(); err != nil {
		return nil, err
	}
	return i, nil
}

func newSessionID() string {
	var buf [16]byte
	io.ReadFull(rand.Reader, buf[:])
	return fmt.Sprintf("%x", buf[:])
}

// instance implements the Instance interface.
type instance struct {
	opts           *Options
	child          *exec.Cmd
	apiURL         *url.URL // base URL of API HTTP server
	adminURL       string   // base URL of admin HTTP server
	appDir         string
	appID          string
	startupTimeout time.Duration
}

// NewRequest returns an *http.Request associated with this instance.
func (i *instance) NewRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, urlStr, body)
	if err != nil {
		return nil, err
	}

	// Associate this request.
	return internal.RegisterTestRequest(req, i.apiURL, "dev~"+i.appID), nil
}

// Close kills the child api_server.py process, releasing its resources.
func (i *instance) Close() (err error) {
	child := i.child
	if child == nil {
		return nil
	}
	defer func() {
		i.child = nil
		err1 := os.RemoveAll(i.appDir)
		if err == nil {
			err = err1
		}
	}()

	if p := child.Process; p != nil {
		errc := make(chan error, 1)
		go func() {
			errc <- child.Wait()
		}()

		// Call the quit handler on the admin server.
		res, err := http.Get(i.adminURL + "/quit")
		if err != nil {
			p.Kill()
			return fmt.Errorf("unable to call /quit handler: %v", err)
		}
		res.Body.Close()
		select {
		case <-time.After(15 * time.Second):
			p.Kill()
			return errors.New("timeout killing child process")
		case err = <-errc:
			// Do nothing.
		}
	}
	return
}

func fileExists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

func findPython() (path string, err error) {
	for _, name := range []string{"python2.7", "python"} {
		path, err = exec.LookPath(name)
		if err == nil {
			return
		}
	}
	return
}

func findDevAppserver() (string, error) {
	if p := os.Getenv("APPENGINE_DEV_APPSERVER"); p != "" {
		if fileExists(p) {
			return p, nil
		}
		return "", fmt.Errorf("invalid APPENGINE_DEV_APPSERVER environment variable; path %q doesn't exist", p)
	}
	return exec.LookPath("dev_appserver.py")
}

var apiServerAddrRE = regexp.MustCompile(`Starting API server at: (\S+)`)
var adminServerAddrRE = regexp.MustCompile(`Starting admin server at: (\S+)`)

func (i *instance) startChild() (err error) {
	if PrepareDevAppserver != nil {
		if err := PrepareDevAppserver(); err != nil {
			return err
		}
	}
	executable := os.Getenv("APPENGINE_DEV_APPSERVER_BINARY")
	var appserverArgs []string
	if len(executable) == 0 {
		executable, err = findPython()
		if err != nil {
			return fmt.Errorf("Could not find python interpreter: %v", err)
		}
		devAppserver, err := findDevAppserver()
		if err != nil {
			return fmt.Errorf("Could not find dev_appserver.py: %v", err)
		}
		appserverArgs = append(appserverArgs, devAppserver)
	}

	i.appDir, err = ioutil.TempDir("", "appengine-aetest")
	if err != nil {
		return err
	}
	defer func() {
		if err != nil {
			os.RemoveAll(i.appDir)
		}
	}()
	err = os.Mkdir(filepath.Join(i.appDir, "app"), 0755)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(i.appDir, "app", "app.yaml"), []byte(i.appYAML()), 0644)
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(filepath.Join(i.appDir, "app", "stubapp.go"), []byte(appSource), 0644)
	if err != nil {
		return err
	}

	datastorePath := os.Getenv("APPENGINE_DEV_APPSERVER_DATASTORE_PATH")
	if len(datastorePath) == 0 {
		datastorePath = filepath.Join(i.appDir, "datastore")
	}

	appserverArgs = append(appserverArgs,
		"--port=0",
		"--api_port=0",
		"--admin_port=0",
		"--automatic_restart=false",
		"--skip_sdk_update_check=true",
		"--clear_datastore=true",
		"--clear_search_indexes=true",
		"--datastore_path", datastorePath,
	)
	if i.opts != nil && i.opts.StronglyConsistentDatastore {
		appserverArgs = append(appserverArgs, "--datastore_consistency_policy=consistent")
	}
	if i.opts != nil && i.opts.SupportDatastoreEmulator != nil {
		appserverArgs = append(appserverArgs, fmt.Sprintf("--support_datastore_emulator=%t", *i.opts.SupportDatastoreEmulator))
	}
	appserverArgs = append(appserverArgs, filepath.Join(i.appDir, "app"))

	i.child = exec.Command(executable, appserverArgs...)

	i.child.Stdout = os.Stdout
	var stderr io.Reader
	stderr, err = i.child.StderrPipe()
	if err != nil {
		return err
	}

	if err = i.child.Start(); err != nil {
		return err
	}

	// Read stderr until we have read the URLs of the API server and admin interface.
	errc := make(chan error, 1)
	go func() {
		s := bufio.NewScanner(stderr)
		for s.Scan() {
			// Pass stderr along as we go so the user can see it.
			if !(i.opts != nil && i.opts.SuppressDevAppServerLog) {
				fmt.Fprintln(os.Stderr, s.Text())
			}
			if match := apiServerAddrRE.FindStringSubmatch(s.Text()); match != nil {
				u, err := url.Parse(match[1])
				if err != nil {
					errc <- fmt.Errorf("failed to parse API URL %q: %v", match[1], err)
					return
				}
				i.apiURL = u
			}
			if match := adminServerAddrRE.FindStringSubmatch(s.Text()); match != nil {
				i.adminURL = match[1]
			}
			if i.adminURL != "" && i.apiURL != nil {
				// Pass along stderr to the user after we're done with it.
				if !(i.opts != nil && i.opts.SuppressDevAppServerLog) {
					go io.Copy(os.Stderr, stderr)
				}
				break
			}
		}
		errc <- s.Err()
	}()

	select {
	case <-time.After(i.startupTimeout):
		if p := i.child.Process; p != nil {
			p.Kill()
		}
		return errors.New("timeout starting child process")
	case err := <-errc:
		if err != nil {
			return fmt.Errorf("error reading child process stderr: %v", err)
		}
	}
	if i.adminURL == "" {
		return errors.New("unable to find admin server URL")
	}
	if i.apiURL == nil {
		return errors.New("unable to find API server URL")
	}
	return nil
}

func (i *instance) appYAML() string {
	return fmt.Sprintf(appYAMLTemplate, i.appID)
}

const appYAMLTemplate = `
application: %s
version: 1
runtime: go111

handlers:
- url: /.*
  script: _go_app
`

const appSource = `
package main
import "google.golang.org/appengine/v2"
func main() { appengine.Main() }
`
<|endoftext|>


<commit_before>package metrics

import (
	"context"
	"fmt"
	"io"
	"log"
	"math"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/DataDog/datadog-api-client-go/api/v1/datadog"
	"github.com/DataDog/datadog-go/statsd"
)

// DatadogClient is a metrics.Client that emits directly to Datadog.
type DatadogClient struct {
	StatsdClient
	cfg *datadog.Configuration
}

var _ Client = (*DatadogClient)(nil)
var _ io.Closer = (*DatadogClient)(nil)

func NewDatadogClient(apiKey, appKey string, opts ...DatadogClientOpt) *DatadogClient {
	cfg := datadog.NewConfiguration()
	ds := newDatadogStatsd(cfg, apiKey, appKey)
	c := &DatadogClient{
		cfg: cfg,
		StatsdClient: StatsdClient{
			client: ds,
		},
	}
	for _, o := range opts {
		o(c)
	}

	// start the client's loop after processing options, so tests can shorten the interval
	submitLoopCtx, cancel := context.WithCancel(context.Background())
	ds.cancelSubmitLoop = cancel
	go ds.submitLoop(submitLoopCtx)
	return c
}

type DatadogClientOpt func(*DatadogClient)

func WithDatadogTags(tags []string) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).tags = tags
	}
}

func WithDatadogURL(datadog url.URL) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.cfg.Host = datadog.Host
		c.cfg.Scheme = datadog.Scheme
	}
}

func WithDatadogFrozenClock(frozenTime float64) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).now = func() float64 { return frozenTime }
	}
}

func WithDatadogSubmitInterval(dur time.Duration) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).submitInterval = dur
	}
}

func (d *DatadogClient) Close() error {
	d.client.(*datadogStatsd).Close()
	return nil
}

// datadogStatsd is an alternative statsd.Client that transmits directly to Datadog
type datadogStatsd struct {
	authCtx          context.Context
	cancelSubmitLoop context.CancelFunc
	submitInterval   time.Duration

	metrics *datadog.MetricsApiService
	events  *datadog.EventsApiService
	tags    []string
	now     func() float64

	mu     sync.Mutex
	series []*datadog.Series
	// map of timingKey() to timestamp, to values
	durationData map[string]map[float64][]float64
}

var _ statsdClient = (*datadogStatsd)(nil)

const submitTimeout = 3 * time.Second

func newDatadogStatsd(cfg *datadog.Configuration, apiKey, appKey string) *datadogStatsd {
	client := datadog.NewAPIClient(cfg)
	keys := map[string]datadog.APIKey{
		"apiKeyAuth": {Key: apiKey},
		"appKeyAuth": {Key: appKey},
	}
	return &datadogStatsd{
		authCtx:        context.WithValue(context.Background(), datadog.ContextAPIKeys, keys),
		metrics:        client.MetricsApi,
		events:         client.EventsApi,
		now:            func() float64 { return float64(time.Now().Unix()) },
		durationData:   make(map[string]map[float64][]float64),
		submitInterval: 2 * time.Second,
	}
}

const (
	durationType = "gauge"
	countType    = "count"
)

func (d *datadogStatsd) Incr(metric string, tags []string, _ float64) error {
	tags = append(d.tags, tags...)
	now := d.now()

	d.mu.Lock()
	defer d.mu.Unlock()

	if existing := d.findSeries(metric, tags); existing != nil {
		for i, p := range existing.Points {
			if p[0] == now {
				existing.Points[i][1]++
				return nil
			}
		}
		existing.Points = append(existing.Points, []float64{now, 1})
		return nil
	}

	// Not found, create
	s := datadog.NewSeries(metric, [][]float64{{now, 1}})
	s.SetType(countType)
	s.SetTags(tags)
	d.series = append(d.series, s)
	return nil
}

func (d *datadogStatsd) Timing(metric string, dur time.Duration, tags []string, _ float64) error {
	tags = append(d.tags, tags...)
	now := d.now()
	val := float64(dur.Milliseconds())
	tk := timingKey(metric, tags)

	d.mu.Lock()
	defer d.mu.Unlock()

	if existing := d.findSeries(metric, tags); existing != nil {
		d.durationData[tk][now] = append(d.durationData[tk][now], val)
		return nil
	}

	// Not found, create
	s := datadog.NewSeries(metric, nil)
	s.SetType(durationType)
	s.SetTags(tags)
	s.SetInterval(1)
	d.series = append(d.series, s)
	d.durationData[tk] = map[float64][]float64{now: {val}}
	return nil
}

func timingKey(metric string, tags []string) string {
	return fmt.Sprintf("%s %s", metric, strings.Join(tags, ","))
}

func (d *datadogStatsd) Event(e *statsd.Event) error {
	ctx, cancel := context.WithTimeout(d.authCtx, submitTimeout)
	defer cancel()

	ddEvent := datadog.NewEventCreateRequest(e.Text, e.Title)
	ddEvent.SetAlertType(datadog.EventAlertType(e.AlertType))
	ddEvent.SetAggregationKey(e.AggregationKey)
	ddEvent.SetPriority(datadog.EventPriority(e.Priority))
	ddEvent.SetTags(append(d.tags, e.Tags...))
	if e.Timestamp.IsZero() {
		ddEvent.SetDateHappened(int64(d.now()))
	} else {
		ddEvent.SetDateHappened(e.Timestamp.Unix())
	}

	if _, _, err := d.events.CreateEvent(ctx, *ddEvent); err != nil {
		log.Println("error submitting event to datadog", err)
	}
	return nil
}

func (d *datadogStatsd) Close() {
	d.cancelSubmitLoop()
	d.submit()
}

func (d *datadogStatsd) submitLoop(ctx context.Context) {
	t := time.NewTicker(d.submitInterval)
	defer t.Stop()

	for {
		select {
		case <-t.C:
			d.submit()
		case <-ctx.Done():
			return
		}
	}
}

func (d *datadogStatsd) submit() {
	series := d.flushSeries()
	if len(series) == 0 {
		return
	}

	ctx, cancel := context.WithTimeout(d.authCtx, submitTimeout)
	defer cancel()
	if _, _, err := d.metrics.SubmitMetrics(ctx, *datadog.NewMetricsPayload(series)); err != nil {
		log.Println("error submitting metrics to datadog", err)
	}
}

func (d *datadogStatsd) flushSeries() []datadog.Series {
	d.mu.Lock()
	defer d.mu.Unlock()

	series := make([]datadog.Series, 0, len(d.series))
	for _, s := range d.series {
		if s.GetType() != durationType {
			series = append(series, *s)
			continue
		}

		// Construct series from captured samples:
		tk := timingKey(s.GetMetric(), s.GetTags())
		var counts, averages, p95s, p99s, maxes [][]float64
		for ts, data := range d.durationData[tk] {
			sort.Float64s(data)
			var sum float64
			for _, f := range data {
				sum += f
			}
			count := float64(len(data))
			counts = append(counts, []float64{ts, count})
			averages = append(averages, []float64{ts, sum / count})
			p95s = append(p95s, []float64{ts, percentile(data, 0.95)})
			p99s = append(p99s, []float64{ts, percentile(data, 0.99)})
			maxes = append(maxes, []float64{ts, data[len(data)-1]})
		}
		series = append(series, cloneSeries(s, "count", counts))
		series = append(series, cloneSeries(s, "avg", averages))
		series = append(series, cloneSeries(s, "95percentile", p95s))
		series = append(series, cloneSeries(s, "99percentile", p99s))
		series = append(series, cloneSeries(s, "max", maxes))
	}
	d.series = nil
	d.durationData = make(map[string]map[float64][]float64)
	return series
}

func cloneSeries(s *datadog.Series, suffix string, points [][]float64) datadog.Series {
	return datadog.Series{
		Host:           s.Host,
		Interval:       s.Interval,
		Metric:         fmt.Sprintf("%s.%s", s.GetMetric(), suffix),
		Points:         points,
		Tags:           s.Tags,
		Type:           s.Type,
		UnparsedObject: s.UnparsedObject,
	}
}

func percentile(data []float64, p float64) float64 {
	pos := float64(len(data)) * p
	if math.Round(pos) == pos {
		// Return exact value at percentile
		return data[int(pos)-1]
	}

	return (data[int(pos-1)] + data[int(pos)]) / 2
}

func (d *datadogStatsd) findSeries(metric string, tags []string) *datadog.Series {
	sort.Strings(tags)
seriesLoop:
	for _, s := range d.series {
		if s.GetMetric() != metric {
			continue
		}
		sTags := s.GetTags()
		if len(sTags) != len(tags) {
			continue
		}
		sort.Strings(sTags)
		for i, t := range sTags {
			if tags[i] != t {
				continue seriesLoop
			}
		}
		return s
	}
	return nil
}
<commit_msg>datadog series: comments and extracted func<commit_after>package metrics

import (
	"context"
	"fmt"
	"io"
	"log"
	"math"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/DataDog/datadog-api-client-go/api/v1/datadog"
	"github.com/DataDog/datadog-go/statsd"
)

// DatadogClient is a metrics.Client that emits directly to Datadog.
type DatadogClient struct {
	StatsdClient
	cfg *datadog.Configuration
}

var _ Client = (*DatadogClient)(nil)
var _ io.Closer = (*DatadogClient)(nil)

func NewDatadogClient(apiKey, appKey string, opts ...DatadogClientOpt) *DatadogClient {
	cfg := datadog.NewConfiguration()
	ds := newDatadogStatsd(cfg, apiKey, appKey)
	c := &DatadogClient{
		cfg: cfg,
		StatsdClient: StatsdClient{
			client: ds,
		},
	}
	for _, o := range opts {
		o(c)
	}

	// start the submission loop after processing options, so tests can shorten the interval
	submitLoopCtx, cancel := context.WithCancel(context.Background())
	ds.cancelSubmitLoop = cancel
	go ds.submitLoop(submitLoopCtx)
	return c
}

type DatadogClientOpt func(*DatadogClient)

func WithDatadogTags(tags []string) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).tags = tags
	}
}

func WithDatadogURL(datadog url.URL) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.cfg.Host = datadog.Host
		c.cfg.Scheme = datadog.Scheme
	}
}

func WithDatadogFrozenClock(frozenTime float64) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).now = func() float64 { return frozenTime }
	}
}

func WithDatadogSubmitInterval(dur time.Duration) DatadogClientOpt {
	return func(c *DatadogClient) {
		c.client.(*datadogStatsd).submitInterval = dur
	}
}

func (d *DatadogClient) Close() error {
	d.client.(*datadogStatsd).Close()
	return nil
}

// datadogStatsd is an alternative statsd.Client that aggregates in memory, with periodic submission to Datadog API.
type datadogStatsd struct {
	authCtx          context.Context
	cancelSubmitLoop context.CancelFunc
	submitInterval   time.Duration

	metrics *datadog.MetricsApiService
	events  *datadog.EventsApiService
	tags    []string
	now     func() float64

	// series tracks all metrics, but timingData is aggregated separately to prefilter calculations for summary metrics (e.g. p95)
	mu         sync.Mutex
	series     []*datadog.Series
	timingData map[string]map[float64][]float64
}

var _ statsdClient = (*datadogStatsd)(nil)

const submitTimeout = 3 * time.Second

func newDatadogStatsd(cfg *datadog.Configuration, apiKey, appKey string) *datadogStatsd {
	client := datadog.NewAPIClient(cfg)
	keys := map[string]datadog.APIKey{
		"apiKeyAuth": {Key: apiKey},
		"appKeyAuth": {Key: appKey},
	}
	return &datadogStatsd{
		authCtx:        context.WithValue(context.Background(), datadog.ContextAPIKeys, keys),
		metrics:        client.MetricsApi,
		events:         client.EventsApi,
		now:            func() float64 { return float64(time.Now().Unix()) },
		timingData:     make(map[string]map[float64][]float64),
		submitInterval: 2 * time.Second,
	}
}

const (
	timingType = "gauge"
	countType  = "count"
)

func (d *datadogStatsd) Incr(metric string, tags []string, _ float64) error {
	tags = append(d.tags, tags...)
	now := d.now()

	d.mu.Lock()
	defer d.mu.Unlock()

	if existing := d.findSeries(metric, tags); existing != nil {
		// Series exists: increment or add the per-timestamp count
		for i, p := range existing.Points {
			if p[0] == now {
				existing.Points[i][1]++
				return nil
			}
		}
		existing.Points = append(existing.Points, []float64{now, 1})
		return nil
	}

	// Not found, create
	s := datadog.NewSeries(metric, [][]float64{{now, 1}})
	s.SetType(countType)
	s.SetTags(tags)
	d.series = append(d.series, s)
	return nil
}

func (d *datadogStatsd) Timing(metric string, dur time.Duration, tags []string, _ float64) error {
	tags = append(d.tags, tags...)
	now := d.now()
	val := float64(dur.Milliseconds())
	tk := timingKey(metric, tags)

	d.mu.Lock()
	defer d.mu.Unlock()

	if existing := d.findSeries(metric, tags); existing != nil {
		// Series exists, we only need to aggregate timing data:
		d.timingData[tk][now] = append(d.timingData[tk][now], val)
		return nil
	}

	// Not found, create
	s := datadog.NewSeries(metric, nil)
	s.SetType(timingType)
	s.SetTags(tags)
	s.SetInterval(1)
	d.series = append(d.series, s)
	d.timingData[tk] = map[float64][]float64{now: {val}}
	return nil
}

// timingKey serializes metric+tags as a key for timingData
func timingKey(metric string, tags []string) string {
	return fmt.Sprintf("%s %s", metric, strings.Join(tags, ","))
}

func (d *datadogStatsd) Event(e *statsd.Event) error {
	ctx, cancel := context.WithTimeout(d.authCtx, submitTimeout)
	defer cancel()

	ddEvent := datadog.NewEventCreateRequest(e.Text, e.Title)
	ddEvent.SetAlertType(datadog.EventAlertType(e.AlertType))
	ddEvent.SetAggregationKey(e.AggregationKey)
	ddEvent.SetPriority(datadog.EventPriority(e.Priority))
	ddEvent.SetTags(append(d.tags, e.Tags...))
	if e.Timestamp.IsZero() {
		ddEvent.SetDateHappened(int64(d.now()))
	} else {
		ddEvent.SetDateHappened(e.Timestamp.Unix())
	}

	if _, _, err := d.events.CreateEvent(ctx, *ddEvent); err != nil {
		log.Println("error submitting event to datadog", err)
	}
	return nil
}

func (d *datadogStatsd) Close() {
	d.cancelSubmitLoop()
	// flush any buffered metrics. this may block on mutex until the submitLoop finishes
	d.submit()
}

func (d *datadogStatsd) submitLoop(ctx context.Context) {
	t := time.NewTicker(d.submitInterval)
	defer t.Stop()

	for {
		select {
		case <-t.C:
			d.submit()
		case <-ctx.Done():
			return
		}
	}
}

func (d *datadogStatsd) submit() {
	series := d.flushSeries()
	if len(series) == 0 {
		return
	}

	ctx, cancel := context.WithTimeout(d.authCtx, submitTimeout)
	defer cancel()
	if _, _, err := d.metrics.SubmitMetrics(ctx, *datadog.NewMetricsPayload(series)); err != nil {
		log.Println("error submitting metrics to datadog", err)
	}
}

func (d *datadogStatsd) flushSeries() []datadog.Series {
	d.mu.Lock()
	defer d.mu.Unlock()

	// most series map 1:1, but timing expands to 5 series
	series := make([]datadog.Series, 0, len(d.series)+len(d.timingData)*4)
	for _, s := range d.series {
		if s.GetType() == timingType {
			series = append(series, d.deriveTimingSeries(s)...)
			continue
		}
		series = append(series, *s)
	}
	d.series = nil
	d.timingData = make(map[string]map[float64][]float64)
	return series
}

func (d *datadogStatsd) deriveTimingSeries(s *datadog.Series) []datadog.Series {
	tk := timingKey(s.GetMetric(), s.GetTags())
	var counts, averages, p95s, p99s, maxes [][]float64
	for ts, data := range d.timingData[tk] {
		sort.Float64s(data)
		var sum float64
		for _, f := range data {
			sum += f
		}
		count := float64(len(data))
		counts = append(counts, []float64{ts, count})
		averages = append(averages, []float64{ts, sum / count})
		p95s = append(p95s, []float64{ts, percentile(data, 0.95)})
		p99s = append(p99s, []float64{ts, percentile(data, 0.99)})
		maxes = append(maxes, []float64{ts, data[len(data)-1]})
	}

	return []datadog.Series{
		cloneSeries(s, "count", counts),
		cloneSeries(s, "avg", averages),
		cloneSeries(s, "95percentile", p95s),
		cloneSeries(s, "99percentile", p99s),
		cloneSeries(s, "max", maxes),
	}
}

func cloneSeries(s *datadog.Series, suffix string, points [][]float64) datadog.Series {
	return datadog.Series{
		Host:           s.Host,
		Interval:       s.Interval,
		Metric:         fmt.Sprintf("%s.%s", s.GetMetric(), suffix),
		Points:         points,
		Tags:           s.Tags,
		Type:           s.Type,
		UnparsedObject: s.UnparsedObject,
	}
}

func percentile(data []float64, p float64) float64 {
	pos := float64(len(data)) * p
	if math.Round(pos) == pos {
		// Return exact value at percentile
		return data[int(pos)-1]
	}

	return (data[int(pos-1)] + data[int(pos)]) / 2
}

func (d *datadogStatsd) findSeries(metric string, tags []string) *datadog.Series {
	sort.Strings(tags)
seriesLoop:
	for _, s := range d.series {
		if s.GetMetric() != metric {
			continue
		}
		sTags := s.GetTags()
		if len(sTags) != len(tags) {
			continue
		}
		sort.Strings(sTags)
		for i, t := range sTags {
			if tags[i] != t {
				continue seriesLoop
			}
		}
		return s
	}
	return nil
}
<|endoftext|>


<commit_before>package compiler

import (
	"github.com/davyxu/tabtoy/v3/helper"
	"github.com/davyxu/tabtoy/v3/model"
	"github.com/davyxu/tabtoy/v3/report"
	"strings"
)

func Loadheader(sheet helper.TableSheet, tab *model.DataTable, resolveTableType string, typeTab *model.TypeTable) {
	// 读取表头

	for col := 0; ; col++ {

		headerValue := sheet.GetValue(0, col, false)

		// 空列，终止
		if headerValue == "" {
			break
		}
		// 列头带#时，本列忽略
		if strings.HasPrefix(headerValue, "#") {
			continue
		}

		header := tab.MustGetHeader(col)
		header.Cell.CopyFrom(&model.Cell{
			Value: headerValue,
			Col:   col,
			Row:   0,
			Table: tab,
		})

	}

	resolveHeaderFields(tab, resolveTableType, typeTab)

	checkHeaderTypes(tab, typeTab)
}

func checkHeaderTypes(tab *model.DataTable, typeTab *model.TypeTable) {

	for _, header := range tab.Headers {

		if header.TypeInfo == nil {
			continue
		}

		// 原始类型检查
		if !model.PrimitiveExists(header.TypeInfo.FieldType) &&
			!typeTab.ObjectExists(header.TypeInfo.FieldType) { // 对象检查

			report.ReportError("UnknownFieldType", header.Cell.String())
		}
	}

}

func headerValueExists(offset int, name string, headers []*model.HeaderField) bool {

	for i := offset; i < len(headers); i++ {
		if headers[i].Cell.Value == name {
			return true
		}
	}

	return false
}

func resolveHeaderFields(tab *model.DataTable, tableObjectType string, typeTab *model.TypeTable) {

	tab.OriginalHeaderType = tableObjectType
	for index, header := range tab.Headers {

		if header.Cell.Value == "" {
			continue
		}

		tf := typeTab.FieldByName(tableObjectType, header.Cell.Value)
		if tf == nil {
			report.ReportError("HeaderFieldNotDefined", header.Cell.String())
		}

		if headerValueExists(index+1, header.Cell.Value, tab.Headers) && !tf.IsArray() {
			report.ReportError("DuplicateHeaderField", header.Cell.String())
		}

		// 解析好的类型
		header.TypeInfo = tf
	}

}
<commit_msg>fix 调整报错提示<commit_after>package compiler

import (
	"github.com/davyxu/tabtoy/v3/helper"
	"github.com/davyxu/tabtoy/v3/model"
	"github.com/davyxu/tabtoy/v3/report"
	"strings"
)

func Loadheader(sheet helper.TableSheet, tab *model.DataTable, resolveTableType string, typeTab *model.TypeTable) {
	// 读取表头

	for col := 0; ; col++ {

		headerValue := sheet.GetValue(0, col, false)

		// 空列，终止
		if headerValue == "" {
			break
		}
		// 列头带#时，本列忽略
		if strings.HasPrefix(headerValue, "#") {
			continue
		}

		header := tab.MustGetHeader(col)
		header.Cell.CopyFrom(&model.Cell{
			Value: headerValue,
			Col:   col,
			Row:   0,
			Table: tab,
		})

	}

	resolveHeaderFields(tab, resolveTableType, typeTab)

	checkHeaderTypes(tab, typeTab)
}

func checkHeaderTypes(tab *model.DataTable, typeTab *model.TypeTable) {

	for _, header := range tab.Headers {

		if header.TypeInfo == nil {
			continue
		}

		// 原始类型检查
		if !model.PrimitiveExists(header.TypeInfo.FieldType) &&
			!typeTab.ObjectExists(header.TypeInfo.FieldType) { // 对象检查

			report.ReportError("UnknownFieldType", header.TypeInfo.FieldType, header.Cell.String())
		}
	}

}

func headerValueExists(offset int, name string, headers []*model.HeaderField) bool {

	for i := offset; i < len(headers); i++ {
		if headers[i].Cell.Value == name {
			return true
		}
	}

	return false
}

func resolveHeaderFields(tab *model.DataTable, tableObjectType string, typeTab *model.TypeTable) {

	tab.OriginalHeaderType = tableObjectType
	for index, header := range tab.Headers {

		if header.Cell.Value == "" {
			continue
		}

		tf := typeTab.FieldByName(tableObjectType, header.Cell.Value)
		if tf == nil {
			report.ReportError("HeaderFieldNotDefined", header.Cell.String())
		}

		if headerValueExists(index+1, header.Cell.Value, tab.Headers) && !tf.IsArray() {
			report.ReportError("DuplicateHeaderField", header.Cell.String())
		}

		// 解析好的类型
		header.TypeInfo = tf
	}

}
<|endoftext|>


<commit_before>package main

import (
  "math"
  "math/big"
  "fmt"
  "strconv"
  _ "encoding/hex"
)

// Op codes
const (
  oSTOP       int = 0x00
  oADD        int = 0x10
  oSUB        int = 0x11
  oMUL        int = 0x12
  oDIV        int = 0x13
  oSDIV       int = 0x14
  oMOD        int = 0x15
  oSMOD       int = 0x16
  oEXP        int = 0x17
  oNEG        int = 0x18
  oLT         int = 0x20
  oLE         int = 0x21
  oGT         int = 0x22
  oGE         int = 0x23
  oEQ         int = 0x24
  oNOT        int = 0x25
  oSHA256     int = 0x30
  oRIPEMD160  int = 0x31
  oECMUL      int = 0x32
  oECADD      int = 0x33
  oSIGN       int = 0x34
  oRECOVER    int = 0x35
  oCOPY       int = 0x40
  oST         int = 0x41
  oLD         int = 0x42
  oSET        int = 0x43
  oJMP        int = 0x50
  oJMPI       int = 0x51
  oIND        int = 0x52
  oEXTRO      int = 0x60
  oBALANCE    int = 0x61
  oMKTX       int = 0x70
  oDATA       int = 0x80
  oDATAN      int = 0x81
  oMYADDRESS  int = 0x90
  oSUICIDE    int = 0xff
)

type OpType int
const (
  tNorm = iota
  tData
  tExtro
  tCrypto
)
type TxCallback func(opType OpType) bool

type Vm struct {
  // Memory stack
  stack map[string]string
  // Index ptr
  iptr int
  memory map[string]map[string]string
}

func NewVm() *Vm {
  fmt.Println("init Ethereum VM")

  stackSize := uint(256)
  fmt.Println("stack size =", stackSize)

  return &Vm{make(map[string]string), 0, make(map[string]map[string]string)}
}

func (vm *Vm) RunTransaction(tx *Transaction, cb TxCallback) {
  fmt.Printf(`
# processing Tx (%v)
# fee = %f, ops = %d, sender = %s, value = %d
`, tx.addr, float32(tx.fee) / 1e8, len(tx.data), tx.sender, tx.value)

  vm.stack = make(map[string]string)
  vm.stack["0"] = tx.sender
  vm.stack["1"] = "100"  //int(tx.value)
  vm.stack["1"] = "1000" //int(tx.fee)

  //vm.memory[tx.addr] = make([]int, 256)
  vm.memory[tx.addr] = make(map[string]string)

  // Define instruction 'accessors' for the instruction, which makes it more readable
  // also called register values, shorthanded as Rx/y/z. Memory address are shorthanded as Mx/y/z.
  // Instructions are shorthanded as Ix/y/z
  x := 0; y := 1; z := 2; //a := 3; b := 4; c := 5
out:
  for vm.iptr < len(tx.data) {
    // The base big int for all calculations. Use this for any results.
    base := new(big.Int)
    // XXX Should Instr return big int slice instead of string slice?
    op, args, _ := Instr(tx.data[vm.iptr])

    fmt.Printf("%-3d %d %v\n", vm.iptr, op, args)

    opType     := OpType(tNorm)
    // Determine the op type (used for calculating fees by the block manager)
    switch op {
    case oEXTRO, oBALANCE:
      opType = tExtro
    case oSHA256, oRIPEMD160, oECMUL, oECADD: // TODO add rest
      opType = tCrypto
    }

    // If the callback yielded a negative result abort execution
    if !cb(opType) { break out }

    nptr := vm.iptr
    switch op {
    case oSTOP:
      fmt.Println("exiting (oSTOP), idx =", nptr)

      break out
    case oADD:
      // (Rx + Ry) % 2 ** 256
      base.Add(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oSUB:
      // (Rx - Ry) % 2 ** 256
      base.Sub(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oMUL:
      // (Rx * Ry) % 2 ** 256
      base.Mul(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oDIV:
      // floor(Rx / Ry)
      base.Div(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oSET:
      // Set the (numeric) value at Iy to Rx
      vm.stack[args[ x ]] = args[ y ]
    case oLD:
      // Load the value at Mx to Ry
      vm.stack[args[ y ]] = vm.memory[tx.addr][vm.stack[args[ x ]]]
    case oLT:
      cmp := Big(vm.stack[args[ x ]]).Cmp( Big(vm.stack[args[ y ]]) )
      // Set the result as "boolean" value to Rz
      if  cmp < 0 { // a < b
        vm.stack[args[ z ]] = "1"
      } else {
        vm.stack[args[ z ]] = "0"
      }
    case oJMP:
      // Set the instruction pointer to the value at Rx
      ptr, _ := strconv.Atoi( vm.stack[args[ x ]] )
      nptr = ptr
    case oJMPI:
      // Set the instruction pointer to the value at Ry if Rx yields true
      if vm.stack[args[ x ]] != "0" {
        ptr, _ := strconv.Atoi( vm.stack[args[ y ]] )
        nptr = ptr
      }
    default:
      fmt.Println("Error op", op)
      break
    }

    if vm.iptr == nptr {
      vm.iptr++
    } else {
      vm.iptr = nptr
      fmt.Println("... JMP", nptr, "...")
    }
  }
  fmt.Println("# finished processing Tx\n")
}
<commit_msg>Reset stack pointer on run<commit_after>package main

import (
  "math"
  "math/big"
  "fmt"
  "strconv"
  _ "encoding/hex"
)

// Op codes
const (
  oSTOP       int = 0x00
  oADD        int = 0x10
  oSUB        int = 0x11
  oMUL        int = 0x12
  oDIV        int = 0x13
  oSDIV       int = 0x14
  oMOD        int = 0x15
  oSMOD       int = 0x16
  oEXP        int = 0x17
  oNEG        int = 0x18
  oLT         int = 0x20
  oLE         int = 0x21
  oGT         int = 0x22
  oGE         int = 0x23
  oEQ         int = 0x24
  oNOT        int = 0x25
  oSHA256     int = 0x30
  oRIPEMD160  int = 0x31
  oECMUL      int = 0x32
  oECADD      int = 0x33
  oSIGN       int = 0x34
  oRECOVER    int = 0x35
  oCOPY       int = 0x40
  oST         int = 0x41
  oLD         int = 0x42
  oSET        int = 0x43
  oJMP        int = 0x50
  oJMPI       int = 0x51
  oIND        int = 0x52
  oEXTRO      int = 0x60
  oBALANCE    int = 0x61
  oMKTX       int = 0x70
  oDATA       int = 0x80
  oDATAN      int = 0x81
  oMYADDRESS  int = 0x90
  oSUICIDE    int = 0xff
)

type OpType int
const (
  tNorm = iota
  tData
  tExtro
  tCrypto
)
type TxCallback func(opType OpType) bool

type Vm struct {
  // Memory stack
  stack map[string]string
  memory map[string]map[string]string
}

func NewVm() *Vm {
  fmt.Println("init Ethereum VM")

  stackSize := uint(256)
  fmt.Println("stack size =", stackSize)

  return &Vm{
    stack: make(map[string]string),
    memory: make(map[string]map[string]string),
  }
}

func (vm *Vm) RunTransaction(tx *Transaction, cb TxCallback) {
  fmt.Printf(`
# processing Tx (%v)
# fee = %f, ops = %d, sender = %s, value = %d
`, tx.addr, float32(tx.fee) / 1e8, len(tx.data), tx.sender, tx.value)

  vm.stack = make(map[string]string)
  vm.stack["0"] = tx.sender
  vm.stack["1"] = "100"  //int(tx.value)
  vm.stack["1"] = "1000" //int(tx.fee)
  // Stack pointer
  stPtr := 0

  //vm.memory[tx.addr] = make([]int, 256)
  vm.memory[tx.addr] = make(map[string]string)

  // Define instruction 'accessors' for the instruction, which makes it more readable
  // also called register values, shorthanded as Rx/y/z. Memory address are shorthanded as Mx/y/z.
  // Instructions are shorthanded as Ix/y/z
  x := 0; y := 1; z := 2; //a := 3; b := 4; c := 5
out:
  for stPtr < len(tx.data) {
    // The base big int for all calculations. Use this for any results.
    base := new(big.Int)
    // XXX Should Instr return big int slice instead of string slice?
    op, args, _ := Instr(tx.data[stPtr])

    fmt.Printf("%-3d %d %v\n", stPtr, op, args)

    opType     := OpType(tNorm)
    // Determine the op type (used for calculating fees by the block manager)
    switch op {
    case oEXTRO, oBALANCE:
      opType = tExtro
    case oSHA256, oRIPEMD160, oECMUL, oECADD: // TODO add rest
      opType = tCrypto
    }

    // If the callback yielded a negative result abort execution
    if !cb(opType) { break out }

    nptr := stPtr
    switch op {
    case oSTOP:
      fmt.Println("exiting (oSTOP), idx =", nptr)

      break out
    case oADD:
      // (Rx + Ry) % 2 ** 256
      base.Add(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oSUB:
      // (Rx - Ry) % 2 ** 256
      base.Sub(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oMUL:
      // (Rx * Ry) % 2 ** 256
      base.Mul(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      base.Mod(base, big.NewInt(int64(math.Pow(2, 256))))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oDIV:
      // floor(Rx / Ry)
      base.Div(Big(vm.stack[args[ x ]]), Big(vm.stack[args[ y ]]))
      // Set the result to Rz
      vm.stack[args[ z ]] = base.String()
    case oSET:
      // Set the (numeric) value at Iy to Rx
      vm.stack[args[ x ]] = args[ y ]
    case oLD:
      // Load the value at Mx to Ry
      vm.stack[args[ y ]] = vm.memory[tx.addr][vm.stack[args[ x ]]]
    case oLT:
      cmp := Big(vm.stack[args[ x ]]).Cmp( Big(vm.stack[args[ y ]]) )
      // Set the result as "boolean" value to Rz
      if  cmp < 0 { // a < b
        vm.stack[args[ z ]] = "1"
      } else {
        vm.stack[args[ z ]] = "0"
      }
    case oJMP:
      // Set the instruction pointer to the value at Rx
      ptr, _ := strconv.Atoi( vm.stack[args[ x ]] )
      nptr = ptr
    case oJMPI:
      // Set the instruction pointer to the value at Ry if Rx yields true
      if vm.stack[args[ x ]] != "0" {
        ptr, _ := strconv.Atoi( vm.stack[args[ y ]] )
        nptr = ptr
      }
    default:
      fmt.Println("Error op", op)
      break
    }

    if stPtr == nptr {
      stPtr++
    } else {
      stPtr = nptr
      fmt.Println("... JMP", nptr, "...")
    }
  }
  fmt.Println("# finished processing Tx\n")
}
<|endoftext|>


<commit_before>package web_test

import (
	"fmt"
	"time"

	"github.com/concourse/atc"
	"github.com/concourse/testflight/gitserver"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("BuildsView", func() {
	var build atc.Build

	Context("with a job in the configuration", func() {
		var originGitServer *gitserver.Server

		BeforeEach(func() {
			originGitServer = gitserver.Start(client)
			originGitServer.CommitResource()

			_, _, _, err := team.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "some-job",
						Plan: atc.PlanSequence{
							{
								Get: "some-input-resource",
							},
							{
								Task: "some-task",
								TaskConfig: &atc.TaskConfig{
									Platform: "linux",
									ImageResource: &atc.ImageResource{
										Type:   "docker-image",
										Source: atc.Source{"repository": "busybox"},
									},
									Inputs: []atc.TaskInputConfig{
										{Name: "some-input-resource"},
									},
									Outputs: []atc.TaskOutputConfig{
										{Name: "some-output-src"},
									},
									Run: atc.TaskRunConfig{
										Path: "cp",
										Args: []string{"-r", "some-input-resource/.", "some-output-src"},
									},
								},
							},
							{
								Put:    "some-output-resource",
								Params: atc.Params{"repository": "some-output-src"},
							},
						},
					},
				},
				Resources: []atc.ResourceConfig{
					{
						Name: "some-input-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
					{
						Name: "some-output-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			build, err = team.CreateJobBuild(pipelineName, "some-job")
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			originGitServer.Stop()
		})

		It("can view resource information of a job build", func() {
			url := atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job", teamName, pipelineName))

			Expect(page.Navigate(url)).To(Succeed())
			Eventually(page.Find(".build-header.succeeded")).Should(BeFound())

			Eventually(page.All(".builds-list li")).Should(HaveCount(1))

			Expect(page.Find(".builds-list li:first-child a")).To(HaveText("#1"))
			Eventually(page.Find(".builds-list li:first-child a.succeeded"), 10*time.Second).Should(BeFound())

			buildTimes, err := page.Find(".builds-list li:first-child .build-duration").Text()
			Expect(err).NotTo(HaveOccurred())
			Expect(buildTimes).To(ContainSubstring("started"))
			Expect(buildTimes).To(MatchRegexp("started \\d+s ago"))
			Expect(buildTimes).To(MatchRegexp("finished \\d+s ago"))
			Expect(buildTimes).To(MatchRegexp("duration \\d+s"))

			Eventually(page.Find(".builds-list li:first-child .inputs .resource-name"), 10*time.Second).Should(BeFound())
			Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("some-input-resource"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(MatchText("[0-9a-f]{40}"))

			Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output-resource"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(MatchText("[0-9a-z]{40}"))

			// button should not have the boolean attribute "disabled" set. agouti currently returns
			// an empty string in that case.
			Expect(page.Find("button.build-action")).ToNot(BeNil())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", ""))
		})

		Describe("paused pipeline", func() {
			BeforeEach(func() {
				_, err := team.PausePipeline(pipelineName)
				Expect(err).NotTo(HaveOccurred())
			})

			It("displays a blue header", func() {
				Expect(page.Navigate(atcRoute(build.URL))).To(Succeed())

				Expect(page.Navigate(atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job/builds/%s", teamName, pipelineName, build.Name)))).To(Succeed())

				// top bar should show the pipeline is paused
				Eventually(page.Find(".top-bar.test.paused"), 10*time.Second).Should(BeFound())
			})
		})
	})

	Context("when manual triggering of the job is disabled", func() {
		var manualTriggerDisabledBuild atc.Build

		BeforeEach(func() {
			_, _, _, err := team.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "job-manual-trigger-disabled",
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			manualTriggerDisabledBuild, err = team.CreateJobBuild(pipelineName, "job-manual-trigger-disabled")
			Expect(err).NotTo(HaveOccurred())

			_, _, pipelineVersion, _, err := team.PipelineConfig(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, pipelineVersion, atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name:                 "job-manual-trigger-disabled",
						DisableManualTrigger: true,
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		It("should have a disabled button in the build details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))
			Eventually(page.Find("button.build-action"), 10*time.Second).Should(HaveAttribute("disabled", "true"))
		})

		It("should have a disabled button in the job details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))

			Eventually(page.Find("h1 a"), 10*time.Second).Should(BeFound())
			Expect(page.Find("h1 a").Click()).To(Succeed())
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled",
				teamName,
				pipelineName,
			))))

			Eventually(page.Find("button.build-action")).Should(BeFound())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", "true"))
		})
	})
})
<commit_msg>remove testing unrelated to example description<commit_after>package web_test

import (
	"fmt"
	"time"

	"github.com/concourse/atc"
	"github.com/concourse/testflight/gitserver"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/sclevine/agouti/matchers"
)

var _ = Describe("BuildsView", func() {
	var build atc.Build

	Context("with a job in the configuration", func() {
		var originGitServer *gitserver.Server

		BeforeEach(func() {
			originGitServer = gitserver.Start(client)
			originGitServer.CommitResource()

			_, _, _, err := team.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "some-job",
						Plan: atc.PlanSequence{
							{
								Get: "some-input-resource",
							},
							{
								Task: "some-task",
								TaskConfig: &atc.TaskConfig{
									Platform: "linux",
									ImageResource: &atc.ImageResource{
										Type:   "docker-image",
										Source: atc.Source{"repository": "busybox"},
									},
									Inputs: []atc.TaskInputConfig{
										{Name: "some-input-resource"},
									},
									Outputs: []atc.TaskOutputConfig{
										{Name: "some-output-src"},
									},
									Run: atc.TaskRunConfig{
										Path: "cp",
										Args: []string{"-r", "some-input-resource/.", "some-output-src"},
									},
								},
							},
							{
								Put:    "some-output-resource",
								Params: atc.Params{"repository": "some-output-src"},
							},
						},
					},
				},
				Resources: []atc.ResourceConfig{
					{
						Name: "some-input-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
					{
						Name: "some-output-resource",
						Type: "git",
						Source: atc.Source{
							"branch": "master",
							"uri":    originGitServer.URI(),
						},
						CheckEvery: "",
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			build, err = team.CreateJobBuild(pipelineName, "some-job")
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			originGitServer.Stop()
		})

		It("can view resource information of a job build", func() {
			url := atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job", teamName, pipelineName))

			Expect(page.Navigate(url)).To(Succeed())
			Eventually(page.Find(".build-header.succeeded")).Should(BeFound())

			Eventually(page.All(".builds-list li")).Should(HaveCount(1))

			Expect(page.Find(".builds-list li:first-child a")).To(HaveText("#1"))
			Eventually(page.Find(".builds-list li:first-child a.succeeded"), 10*time.Second).Should(BeFound())

			Eventually(page.Find(".builds-list li:first-child .inputs .resource-name"), 10*time.Second).Should(BeFound())
			Expect(page.Find(".builds-list li:first-child .inputs .resource-name")).To(HaveText("some-input-resource"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .inputs .resource-version .dict-value")).To(MatchText("[0-9a-f]{40}"))

			Expect(page.Find(".builds-list li:first-child .outputs .resource-name")).To(HaveText("some-output-resource"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-key")).To(HaveText("ref"))
			Expect(page.Find(".builds-list li:first-child .outputs .resource-version .dict-value")).To(MatchText("[0-9a-z]{40}"))

			// button should not have the boolean attribute "disabled" set. agouti currently returns
			// an empty string in that case.
			Expect(page.Find("button.build-action")).ToNot(BeNil())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", ""))
		})

		Describe("paused pipeline", func() {
			BeforeEach(func() {
				_, err := team.PausePipeline(pipelineName)
				Expect(err).NotTo(HaveOccurred())
			})

			It("displays a blue header", func() {
				Expect(page.Navigate(atcRoute(build.URL))).To(Succeed())

				Expect(page.Navigate(atcRoute(fmt.Sprintf("/teams/%s/pipelines/%s/jobs/some-job/builds/%s", teamName, pipelineName, build.Name)))).To(Succeed())

				// top bar should show the pipeline is paused
				Eventually(page.Find(".top-bar.test.paused"), 10*time.Second).Should(BeFound())
			})
		})
	})

	Context("when manual triggering of the job is disabled", func() {
		var manualTriggerDisabledBuild atc.Build

		BeforeEach(func() {
			_, _, _, err := team.CreateOrUpdatePipelineConfig(pipelineName, "0", atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name: "job-manual-trigger-disabled",
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())

			_, err = team.UnpausePipeline(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			manualTriggerDisabledBuild, err = team.CreateJobBuild(pipelineName, "job-manual-trigger-disabled")
			Expect(err).NotTo(HaveOccurred())

			_, _, pipelineVersion, _, err := team.PipelineConfig(pipelineName)
			Expect(err).NotTo(HaveOccurred())

			_, _, _, err = team.CreateOrUpdatePipelineConfig(pipelineName, pipelineVersion, atc.Config{
				Jobs: []atc.JobConfig{
					{
						Name:                 "job-manual-trigger-disabled",
						DisableManualTrigger: true,
					},
				},
			})
			Expect(err).NotTo(HaveOccurred())
		})

		It("should have a disabled button in the build details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))
			Eventually(page.Find("button.build-action"), 10*time.Second).Should(HaveAttribute("disabled", "true"))
		})

		It("should have a disabled button in the job details view", func() {
			Expect(page.Navigate(atcRoute(manualTriggerDisabledBuild.URL))).To(Succeed())

			// job detail w/build info -> job detail
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled/builds/%s",
				teamName,
				pipelineName,
				manualTriggerDisabledBuild.Name,
			))))

			Eventually(page.Find("h1 a"), 10*time.Second).Should(BeFound())
			Expect(page.Find("h1 a").Click()).To(Succeed())
			Eventually(page, 10*time.Second).Should(HaveURL(atcRoute(fmt.Sprintf(
				"/teams/%s/pipelines/%s/jobs/job-manual-trigger-disabled",
				teamName,
				pipelineName,
			))))

			Eventually(page.Find("button.build-action")).Should(BeFound())
			Expect(page.Find("button.build-action")).To(HaveAttribute("disabled", "true"))
		})
	})
})
<|endoftext|>


<commit_before>package common

import (
    "os"
    "io"
    "time"
    "strings"
    "strconv"
    "net/http"
)

const (
    logFile = "access_log.txt"
)

type accessLog struct {
    ip                    string
    time                  time.Time
    method, uri, protocol string
    status                int
    responseBytes         int64
    elapsedTime           time.Duration
}

func LogAccess(w http.ResponseWriter, req *http.Request, duration time.Duration) {
    clientIP := req.RemoteAddr

    if colon := strings.LastIndex(clientIP, ":"); colon != -1 {
        clientIP = clientIP[:colon]
    }

    record := &accessLog{
        ip:             clientIP,
        time:           time.Time{},
        method:         req.Method,
        uri:            req.RequestURI,
        protocol:       req.Proto,
        status:         http.StatusOK,
        elapsedTime:    duration,
    }

    writeAccessLog(record)
}

func writeAccessLog(record *accessLog) {
    logRecord := "["+record.time.Format("02/Jan/2006 03:04:05")+"] "+record.ip+" "+record.protocol+" "+record.method+": "+record.uri+" (load time: "+strconv.FormatFloat(record.elapsedTime.Seconds(), 'f', 5, 64)+" seconds)\n"

    file, err := os.OpenFile(logFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0666)
    CheckError(err)
    io.WriteString(file, logRecord)
    file.Close()
}<commit_msg>Added 'inspiration' comment<commit_after>// Note: inspiration for this from https://gist.github.com/cespare/3985516

package common

import (
    "os"
    "io"
    "time"
    "strings"
    "strconv"
    "net/http"
)

const (
    logFile = "access_log.txt"
)

type accessLog struct {
    ip                    string
    time                  time.Time
    method, uri, protocol string
    status                int
    responseBytes         int64
    elapsedTime           time.Duration
}

func LogAccess(w http.ResponseWriter, req *http.Request, duration time.Duration) {
    clientIP := req.RemoteAddr

    if colon := strings.LastIndex(clientIP, ":"); colon != -1 {
        clientIP = clientIP[:colon]
    }

    record := &accessLog{
        ip:             clientIP,
        time:           time.Time{},
        method:         req.Method,
        uri:            req.RequestURI,
        protocol:       req.Proto,
        status:         http.StatusOK,
        elapsedTime:    duration,
    }

    writeAccessLog(record)
}

func writeAccessLog(record *accessLog) {
    logRecord := "["+record.time.Format("02/Jan/2006 03:04:05")+"] "+record.ip+" "+record.protocol+" "+record.method+": "+record.uri+" (load time: "+strconv.FormatFloat(record.elapsedTime.Seconds(), 'f', 5, 64)+" seconds)\n"

    file, err := os.OpenFile(logFile, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0666)
    CheckError(err)
    io.WriteString(file, logRecord)
    file.Close()
}<|endoftext|>


<commit_before>package machine

import (
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"

	// Force to load docker executor
	"errors"
	_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/docker"
)

type machineExecutor struct {
	provider *machineProvider
	executor common.Executor
	data     common.ExecutorData
	config   common.RunnerConfig
}

func (e *machineExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) (err error) {
	// Use the machine
	e.config, e.data, err = e.provider.Use(config, build.ExecutorData)
	if err != nil {
		return err
	}

	// TODO: Currently the docker-machine doesn't support multiple builds
	build.ProjectRunnerID = 0
	if details, _ := build.ExecutorData.(*machineDetails); details != nil {
		build.Hostname = details.Name
	} else if details, _ := e.data.(*machineDetails); details != nil {
		build.Hostname = details.Name
	}

	// Create original executor
	e.executor = e.provider.provider.Create()
	if e.executor == nil {
		return errors.New("failed to create an executor")
	}
	return e.executor.Prepare(globalConfig, &e.config, build)
}

func (e *machineExecutor) Start() error {
	if e.executor == nil {
		return errors.New("missing executor")
	}
	return e.executor.Start()
}

func (e *machineExecutor) Wait() error {
	if e.executor == nil {
		return errors.New("missing executor")
	}
	return e.executor.Wait()
}

func (e *machineExecutor) Finish(err error) {
	if e.executor != nil {
		e.executor.Finish(err)
	}
}

func (e *machineExecutor) Cleanup() {
	// Cleanup executor if were created
	if e.executor != nil {
		e.executor.Cleanup()
	}

	// Release allocated machine
	if e.data != "" {
		e.provider.Release(&e.config, e.data)
		e.data = nil
	}
}

func init() {
	common.RegisterExecutor("docker+machine", newMachineProvider("docker"))
	common.RegisterExecutor("docker-ssh+machine", newMachineProvider("docker-ssh"))
}
<commit_msg>Fix imports<commit_after>package machine

import (
	"errors"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"

	// Force to load docker executor
	_ "gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors/docker"
)

type machineExecutor struct {
	provider *machineProvider
	executor common.Executor
	data     common.ExecutorData
	config   common.RunnerConfig
}

func (e *machineExecutor) Prepare(globalConfig *common.Config, config *common.RunnerConfig, build *common.Build) (err error) {
	// Use the machine
	e.config, e.data, err = e.provider.Use(config, build.ExecutorData)
	if err != nil {
		return err
	}

	// TODO: Currently the docker-machine doesn't support multiple builds
	build.ProjectRunnerID = 0
	if details, _ := build.ExecutorData.(*machineDetails); details != nil {
		build.Hostname = details.Name
	} else if details, _ := e.data.(*machineDetails); details != nil {
		build.Hostname = details.Name
	}

	// Create original executor
	e.executor = e.provider.provider.Create()
	if e.executor == nil {
		return errors.New("failed to create an executor")
	}
	return e.executor.Prepare(globalConfig, &e.config, build)
}

func (e *machineExecutor) Start() error {
	if e.executor == nil {
		return errors.New("missing executor")
	}
	return e.executor.Start()
}

func (e *machineExecutor) Wait() error {
	if e.executor == nil {
		return errors.New("missing executor")
	}
	return e.executor.Wait()
}

func (e *machineExecutor) Finish(err error) {
	if e.executor != nil {
		e.executor.Finish(err)
	}
}

func (e *machineExecutor) Cleanup() {
	// Cleanup executor if were created
	if e.executor != nil {
		e.executor.Cleanup()
	}

	// Release allocated machine
	if e.data != "" {
		e.provider.Release(&e.config, e.data)
		e.data = nil
	}
}

func init() {
	common.RegisterExecutor("docker+machine", newMachineProvider("docker"))
	common.RegisterExecutor("docker-ssh+machine", newMachineProvider("docker-ssh"))
}
<|endoftext|>


<commit_before>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"fmt"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex, err: err}
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(machines[0], 50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		key := bucketsKey[1]

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			val := dynamicBucket.GetKeys()[key]
			valType := reflect.TypeOf(val).Kind()
			if valType == reflect.Slice {
				data, err := ffjson.Marshal(val)
				if err != nil {
				    log.Error("Failed decoding from JSON")
				} else {
					vars[key] = string(data[:])
				}
			} else {
				vars[key] = fmt.Sprint(val)
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse)
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<commit_msg>Convert to string correctly<commit_after>package config_service

import (
	"strings"
	cfgsvc "github.com/Flipkart/config-service/client-go"
	"github.com/kelseyhightower/confd/log"
	"errors"
	"reflect"
	"github.com/pquerna/ffjson/ffjson"
	"strconv"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *cfgsvc.ConfigServiceClient
}

type BucketListener struct{
	watchResp chan *watchResponse
	currentIndex uint64
}

type watchResponse struct {
	waitIndex uint64
	err       error
}

func (this *BucketListener) Connected(bucketName string) {
	log.Info("Connected! " + bucketName)
}

func (this *BucketListener) Disconnected(bucketName string, err error) {
	log.Info("Disconnected! " + bucketName)
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex, err: err}
}

func (this *BucketListener) Deleted(bucketName string) {
	log.Info("deleted " + bucketName)
	this.watchResp <- &watchResponse{waitIndex: 0, err: errors.New(bucketName + " was deleted")}
}

func (this *BucketListener) Updated(oldBucket *cfgsvc.Bucket, newBucket *cfgsvc.Bucket) {
	this.watchResp <- &watchResponse{waitIndex:this.currentIndex+1, err: nil}
}

func NewConfigClient(machines []string) (*Client, error) {
	c, err := cfgsvc.NewConfigServiceClient(machines[0], 50) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		bucketsKey := strings.Split(strings.TrimPrefix(v, "/"), "/")
		buckets := strings.Split(bucketsKey[0], ",")
		key := bucketsKey[1]

		dynamicBuckets, err := c.getDynamicBuckets(buckets)
		if err != nil {
			return vars, err
		}

		for _, dynamicBucket := range dynamicBuckets {
			val := dynamicBucket.GetKeys()[key]
			valType := reflect.TypeOf(val).Kind()
			if valType == reflect.Slice {
				data, err := ffjson.Marshal(val)
				if err != nil {
				    log.Error("Failed decoding from JSON")
				} else {
					vars[key] = string(data[:])
				}
			} else {
				switch val.(type) {
					case int,int64:
					vars[key] = strconv.FormatInt(val.(int64), 64)
					case string:
					vars[key] = val.(string)
					case bool:
					vars[key] = strconv.FormatBool(val.(bool))
					case float32,float64:
					vars[key] = strconv.FormatFloat(val.(float64), 'f', -1, 64)
				}
			}
		}

	}
	return vars, nil
}

func (c *Client) getDynamicBuckets(buckets []string) ([]*cfgsvc.DynamicBucket, error) {
	var dynamicBuckets []*cfgsvc.DynamicBucket
	for _, bucket := range buckets {
		bucketName := strings.TrimSpace(bucket)
		dynamicBucket, err := c.client.GetDynamicBucket(bucketName)
		if err != nil {
			return dynamicBuckets, err
		}
		dynamicBuckets = append(dynamicBuckets, dynamicBucket)
	}
	return dynamicBuckets, nil
}

func setupDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.AddListeners(bucketListener)
	}
}

func removeDynamicBucketListeners(dynamicBuckets []*cfgsvc.DynamicBucket, bucketListener *BucketListener) {
	for _, dynamicBucket := range dynamicBuckets {
		dynamicBucket.RemoveListeners(bucketListener)
	}
}

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	prefix = strings.TrimPrefix(prefix, "/")
	prefixes := strings.Split(prefix, ",")
	dynamicBuckets, err := c.getDynamicBuckets(prefixes)
	if err != nil {
		return waitIndex, err
	}

	if waitIndex == 0 {
		return waitIndex+1, nil
	}  else {
		watchResp := make(chan *watchResponse)
		bucketListener := &BucketListener{watchResp: watchResp, currentIndex: waitIndex}
		setupDynamicBucketListeners(dynamicBuckets, bucketListener)
		select {
			case watchResp := <- watchResp:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
		 		return watchResp.waitIndex, watchResp.err
		    case <-stopChan:
				removeDynamicBucketListeners(dynamicBuckets, bucketListener)
				return 0, nil
		}
	}
}

<|endoftext|>