<commit_before>package brokers

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
)

// AWSSQSBroker represents a AWS SQS broker
// There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html
type AWSSQSBroker struct {
	Broker
	processingWG      sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal
	receivingWG       sync.WaitGroup
	stopReceivingChan chan int
	sess              *session.Session
	service           sqsiface.SQSAPI
}

// NewAWSSQSBroker creates new Broker instance
func NewAWSSQSBroker(cnf *config.Config) Interface {
	b := &AWSSQSBroker{Broker: New(cnf)}
	if cnf.SQS != nil && cnf.SQS.Client != nil {
		// Use provided *SQS client
		b.service = cnf.SQS.Client
	} else {
		// Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials.
		// See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
		// Also, env AWS_REGION is also required
		b.sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
		b.service = sqs.New(b.sess)
	}

	return b
}

// GetPendingTasks returns a slice of task.Signatures waiting in the queue
func (b *AWSSQSBroker) GetPendingTasks(queue string) ([]*tasks.Signature, error) {
	return nil, errors.New("Not implemented")
}

// StartConsuming enters a loop and waits for incoming messages
func (b *AWSSQSBroker) StartConsuming(consumerTag string, concurrency int, taskProcessor TaskProcessor) (bool, error) {
	b.startConsuming(consumerTag, taskProcessor)
	qURL := b.defaultQueueURL()
	deliveries := make(chan *sqs.ReceiveMessageOutput)

	b.stopReceivingChan = make(chan int)
	b.receivingWG.Add(1)

	go func() {
		defer b.receivingWG.Done()

		log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C")

		for {
			select {
			// A way to stop this goroutine from b.StopConsuming
			case <-b.stopReceivingChan:
				return
			default:
				output, err := b.receiveMessage(qURL)
				if err != nil {
					log.ERROR.Printf("Queue consume error: %s", err)
					continue
				}
				if len(output.Messages) == 0 {
					continue
				}

				deliveries <- output
			}

			whetherContinue, err := b.continueReceivingMessages(qURL, deliveries)
			if err != nil {
				log.ERROR.Printf("Error when receiving messages. Error: %v", err)
			}
			if whetherContinue == false {
				return
			}
		}
	}()

	if err := b.consume(deliveries, concurrency, taskProcessor); err != nil {
		return b.retry, err
	}

	return b.retry, nil
}

// StopConsuming quits the loop
func (b *AWSSQSBroker) StopConsuming() {
	b.stopConsuming()

	b.stopReceiving()

	// Waiting for any tasks being processed to finish
	b.processingWG.Wait()

	// Waiting for the receiving goroutine to have stopped
	b.receivingWG.Wait()
}

// Publish places a new message on the default queue
func (b *AWSSQSBroker) Publish(signature *tasks.Signature) error {

	msg, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON marshal error: %s", err)
	}

	// Check that signature.RoutingKey is set, if not switch to DefaultQueue
	AdjustRoutingKey(b, signature)

	MsgInput := &sqs.SendMessageInput{
		MessageBody: aws.String(string(msg)),
		QueueUrl:    aws.String(b.cnf.Broker + "/" + signature.RoutingKey),
	}

	// if this is a fifo queue, there needs to be some additional parameters.
	if strings.HasSuffix(signature.RoutingKey, ".fifo") {
		// Use Machinery's signature Task UUID as SQS Message Group ID.
		MsgDedupID := signature.UUID
		MsgInput.MessageDeduplicationId = aws.String(MsgDedupID)

		// Use Machinery's signature Group UUID as SQS Message Group ID.
		MsgGroupID := signature.GroupUUID
		MsgInput.MessageGroupId = aws.String(MsgGroupID)
	}

	// Check the ETA signature field, if it is set and it is in the future,
	// and is not a fifo queue, set a delay in seconds for the task.
	if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") {
		now := time.Now().UTC()

		if signature.ETA.After(now) {
			MsgInput.DelaySeconds = aws.Int64(signature.ETA.Unix() - now.Unix())
		}
	}

	result, err := b.service.SendMessage(MsgInput)

	if err != nil {
		log.ERROR.Printf("Error when sending a message: %v", err)
		return err

	}
	log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId)
	return nil

}

// consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal
func (b *AWSSQSBroker) consume(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor TaskProcessor) error {
	pool := make(chan struct{}, concurrency)

	// initialize worker pool with maxWorkers workers
	go func() {
		b.initializePool(pool, concurrency)
	}()

	errorsChan := make(chan error)

	for {
		whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan)
		if err != nil {
			return err
		}
		if whetherContinue == false {
			return nil
		}
	}
}

// consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS
func (b *AWSSQSBroker) consumeOne(delivery *sqs.ReceiveMessageOutput, taskProcessor TaskProcessor) error {
	sig := new(tasks.Signature)
	if len(delivery.Messages) == 0 {
		log.ERROR.Printf("received an empty message, the delivery was %v", delivery)
		return errors.New("received empty message, the delivery is " + delivery.GoString())
	}
	msg := delivery.Messages[0].Body
	if err := json.Unmarshal([]byte(*msg), sig); err != nil {
		log.ERROR.Printf("unmarshal error. the delivery is %v", delivery)
		return err
	}

	// Delete message after consuming successfully
	err := b.deleteOne(delivery)
	if err != nil {
		log.ERROR.Printf("error when deleting the delivery. the delivery is %v", delivery)
	}

	// If the task is not registered, we requeue it,
	// there might be different workers for processing specific tasks
	if !b.IsTaskRegistered(sig.Name) {
		err := b.Publish(sig)
		if err != nil {
			return err
		}
		log.INFO.Printf("requeue a task to default queue: %v", sig)
		return nil
	}
	return taskProcessor.Process(sig)
}

// deleteOne is a method delete a delivery from AWS SQS
func (b *AWSSQSBroker) deleteOne(delivery *sqs.ReceiveMessageOutput) error {
	qURL := b.defaultQueueURL()
	_, err := b.service.DeleteMessage(&sqs.DeleteMessageInput{
		QueueUrl:      qURL,
		ReceiptHandle: delivery.Messages[0].ReceiptHandle,
	})

	if err != nil {
		return err
	}
	return nil
}

// defaultQueueURL is a method returns the default queue url
func (b *AWSSQSBroker) defaultQueueURL() *string {
	return aws.String(b.cnf.Broker + "/" + b.cnf.DefaultQueue)
}

// receiveMessage is a method receives a message from specified queue url
func (b *AWSSQSBroker) receiveMessage(qURL *string) (*sqs.ReceiveMessageOutput, error) {
	var waitTimeSeconds int
	if b.cnf.SQS != nil {
		waitTimeSeconds = b.cnf.SQS.WaitTimeSeconds
	} else {
		waitTimeSeconds = 0
	}
	result, err := b.service.ReceiveMessage(&sqs.ReceiveMessageInput{
		AttributeNames: []*string{
			aws.String(sqs.MessageSystemAttributeNameSentTimestamp),
		},
		MessageAttributeNames: []*string{
			aws.String(sqs.QueueAttributeNameAll),
		},
		QueueUrl:            qURL,
		MaxNumberOfMessages: aws.Int64(1),
		VisibilityTimeout:   aws.Int64(int64(b.cnf.ResultsExpireIn)), // 10 hours
		WaitTimeSeconds:     aws.Int64(int64(waitTimeSeconds)),
	})
	if err != nil {
		return nil, err
	}
	return result, err
}

// initializePool is a method which initializes concurrency pool
func (b *AWSSQSBroker) initializePool(pool chan struct{}, concurrency int) {
	for i := 0; i < concurrency; i++ {
		pool <- struct{}{}
	}
}

// consumeDeliveries is a method consuming deliveries from deliveries channel
func (b *AWSSQSBroker) consumeDeliveries(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) {
	select {
	case err := <-errorsChan:
		return false, err
	case d := <-deliveries:
		if concurrency > 0 {
			// get worker from pool (blocks until one is available)
			<-pool
		}

		b.processingWG.Add(1)

		// Consume the task inside a goroutine so multiple tasks
		// can be processed concurrently
		go func() {

			if err := b.consumeOne(d, taskProcessor); err != nil {
				errorsChan <- err
			}

			b.processingWG.Done()

			if concurrency > 0 {
				// give worker back to pool
				pool <- struct{}{}
			}
		}()
	case <-b.stopChan:
		return false, nil
	}
	return true, nil
}

// continueReceivingMessages is a method returns a continue signal
func (b *AWSSQSBroker) continueReceivingMessages(qURL *string, deliveries chan *sqs.ReceiveMessageOutput) (bool, error) {
	select {
	// A way to stop this goroutine from b.StopConsuming
	case <-b.stopReceivingChan:
		return false, nil
	default:
		output, err := b.receiveMessage(qURL)
		if err != nil {
			return true, err
		}
		if len(output.Messages) == 0 {
			return true, nil
		}
		go func() { deliveries <- output }()
	}
	return true, nil
}

// stopReceiving is a method sending a signal to stopReceivingChan
func (b *AWSSQSBroker) stopReceiving() {
	// Stop the receiving goroutine
	b.stopReceivingChan <- 1
}
<commit_msg>gofmt<commit_after>package brokers

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/log"
	"github.com/RichardKnop/machinery/v1/tasks"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
)

// AWSSQSBroker represents a AWS SQS broker
// There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html
type AWSSQSBroker struct {
	Broker
	processingWG      sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal
	receivingWG       sync.WaitGroup
	stopReceivingChan chan int
	sess              *session.Session
	service           sqsiface.SQSAPI
}

// NewAWSSQSBroker creates new Broker instance
func NewAWSSQSBroker(cnf *config.Config) Interface {
	b := &AWSSQSBroker{Broker: New(cnf)}
	if cnf.SQS != nil && cnf.SQS.Client != nil {
		// Use provided *SQS client
		b.service = cnf.SQS.Client
	} else {
		// Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials.
		// See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
		// Also, env AWS_REGION is also required
		b.sess = session.Must(session.NewSessionWithOptions(session.Options{
			SharedConfigState: session.SharedConfigEnable,
		}))
		b.service = sqs.New(b.sess)
	}

	return b
}

// GetPendingTasks returns a slice of task.Signatures waiting in the queue
func (b *AWSSQSBroker) GetPendingTasks(queue string) ([]*tasks.Signature, error) {
	return nil, errors.New("Not implemented")
}

// StartConsuming enters a loop and waits for incoming messages
func (b *AWSSQSBroker) StartConsuming(consumerTag string, concurrency int, taskProcessor TaskProcessor) (bool, error) {
	b.startConsuming(consumerTag, taskProcessor)
	qURL := b.defaultQueueURL()
	deliveries := make(chan *sqs.ReceiveMessageOutput)

	b.stopReceivingChan = make(chan int)
	b.receivingWG.Add(1)

	go func() {
		defer b.receivingWG.Done()

		log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C")

		for {
			select {
			// A way to stop this goroutine from b.StopConsuming
			case <-b.stopReceivingChan:
				return
			default:
				output, err := b.receiveMessage(qURL)
				if err != nil {
					log.ERROR.Printf("Queue consume error: %s", err)
					continue
				}
				if len(output.Messages) == 0 {
					continue
				}

				deliveries <- output
			}

			whetherContinue, err := b.continueReceivingMessages(qURL, deliveries)
			if err != nil {
				log.ERROR.Printf("Error when receiving messages. Error: %v", err)
			}
			if whetherContinue == false {
				return
			}
		}
	}()

	if err := b.consume(deliveries, concurrency, taskProcessor); err != nil {
		return b.retry, err
	}

	return b.retry, nil
}

// StopConsuming quits the loop
func (b *AWSSQSBroker) StopConsuming() {
	b.stopConsuming()

	b.stopReceiving()

	// Waiting for any tasks being processed to finish
	b.processingWG.Wait()

	// Waiting for the receiving goroutine to have stopped
	b.receivingWG.Wait()
}

// Publish places a new message on the default queue
func (b *AWSSQSBroker) Publish(signature *tasks.Signature) error {

	msg, err := json.Marshal(signature)
	if err != nil {
		return fmt.Errorf("JSON marshal error: %s", err)
	}

	// Check that signature.RoutingKey is set, if not switch to DefaultQueue
	AdjustRoutingKey(b, signature)

	MsgInput := &sqs.SendMessageInput{
		MessageBody: aws.String(string(msg)),
		QueueUrl:    aws.String(b.cnf.Broker + "/" + signature.RoutingKey),
	}

	// if this is a fifo queue, there needs to be some additional parameters.
	if strings.HasSuffix(signature.RoutingKey, ".fifo") {
		// Use Machinery's signature Task UUID as SQS Message Group ID.
		MsgDedupID := signature.UUID
		MsgInput.MessageDeduplicationId = aws.String(MsgDedupID)

		// Use Machinery's signature Group UUID as SQS Message Group ID.
		MsgGroupID := signature.GroupUUID
		MsgInput.MessageGroupId = aws.String(MsgGroupID)
	}

	// Check the ETA signature field, if it is set and it is in the future,
	// and is not a fifo queue, set a delay in seconds for the task.
	if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") {
		now := time.Now().UTC()

		if signature.ETA.After(now) {
			MsgInput.DelaySeconds = aws.Int64(signature.ETA.Unix() - now.Unix())
		}
	}

	result, err := b.service.SendMessage(MsgInput)

	if err != nil {
		log.ERROR.Printf("Error when sending a message: %v", err)
		return err

	}
	log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId)
	return nil

}

// consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal
func (b *AWSSQSBroker) consume(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor TaskProcessor) error {
	pool := make(chan struct{}, concurrency)

	// initialize worker pool with maxWorkers workers
	go func() {
		b.initializePool(pool, concurrency)
	}()

	errorsChan := make(chan error)

	for {
		whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan)
		if err != nil {
			return err
		}
		if whetherContinue == false {
			return nil
		}
	}
}

// consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS
func (b *AWSSQSBroker) consumeOne(delivery *sqs.ReceiveMessageOutput, taskProcessor TaskProcessor) error {
	sig := new(tasks.Signature)
	if len(delivery.Messages) == 0 {
		log.ERROR.Printf("received an empty message, the delivery was %v", delivery)
		return errors.New("received empty message, the delivery is " + delivery.GoString())
	}
	msg := delivery.Messages[0].Body
	if err := json.Unmarshal([]byte(*msg), sig); err != nil {
		log.ERROR.Printf("unmarshal error. the delivery is %v", delivery)
		return err
	}

	// Delete message after consuming successfully
	err := b.deleteOne(delivery)
	if err != nil {
		log.ERROR.Printf("error when deleting the delivery. the delivery is %v", delivery)
	}

	// If the task is not registered, we requeue it,
	// there might be different workers for processing specific tasks
	if !b.IsTaskRegistered(sig.Name) {
		err := b.Publish(sig)
		if err != nil {
			return err
		}
		log.INFO.Printf("requeue a task to default queue: %v", sig)
		return nil
	}
	return taskProcessor.Process(sig)
}

// deleteOne is a method delete a delivery from AWS SQS
func (b *AWSSQSBroker) deleteOne(delivery *sqs.ReceiveMessageOutput) error {
	qURL := b.defaultQueueURL()
	_, err := b.service.DeleteMessage(&sqs.DeleteMessageInput{
		QueueUrl:      qURL,
		ReceiptHandle: delivery.Messages[0].ReceiptHandle,
	})

	if err != nil {
		return err
	}
	return nil
}

// defaultQueueURL is a method returns the default queue url
func (b *AWSSQSBroker) defaultQueueURL() *string {
	return aws.String(b.cnf.Broker + "/" + b.cnf.DefaultQueue)
}

// receiveMessage is a method receives a message from specified queue url
func (b *AWSSQSBroker) receiveMessage(qURL *string) (*sqs.ReceiveMessageOutput, error) {
	var waitTimeSeconds int
	if b.cnf.SQS != nil {
		waitTimeSeconds = b.cnf.SQS.WaitTimeSeconds
	} else {
		waitTimeSeconds = 0
	}
	result, err := b.service.ReceiveMessage(&sqs.ReceiveMessageInput{
		AttributeNames: []*string{
			aws.String(sqs.MessageSystemAttributeNameSentTimestamp),
		},
		MessageAttributeNames: []*string{
			aws.String(sqs.QueueAttributeNameAll),
		},
		QueueUrl:            qURL,
		MaxNumberOfMessages: aws.Int64(1),
		VisibilityTimeout:   aws.Int64(int64(b.cnf.ResultsExpireIn)), // 10 hours
		WaitTimeSeconds:     aws.Int64(int64(waitTimeSeconds)),
	})
	if err != nil {
		return nil, err
	}
	return result, err
}

// initializePool is a method which initializes concurrency pool
func (b *AWSSQSBroker) initializePool(pool chan struct{}, concurrency int) {
	for i := 0; i < concurrency; i++ {
		pool <- struct{}{}
	}
}

// consumeDeliveries is a method consuming deliveries from deliveries channel
func (b *AWSSQSBroker) consumeDeliveries(deliveries <-chan *sqs.ReceiveMessageOutput, concurrency int, taskProcessor TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) {
	select {
	case err := <-errorsChan:
		return false, err
	case d := <-deliveries:
		if concurrency > 0 {
			// get worker from pool (blocks until one is available)
			<-pool
		}

		b.processingWG.Add(1)

		// Consume the task inside a goroutine so multiple tasks
		// can be processed concurrently
		go func() {

			if err := b.consumeOne(d, taskProcessor); err != nil {
				errorsChan <- err
			}

			b.processingWG.Done()

			if concurrency > 0 {
				// give worker back to pool
				pool <- struct{}{}
			}
		}()
	case <-b.stopChan:
		return false, nil
	}
	return true, nil
}

// continueReceivingMessages is a method returns a continue signal
func (b *AWSSQSBroker) continueReceivingMessages(qURL *string, deliveries chan *sqs.ReceiveMessageOutput) (bool, error) {
	select {
	// A way to stop this goroutine from b.StopConsuming
	case <-b.stopReceivingChan:
		return false, nil
	default:
		output, err := b.receiveMessage(qURL)
		if err != nil {
			return true, err
		}
		if len(output.Messages) == 0 {
			return true, nil
		}
		go func() { deliveries <- output }()
	}
	return true, nil
}

// stopReceiving is a method sending a signal to stopReceivingChan
func (b *AWSSQSBroker) stopReceiving() {
	// Stop the receiving goroutine
	b.stopReceivingChan <- 1
}
<|endoftext|>


<commit_before>package parser

// line parsers are dispatch calls that parse a single unit of text into a
// Node object which contains the whole statement. Dockerfiles have varied
// (but not usually unique, see ONBUILD for a unique example) parsing rules
// per-command, and these unify the processing in a way that makes it
// manageable.

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/docker/docker/builder/dockerfile/command"
)

var (
	errDockerfileNotStringArray = errors.New("When using JSON array syntax, arrays must be comprised of strings only.")
)

const (
	commandLabel = "LABEL"
)

// ignore the current argument. This will still leave a command parsed, but
// will not incorporate the arguments into the ast.
func parseIgnore(rest string, d *Directive) (*Node, map[string]bool, error) {
	return &Node{}, nil, nil
}

// used for onbuild. Could potentially be used for anything that represents a
// statement with sub-statements.
//
// ONBUILD RUN foo bar -> (onbuild (run foo bar))
//
func parseSubCommand(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	child, err := newNodeFromLine(rest, d)
	if err != nil {
		return nil, nil, err
	}

	return &Node{Children: []*Node{child}}, nil, nil
}

// helper to parse words (i.e space delimited or quoted strings) in a statement.
// The quotes are preserved as part of this function and they are stripped later
// as part of processWords().
func parseWords(rest string, d *Directive) []string {
	const (
		inSpaces = iota // looking for start of a word
		inWord
		inQuote
	)

	words := []string{}
	phase := inSpaces
	word := ""
	quote := '\000'
	blankOK := false
	var ch rune
	var chWidth int

	for pos := 0; pos <= len(rest); pos += chWidth {
		if pos != len(rest) {
			ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
		}

		if phase == inSpaces { // Looking for start of word
			if pos == len(rest) { // end of input
				break
			}
			if unicode.IsSpace(ch) { // skip spaces
				continue
			}
			phase = inWord // found it, fall through
		}
		if (phase == inWord || phase == inQuote) && (pos == len(rest)) {
			if blankOK || len(word) > 0 {
				words = append(words, word)
			}
			break
		}
		if phase == inWord {
			if unicode.IsSpace(ch) {
				phase = inSpaces
				if blankOK || len(word) > 0 {
					words = append(words, word)
				}
				word = ""
				blankOK = false
				continue
			}
			if ch == '\'' || ch == '"' {
				quote = ch
				blankOK = true
				phase = inQuote
			}
			if ch == d.escapeToken {
				if pos+chWidth == len(rest) {
					continue // just skip an escape token at end of line
				}
				// If we're not quoted and we see an escape token, then always just
				// add the escape token plus the char to the word, even if the char
				// is a quote.
				word += string(ch)
				pos += chWidth
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
			continue
		}
		if phase == inQuote {
			if ch == quote {
				phase = inWord
			}
			// The escape token is special except for ' quotes - can't escape anything for '
			if ch == d.escapeToken && quote != '\'' {
				if pos+chWidth == len(rest) {
					phase = inWord
					continue // just skip the escape token at end
				}
				pos += chWidth
				word += string(ch)
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
		}
	}

	return words
}

// parse environment like statements. Note that this does *not* handle
// variable interpolation, which will be handled in the evaluator.
func parseNameVal(rest string, key string, d *Directive) (*Node, error) {
	// This is kind of tricky because we need to support the old
	// variant:   KEY name value
	// as well as the new one:    KEY name=value ...
	// The trigger to know which one is being used will be whether we hit
	// a space or = first.  space ==> old, "=" ==> new

	words := parseWords(rest, d)
	if len(words) == 0 {
		return nil, nil
	}

	// Old format (KEY name value)
	if !strings.Contains(words[0], "=") {
		parts := tokenWhitespace.Split(rest, 2)
		if len(parts) < 2 {
			return nil, fmt.Errorf(key + " must have two arguments")
		}
		return newKeyValueNode(parts[0], parts[1]), nil
	}

	var rootNode *Node
	var prevNode *Node
	for _, word := range words {
		if !strings.Contains(word, "=") {
			return nil, fmt.Errorf("Syntax error - can't find = in %q. Must be of the form: name=value", word)
		}

		parts := strings.SplitN(word, "=", 2)
		node := newKeyValueNode(parts[0], parts[1])
		rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode)
	}

	return rootNode, nil
}

func newKeyValueNode(key, value string) *Node {
	return &Node{
		Value: key,
		Next:  &Node{Value: value},
	}
}

func appendKeyValueNode(node, rootNode, prevNode *Node) (*Node, *Node) {
	if rootNode == nil {
		rootNode = node
	}
	if prevNode != nil {
		prevNode.Next = node
	}

	prevNode = node.Next
	return rootNode, prevNode
}

func parseEnv(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, err := parseNameVal(rest, "ENV", d)
	return node, nil, err
}

func parseLabel(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, err := parseNameVal(rest, commandLabel, d)
	return node, nil, err
}

// NodeFromLabels returns a Node for the injected labels
func NodeFromLabels(labels map[string]string) *Node {
	keys := []string{}
	for key := range labels {
		keys = append(keys, key)
	}
	// Sort the label to have a repeatable order
	sort.Strings(keys)

	labelPairs := []string{}
	var rootNode *Node
	var prevNode *Node
	for _, key := range keys {
		value := labels[key]
		labelPairs = append(labelPairs, fmt.Sprintf("%q='%s'", key, value))
		// Value must be single quoted to prevent env variable expansion
		// See https://github.com/docker/docker/issues/26027
		node := newKeyValueNode(key, "'"+value+"'")
		rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode)
	}

	return &Node{
		Value:    command.Label,
		Original: commandLabel + " " + strings.Join(labelPairs, " "),
		Next:     rootNode,
	}
}

// parses a statement containing one or more keyword definition(s) and/or
// value assignments, like `name1 name2= name3="" name4=value`.
// Note that this is a stricter format than the old format of assignment,
// allowed by parseNameVal(), in a way that this only allows assignment of the
// form `keyword=[<value>]` like  `name2=`, `name3=""`, and `name4=value` above.
// In addition, a keyword definition alone is of the form `keyword` like `name1`
// above. And the assignments `name2=` and `name3=""` are equivalent and
// assign an empty value to the respective keywords.
func parseNameOrNameVal(rest string, d *Directive) (*Node, map[string]bool, error) {
	words := parseWords(rest, d)
	if len(words) == 0 {
		return nil, nil, nil
	}

	var (
		rootnode *Node
		prevNode *Node
	)
	for i, word := range words {
		node := &Node{}
		node.Value = word
		if i == 0 {
			rootnode = node
		} else {
			prevNode.Next = node
		}
		prevNode = node
	}

	return rootnode, nil, nil
}

// parses a whitespace-delimited set of arguments. The result is effectively a
// linked list of string arguments.
func parseStringsWhitespaceDelimited(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	node := &Node{}
	rootnode := node
	prevnode := node
	for _, str := range tokenWhitespace.Split(rest, -1) { // use regexp
		prevnode = node
		node.Value = str
		node.Next = &Node{}
		node = node.Next
	}

	// XXX to get around regexp.Split *always* providing an empty string at the
	// end due to how our loop is constructed, nil out the last node in the
	// chain.
	prevnode.Next = nil

	return rootnode, nil, nil
}

// parseString just wraps the string in quotes and returns a working node.
func parseString(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}
	n := &Node{}
	n.Value = rest
	return n, nil, nil
}

// parseJSON converts JSON arrays to an AST.
func parseJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
	if !strings.HasPrefix(rest, "[") {
		return nil, nil, fmt.Errorf(`Error parsing "%s" as a JSON array`, rest)
	}

	var myJSON []interface{}
	if err := json.NewDecoder(strings.NewReader(rest)).Decode(&myJSON); err != nil {
		return nil, nil, err
	}

	var top, prev *Node
	for _, str := range myJSON {
		s, ok := str.(string)
		if !ok {
			return nil, nil, errDockerfileNotStringArray
		}

		node := &Node{Value: s}
		if prev == nil {
			top = node
		} else {
			prev.Next = node
		}
		prev = node
	}

	return top, map[string]bool{"json": true}, nil
}

// parseMaybeJSON determines if the argument appears to be a JSON array. If
// so, passes to parseJSON; if not, quotes the result and returns a single
// node.
func parseMaybeJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	node, attrs, err := parseJSON(rest, d)

	if err == nil {
		return node, attrs, nil
	}
	if err == errDockerfileNotStringArray {
		return nil, nil, err
	}

	node = &Node{}
	node.Value = rest
	return node, nil, nil
}

// parseMaybeJSONToList determines if the argument appears to be a JSON array. If
// so, passes to parseJSON; if not, attempts to parse it as a whitespace
// delimited string.
func parseMaybeJSONToList(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, attrs, err := parseJSON(rest, d)

	if err == nil {
		return node, attrs, nil
	}
	if err == errDockerfileNotStringArray {
		return nil, nil, err
	}

	return parseStringsWhitespaceDelimited(rest, d)
}

// The HEALTHCHECK command is like parseMaybeJSON, but has an extra type argument.
func parseHealthConfig(rest string, d *Directive) (*Node, map[string]bool, error) {
	// Find end of first argument
	var sep int
	for ; sep < len(rest); sep++ {
		if unicode.IsSpace(rune(rest[sep])) {
			break
		}
	}
	next := sep
	for ; next < len(rest); next++ {
		if !unicode.IsSpace(rune(rest[next])) {
			break
		}
	}

	if sep == 0 {
		return nil, nil, nil
	}

	typ := rest[:sep]
	cmd, attrs, err := parseMaybeJSON(rest[next:], d)
	if err != nil {
		return nil, nil, err
	}

	return &Node{Value: typ, Next: cmd}, attrs, err
}
<commit_msg>Fix golint errors.<commit_after>package parser

// line parsers are dispatch calls that parse a single unit of text into a
// Node object which contains the whole statement. Dockerfiles have varied
// (but not usually unique, see ONBUILD for a unique example) parsing rules
// per-command, and these unify the processing in a way that makes it
// manageable.

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/docker/docker/builder/dockerfile/command"
)

var (
	errDockerfileNotStringArray = errors.New("when using JSON array syntax, arrays must be comprised of strings only")
)

const (
	commandLabel = "LABEL"
)

// ignore the current argument. This will still leave a command parsed, but
// will not incorporate the arguments into the ast.
func parseIgnore(rest string, d *Directive) (*Node, map[string]bool, error) {
	return &Node{}, nil, nil
}

// used for onbuild. Could potentially be used for anything that represents a
// statement with sub-statements.
//
// ONBUILD RUN foo bar -> (onbuild (run foo bar))
//
func parseSubCommand(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	child, err := newNodeFromLine(rest, d)
	if err != nil {
		return nil, nil, err
	}

	return &Node{Children: []*Node{child}}, nil, nil
}

// helper to parse words (i.e space delimited or quoted strings) in a statement.
// The quotes are preserved as part of this function and they are stripped later
// as part of processWords().
func parseWords(rest string, d *Directive) []string {
	const (
		inSpaces = iota // looking for start of a word
		inWord
		inQuote
	)

	words := []string{}
	phase := inSpaces
	word := ""
	quote := '\000'
	blankOK := false
	var ch rune
	var chWidth int

	for pos := 0; pos <= len(rest); pos += chWidth {
		if pos != len(rest) {
			ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
		}

		if phase == inSpaces { // Looking for start of word
			if pos == len(rest) { // end of input
				break
			}
			if unicode.IsSpace(ch) { // skip spaces
				continue
			}
			phase = inWord // found it, fall through
		}
		if (phase == inWord || phase == inQuote) && (pos == len(rest)) {
			if blankOK || len(word) > 0 {
				words = append(words, word)
			}
			break
		}
		if phase == inWord {
			if unicode.IsSpace(ch) {
				phase = inSpaces
				if blankOK || len(word) > 0 {
					words = append(words, word)
				}
				word = ""
				blankOK = false
				continue
			}
			if ch == '\'' || ch == '"' {
				quote = ch
				blankOK = true
				phase = inQuote
			}
			if ch == d.escapeToken {
				if pos+chWidth == len(rest) {
					continue // just skip an escape token at end of line
				}
				// If we're not quoted and we see an escape token, then always just
				// add the escape token plus the char to the word, even if the char
				// is a quote.
				word += string(ch)
				pos += chWidth
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
			continue
		}
		if phase == inQuote {
			if ch == quote {
				phase = inWord
			}
			// The escape token is special except for ' quotes - can't escape anything for '
			if ch == d.escapeToken && quote != '\'' {
				if pos+chWidth == len(rest) {
					phase = inWord
					continue // just skip the escape token at end
				}
				pos += chWidth
				word += string(ch)
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
		}
	}

	return words
}

// parse environment like statements. Note that this does *not* handle
// variable interpolation, which will be handled in the evaluator.
func parseNameVal(rest string, key string, d *Directive) (*Node, error) {
	// This is kind of tricky because we need to support the old
	// variant:   KEY name value
	// as well as the new one:    KEY name=value ...
	// The trigger to know which one is being used will be whether we hit
	// a space or = first.  space ==> old, "=" ==> new

	words := parseWords(rest, d)
	if len(words) == 0 {
		return nil, nil
	}

	// Old format (KEY name value)
	if !strings.Contains(words[0], "=") {
		parts := tokenWhitespace.Split(rest, 2)
		if len(parts) < 2 {
			return nil, fmt.Errorf(key + " must have two arguments")
		}
		return newKeyValueNode(parts[0], parts[1]), nil
	}

	var rootNode *Node
	var prevNode *Node
	for _, word := range words {
		if !strings.Contains(word, "=") {
			return nil, fmt.Errorf("Syntax error - can't find = in %q. Must be of the form: name=value", word)
		}

		parts := strings.SplitN(word, "=", 2)
		node := newKeyValueNode(parts[0], parts[1])
		rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode)
	}

	return rootNode, nil
}

func newKeyValueNode(key, value string) *Node {
	return &Node{
		Value: key,
		Next:  &Node{Value: value},
	}
}

func appendKeyValueNode(node, rootNode, prevNode *Node) (*Node, *Node) {
	if rootNode == nil {
		rootNode = node
	}
	if prevNode != nil {
		prevNode.Next = node
	}

	prevNode = node.Next
	return rootNode, prevNode
}

func parseEnv(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, err := parseNameVal(rest, "ENV", d)
	return node, nil, err
}

func parseLabel(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, err := parseNameVal(rest, commandLabel, d)
	return node, nil, err
}

// NodeFromLabels returns a Node for the injected labels
func NodeFromLabels(labels map[string]string) *Node {
	keys := []string{}
	for key := range labels {
		keys = append(keys, key)
	}
	// Sort the label to have a repeatable order
	sort.Strings(keys)

	labelPairs := []string{}
	var rootNode *Node
	var prevNode *Node
	for _, key := range keys {
		value := labels[key]
		labelPairs = append(labelPairs, fmt.Sprintf("%q='%s'", key, value))
		// Value must be single quoted to prevent env variable expansion
		// See https://github.com/docker/docker/issues/26027
		node := newKeyValueNode(key, "'"+value+"'")
		rootNode, prevNode = appendKeyValueNode(node, rootNode, prevNode)
	}

	return &Node{
		Value:    command.Label,
		Original: commandLabel + " " + strings.Join(labelPairs, " "),
		Next:     rootNode,
	}
}

// parses a statement containing one or more keyword definition(s) and/or
// value assignments, like `name1 name2= name3="" name4=value`.
// Note that this is a stricter format than the old format of assignment,
// allowed by parseNameVal(), in a way that this only allows assignment of the
// form `keyword=[<value>]` like  `name2=`, `name3=""`, and `name4=value` above.
// In addition, a keyword definition alone is of the form `keyword` like `name1`
// above. And the assignments `name2=` and `name3=""` are equivalent and
// assign an empty value to the respective keywords.
func parseNameOrNameVal(rest string, d *Directive) (*Node, map[string]bool, error) {
	words := parseWords(rest, d)
	if len(words) == 0 {
		return nil, nil, nil
	}

	var (
		rootnode *Node
		prevNode *Node
	)
	for i, word := range words {
		node := &Node{}
		node.Value = word
		if i == 0 {
			rootnode = node
		} else {
			prevNode.Next = node
		}
		prevNode = node
	}

	return rootnode, nil, nil
}

// parses a whitespace-delimited set of arguments. The result is effectively a
// linked list of string arguments.
func parseStringsWhitespaceDelimited(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	node := &Node{}
	rootnode := node
	prevnode := node
	for _, str := range tokenWhitespace.Split(rest, -1) { // use regexp
		prevnode = node
		node.Value = str
		node.Next = &Node{}
		node = node.Next
	}

	// XXX to get around regexp.Split *always* providing an empty string at the
	// end due to how our loop is constructed, nil out the last node in the
	// chain.
	prevnode.Next = nil

	return rootnode, nil, nil
}

// parseString just wraps the string in quotes and returns a working node.
func parseString(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}
	n := &Node{}
	n.Value = rest
	return n, nil, nil
}

// parseJSON converts JSON arrays to an AST.
func parseJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
	if !strings.HasPrefix(rest, "[") {
		return nil, nil, fmt.Errorf(`Error parsing "%s" as a JSON array`, rest)
	}

	var myJSON []interface{}
	if err := json.NewDecoder(strings.NewReader(rest)).Decode(&myJSON); err != nil {
		return nil, nil, err
	}

	var top, prev *Node
	for _, str := range myJSON {
		s, ok := str.(string)
		if !ok {
			return nil, nil, errDockerfileNotStringArray
		}

		node := &Node{Value: s}
		if prev == nil {
			top = node
		} else {
			prev.Next = node
		}
		prev = node
	}

	return top, map[string]bool{"json": true}, nil
}

// parseMaybeJSON determines if the argument appears to be a JSON array. If
// so, passes to parseJSON; if not, quotes the result and returns a single
// node.
func parseMaybeJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
	if rest == "" {
		return nil, nil, nil
	}

	node, attrs, err := parseJSON(rest, d)

	if err == nil {
		return node, attrs, nil
	}
	if err == errDockerfileNotStringArray {
		return nil, nil, err
	}

	node = &Node{}
	node.Value = rest
	return node, nil, nil
}

// parseMaybeJSONToList determines if the argument appears to be a JSON array. If
// so, passes to parseJSON; if not, attempts to parse it as a whitespace
// delimited string.
func parseMaybeJSONToList(rest string, d *Directive) (*Node, map[string]bool, error) {
	node, attrs, err := parseJSON(rest, d)

	if err == nil {
		return node, attrs, nil
	}
	if err == errDockerfileNotStringArray {
		return nil, nil, err
	}

	return parseStringsWhitespaceDelimited(rest, d)
}

// The HEALTHCHECK command is like parseMaybeJSON, but has an extra type argument.
func parseHealthConfig(rest string, d *Directive) (*Node, map[string]bool, error) {
	// Find end of first argument
	var sep int
	for ; sep < len(rest); sep++ {
		if unicode.IsSpace(rune(rest[sep])) {
			break
		}
	}
	next := sep
	for ; next < len(rest); next++ {
		if !unicode.IsSpace(rune(rest[next])) {
			break
		}
	}

	if sep == 0 {
		return nil, nil, nil
	}

	typ := rest[:sep]
	cmd, attrs, err := parseMaybeJSON(rest[next:], d)
	if err != nil {
		return nil, nil, err
	}

	return &Node{Value: typ, Next: cmd}, attrs, err
}
<|endoftext|>


<commit_before>// Generate Plugins is a small program that updates the lists of plugins in
// command/plugin.go so they will be compiled into the main packer binary.
//
// See https://github.com/mitchellh/packer/pull/2608 for details.
package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

const target = "command/plugin.go"

func main() {
	wd, _ := os.Getwd()
	if filepath.Base(wd) != "packer" {
		os.Chdir("..")
		wd, _ = os.Getwd()
		if filepath.Base(wd) != "packer" {
			log.Fatalf("This program must be invoked in the packer project root; in %s", wd)
		}
	}

	builders, err := discoverBuilders()
	if err != nil {
		log.Fatalf("Failed to discover builders: %s", err)
	}

	provisioners, err := discoverProvisioners()
	if err != nil {
		log.Fatalf("Failed to discover provisioners: %s", err)
	}

	postProcessors, err := discoverPostProcessors()
	if err != nil {
		log.Fatalf("Failed to discover post processors: %s", err)
	}

	output := source
	output = strings.Replace(output, "IMPORTS", makeImports(builders, provisioners, postProcessors), 1)
	output = strings.Replace(output, "BUILDERS", makeMap("Builders", "Builder", builders), 1)
	output = strings.Replace(output, "PROVISIONERS", makeMap("Provisioners", "Provisioner", provisioners), 1)
	output = strings.Replace(output, "POSTPROCESSORS", makeMap("PostProcessors", "PostProcessor", postProcessors), 1)

	// TODO sort the lists of plugins so we are not subjected to random OS ordering of the plugin lists
	// TODO format the file

	file, err := os.Create(target)
	if err != nil {
		log.Fatalf("Failed to open %s for writing: %s", target, err)
	}
	file.WriteString(output)
	file.Close()

	log.Printf("Generated %s", target)
}

type plugin struct {
	Package    string
	PluginName string
	TypeName   string
	Path       string
	ImportName string
}

// makeMap creates a map named Name with type packer.Name that looks something
// like this:
//
// var Builders = map[string]packer.Builder{
// 	"amazon-chroot":   new(chroot.Builder),
// 	"amazon-ebs":      new(ebs.Builder),
// 	"amazon-instance": new(instance.Builder),
func makeMap(varName, varType string, items []plugin) string {
	output := ""

	output += fmt.Sprintf("var %s = map[string]packer.%s{\n", varName, varType)
	for _, item := range items {
		output += fmt.Sprintf("\t\"%s\":   new(%s.%s),\n", item.PluginName, item.ImportName, item.TypeName)
	}
	output += "}\n"
	return output
}

func makeImports(builders, provisioners, postProcessors []plugin) string {
	plugins := []string{}

	for _, builder := range builders {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", builder.ImportName, builder.Path))
	}

	for _, provisioner := range provisioners {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", provisioner.ImportName, provisioner.Path))
	}

	for _, postProcessor := range postProcessors {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", postProcessor.ImportName, postProcessor.Path))
	}

	// Make things pretty
	sort.Strings(plugins)

	return strings.Join(plugins, "")
}

// listDirectories recursively lists directories under the specified path
func listDirectories(path string) ([]string, error) {
	names := []string{}
	items, err := ioutil.ReadDir(path)
	if err != nil {
		return names, err
	}

	for _, item := range items {
		// We only want directories
		if item.IsDir() {
			currentDir := filepath.Join(path, item.Name())
			names = append(names, currentDir)

			// Do some recursion
			subNames, err := listDirectories(currentDir)
			if err == nil {
				names = append(names, subNames...)
			}
		}
	}

	return names, nil
}

// deriveName determines the name of the plugin (what you'll see in a packer
// template) based on the filesystem path. We use two rules:
//
// Start with                     -> builder/virtualbox/iso
//
// 1. Strip the root              -> virtualbox/iso
// 2. Switch slash / to dash -    -> virtualbox-iso
func deriveName(root, full string) string {
	short, _ := filepath.Rel(root, full)
	bits := strings.Split(short, string(os.PathSeparator))
	return strings.Join(bits, "-")
}

// deriveImport will build a unique import identifier based on packageName and
// the result of deriveName()
//
// This will be something like    -> virtualboxisobuilder
//
// Which is long, but deterministic and unique.
func deriveImport(typeName, derivedName string) string {
	return strings.Replace(derivedName, "-", "", -1) + strings.ToLower(typeName)
}

// discoverTypesInPath searches for types of typeID in path and returns a list
// of plugins it finds.
func discoverTypesInPath(path, typeID string) ([]plugin, error) {
	postProcessors := []plugin{}

	dirs, err := listDirectories(path)
	if err != nil {
		return postProcessors, err
	}

	for _, dir := range dirs {
		fset := token.NewFileSet()
		goPackages, err := parser.ParseDir(fset, dir, nil, parser.AllErrors)
		if err != nil {
			return postProcessors, fmt.Errorf("Failed parsing directory %s: %s", dir, err)
		}

		for _, goPackage := range goPackages {
			ast.PackageExports(goPackage)
			ast.Inspect(goPackage, func(n ast.Node) bool {
				switch x := n.(type) {
				case *ast.TypeSpec:
					if x.Name.Name == typeID {
						derivedName := deriveName(path, dir)
						postProcessors = append(postProcessors, plugin{
							Package:    goPackage.Name,
							PluginName: derivedName,
							ImportName: deriveImport(x.Name.Name, derivedName),
							TypeName:   x.Name.Name,
							Path:       dir,
						})
						// The AST stops parsing when we return false. Once we
						// find the symbol we want we can stop parsing.

						// DEBUG:
						// fmt.Printf("package %#v\n", goPackage)
						return false
					}
				}
				return true
			})
		}
	}

	return postProcessors, nil
}

func discoverBuilders() ([]plugin, error) {
	path := "./builder"
	typeID := "Builder"
	return discoverTypesInPath(path, typeID)
}

func discoverProvisioners() ([]plugin, error) {
	path := "./provisioner"
	typeID := "Provisioner"
	return discoverTypesInPath(path, typeID)
}

func discoverPostProcessors() ([]plugin, error) {
	path := "./post-processor"
	typeID := "PostProcessor"
	return discoverTypesInPath(path, typeID)
}

const source = `//
// This file is automatically generated by scripts/generate-plugins.go -- Do not edit!
//

package command

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"

IMPORTS
)

type PluginCommand struct {
	Meta
}

BUILDERS

PROVISIONERS

POSTPROCESSORS

var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")

func (c *PluginCommand) Run(args []string) int {
	// This is an internal call (users should not call this directly) so we're
	// not going to do much input validation. If there's a problem we'll often
	// just crash. Error handling should be added to facilitate debugging.
	log.Printf("args: %#v", args)
	if len(args) != 1 {
		c.Ui.Error("Wrong number of args")
		return 1
	}

	// Plugin will match something like "packer-builder-amazon-ebs"
	parts := pluginRegexp.FindStringSubmatch(args[0])
	if len(parts) != 3 {
		c.Ui.Error(fmt.Sprintf("Error parsing plugin argument [DEBUG]: %#v", parts))
		return 1
	}
	pluginType := parts[1] // capture group 1 (builder|post-processor|provisioner)
	pluginName := parts[2] // capture group 2 (.+)

	server, err := plugin.Server()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
		return 1
	}

	switch pluginType {
	case "builder":
		builder, found := Builders[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
			return 1
		}
		server.RegisterBuilder(builder)
	case "provisioner":
		provisioner, found := Provisioners[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
			return 1
		}
		server.RegisterProvisioner(provisioner)
	case "post-processor":
		postProcessor, found := PostProcessors[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
			return 1
		}
		server.RegisterPostProcessor(postProcessor)
	}

	server.Serve()

	return 0
}

func (*PluginCommand) Help() string {
	helpText := ` + "`" + `
Usage: packer plugin PLUGIN

  Runs an internally-compiled version of a plugin from the packer binary.

  NOTE: this is an internal command and you should not call it yourself.
` + "`" + `

	return strings.TrimSpace(helpText)
}

func (c *PluginCommand) Synopsis() string {
	return "internal plugin command"
}
`
<commit_msg>Add some documentation to generate-plugins<commit_after>// Generate Plugins is a small program that updates the lists of plugins in
// command/plugin.go so they will be compiled into the main packer binary.
//
// See https://github.com/mitchellh/packer/pull/2608 for details.
package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

const target = "command/plugin.go"

func main() {
	// Normally this is run via go:generate from the command folder so we need
	// to cd .. first. But when developing it's easier to use go run, so we'll
	// support that too.
	wd, _ := os.Getwd()
	if filepath.Base(wd) != "packer" {
		os.Chdir("..")
		wd, _ = os.Getwd()
		if filepath.Base(wd) != "packer" {
			log.Fatalf("This program must be invoked in the packer project root; in %s", wd)
		}
	}

	// Collect all of the data we need about plugins we have in the project
	builders, err := discoverBuilders()
	if err != nil {
		log.Fatalf("Failed to discover builders: %s", err)
	}

	provisioners, err := discoverProvisioners()
	if err != nil {
		log.Fatalf("Failed to discover provisioners: %s", err)
	}

	postProcessors, err := discoverPostProcessors()
	if err != nil {
		log.Fatalf("Failed to discover post processors: %s", err)
	}

	// Do some simple code generation and templating
	output := source
	output = strings.Replace(output, "IMPORTS", makeImports(builders, provisioners, postProcessors), 1)
	output = strings.Replace(output, "BUILDERS", makeMap("Builders", "Builder", builders), 1)
	output = strings.Replace(output, "PROVISIONERS", makeMap("Provisioners", "Provisioner", provisioners), 1)
	output = strings.Replace(output, "POSTPROCESSORS", makeMap("PostProcessors", "PostProcessor", postProcessors), 1)

	// TODO sort the lists of plugins so we are not subjected to random OS ordering of the plugin lists
	// TODO format the file

	// Write our generated code to the command/plugin.go file
	file, err := os.Create(target)
	defer file.Close()
	if err != nil {
		log.Fatalf("Failed to open %s for writing: %s", target, err)
	}

	_, err = file.WriteString(output)
	if err != nil {
		log.Fatalf("Failed writing to %s: %s", target, err)
	}

	log.Printf("Generated %s", target)
}

type plugin struct {
	Package    string // This plugin's package name (iso)
	PluginName string // Name of plugin (vmware-iso)
	TypeName   string // Type of plugin (builder)
	Path       string // Path relative to packer root (builder/vmware/iso)
	ImportName string // PluginName+TypeName (vmwareisobuilder)
}

// makeMap creates a map named Name with type packer.Name that looks something
// like this:
//
// var Builders = map[string]packer.Builder{
// 	"amazon-chroot":   new(chroot.Builder),
// 	"amazon-ebs":      new(ebs.Builder),
// 	"amazon-instance": new(instance.Builder),
func makeMap(varName, varType string, items []plugin) string {
	output := ""

	output += fmt.Sprintf("var %s = map[string]packer.%s{\n", varName, varType)
	for _, item := range items {
		output += fmt.Sprintf("\t\"%s\":   new(%s.%s),\n", item.PluginName, item.ImportName, item.TypeName)
	}
	output += "}\n"
	return output
}

func makeImports(builders, provisioners, postProcessors []plugin) string {
	plugins := []string{}

	for _, builder := range builders {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", builder.ImportName, builder.Path))
	}

	for _, provisioner := range provisioners {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", provisioner.ImportName, provisioner.Path))
	}

	for _, postProcessor := range postProcessors {
		plugins = append(plugins, fmt.Sprintf("\t%s \"github.com/mitchellh/packer/%s\"\n", postProcessor.ImportName, postProcessor.Path))
	}

	// Make things pretty
	sort.Strings(plugins)

	return strings.Join(plugins, "")
}

// listDirectories recursively lists directories under the specified path
func listDirectories(path string) ([]string, error) {
	names := []string{}
	items, err := ioutil.ReadDir(path)
	if err != nil {
		return names, err
	}

	for _, item := range items {
		// We only want directories
		if item.IsDir() {
			currentDir := filepath.Join(path, item.Name())
			names = append(names, currentDir)

			// Do some recursion
			subNames, err := listDirectories(currentDir)
			if err == nil {
				names = append(names, subNames...)
			}
		}
	}

	return names, nil
}

// deriveName determines the name of the plugin (what you'll see in a packer
// template) based on the filesystem path. We use two rules:
//
// Start with                     -> builder/virtualbox/iso
//
// 1. Strip the root              -> virtualbox/iso
// 2. Switch slash / to dash -    -> virtualbox-iso
func deriveName(root, full string) string {
	short, _ := filepath.Rel(root, full)
	bits := strings.Split(short, string(os.PathSeparator))
	return strings.Join(bits, "-")
}

// deriveImport will build a unique import identifier based on packageName and
// the result of deriveName()
//
// This will be something like    -> virtualboxisobuilder
//
// Which is long, but deterministic and unique.
func deriveImport(typeName, derivedName string) string {
	return strings.Replace(derivedName, "-", "", -1) + strings.ToLower(typeName)
}

// discoverTypesInPath searches for types of typeID in path and returns a list
// of plugins it finds.
func discoverTypesInPath(path, typeID string) ([]plugin, error) {
	postProcessors := []plugin{}

	dirs, err := listDirectories(path)
	if err != nil {
		return postProcessors, err
	}

	for _, dir := range dirs {
		fset := token.NewFileSet()
		goPackages, err := parser.ParseDir(fset, dir, nil, parser.AllErrors)
		if err != nil {
			return postProcessors, fmt.Errorf("Failed parsing directory %s: %s", dir, err)
		}

		for _, goPackage := range goPackages {
			ast.PackageExports(goPackage)
			ast.Inspect(goPackage, func(n ast.Node) bool {
				switch x := n.(type) {
				case *ast.TypeSpec:
					if x.Name.Name == typeID {
						derivedName := deriveName(path, dir)
						postProcessors = append(postProcessors, plugin{
							Package:    goPackage.Name,
							PluginName: derivedName,
							ImportName: deriveImport(x.Name.Name, derivedName),
							TypeName:   x.Name.Name,
							Path:       dir,
						})
						// The AST stops parsing when we return false. Once we
						// find the symbol we want we can stop parsing.

						// DEBUG:
						// fmt.Printf("package %#v\n", goPackage)
						return false
					}
				}
				return true
			})
		}
	}

	return postProcessors, nil
}

func discoverBuilders() ([]plugin, error) {
	path := "./builder"
	typeID := "Builder"
	return discoverTypesInPath(path, typeID)
}

func discoverProvisioners() ([]plugin, error) {
	path := "./provisioner"
	typeID := "Provisioner"
	return discoverTypesInPath(path, typeID)
}

func discoverPostProcessors() ([]plugin, error) {
	path := "./post-processor"
	typeID := "PostProcessor"
	return discoverTypesInPath(path, typeID)
}

const source = `//
// This file is automatically generated by scripts/generate-plugins.go -- Do not edit!
//

package command

import (
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/mitchellh/packer/packer"
	"github.com/mitchellh/packer/packer/plugin"

IMPORTS
)

type PluginCommand struct {
	Meta
}

BUILDERS

PROVISIONERS

POSTPROCESSORS

var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)")

func (c *PluginCommand) Run(args []string) int {
	// This is an internal call (users should not call this directly) so we're
	// not going to do much input validation. If there's a problem we'll often
	// just crash. Error handling should be added to facilitate debugging.
	log.Printf("args: %#v", args)
	if len(args) != 1 {
		c.Ui.Error("Wrong number of args")
		return 1
	}

	// Plugin will match something like "packer-builder-amazon-ebs"
	parts := pluginRegexp.FindStringSubmatch(args[0])
	if len(parts) != 3 {
		c.Ui.Error(fmt.Sprintf("Error parsing plugin argument [DEBUG]: %#v", parts))
		return 1
	}
	pluginType := parts[1] // capture group 1 (builder|post-processor|provisioner)
	pluginName := parts[2] // capture group 2 (.+)

	server, err := plugin.Server()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error starting plugin server: %s", err))
		return 1
	}

	switch pluginType {
	case "builder":
		builder, found := Builders[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load builder: %s", pluginName))
			return 1
		}
		server.RegisterBuilder(builder)
	case "provisioner":
		provisioner, found := Provisioners[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load provisioner: %s", pluginName))
			return 1
		}
		server.RegisterProvisioner(provisioner)
	case "post-processor":
		postProcessor, found := PostProcessors[pluginName]
		if !found {
			c.Ui.Error(fmt.Sprintf("Could not load post-processor: %s", pluginName))
			return 1
		}
		server.RegisterPostProcessor(postProcessor)
	}

	server.Serve()

	return 0
}

func (*PluginCommand) Help() string {
	helpText := ` + "`" + `
Usage: packer plugin PLUGIN

  Runs an internally-compiled version of a plugin from the packer binary.

  NOTE: this is an internal command and you should not call it yourself.
` + "`" + `

	return strings.TrimSpace(helpText)
}

func (c *PluginCommand) Synopsis() string {
	return "internal plugin command"
}
`
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

// Count UTF-8 sequences in s.
// Assumes s is well-formed.
export func utflen(s string) int {
	n := 0;
	for i := 0; i < len(s); i++ {
		if s[i]&0xC0 != 0x80 {
			n++
		}
	}
	return n
}

// Split string into array of UTF-8 sequences (still strings)
export func explode(s string) *[]string {
	a := new([]string, utflen(s));
	j := 0;
	for i := 0; i < len(a); i++ {
		ej := j;
		ej++;
		for ej < len(s) && (s[ej]&0xC0) == 0x80 {
			ej++
		}
		a[i] = s[j:ej];
		j = ej
	}
	return a
}

// Count non-overlapping instances of sep in s.
export func count(s, sep string) int {
	if sep == "" {
		return utflen(s)+1
	}
	c := sep[0];
	n := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++;
			i += len(sep)-1
		}
	}
	return n
}

// Return index of first instance of sep in s.
export func index(s, sep string) int {
	if sep == "" {
		return 0
	}
	c := sep[0];
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			return i
		}
	}
	return -1
}

// Split string into list of strings at separators
export func split(s, sep string) *[]string {
	if sep == "" {
		return explode(s)
	}
	c := sep[0];
	start := 0;
	n := count(s, sep)+1;
	a := new([]string, n);
	na := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start:i];
			na++;
			start = i+len(sep);
			i += len(sep)-1
		}
	}
	a[na] = s[start:len(s)];
	return a
}

// Join list of strings with separators between them.
export func join(a *[]string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a)-1);
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := new([]byte, n);
	bp := 0;
	for i := 0; i < len(a); i++ {
		s := a[i];
		for j := 0; j < len(s); j++ {
			b[bp] = s[j];
			bp++
		}
		if i + 1 < len(a) {
			s = sep;
			for j := 0; j < len(s); j++ {
				b[bp] = s[j];
				bp++
			}
		}
	}
	return string(b)
}

// Convert decimal string to integer.
// TODO: Doesn't check for overflow.
export func atol(s string) (i int64, ok bool) {
	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off leading sign
	neg := false;
	if s[0] == '+' {
		s = s[1:len(s)]
	} else if s[0] == '-' {
		neg = true;
		s = s[1:len(s)]
	}

	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off zero
	if s == "0" {
		return 0, true
	}

	// otherwise, leading zero bad
	if s[0] == '0' {
		return 0, false
	}

	// parse number
	n := int64(0);
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return 0, false
		}
		n = n*10 + int64(s[i] - '0')
	}
	if neg {
		n = -n
	}
	return n, true
}

export func atoi(s string) (i int, ok bool) {
	ii, okok := atoi(s);
	i = int32(ii);
	return i, okok
}

export func itol(i int64) string {
	if i == 0 {
		return "0"
	}

	neg := false;	// negative
	u := uint(i);
	if i < 0 {
		neg = true;
		u = -u;
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0; u /= 10 {
		bp--;
		b[bp] = byte(u%10) + '0'
	}
	if neg {	// add sign
		bp--;
		b[bp] = '-'
	}

	// BUG return string(b[bp:len(b)])
	return string((&b)[bp:len(b)])
}

export func itoa(i int) string {
	return itol(int64(i));
}
<commit_msg>fix typo causing infinite recursion<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strings

// Count UTF-8 sequences in s.
// Assumes s is well-formed.
export func utflen(s string) int {
	n := 0;
	for i := 0; i < len(s); i++ {
		if s[i]&0xC0 != 0x80 {
			n++
		}
	}
	return n
}

// Split string into array of UTF-8 sequences (still strings)
export func explode(s string) *[]string {
	a := new([]string, utflen(s));
	j := 0;
	for i := 0; i < len(a); i++ {
		ej := j;
		ej++;
		for ej < len(s) && (s[ej]&0xC0) == 0x80 {
			ej++
		}
		a[i] = s[j:ej];
		j = ej
	}
	return a
}

// Count non-overlapping instances of sep in s.
export func count(s, sep string) int {
	if sep == "" {
		return utflen(s)+1
	}
	c := sep[0];
	n := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			n++;
			i += len(sep)-1
		}
	}
	return n
}

// Return index of first instance of sep in s.
export func index(s, sep string) int {
	if sep == "" {
		return 0
	}
	c := sep[0];
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			return i
		}
	}
	return -1
}

// Split string into list of strings at separators
export func split(s, sep string) *[]string {
	if sep == "" {
		return explode(s)
	}
	c := sep[0];
	start := 0;
	n := count(s, sep)+1;
	a := new([]string, n);
	na := 0;
	for i := 0; i+len(sep) <= len(s); i++ {
		if s[i] == c && (len(sep) == 1 || s[i:i+len(sep)] == sep) {
			a[na] = s[start:i];
			na++;
			start = i+len(sep);
			i += len(sep)-1
		}
	}
	a[na] = s[start:len(s)];
	return a
}

// Join list of strings with separators between them.
export func join(a *[]string, sep string) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a)-1);
	for i := 0; i < len(a); i++ {
		n += len(a[i])
	}

	b := new([]byte, n);
	bp := 0;
	for i := 0; i < len(a); i++ {
		s := a[i];
		for j := 0; j < len(s); j++ {
			b[bp] = s[j];
			bp++
		}
		if i + 1 < len(a) {
			s = sep;
			for j := 0; j < len(s); j++ {
				b[bp] = s[j];
				bp++
			}
		}
	}
	return string(b)
}

// Convert decimal string to integer.
// TODO: Doesn't check for overflow.
export func atol(s string) (i int64, ok bool) {
	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off leading sign
	neg := false;
	if s[0] == '+' {
		s = s[1:len(s)]
	} else if s[0] == '-' {
		neg = true;
		s = s[1:len(s)]
	}

	// empty string bad
	if len(s) == 0 {
		return 0, false
	}

	// pick off zero
	if s == "0" {
		return 0, true
	}

	// otherwise, leading zero bad
	if s[0] == '0' {
		return 0, false
	}

	// parse number
	n := int64(0);
	for i := 0; i < len(s); i++ {
		if s[i] < '0' || s[i] > '9' {
			return 0, false
		}
		n = n*10 + int64(s[i] - '0')
	}
	if neg {
		n = -n
	}
	return n, true
}

export func atoi(s string) (i int, ok bool) {
	ii, okok := atol(s);
	i = int32(ii);
	return i, okok
}

export func itol(i int64) string {
	if i == 0 {
		return "0"
	}

	neg := false;	// negative
	u := uint(i);
	if i < 0 {
		neg = true;
		u = -u;
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0; u /= 10 {
		bp--;
		b[bp] = byte(u%10) + '0'
	}
	if neg {	// add sign
		bp--;
		b[bp] = '-'
	}

	// BUG return string(b[bp:len(b)])
	return string((&b)[bp:len(b)])
}

export func itoa(i int) string {
	return itol(int64(i));
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"log"

	"github.com/aws/aws-lambda-go/lambda"
	alexa "github.com/ericdaugherty/alexa-skills-kit-golang"
)

var a = &alexa.Alexa{ApplicationID: "amzn1.ask.skill.<SKILL_ID>", RequestHandler: &HelloWorld{}, IgnoreApplicationID: true, IgnoreTimestamp: true}

const cardTitle = "HelloWorld"

// HelloWorld handles reqeusts from the HelloWorld skill.
type HelloWorld struct{}

// Handle processes calls from Lambda
func Handle(ctx context.Context, requestEnv *alexa.RequestEnvelope) (interface{}, error) {
	return a.ProcessRequest(requestEnv)
}

// OnSessionStarted called when a new session is created.
func (h *HelloWorld) OnSessionStarted(request *alexa.Request, session *alexa.Session, response *alexa.Response) error {

	log.Printf("OnSessionStarted requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	return nil
}

// OnLaunch called with a reqeust is received of type LaunchRequest
func (h *HelloWorld) OnLaunch(request *alexa.Request, session *alexa.Session, response *alexa.Response) error {
	speechText := "Welcome to the Alexa Skills Kit, you can say hello"

	log.Printf("OnLaunch requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	response.SetSimpleCard(cardTitle, speechText)
	response.SetOutputText(speechText)
	response.SetRepromptText(speechText)

	response.ShouldSessionEnd = true

	return nil
}

// OnIntent called with a reqeust is received of type IntentRequest
func (h *HelloWorld) OnIntent(request *alexa.Request, session *alexa.Session, response *alexa.Response) error {

	log.Printf("OnIntent requestId=%s, sessionId=%s, intent=%s", request.RequestID, session.SessionID, request.Intent.Name)

	switch request.Intent.Name {
	case "HelloWorldIntent":
		log.Println("HelloWorldIntent triggered")
		speechText := "Hello World"

		response.SetSimpleCard(cardTitle, speechText)
		response.SetOutputText(speechText)

		log.Printf("Set Output speech, value now: %s", response.OutputSpeech.Text)
	case "AMAZON.HelpIntent":
		log.Println("AMAZON.HelpIntent triggered")
		speechText := "You can say hello to me!"

		response.SetSimpleCard("HelloWorld", speechText)
		response.SetOutputText(speechText)
		response.SetRepromptText(speechText)
	default:
		return errors.New("Invalid Intent")
	}

	return nil
}

// OnSessionEnded called with a reqeust is received of type SessionEndedRequest
func (h *HelloWorld) OnSessionEnded(request *alexa.Request, session *alexa.Session, response *alexa.Response) error {

	log.Printf("OnSessionEnded requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	return nil
}

func main() {
	lambda.Start(Handle)
}
<commit_msg>Fixed errors in HelloWorld sample. The code was not compiling as the context parameter was missing on the On functions. This is due to a change in alexa.go<commit_after>package main

import (
	"context"
	"errors"
	"log"

	"github.com/aws/aws-lambda-go/lambda"
	alexa "github.com/ericdaugherty/alexa-skills-kit-golang"
)

var a = &alexa.Alexa{ApplicationID: "amzn1.ask.skill.<SKILL_ID>", RequestHandler: &HelloWorld{}, IgnoreApplicationID: true, IgnoreTimestamp: true}

const cardTitle = "HelloWorld"

// HelloWorld handles reqeusts from the HelloWorld skill.
type HelloWorld struct{}

// Handle processes calls from Lambda
func Handle(ctx context.Context, requestEnv *alexa.RequestEnvelope) (interface{}, error) {
	return a.ProcessRequest(ctx, requestEnv)
}

// OnSessionStarted called when a new session is created.
func (h *HelloWorld) OnSessionStarted(ctx context.Context,request *alexa.Request, session *alexa.Session, ctx_ptr *alexa.Context,response *alexa.Response) error {

	log.Printf("OnSessionStarted requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	return nil
}

// OnLaunch called with a reqeust is received of type LaunchRequest
func (h *HelloWorld) OnLaunch(ctx context.Context,request *alexa.Request, session *alexa.Session, ctx_ptr *alexa.Context,response *alexa.Response) error {
	speechText := "Welcome to the Alexa Skills Kit, you can say hello"

	log.Printf("OnLaunch requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	response.SetSimpleCard(cardTitle, speechText)
	response.SetOutputText(speechText)
	response.SetRepromptText(speechText)

	response.ShouldSessionEnd = true

	return nil
}

// OnIntent called with a reqeust is received of type IntentRequest
func (h *HelloWorld) OnIntent(ctx context.Context,request *alexa.Request, session *alexa.Session, ctx_ptr *alexa.Context,response *alexa.Response) error {

	log.Printf("OnIntent requestId=%s, sessionId=%s, intent=%s", request.RequestID, session.SessionID, request.Intent.Name)

	switch request.Intent.Name {
	case "HelloWorldIntent":
		log.Println("HelloWorldIntent triggered")
		speechText := "Hello World"

		response.SetSimpleCard(cardTitle, speechText)
		response.SetOutputText(speechText)

		log.Printf("Set Output speech, value now: %s", response.OutputSpeech.Text)
	case "AMAZON.HelpIntent":
		log.Println("AMAZON.HelpIntent triggered")
		speechText := "You can say hello to me!"

		response.SetSimpleCard("HelloWorld", speechText)
		response.SetOutputText(speechText)
		response.SetRepromptText(speechText)
	default:
		return errors.New("Invalid Intent")
	}

	return nil
}

// OnSessionEnded called with a reqeust is received of type SessionEndedRequest
func (h *HelloWorld) OnSessionEnded(ctx context.Context,request *alexa.Request, session *alexa.Session, ctx_ptr *alexa.Context,response *alexa.Response) error {

	log.Printf("OnSessionEnded requestId=%s, sessionId=%s", request.RequestID, session.SessionID)

	return nil
}

func main() {
	lambda.Start(Handle)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	debugging   = false
	erroring    = false
	maxprocs    = 4
	testPattern = regexp.MustCompile(`test/test-([a-z\-]+)\.sh$`)
)

func mainIntegration() {
	if len(os.Getenv("DEBUG")) > 0 {
		debugging = true
	}

	if maxprocs < 1 {
		maxprocs = 1
	}

	files := testFiles()

	if len(files) == 0 {
		fmt.Println("no tests to run")
		os.Exit(1)
	}

	var wg sync.WaitGroup
	tests := make(chan string, len(files))
	output := make(chan string, len(files))

	for _, file := range files {
		tests <- file
	}

	go printOutput(output)
	for i := 0; i < maxprocs; i++ {
		wg.Add(1)
		go worker(tests, output, &wg)
	}

	close(tests)
	wg.Wait()
	close(output)
	printOutput(output)

	if erroring {
		os.Exit(1)
	}
}

func runTest(output chan string, test string) {
	out, err := exec.Command("/bin/bash", test).CombinedOutput()
	if err != nil {
		erroring = true
	}

	output <- strings.TrimSpace(string(out))
}

func printOutput(output <-chan string) {
	for {
		select {
		case out, ok := <-output:
			if !ok {
				return
			}

			fmt.Println(out)
		}
	}
}

func worker(tests <-chan string, output chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		select {
		case testname, ok := <-tests:
			if !ok {
				return
			}
			runTest(output, testname)
		}
	}
}

func testFiles() []string {
	if len(os.Args) < 4 {
		return allTestFiles()
	}

	fileMap := make(map[string]bool)
	for _, file := range allTestFiles() {
		fileMap[file] = true
	}

	files := make([]string, 0, len(os.Args)-3)
	for _, arg := range os.Args {
		fullname := "test/test-" + arg + ".sh"
		if fileMap[fullname] {
			files = append(files, fullname)
		}
	}

	return files
}

func allTestFiles() []string {
	files := make([]string, 0, 100)
	filepath.Walk("test", func(path string, info os.FileInfo, err error) error {
		if debugging {
			fmt.Println("FOUND:", path)
		}
		if err != nil || info.IsDir() || !testPattern.MatchString(path) {
			return nil
		}

		if debugging {
			fmt.Println("MATCHING:", path)
		}
		files = append(files, path)
		return nil
	})
	return files
}
<commit_msg>ラララララ ラー ウウウ フフフ<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

var (
	debugging   = false
	erroring    = false
	maxprocs    = 4
	testPattern = regexp.MustCompile(`test[/\\]test-([a-z\-]+)\.sh$`)
)

func mainIntegration() {
	if len(os.Getenv("DEBUG")) > 0 {
		debugging = true
	}

	if maxprocs < 1 {
		maxprocs = 1
	}

	files := testFiles()

	if len(files) == 0 {
		fmt.Println("no tests to run")
		os.Exit(1)
	}

	var wg sync.WaitGroup
	tests := make(chan string, len(files))
	output := make(chan string, len(files))

	for _, file := range files {
		tests <- file
	}

	go printOutput(output)
	for i := 0; i < maxprocs; i++ {
		wg.Add(1)
		go worker(tests, output, &wg)
	}

	close(tests)
	wg.Wait()
	close(output)
	printOutput(output)

	if erroring {
		os.Exit(1)
	}
}

func runTest(output chan string, test string) {
	out, err := exec.Command("/bin/bash", test).CombinedOutput()
	if err != nil {
		erroring = true
	}

	output <- strings.TrimSpace(string(out))
}

func printOutput(output <-chan string) {
	for {
		select {
		case out, ok := <-output:
			if !ok {
				return
			}

			fmt.Println(out)
		}
	}
}

func worker(tests <-chan string, output chan string, wg *sync.WaitGroup) {
	defer wg.Done()
	for {
		select {
		case testname, ok := <-tests:
			if !ok {
				return
			}
			runTest(output, testname)
		}
	}
}

func testFiles() []string {
	if len(os.Args) < 4 {
		return allTestFiles()
	}

	fileMap := make(map[string]bool)
	for _, file := range allTestFiles() {
		fileMap[file] = true
	}

	files := make([]string, 0, len(os.Args)-3)
	for _, arg := range os.Args {
		fullname := "test/test-" + arg + ".sh"
		if fileMap[fullname] {
			files = append(files, fullname)
		}
	}

	return files
}

func allTestFiles() []string {
	files := make([]string, 0, 100)
	filepath.Walk("test", func(path string, info os.FileInfo, err error) error {
		if debugging {
			fmt.Println("FOUND:", path)
		}
		if err != nil || info.IsDir() || !testPattern.MatchString(path) {
			return nil
		}

		if debugging {
			fmt.Println("MATCHING:", path)
		}
		files = append(files, path)
		return nil
	})
	return files
}
<|endoftext|>


<commit_before>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	labels = make([]string, 6)
	//labels = tmpdata[0]
	val data []Record
	for i := 1; i<len(tmpdata)-1; i++ {
		rec := new(Record)
		rec[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		rec[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Power, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].Null = true
		}
		data = append(data, rec)
	}
	data = fillRecords (data)
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return
}
<commit_msg>typos...<commit_after>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	labels = make([]string, 6)
	//labels = tmpdata[0]
	var data []Record
	for i := 1; i<len(tmpdata)-1; i++ {
		rec := new(Record)
		rec[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		rec[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		rec[i-1].Power, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].Null = true
		}
		data = append(data, rec)
	}
	data = fillRecords (data)
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return
}
<|endoftext|>