<commit_before>package collector

import (
	"encoding/json"
	"strings"

	flattener "github.com/karimra/go-map-flattener"
	"github.com/openconfig/gnmi/proto/gnmi"
)

// EventMsg //
type EventMsg struct {
	Name      string                 `json:"name,omitempty"` // measurement name
	Timestamp int64                  `json:"timestamp,omitempty"`
	Tags      map[string]string      `json:"tags,omitempty"`
	Values    map[string]interface{} `json:"values,omitempty"`
	Deletes   []string               `json:"deletes,omitempty"`
}

// ResponseToEventMsgs //
func ResponseToEventMsgs(name string, rsp *gnmi.SubscribeResponse, meta map[string]string) ([]*EventMsg, error) {
	if rsp == nil {
		return nil, nil
	}
	evs := make([]*EventMsg, 0)
	switch rsp := rsp.Response.(type) {
	case *gnmi.SubscribeResponse_Update:
		tags := make(map[string]string)
		namePrefix, prefixTags := TagsFromGNMIPath(rsp.Update.Prefix)
		for k, v := range prefixTags {
			if _, ok := tags[k]; ok {
				tags[strings.Join([]string{namePrefix, k}, "/")] = v
				continue
			}
			tags[k] = v
		}

		for _, upd := range rsp.Update.Update {
			e := &EventMsg{
				Tags:   make(map[string]string),
				Values: make(map[string]interface{}),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			e.Values = make(map[string]interface{})
			pathName, pTags := TagsFromGNMIPath(upd.Path)
			pathName = strings.TrimRight(namePrefix, "/") + "/" + strings.TrimLeft(pathName, "/")
			for k, v := range pTags {
				if _, ok := e.Tags[k]; ok {
					e.Tags[strings.Join([]string{pathName, k}, "/")] = v
					continue
				}
				e.Tags[k] = v
			}
			vs, err := parseUpdate(pathName, upd)
			if err != nil {
				return nil, err
			}
			for k, v := range vs {
				e.Values[k] = v
			}
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			evs = append(evs, e)
		}

		if len(rsp.Update.Delete) > 0 {
			e := &EventMsg{
				Deletes: make([]string, 0, len(rsp.Update.Delete)),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			for _, del := range rsp.Update.Delete {
				e.Deletes = append(e.Deletes, gnmiPathToXPath(del))
			}
			evs = append(evs, e)
		}
	}
	return evs, nil
}

// TagsFromGNMIPath //
func TagsFromGNMIPath(p *gnmi.Path) (string, map[string]string) {
	if p == nil {
		return "", nil
	}
	tags := make(map[string]string)
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.Write([]byte(":"))
	}
	for _, e := range p.Elem {
		if e.Name != "" {
			sb.Write([]byte("/"))
			sb.WriteString(e.Name)
		}
		if e.Key != nil {
			for k, v := range e.Key {
				tags[k] = v
			}
		}
	}
	return sb.String(), tags
}

// parseUpdate //
func parseUpdate(prefix string, upd *gnmi.Update) (map[string]interface{}, error) {
	val, err := getValueFlat(prefix, upd.GetVal())
	if err != nil {
		return nil, err
	}
	return val, nil
}

func getValueFlat(prefix string, updValue *gnmi.TypedValue) (map[string]interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var jsondata []byte
	values := make(map[string]interface{})
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		values[prefix] = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		values[prefix] = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		values[prefix] = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		values[prefix] = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		values[prefix] = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		values[prefix] = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		values[prefix] = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		values[prefix] = updValue.GetUintVal()
	case *gnmi.TypedValue_LeaflistVal:
		values[prefix] = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		values[prefix] = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		values[prefix] = updValue.GetAnyVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	}
	if len(jsondata) != 0 {
		var value interface{}
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
		switch value := value.(type) {
		case map[string]interface{}:
			f := flattener.NewFlattener()
			values, err = f.Flatten(value)
		default:
			values[prefix] = value
		}
		if err != nil {
			return nil, err
		}
	}
	return values, nil
}
<commit_msg>remove extra func<commit_after>package collector

import (
	"encoding/json"
	"strings"

	flattener "github.com/karimra/go-map-flattener"
	"github.com/openconfig/gnmi/proto/gnmi"
)

// EventMsg //
type EventMsg struct {
	Name      string                 `json:"name,omitempty"` // measurement name
	Timestamp int64                  `json:"timestamp,omitempty"`
	Tags      map[string]string      `json:"tags,omitempty"`
	Values    map[string]interface{} `json:"values,omitempty"`
	Deletes   []string               `json:"deletes,omitempty"`
}

// ResponseToEventMsgs //
func ResponseToEventMsgs(name string, rsp *gnmi.SubscribeResponse, meta map[string]string) ([]*EventMsg, error) {
	if rsp == nil {
		return nil, nil
	}
	evs := make([]*EventMsg, 0)
	switch rsp := rsp.Response.(type) {
	case *gnmi.SubscribeResponse_Update:
		tags := make(map[string]string)
		namePrefix, prefixTags := TagsFromGNMIPath(rsp.Update.Prefix)
		for k, v := range prefixTags {
			if _, ok := tags[k]; ok {
				tags[strings.Join([]string{namePrefix, k}, "/")] = v
				continue
			}
			tags[k] = v
		}

		for _, upd := range rsp.Update.Update {
			e := &EventMsg{
				Tags:   make(map[string]string),
				Values: make(map[string]interface{}),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			e.Values = make(map[string]interface{})
			pathName, pTags := TagsFromGNMIPath(upd.Path)
			pathName = strings.TrimRight(namePrefix, "/") + "/" + strings.TrimLeft(pathName, "/")
			for k, v := range pTags {
				if _, ok := e.Tags[k]; ok {
					e.Tags[strings.Join([]string{pathName, k}, "/")] = v
					continue
				}
				e.Tags[k] = v
			}
			vs, err := getValueFlat(pathName, upd.GetVal())
			if err != nil {
				return nil, err
			}
			for k, v := range vs {
				e.Values[k] = v
			}
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			evs = append(evs, e)
		}

		if len(rsp.Update.Delete) > 0 {
			e := &EventMsg{
				Deletes: make([]string, 0, len(rsp.Update.Delete)),
			}
			e.Timestamp = rsp.Update.Timestamp
			e.Name = name
			e.Tags = tags
			for k, v := range meta {
				if k == "format" {
					continue
				}
				if _, ok := e.Tags[k]; ok {
					e.Tags["meta:"+k] = v
					continue
				}
				e.Tags[k] = v
			}
			for _, del := range rsp.Update.Delete {
				e.Deletes = append(e.Deletes, gnmiPathToXPath(del))
			}
			evs = append(evs, e)
		}
	}
	return evs, nil
}

// TagsFromGNMIPath //
func TagsFromGNMIPath(p *gnmi.Path) (string, map[string]string) {
	if p == nil {
		return "", nil
	}
	tags := make(map[string]string)
	sb := strings.Builder{}
	if p.Origin != "" {
		sb.WriteString(p.Origin)
		sb.Write([]byte(":"))
	}
	for _, e := range p.Elem {
		if e.Name != "" {
			sb.Write([]byte("/"))
			sb.WriteString(e.Name)
		}
		if e.Key != nil {
			for k, v := range e.Key {
				tags[k] = v
			}
		}
	}
	return sb.String(), tags
}

func getValueFlat(prefix string, updValue *gnmi.TypedValue) (map[string]interface{}, error) {
	if updValue == nil {
		return nil, nil
	}
	var jsondata []byte
	values := make(map[string]interface{})
	switch updValue.Value.(type) {
	case *gnmi.TypedValue_AsciiVal:
		values[prefix] = updValue.GetAsciiVal()
	case *gnmi.TypedValue_BoolVal:
		values[prefix] = updValue.GetBoolVal()
	case *gnmi.TypedValue_BytesVal:
		values[prefix] = updValue.GetBytesVal()
	case *gnmi.TypedValue_DecimalVal:
		values[prefix] = updValue.GetDecimalVal()
	case *gnmi.TypedValue_FloatVal:
		values[prefix] = updValue.GetFloatVal()
	case *gnmi.TypedValue_IntVal:
		values[prefix] = updValue.GetIntVal()
	case *gnmi.TypedValue_StringVal:
		values[prefix] = updValue.GetStringVal()
	case *gnmi.TypedValue_UintVal:
		values[prefix] = updValue.GetUintVal()
	case *gnmi.TypedValue_LeaflistVal:
		values[prefix] = updValue.GetLeaflistVal()
	case *gnmi.TypedValue_ProtoBytes:
		values[prefix] = updValue.GetProtoBytes()
	case *gnmi.TypedValue_AnyVal:
		values[prefix] = updValue.GetAnyVal()
	case *gnmi.TypedValue_JsonIetfVal:
		jsondata = updValue.GetJsonIetfVal()
	case *gnmi.TypedValue_JsonVal:
		jsondata = updValue.GetJsonVal()
	}
	if len(jsondata) != 0 {
		var value interface{}
		err := json.Unmarshal(jsondata, &value)
		if err != nil {
			return nil, err
		}
		switch value := value.(type) {
		case map[string]interface{}:
			f := flattener.NewFlattener()
			values, err = f.Flatten(value)
		default:
			values[prefix] = value
		}
		if err != nil {
			return nil, err
		}
	}
	return values, nil
}
<|endoftext|>


<commit_before>package cli

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/kopia/kopia/block"

	"github.com/kopia/kopia/repo"
	"github.com/rs/zerolog/log"
)

var (
	blockRewriteCommand       = blockCommands.Command("rewrite", "Rewrite blocks using most recent format")
	blockRewriteIDs           = blockRewriteCommand.Arg("blockID", "Identifiers of blocks to rewrite").Strings()
	blockRewriteShortPacks    = blockRewriteCommand.Flag("short", "Rewrite blocks from short packs").Bool()
	blockRewriteFormatVersion = blockRewriteCommand.Flag("format-version", "Rewrite blocks using the provided format version").Default("-1").Int()
	blockRewritePackPrefix    = blockRewriteCommand.Flag("pack-prefix", "Only rewrite pack blocks with a given prefix").String()
	blockRewriteDryRun        = blockRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool()
)

func runRewriteBlocksAction(ctx context.Context, rep *repo.Repository) error {
	blocks, err := getBlocksToRewrite(ctx, rep)
	if err != nil {
		return fmt.Errorf("unable to determine blocks to rewrite: %v", err)
	}

	var totalBytes int64
	failedCount := 0
	for _, b := range blocks {
		var optDeleted string
		if b.Deleted {
			optDeleted = " (deleted)"
		}
		fmt.Fprintf(os.Stderr, "Rewriting block %v (%v bytes) from pack %v%v\n", b.BlockID, b.Length, b.PackFile, optDeleted)
		totalBytes += int64(b.Length)
		if *blockRewriteDryRun {
			continue
		}
		if err := rep.Blocks.RewriteBlock(ctx, b.BlockID); err != nil {
			log.Warn().Msgf("unable to rewrite block %q: %v", b.BlockID, err)
			failedCount++
		}
	}

	fmt.Fprintf(os.Stderr, "Total bytes rewritten %v\n", totalBytes)

	if failedCount == 0 {
		return nil
	}

	return fmt.Errorf("failed to rewrite %v blocks", failedCount)
}

func getBlocksToRewrite(ctx context.Context, rep *repo.Repository) ([]block.Info, error) {
	// get blocks listed on command line
	result, err := getBlockInfos(ctx, rep, *blockRewriteIDs)
	if err != nil {
		return nil, err
	}

	// add all blocks from short packs
	if *blockRewriteShortPacks {
		threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10)
		info, err := getBlocksInShortPacks(ctx, rep, threshold)
		if err != nil {
			return nil, err
		}

		result = append(result, info...)
	}

	// add all blocks with given format version
	if *blockRewriteFormatVersion != -1 {
		info, err := getBlocksWithFormatVersion(ctx, rep, *blockRewriteFormatVersion)
		if err != nil {
			return nil, err
		}

		result = append(result, info...)
	}

	return result, nil
}

func getBlockInfos(ctx context.Context, rep *repo.Repository, blockIDs []string) ([]block.Info, error) {
	var result []block.Info
	for _, blockID := range blockIDs {
		i, err := rep.Blocks.BlockInfo(ctx, blockID)
		if err != nil {
			return nil, fmt.Errorf("unable to get info for block %q: %v", blockID, err)
		}
		result = append(result, i)
	}
	return result, nil
}

func getBlocksWithFormatVersion(ctx context.Context, rep *repo.Repository, version int) ([]block.Info, error) {
	var result []block.Info

	infos, err := rep.Blocks.ListBlockInfos("", true)
	if err != nil {
		return nil, fmt.Errorf("unable to list index blocks: %v", err)
	}

	for _, b := range infos {
		if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) {
			result = append(result, b)
		}
	}

	return result, nil
}

func getBlocksInShortPacks(ctx context.Context, rep *repo.Repository, threshold uint32) ([]block.Info, error) {
	var result []block.Info

	infos, err := rep.Blocks.ListBlockInfos("", true)
	if err != nil {
		return nil, fmt.Errorf("unable to list index blocks: %v", err)
	}

	shortPackBlocks, err := findShortPackBlocks(infos, threshold)
	if err != nil {
		return nil, fmt.Errorf("unable to find short pack blocks: %v", err)
	}
	log.Printf("found %v short pack blocks", len(shortPackBlocks))

	if len(shortPackBlocks) <= 1 {
		fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks))
	} else {
		for _, b := range infos {
			if shortPackBlocks[b.PackFile] && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) {
				result = append(result, b)
			}
		}
	}
	return result, nil
}

func findShortPackBlocks(infos []block.Info, threshold uint32) (map[string]bool, error) {
	packUsage := map[string]uint32{}

	for _, bi := range infos {
		packUsage[bi.PackFile] += bi.Length
	}

	shortPackBlocks := map[string]bool{}

	for packFile, usage := range packUsage {
		if usage < threshold {
			shortPackBlocks[packFile] = true
		}
	}

	return shortPackBlocks, nil
}

func init() {
	blockRewriteCommand.Action(repositoryAction(runRewriteBlocksAction))
}
<commit_msg>parallelized 'block rewrite'<commit_after>package cli

import (
	"context"
	"fmt"
	"os"
	"strings"
	"sync"

	"github.com/kopia/kopia/block"

	"github.com/kopia/kopia/repo"
	"github.com/rs/zerolog/log"
)

var (
	blockRewriteCommand     = blockCommands.Command("rewrite", "Rewrite blocks using most recent format")
	blockRewriteIDs         = blockRewriteCommand.Arg("blockID", "Identifiers of blocks to rewrite").Strings()
	blockRewriteParallelism = blockRewriteCommand.Flag("parallelism", "Number of parallel workers").Default("16").Int()

	blockRewriteShortPacks    = blockRewriteCommand.Flag("short", "Rewrite blocks from short packs").Bool()
	blockRewriteFormatVersion = blockRewriteCommand.Flag("format-version", "Rewrite blocks using the provided format version").Default("-1").Int()
	blockRewritePackPrefix    = blockRewriteCommand.Flag("pack-prefix", "Only rewrite pack blocks with a given prefix").String()
	blockRewriteDryRun        = blockRewriteCommand.Flag("dry-run", "Do not actually rewrite, only print what would happen").Short('n').Bool()
)

type blockInfoOrError struct {
	block.Info
	err error
}

func runRewriteBlocksAction(ctx context.Context, rep *repo.Repository) error {
	blocks := getBlocksToRewrite(ctx, rep)

	var (
		mu          sync.Mutex
		totalBytes  int64
		failedCount int
	)

	var wg sync.WaitGroup

	for i := 0; i < *blockRewriteParallelism; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for b := range blocks {
				if b.err != nil {
					log.Error().Msgf("got error: %v", b.err)
					mu.Lock()
					failedCount++
					mu.Unlock()
					return
				}

				var optDeleted string
				if b.Deleted {
					optDeleted = " (deleted)"
				}

				fmt.Fprintf(os.Stderr, "Rewriting block %v (%v bytes) from pack %v%v\n", b.BlockID, b.Length, b.PackFile, optDeleted)
				mu.Lock()
				totalBytes += int64(b.Length)
				mu.Unlock()
				if *blockRewriteDryRun {
					continue
				}
				if err := rep.Blocks.RewriteBlock(ctx, b.BlockID); err != nil {
					log.Warn().Msgf("unable to rewrite block %q: %v", b.BlockID, err)
					mu.Lock()
					failedCount++
					mu.Unlock()
				}
			}
		}()
	}

	wg.Wait()

	fmt.Fprintf(os.Stderr, "Total bytes rewritten %v\n", totalBytes)

	if failedCount == 0 {
		return nil
	}

	return fmt.Errorf("failed to rewrite %v blocks", failedCount)
}

func getBlocksToRewrite(ctx context.Context, rep *repo.Repository) <-chan blockInfoOrError {
	ch := make(chan blockInfoOrError)
	go func() {
		defer close(ch)

		// get blocks listed on command line
		findBlockInfos(ctx, rep, ch, *blockRewriteIDs)

		// add all blocks from short packs
		if *blockRewriteShortPacks {
			threshold := uint32(rep.Blocks.Format.MaxPackSize * 6 / 10)
			findBlocksInShortPacks(ctx, rep, ch, threshold)
		}

		// add all blocks with given format version
		if *blockRewriteFormatVersion != -1 {
			findBlocksWithFormatVersion(ctx, rep, ch, *blockRewriteFormatVersion)
		}
	}()

	return ch
}

func findBlockInfos(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, blockIDs []string) {
	for _, blockID := range blockIDs {
		i, err := rep.Blocks.BlockInfo(ctx, blockID)
		if err != nil {
			ch <- blockInfoOrError{err: fmt.Errorf("unable to get info for block %q: %v", blockID, err)}
		} else {
			ch <- blockInfoOrError{Info: i}
		}
	}
}

func findBlocksWithFormatVersion(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, version int) {
	infos, err := rep.Blocks.ListBlockInfos("", true)
	if err != nil {
		ch <- blockInfoOrError{err: fmt.Errorf("unable to list index blocks: %v", err)}
		return
	}

	for _, b := range infos {
		if int(b.FormatVersion) == *blockRewriteFormatVersion && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) {
			ch <- blockInfoOrError{Info: b}
		}
	}
}

func findBlocksInShortPacks(ctx context.Context, rep *repo.Repository, ch chan blockInfoOrError, threshold uint32) {
	log.Printf("listing blocks...")
	infos, err := rep.Blocks.ListBlockInfos("", true)
	if err != nil {
		ch <- blockInfoOrError{err: fmt.Errorf("unable to list index blocks: %v", err)}
		return
	}

	log.Printf("finding blocks in short packs...")
	shortPackBlocks, err := findShortPackBlocks(infos, threshold)
	if err != nil {
		ch <- blockInfoOrError{err: fmt.Errorf("unable to find short pack blocks: %v", err)}
		return
	}
	log.Printf("found %v short pack blocks", len(shortPackBlocks))

	if len(shortPackBlocks) <= 1 {
		fmt.Printf("Nothing to do, found %v short pack blocks\n", len(shortPackBlocks))
	} else {
		for _, b := range infos {
			if shortPackBlocks[b.PackFile] && strings.HasPrefix(b.PackFile, *blockRewritePackPrefix) {
				ch <- blockInfoOrError{Info: b}
			}
		}
	}
}

func findShortPackBlocks(infos []block.Info, threshold uint32) (map[string]bool, error) {
	packUsage := map[string]uint32{}

	for _, bi := range infos {
		packUsage[bi.PackFile] += bi.Length
	}

	shortPackBlocks := map[string]bool{}

	for packFile, usage := range packUsage {
		if usage < threshold {
			shortPackBlocks[packFile] = true
		}
	}

	return shortPackBlocks, nil
}

func init() {
	blockRewriteCommand.Action(repositoryAction(runRewriteBlocksAction))
}
<|endoftext|>


<commit_before>package command

import (
	"strings"
	"sync"

	"github.com/igungor/tlbot"
)

// A Command is an implementation of a bot command.
type Command struct {
	// Name of the command
	Name string

	// Short description of the command
	ShortLine string

	// Some commands would like to stay private, such as easter eggs or
	// built-in commands. Respect to their choices.
	Private bool

	// Run runs the command.
	Run func(bot *tlbot.Bot, msg *tlbot.Message)
}

var (
	mu       sync.Mutex
	commands = make(map[string]*Command)
)

func register(cmd *Command) {
	mu.Lock()
	defer mu.Unlock()

	commands[cmd.Name] = cmd
}

// Lookup looks-up name from registered commands and returns
// corresponding Command if any.
func Lookup(cmdname string) *Command {
	mu.Lock()
	defer mu.Unlock()

	cmdname = strings.TrimSuffix(cmdname, "@ilberbot")
	cmd, ok := commands[cmdname]
	if !ok {
		// TODO: may be add an autocorrect mechanism here? /idbm -> /imdb
		return nil
	}
	return cmd
}
<commit_msg>Add autocorrect mechanism. Pointless coding sessions.<commit_after>package command

import (
	"strings"
	"sync"

	"github.com/igungor/tlbot"
)

// A Command is an implementation of a bot command.
type Command struct {
	// Name of the command
	Name string

	// Short description of the command
	ShortLine string

	// Some commands would like to stay private, such as easter eggs or
	// built-in commands. Respect to their choices.
	Private bool

	// Run runs the command.
	Run func(bot *tlbot.Bot, msg *tlbot.Message)
}

var (
	mu       sync.Mutex
	commands = make(map[string]*Command)
)

func register(cmd *Command) {
	mu.Lock()
	defer mu.Unlock()

	commands[cmd.Name] = cmd
}

// Lookup looks-up name from registered commands and returns
// corresponding Command if any.
func Lookup(cmdname string) *Command {
	mu.Lock()
	defer mu.Unlock()

	cmdname = strings.TrimSuffix(cmdname, "@ilberbot")
	cmd, ok := commands[cmdname]
	if ok {
		return cmd
	}

	//
	// we don't have an exact match. try to guess the input.
	//

	// don't even bother single letter command inputs
	if len(cmdname) < 2 {
		return nil
	}
	// autocorrect based on levenshtein distance, if possible
	for k := range commands {
		if distance(cmdname, k) <= 2 {
			return commands[k]
		}
	}

	// at least we tried.
	return nil
}

// distance returns the levenshtein distance between given strings.
func distance(s1, s2 string) int {
	var cost, lastdiag, olddiag int
	len_s1 := len([]rune(s1))
	len_s2 := len([]rune(s2))

	column := make([]int, len_s1+1)

	for y := 1; y <= len_s1; y++ {
		column[y] = y
	}

	for x := 1; x <= len_s2; x++ {
		column[0] = x
		lastdiag = x - 1
		for y := 1; y <= len_s1; y++ {
			olddiag = column[y]
			cost = 0
			if s1[y-1] != s2[x-1] {
				cost = 1
			}
			column[y] = min(column[y]+1, min(column[y-1]+1, lastdiag+cost))
			lastdiag = olddiag
		}
	}
	return column[len_s1]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>// Package command implements functions for responding to user
// input and dispatching to appropriate backend commands.
package command

import (
	"bufio"
	"debug/gosym"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/derekparker/delve/proctl"
)

type cmdfunc func(proc *proctl.DebuggedProcess, args ...string) error

type command struct {
	aliases []string
	helpMsg string
	cmdFn   cmdfunc
}

// Returns true if the command string matches one of the aliases for this command
func (c command) match(cmdstr string) bool {
	for _, v := range c.aliases {
		if v == cmdstr {
			return true
		}
	}
	return false
}

type Commands struct {
	cmds    []command
	lastCmd cmdfunc
}

// Returns a Commands struct with default commands defined.
func DebugCommands() *Commands {
	c := &Commands{}

	c.cmds = []command{
		command{aliases: []string{"help"}, cmdFn: c.help, helpMsg: "Prints the help message."},
		command{aliases: []string{"break", "b"}, cmdFn: breakpoint, helpMsg: "Set break point at the entry point of a function, or at a specific file/line. Example: break foo.go:13"},
		command{aliases: []string{"continue", "c"}, cmdFn: cont, helpMsg: "Run until breakpoint or program termination."},
		command{aliases: []string{"step", "si"}, cmdFn: step, helpMsg: "Single step through program."},
		command{aliases: []string{"next", "n"}, cmdFn: next, helpMsg: "Step over to next source line."},
		command{aliases: []string{"threads"}, cmdFn: threads, helpMsg: "Print out info for every traced thread."},
		command{aliases: []string{"clear"}, cmdFn: clear, helpMsg: "Deletes breakpoint."},
		command{aliases: []string{"goroutines"}, cmdFn: goroutines, helpMsg: "Print out info for every goroutine."},
		command{aliases: []string{"print", "p"}, cmdFn: printVar, helpMsg: "Evaluate a variable."},
		command{aliases: []string{"exit"}, cmdFn: nullCommand, helpMsg: "Exit the debugger."},
	}

	return c
}

// Register custom commands. Expects cf to be a func of type cmdfunc,
// returning only an error.
func (c *Commands) Register(cmdstr string, cf cmdfunc, helpMsg string) {
	for _, v := range c.cmds {
		if v.match(cmdstr) {
			v.cmdFn = cf
			return
		}
	}

	c.cmds = append(c.cmds, command{aliases: []string{cmdstr}, cmdFn: cf, helpMsg: helpMsg})
}

// Find will look up the command function for the given command input.
// If it cannot find the command it will defualt to noCmdAvailable().
// If the command is an empty string it will replay the last command.
func (c *Commands) Find(cmdstr string) cmdfunc {
	// If <enter> use last command, if there was one.
	if cmdstr == "" {
		if c.lastCmd != nil {
			return c.lastCmd
		}
		return nullCommand
	}

	for _, v := range c.cmds {
		if v.match(cmdstr) {
			c.lastCmd = v.cmdFn
			return v.cmdFn
		}
	}

	return noCmdAvailable
}

func CommandFunc(fn func() error) cmdfunc {
	return func(p *proctl.DebuggedProcess, args ...string) error {
		return fn()
	}
}

func noCmdAvailable(p *proctl.DebuggedProcess, ars ...string) error {
	return fmt.Errorf("command not available")
}

func nullCommand(p *proctl.DebuggedProcess, ars ...string) error {
	return nil
}

func (c *Commands) help(p *proctl.DebuggedProcess, ars ...string) error {
	fmt.Println("The following commands are available:")
	for _, cmd := range c.cmds {
		fmt.Printf("\t%s - %s\n", strings.Join(cmd.aliases, "|"), cmd.helpMsg)
	}
	return nil
}

func threads(p *proctl.DebuggedProcess, ars ...string) error {
	return p.PrintThreadInfo()
}

func goroutines(p *proctl.DebuggedProcess, ars ...string) error {
	return p.PrintGoroutinesInfo()
}

func cont(p *proctl.DebuggedProcess, ars ...string) error {
	err := p.Continue()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func step(p *proctl.DebuggedProcess, args ...string) error {
	err := p.Step()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func next(p *proctl.DebuggedProcess, args ...string) error {
	err := p.Next()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func clear(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	var (
		fn    *gosym.Func
		pc    uint64
		fname = args[0]
	)

	if strings.ContainsRune(fname, ':') {
		fl := strings.Split(fname, ":")

		f, err := filepath.Abs(fl[0])
		if err != nil {
			return err
		}

		l, err := strconv.Atoi(fl[1])
		if err != nil {
			return err
		}

		pc, fn, err = p.GoSymTable.LineToPC(f, l)
		if err != nil {
			return err
		}
	} else {
		fn = p.GoSymTable.LookupFunc(fname)
		if fn == nil {
			return fmt.Errorf("No function named %s", fname)
		}

		pc = fn.Entry
	}

	bp, err := p.Clear(pc)
	if err != nil {
		return err
	}

	fmt.Printf("Breakpoint cleared at %#v for %s %s:%d\n", bp.Addr, bp.FunctionName, bp.File, bp.Line)

	return nil
}

func breakpoint(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	var (
		fn    *gosym.Func
		pc    uint64
		fname = args[0]
	)

	if strings.ContainsRune(fname, ':') {
		fl := strings.Split(fname, ":")

		f, err := filepath.Abs(fl[0])
		if err != nil {
			return err
		}

		l, err := strconv.Atoi(fl[1])
		if err != nil {
			return err
		}

		pc, fn, err = p.GoSymTable.LineToPC(f, l)
		if err != nil {
			return err
		}
	} else {
		fn = p.GoSymTable.LookupFunc(fname)
		if fn == nil {
			return fmt.Errorf("No function named %s", fname)
		}

		pc = fn.Entry
	}

	bp, err := p.Break(uintptr(pc))
	if err != nil {
		return err
	}

	fmt.Printf("Breakpoint set at %#v for %s %s:%d\n", bp.Addr, bp.FunctionName, bp.File, bp.Line)

	return nil
}

func printVar(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	val, err := p.EvalSymbol(args[0])
	if err != nil {
		return err
	}

	fmt.Println(val.Value)
	return nil
}

func printcontext(p *proctl.DebuggedProcess) error {
	var context []string

	regs, err := p.Registers()
	if err != nil {
		return err
	}

	f, l, fn := p.GoSymTable.PCToLine(regs.PC())

	if fn != nil {
		fmt.Printf("current loc: %s %s:%d\n", fn.Name, f, l)
		file, err := os.Open(f)
		if err != nil {
			return err
		}
		defer file.Close()

		buf := bufio.NewReader(file)
		for i := 1; i < l-5; i++ {
			_, err := buf.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
		}

		for i := l - 5; i <= l+5; i++ {
			line, err := buf.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					return err
				}

				if err == io.EOF {
					break
				}
			}

			arrow := "  "
			if i == l {
				arrow = "=>"
			}

			context = append(context, fmt.Sprintf("\033[34m%s %d\033[0m: %s", arrow, i, line))
		}
	} else {
		fmt.Printf("Stopped at: 0x%x\n", regs.PC())
		context = append(context, "\033[34m=>\033[0m    no source available")
	}

	fmt.Println(strings.Join(context, ""))

	return nil
}
<commit_msg>Adding info sources<commit_after>// Package command implements functions for responding to user
// input and dispatching to appropriate backend commands.
package command

import (
	"bufio"
	"debug/gosym"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/derekparker/delve/proctl"
)

type cmdfunc func(proc *proctl.DebuggedProcess, args ...string) error

type command struct {
	aliases []string
	helpMsg string
	cmdFn   cmdfunc
}

// Returns true if the command string matches one of the aliases for this command
func (c command) match(cmdstr string) bool {
	for _, v := range c.aliases {
		if v == cmdstr {
			return true
		}
	}
	return false
}

type Commands struct {
	cmds    []command
	lastCmd cmdfunc
}

// Returns a Commands struct with default commands defined.
func DebugCommands() *Commands {
	c := &Commands{}

	c.cmds = []command{
		command{aliases: []string{"help"}, cmdFn: c.help, helpMsg: "Prints the help message."},
		command{aliases: []string{"break", "b"}, cmdFn: breakpoint, helpMsg: "Set break point at the entry point of a function, or at a specific file/line. Example: break foo.go:13"},
		command{aliases: []string{"continue", "c"}, cmdFn: cont, helpMsg: "Run until breakpoint or program termination."},
		command{aliases: []string{"step", "si"}, cmdFn: step, helpMsg: "Single step through program."},
		command{aliases: []string{"next", "n"}, cmdFn: next, helpMsg: "Step over to next source line."},
		command{aliases: []string{"threads"}, cmdFn: threads, helpMsg: "Print out info for every traced thread."},
		command{aliases: []string{"clear"}, cmdFn: clear, helpMsg: "Deletes breakpoint."},
		command{aliases: []string{"goroutines"}, cmdFn: goroutines, helpMsg: "Print out info for every goroutine."},
		command{aliases: []string{"print", "p"}, cmdFn: printVar, helpMsg: "Evaluate a variable."},
		command{aliases: []string{"info"}, cmdFn: info, helpMsg: "Provides list of source files with symbols."},
		command{aliases: []string{"exit"}, cmdFn: nullCommand, helpMsg: "Exit the debugger."},
	}

	return c
}

// Register custom commands. Expects cf to be a func of type cmdfunc,
// returning only an error.
func (c *Commands) Register(cmdstr string, cf cmdfunc, helpMsg string) {
	for _, v := range c.cmds {
		if v.match(cmdstr) {
			v.cmdFn = cf
			return
		}
	}

	c.cmds = append(c.cmds, command{aliases: []string{cmdstr}, cmdFn: cf, helpMsg: helpMsg})
}

// Find will look up the command function for the given command input.
// If it cannot find the command it will defualt to noCmdAvailable().
// If the command is an empty string it will replay the last command.
func (c *Commands) Find(cmdstr string) cmdfunc {
	// If <enter> use last command, if there was one.
	if cmdstr == "" {
		if c.lastCmd != nil {
			return c.lastCmd
		}
		return nullCommand
	}

	for _, v := range c.cmds {
		if v.match(cmdstr) {
			c.lastCmd = v.cmdFn
			return v.cmdFn
		}
	}

	return noCmdAvailable
}

func CommandFunc(fn func() error) cmdfunc {
	return func(p *proctl.DebuggedProcess, args ...string) error {
		return fn()
	}
}

func noCmdAvailable(p *proctl.DebuggedProcess, ars ...string) error {
	return fmt.Errorf("command not available")
}

func nullCommand(p *proctl.DebuggedProcess, ars ...string) error {
	return nil
}

func (c *Commands) help(p *proctl.DebuggedProcess, ars ...string) error {
	fmt.Println("The following commands are available:")
	for _, cmd := range c.cmds {
		fmt.Printf("\t%s - %s\n", strings.Join(cmd.aliases, "|"), cmd.helpMsg)
	}
	return nil
}

func threads(p *proctl.DebuggedProcess, ars ...string) error {
	return p.PrintThreadInfo()
}

func goroutines(p *proctl.DebuggedProcess, ars ...string) error {
	return p.PrintGoroutinesInfo()
}

func cont(p *proctl.DebuggedProcess, ars ...string) error {
	err := p.Continue()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func step(p *proctl.DebuggedProcess, args ...string) error {
	err := p.Step()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func next(p *proctl.DebuggedProcess, args ...string) error {
	err := p.Next()
	if err != nil {
		return err
	}

	return printcontext(p)
}

func clear(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	var (
		fn    *gosym.Func
		pc    uint64
		fname = args[0]
	)

	if strings.ContainsRune(fname, ':') {
		fl := strings.Split(fname, ":")

		f, err := filepath.Abs(fl[0])
		if err != nil {
			return err
		}

		l, err := strconv.Atoi(fl[1])
		if err != nil {
			return err
		}

		pc, fn, err = p.GoSymTable.LineToPC(f, l)
		if err != nil {
			return err
		}
	} else {
		fn = p.GoSymTable.LookupFunc(fname)
		if fn == nil {
			return fmt.Errorf("No function named %s", fname)
		}

		pc = fn.Entry
	}

	bp, err := p.Clear(pc)
	if err != nil {
		return err
	}

	fmt.Printf("Breakpoint cleared at %#v for %s %s:%d\n", bp.Addr, bp.FunctionName, bp.File, bp.Line)

	return nil
}

func breakpoint(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	var (
		fn    *gosym.Func
		pc    uint64
		fname = args[0]
	)

	if strings.ContainsRune(fname, ':') {
		fl := strings.Split(fname, ":")

		f, err := filepath.Abs(fl[0])
		if err != nil {
			return err
		}

		l, err := strconv.Atoi(fl[1])
		if err != nil {
			return err
		}

		pc, fn, err = p.GoSymTable.LineToPC(f, l)
		if err != nil {
			return err
		}
	} else {
		fn = p.GoSymTable.LookupFunc(fname)
		if fn == nil {
			return fmt.Errorf("No function named %s", fname)
		}

		pc = fn.Entry
	}

	bp, err := p.Break(uintptr(pc))
	if err != nil {
		return err
	}

	fmt.Printf("Breakpoint set at %#v for %s %s:%d\n", bp.Addr, bp.FunctionName, bp.File, bp.Line)

	return nil
}

func printVar(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	val, err := p.EvalSymbol(args[0])
	if err != nil {
		return err
	}

	fmt.Println(val.Value)
	return nil
}

func info(p *proctl.DebuggedProcess, args ...string) error {
	if len(args) == 0 {
		return fmt.Errorf("not enough arguments")
	}

	// Allow for optional regex
	var filter *regexp.Regexp
	if len(args) >= 2 {
		var err error
		if filter, err = regexp.Compile(args[1]); err != nil {
			return fmt.Errorf("invalid filter argument: %s", err.Error())
		}
	}

	switch args[0] {
	case "sources":
		files := make([]string, 0, len(p.GoSymTable.Files))
		for f := range p.GoSymTable.Files {
			if filter == nil || filter.Match([]byte(f)) {
				files = append(files, f)
			}
		}

		sort.Sort(sort.StringSlice(files))

		for _, f := range files {
			fmt.Printf("%s\n", f)
		}

	default:
		return fmt.Errorf("unsupported info type, must be sources")
	}

	return nil
}

func printcontext(p *proctl.DebuggedProcess) error {
	var context []string

	regs, err := p.Registers()
	if err != nil {
		return err
	}

	f, l, fn := p.GoSymTable.PCToLine(regs.PC())

	if fn != nil {
		fmt.Printf("current loc: %s %s:%d\n", fn.Name, f, l)
		file, err := os.Open(f)
		if err != nil {
			return err
		}
		defer file.Close()

		buf := bufio.NewReader(file)
		for i := 1; i < l-5; i++ {
			_, err := buf.ReadString('\n')
			if err != nil && err != io.EOF {
				return err
			}
		}

		for i := l - 5; i <= l+5; i++ {
			line, err := buf.ReadString('\n')
			if err != nil {
				if err != io.EOF {
					return err
				}

				if err == io.EOF {
					break
				}
			}

			arrow := "  "
			if i == l {
				arrow = "=>"
			}

			context = append(context, fmt.Sprintf("\033[34m%s %d\033[0m: %s", arrow, i, line))
		}
	} else {
		fmt.Printf("Stopped at: 0x%x\n", regs.PC())
		context = append(context, "\033[34m=>\033[0m    no source available")
	}

	fmt.Println(strings.Join(context, ""))

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/helper"
	"github.com/kr/pretty"
	"github.com/posener/complete"
)

var (
	// enforceIndexRegex is a regular expression which extracts the enforcement error
	enforceIndexRegex = regexp.MustCompile(`\((Enforcing job modify index.*)\)`)
)

type JobRunCommand struct {
	Meta
	JobGetter
}

func (c *JobRunCommand) Help() string {
	helpText := `
Usage: nomad job run [options] <path>
Alias: nomad run

  Starts running a new job or updates an existing job using
  the specification located at <path>. This is the main command
  used to interact with Nomad.

  If the supplied path is "-", the jobfile is read from stdin. Otherwise
  it is read from the file at the supplied path or downloaded and
  read from URL specified.

  Upon successful job submission, this command will immediately
  enter an interactive monitor. This is useful to watch Nomad's
  internals make scheduling decisions and place the submitted work
  onto nodes. The monitor will end once job placement is done. It
  is safe to exit the monitor early using ctrl+c.

  On successful job submission and scheduling, exit code 0 will be
  returned. If there are job placement issues encountered
  (unsatisfiable constraints, resource exhaustion, etc), then the
  exit code will be 2. Any other errors, including client connection
  issues or internal errors, are indicated by exit code 1.

  If the job has specified the region, the -region flag and NOMAD_REGION
  environment variable are overridden and the job's region is used.

  The run command will set the consul_token of the job based on the following
  precedence, going from highest to lowest: the -consul-token flag, the
  $CONSUL_HTTP_TOKEN environment variable and finally the value in the job file.

  The run command will set the vault_token of the job based on the following
  precedence, going from highest to lowest: the -vault-token flag, the
  $VAULT_TOKEN environment variable and finally the value in the job file.

General Options:

  ` + generalOptionsUsage() + `

Run Options:

  -check-index
    If set, the job is only registered or updated if the passed
    job modify index matches the server side version. If a check-index value of
    zero is passed, the job is only registered if it does not yet exist. If a
    non-zero value is passed, it ensures that the job is being updated from a
    known state. The use of this flag is most common in conjunction with plan
    command.

  -detach
    Return immediately instead of entering monitor mode. After job submission,
    the evaluation ID will be printed to the screen, which can be used to
    examine the evaluation using the eval-status command.

  -output
    Output the JSON that would be submitted to the HTTP API without submitting
    the job.

  -policy-override
    Sets the flag to force override any soft mandatory Sentinel policies.

  -consul-token
    If set, the passed Consul token is stored in the job before sending to the
    Nomad servers. This allows passing the Consul token without storing it in
    the job file. This overrides the token found in $CONSUL_HTTP_TOKEN environment
    variable and that found in the job.

  -vault-token
    If set, the passed Vault token is stored in the job before sending to the
    Nomad servers. This allows passing the Vault token without storing it in
    the job file. This overrides the token found in $VAULT_TOKEN environment
    variable and that found in the job.

  -verbose
    Display full information.
`
	return strings.TrimSpace(helpText)
}

func (c *JobRunCommand) Synopsis() string {
	return "Run a new job or update an existing job"
}

func (c *JobRunCommand) AutocompleteFlags() complete.Flags {
	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
		complete.Flags{
			"-check-index":     complete.PredictNothing,
			"-detach":          complete.PredictNothing,
			"-verbose":         complete.PredictNothing,
			"-consul-token":    complete.PredictNothing,
			"-vault-token":     complete.PredictAnything,
			"-output":          complete.PredictNothing,
			"-policy-override": complete.PredictNothing,
		})
}

func (c *JobRunCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictOr(complete.PredictFiles("*.nomad"), complete.PredictFiles("*.hcl"))
}

func (c *JobRunCommand) Name() string { return "job run" }

func (c *JobRunCommand) Run(args []string) int {
	var detach, verbose, output, override bool
	var checkIndexStr, consulToken, vaultToken string

	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&detach, "detach", false, "")
	flags.BoolVar(&verbose, "verbose", false, "")
	flags.BoolVar(&output, "output", false, "")
	flags.BoolVar(&override, "policy-override", false, "")
	flags.StringVar(&checkIndexStr, "check-index", "", "")
	flags.StringVar(&consulToken, "consul-token", "", "")
	flags.StringVar(&vaultToken, "vault-token", "", "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Truncate the id unless full length is requested
	length := shortId
	if verbose {
		length = fullId
	}

	// Check that we got exactly one argument
	args = flags.Args()
	if len(args) != 1 {
		c.Ui.Error("This command takes one argument: <path>")
		c.Ui.Error(commandErrorText(c))
		return 1
	}

	// Get Job struct from Jobfile
	job, err := c.JobGetter.ApiJob(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err))
		return 1
	}

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Force the region to be that of the job.
	if r := job.Region; r != nil {
		client.SetRegion(*r)
	}

	// Force the namespace to be that of the job.
	if n := job.Namespace; n != nil {
		client.SetNamespace(*n)
	}

	// Check if the job is periodic or is a parameterized job
	periodic := job.IsPeriodic()
	paramjob := job.IsParameterized()

	// Parse the Consul token
	if consulToken == "" {
		// Check the environment variable
		consulToken = os.Getenv("CONSUL_HTTP_TOKEN")
	}

	if consulToken != "" {
		job.ConsulToken = helper.StringToPtr(consulToken)
	}

	// Parse the Vault token
	if vaultToken == "" {
		// Check the environment variable
		vaultToken = os.Getenv("VAULT_TOKEN")
	}

	if vaultToken != "" {
		job.VaultToken = helper.StringToPtr(vaultToken)
	}

	if output {
		req := api.RegisterJobRequest{Job: job}
		buf, err := json.MarshalIndent(req, "", "    ")
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error converting job: %s", err))
			return 1
		}

		c.Ui.Output(string(buf))
		return 0
	}

	// Parse the check-index
	checkIndex, enforce, err := parseCheckIndex(checkIndexStr)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing check-index value %q: %v", checkIndexStr, err))
		return 1
	}

	// Set the register options
	opts := &api.RegisterOptions{}
	if enforce {
		opts.EnforceIndex = true
		opts.ModifyIndex = checkIndex
	}
	if override {
		opts.PolicyOverride = true
	}

	fmt.Printf("========\nJOB IS\n%# v\n========", pretty.Formatter(job))
	// Submit the job
	resp, _, err := client.Jobs().RegisterOpts(job, opts, nil)
	if err != nil {
		if strings.Contains(err.Error(), api.RegisterEnforceIndexErrPrefix) {
			// Format the error specially if the error is due to index
			// enforcement
			matches := enforceIndexRegex.FindStringSubmatch(err.Error())
			if len(matches) == 2 {
				c.Ui.Error(matches[1]) // The matched group
				c.Ui.Error("Job not updated")
				return 1
			}
		}

		c.Ui.Error(fmt.Sprintf("Error submitting job: %s", err))
		return 1
	}

	// Print any warnings if there are any
	if resp.Warnings != "" {
		c.Ui.Output(
			c.Colorize().Color(fmt.Sprintf("[bold][yellow]Job Warnings:\n%s[reset]\n", resp.Warnings)))
	}

	evalID := resp.EvalID

	// Check if we should enter monitor mode
	if detach || periodic || paramjob {
		c.Ui.Output("Job registration successful")
		if periodic && !paramjob {
			loc, err := job.Periodic.GetLocation()
			if err == nil {
				now := time.Now().In(loc)
				next, err := job.Periodic.Next(now)
				if err != nil {
					c.Ui.Error(fmt.Sprintf("Error determining next launch time: %v", err))
				} else {
					c.Ui.Output(fmt.Sprintf("Approximate next launch time: %s (%s from now)",
						formatTime(next), formatTimeDifference(now, next, time.Second)))
				}
			}
		} else if !paramjob {
			c.Ui.Output("Evaluation ID: " + evalID)
		}

		return 0
	}

	// Detach was not specified, so start monitoring
	mon := newMonitor(c.Ui, client, length)
	return mon.monitor(evalID, false)

}

// parseCheckIndex parses the check-index flag and returns the index, whether it
// was set and potentially an error during parsing.
func parseCheckIndex(input string) (uint64, bool, error) {
	if input == "" {
		return 0, false, nil
	}

	u, err := strconv.ParseUint(input, 10, 64)
	return u, true, err
}
<commit_msg>remove logging debug line from cli<commit_after>package command

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/nomad/api"
	"github.com/hashicorp/nomad/helper"
	"github.com/posener/complete"
)

var (
	// enforceIndexRegex is a regular expression which extracts the enforcement error
	enforceIndexRegex = regexp.MustCompile(`\((Enforcing job modify index.*)\)`)
)

type JobRunCommand struct {
	Meta
	JobGetter
}

func (c *JobRunCommand) Help() string {
	helpText := `
Usage: nomad job run [options] <path>
Alias: nomad run

  Starts running a new job or updates an existing job using
  the specification located at <path>. This is the main command
  used to interact with Nomad.

  If the supplied path is "-", the jobfile is read from stdin. Otherwise
  it is read from the file at the supplied path or downloaded and
  read from URL specified.

  Upon successful job submission, this command will immediately
  enter an interactive monitor. This is useful to watch Nomad's
  internals make scheduling decisions and place the submitted work
  onto nodes. The monitor will end once job placement is done. It
  is safe to exit the monitor early using ctrl+c.

  On successful job submission and scheduling, exit code 0 will be
  returned. If there are job placement issues encountered
  (unsatisfiable constraints, resource exhaustion, etc), then the
  exit code will be 2. Any other errors, including client connection
  issues or internal errors, are indicated by exit code 1.

  If the job has specified the region, the -region flag and NOMAD_REGION
  environment variable are overridden and the job's region is used.

  The run command will set the consul_token of the job based on the following
  precedence, going from highest to lowest: the -consul-token flag, the
  $CONSUL_HTTP_TOKEN environment variable and finally the value in the job file.

  The run command will set the vault_token of the job based on the following
  precedence, going from highest to lowest: the -vault-token flag, the
  $VAULT_TOKEN environment variable and finally the value in the job file.

General Options:

  ` + generalOptionsUsage() + `

Run Options:

  -check-index
    If set, the job is only registered or updated if the passed
    job modify index matches the server side version. If a check-index value of
    zero is passed, the job is only registered if it does not yet exist. If a
    non-zero value is passed, it ensures that the job is being updated from a
    known state. The use of this flag is most common in conjunction with plan
    command.

  -detach
    Return immediately instead of entering monitor mode. After job submission,
    the evaluation ID will be printed to the screen, which can be used to
    examine the evaluation using the eval-status command.

  -output
    Output the JSON that would be submitted to the HTTP API without submitting
    the job.

  -policy-override
    Sets the flag to force override any soft mandatory Sentinel policies.

  -consul-token
    If set, the passed Consul token is stored in the job before sending to the
    Nomad servers. This allows passing the Consul token without storing it in
    the job file. This overrides the token found in $CONSUL_HTTP_TOKEN environment
    variable and that found in the job.

  -vault-token
    If set, the passed Vault token is stored in the job before sending to the
    Nomad servers. This allows passing the Vault token without storing it in
    the job file. This overrides the token found in $VAULT_TOKEN environment
    variable and that found in the job.

  -verbose
    Display full information.
`
	return strings.TrimSpace(helpText)
}

func (c *JobRunCommand) Synopsis() string {
	return "Run a new job or update an existing job"
}

func (c *JobRunCommand) AutocompleteFlags() complete.Flags {
	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
		complete.Flags{
			"-check-index":     complete.PredictNothing,
			"-detach":          complete.PredictNothing,
			"-verbose":         complete.PredictNothing,
			"-consul-token":    complete.PredictNothing,
			"-vault-token":     complete.PredictAnything,
			"-output":          complete.PredictNothing,
			"-policy-override": complete.PredictNothing,
		})
}

func (c *JobRunCommand) AutocompleteArgs() complete.Predictor {
	return complete.PredictOr(complete.PredictFiles("*.nomad"), complete.PredictFiles("*.hcl"))
}

func (c *JobRunCommand) Name() string { return "job run" }

func (c *JobRunCommand) Run(args []string) int {
	var detach, verbose, output, override bool
	var checkIndexStr, consulToken, vaultToken string

	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
	flags.Usage = func() { c.Ui.Output(c.Help()) }
	flags.BoolVar(&detach, "detach", false, "")
	flags.BoolVar(&verbose, "verbose", false, "")
	flags.BoolVar(&output, "output", false, "")
	flags.BoolVar(&override, "policy-override", false, "")
	flags.StringVar(&checkIndexStr, "check-index", "", "")
	flags.StringVar(&consulToken, "consul-token", "", "")
	flags.StringVar(&vaultToken, "vault-token", "", "")

	if err := flags.Parse(args); err != nil {
		return 1
	}

	// Truncate the id unless full length is requested
	length := shortId
	if verbose {
		length = fullId
	}

	// Check that we got exactly one argument
	args = flags.Args()
	if len(args) != 1 {
		c.Ui.Error("This command takes one argument: <path>")
		c.Ui.Error(commandErrorText(c))
		return 1
	}

	// Get Job struct from Jobfile
	job, err := c.JobGetter.ApiJob(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err))
		return 1
	}

	// Get the HTTP client
	client, err := c.Meta.Client()
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
		return 1
	}

	// Force the region to be that of the job.
	if r := job.Region; r != nil {
		client.SetRegion(*r)
	}

	// Force the namespace to be that of the job.
	if n := job.Namespace; n != nil {
		client.SetNamespace(*n)
	}

	// Check if the job is periodic or is a parameterized job
	periodic := job.IsPeriodic()
	paramjob := job.IsParameterized()

	// Parse the Consul token
	if consulToken == "" {
		// Check the environment variable
		consulToken = os.Getenv("CONSUL_HTTP_TOKEN")
	}

	if consulToken != "" {
		job.ConsulToken = helper.StringToPtr(consulToken)
	}

	// Parse the Vault token
	if vaultToken == "" {
		// Check the environment variable
		vaultToken = os.Getenv("VAULT_TOKEN")
	}

	if vaultToken != "" {
		job.VaultToken = helper.StringToPtr(vaultToken)
	}

	if output {
		req := api.RegisterJobRequest{Job: job}
		buf, err := json.MarshalIndent(req, "", "    ")
		if err != nil {
			c.Ui.Error(fmt.Sprintf("Error converting job: %s", err))
			return 1
		}

		c.Ui.Output(string(buf))
		return 0
	}

	// Parse the check-index
	checkIndex, enforce, err := parseCheckIndex(checkIndexStr)
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Error parsing check-index value %q: %v", checkIndexStr, err))
		return 1
	}

	// Set the register options
	opts := &api.RegisterOptions{}
	if enforce {
		opts.EnforceIndex = true
		opts.ModifyIndex = checkIndex
	}
	if override {
		opts.PolicyOverride = true
	}

	// Submit the job
	resp, _, err := client.Jobs().RegisterOpts(job, opts, nil)
	if err != nil {
		if strings.Contains(err.Error(), api.RegisterEnforceIndexErrPrefix) {
			// Format the error specially if the error is due to index
			// enforcement
			matches := enforceIndexRegex.FindStringSubmatch(err.Error())
			if len(matches) == 2 {
				c.Ui.Error(matches[1]) // The matched group
				c.Ui.Error("Job not updated")
				return 1
			}
		}

		c.Ui.Error(fmt.Sprintf("Error submitting job: %s", err))
		return 1
	}

	// Print any warnings if there are any
	if resp.Warnings != "" {
		c.Ui.Output(
			c.Colorize().Color(fmt.Sprintf("[bold][yellow]Job Warnings:\n%s[reset]\n", resp.Warnings)))
	}

	evalID := resp.EvalID

	// Check if we should enter monitor mode
	if detach || periodic || paramjob {
		c.Ui.Output("Job registration successful")
		if periodic && !paramjob {
			loc, err := job.Periodic.GetLocation()
			if err == nil {
				now := time.Now().In(loc)
				next, err := job.Periodic.Next(now)
				if err != nil {
					c.Ui.Error(fmt.Sprintf("Error determining next launch time: %v", err))
				} else {
					c.Ui.Output(fmt.Sprintf("Approximate next launch time: %s (%s from now)",
						formatTime(next), formatTimeDifference(now, next, time.Second)))
				}
			}
		} else if !paramjob {
			c.Ui.Output("Evaluation ID: " + evalID)
		}

		return 0
	}

	// Detach was not specified, so start monitoring
	mon := newMonitor(c.Ui, client, length)
	return mon.monitor(evalID, false)

}

// parseCheckIndex parses the check-index flag and returns the index, whether it
// was set and potentially an error during parsing.
func parseCheckIndex(input string) (uint64, bool, error) {
	if input == "" {
		return 0, false, nil
	}

	u, err := strconv.ParseUint(input, 10, 64)
	return u, true, err
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
)

var config Config

// Config type for handling yaml config files
type Config struct {
	Region  string                 `yaml:"region,omitempty"`
	Project string                 `yaml:"project"`
	Bucket  string                 `yaml:"bucket"`
	Global  map[string]interface{} `yaml:"global,omitempty"`
	Stacks  map[string]struct {
		DependsOn  []string               `yaml:"depends_on,omitempty"`
		Parameters []map[string]string    `yaml:"parameters,omitempty"`
		Policy     string                 `yaml:"policy,omitempty"`
		Profile    string                 `yaml:"profile,omitempty"`
		CF         map[string]interface{} `yaml:"cf"`
	} `yaml:"stacks"`
}

// Returns map string of config values
func (c *Config) vars() map[string]interface{} {
	m := make(map[string]interface{})
	m["global"] = c.Global
	m["region"] = c.Region
	m["project"] = c.Project

	for s, v := range c.Stacks {
		m[s] = v.CF
	}

	return m
}

// Adds parameters to given stack based on config
func (c *Config) parameters(s *stack) {

	for stk, val := range c.Stacks {
		if s.name == stk {
			for _, param := range val.Parameters {
				for k, v := range param {
					s.parameters = append(s.parameters, &cloudformation.Parameter{
						ParameterKey:   aws.String(k),
						ParameterValue: aws.String(v),
					})
				}

			}
		}
	}
}

// configReader parses the config YAML file with Viper
func configReader(conf string) error {

	cfg, err := fetchContent(conf)
	if err != nil {
		return err
	}

	if err := yaml.Unmarshal([]byte(cfg), &config); err != nil {
		return err
	}

	Log(fmt.Sprintln("Config File Read:", config), level.debug)

	stacks = make(map[string]*stack)

	// Get Stack Values
	for s, v := range config.Stacks {
		stacks[s] = &stack{}
		stacks[s].name = s
		stacks[s].setStackName()
		stacks[s].dependsOn = v.DependsOn
		stacks[s].policy = v.Policy
		stacks[s].profile = v.Profile

		// set session
		sess, err := manager.GetSess(stacks[s].profile)
		if err != nil {
			return err
		}

		stacks[s].session = sess

		// set parameters, if any
		config.parameters(stacks[s])
	}

	return nil
}
<commit_msg>added source keyword to config<commit_after>package commands

import (
	"fmt"

	yaml "gopkg.in/yaml.v2"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudformation"
)

var config Config

// Config type for handling yaml config files
type Config struct {
	Region  string                 `yaml:"region,omitempty"`
	Project string                 `yaml:"project"`
	Global  map[string]interface{} `yaml:"global,omitempty"`
	Stacks  map[string]struct {
		DependsOn  []string               `yaml:"depends_on,omitempty"`
		Parameters []map[string]string    `yaml:"parameters,omitempty"`
		Policy     string                 `yaml:"policy,omitempty"`
		Profile    string                 `yaml:"profile,omitempty"`
		Source     string                 `yaml:"source"`
		CF         map[string]interface{} `yaml:"cf"`
	} `yaml:"stacks"`
}

// Returns map string of config values
func (c *Config) vars() map[string]interface{} {
	m := make(map[string]interface{})
	m["global"] = c.Global
	m["region"] = c.Region
	m["project"] = c.Project

	for s, v := range c.Stacks {
		m[s] = v.CF
	}

	return m
}

// Adds parameters to given stack based on config
func (c *Config) parameters(s *stack) {

	for stk, val := range c.Stacks {
		if s.name == stk {
			for _, param := range val.Parameters {
				for k, v := range param {
					s.parameters = append(s.parameters, &cloudformation.Parameter{
						ParameterKey:   aws.String(k),
						ParameterValue: aws.String(v),
					})
				}

			}
		}
	}
}

// Read template source and sets the template value in given stack
func (c *Config) getSource(s *stack) error {
	return nil
}

// configReader parses the config YAML file with Viper
func configReader(conf string) error {

	cfg, err := fetchContent(conf)
	if err != nil {
		return err
	}

	if err := yaml.Unmarshal([]byte(cfg), &config); err != nil {
		return err
	}

	Log(fmt.Sprintln("Config File Read:", config), level.debug)

	stacks = make(map[string]*stack)

	// Get Stack Values
	for s, v := range config.Stacks {
		stacks[s] = &stack{}
		stacks[s].name = s
		stacks[s].setStackName()
		stacks[s].dependsOn = v.DependsOn
		stacks[s].policy = v.Policy
		stacks[s].profile = v.Profile
		stacks[s].source = v.Source

		// set session
		sess, err := manager.GetSess(stacks[s].profile)
		if err != nil {
			return err
		}

		stacks[s].session = sess

		// set parameters, if any
		config.parameters(stacks[s])
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package commands

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/openfaas/faas-cli/proxy"
	"github.com/openfaas/faas-cli/stack"
	"github.com/spf13/cobra"
)

// Flags that are to be added to commands.

var (
	envvarOpts  []string
	replace     bool
	update      bool
	constraints []string
	secrets     []string
	labelOpts   []string
)

func init() {
	// Setup flags that are used by multiple commands (variables defined in faas.go)
	deployCmd.Flags().StringVar(&fprocess, "fprocess", "", "Fprocess to be run by the watchdog")
	deployCmd.Flags().StringVarP(&gateway, "gateway", "g", defaultGateway, "Gateway URL starting with http(s)://")
	deployCmd.Flags().StringVar(&handler, "handler", "", "Directory with handler for function, e.g. handler.js")
	deployCmd.Flags().StringVar(&image, "image", "", "Docker image name to build")
	deployCmd.Flags().StringVar(&language, "lang", "", "Programming language template")
	deployCmd.Flags().StringVar(&functionName, "name", "", "Name of the deployed function")
	deployCmd.Flags().StringVar(&network, "network", defaultNetwork, "Name of the network")

	// Setup flags that are used only by this command (variables defined above)
	deployCmd.Flags().StringArrayVarP(&envvarOpts, "env", "e", []string{}, "Set one or more environment variables (ENVVAR=VALUE)")

	deployCmd.Flags().StringArrayVarP(&labelOpts, "label", "l", []string{}, "Set one or more label (LABEL=VALUE)")

	deployCmd.Flags().BoolVar(&replace, "replace", true, "Replace any existing function")
	deployCmd.Flags().BoolVar(&update, "update", false, "Update existing functions")

	deployCmd.Flags().StringArrayVar(&constraints, "constraint", []string{}, "Apply a constraint to the function")
	deployCmd.Flags().StringArrayVar(&secrets, "secret", []string{}, "Give the function access to a secure secret")

	// Set bash-completion.
	_ = deployCmd.Flags().SetAnnotation("handler", cobra.BashCompSubdirsInDir, []string{})

	faasCmd.AddCommand(deployCmd)
}

// deployCmd handles deploying OpenFaaS function containers
var deployCmd = &cobra.Command{
	Use: `deploy -f YAML_FILE [--replace=false]
  faas-cli deploy --image IMAGE_NAME
                  --name FUNCTION_NAME
                  [--lang <ruby|python|node|csharp>]
                  [--gateway GATEWAY_URL]
                  [--network NETWORK_NAME]
                  [--handler HANDLER_DIR]
                  [--fprocess PROCESS]
                  [--env ENVVAR=VALUE ...]
                  [--label LABEL=VALUE ...]
				  [--replace=false]
				  [--update=false]
                  [--constraint PLACEMENT_CONSTRAINT ...]
                  [--regex "REGEX"]
                  [--filter "WILDCARD"]
				  [--secret "SECRET_NAME"]`,

	Short: "Deploy OpenFaaS functions",
	Long: `Deploys OpenFaaS function containers either via the supplied YAML config using
the "--yaml" flag (which may contain multiple function definitions), or directly
via flags. Note: --replace and --update are mutually exclusive.`,
	Example: `  faas-cli deploy -f https://domain/path/myfunctions.yml
  faas-cli deploy -f ./samples.yml
  faas-cli deploy -f ./samples.yml --label canary=true
  faas-cli deploy -f ./samples.yml --filter "*gif*" --secret dockerhuborg
  faas-cli deploy -f ./samples.yml --regex "fn[0-9]_.*"
  faas-cli deploy -f ./samples.yml --replace=false
  faas-cli deploy -f ./samples.yml --update=true
  faas-cli deploy --image=alexellis/faas-url-ping --name=url-ping
  faas-cli deploy --image=my_image --name=my_fn --handler=/path/to/fn/
                  --gateway=http://remote-site.com:8080 --lang=python
                  --env=MYVAR=myval`,
	Run: runDeploy,
}

func runDeploy(cmd *cobra.Command, args []string) {

	if update && replace {
		fmt.Println(`Cannot specify --update and --replace at the same time.
  --replace    removes an existing deployment before re-creating it
  --update     provides a rolling update to a new function image or configuration`)
		return
	}

	var services stack.Services
	if len(yamlFile) > 0 {
		parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter)
		if err != nil {
			log.Fatalln(err.Error())
			return
		}

		parsedServices.Provider.GatewayURL = getGatewayURL(gateway, defaultGateway, parsedServices.Provider.GatewayURL)

		// Override network if passed
		if len(network) > 0 && network != defaultNetwork {
			parsedServices.Provider.Network = network
		}

		if parsedServices != nil {
			services = *parsedServices
		}
	}

	if len(services.Functions) > 0 {
		if len(services.Provider.Network) == 0 {
			services.Provider.Network = defaultNetwork
		}

		for k, function := range services.Functions {

			function.Name = k
			if update {
				fmt.Printf("Updating: %s.\n", function.Name)
			} else {
				fmt.Printf("Deploying: %s.\n", function.Name)
			}

			var functionConstraints []string
			if function.Constraints != nil {
				functionConstraints = *function.Constraints
			} else if len(constraints) > 0 {
				functionConstraints = constraints
			}

			fileEnvironment, err := readFiles(function.EnvironmentFile)
			if err != nil {
				log.Fatalln(err)
			}

			labelMap := map[string]string{}
			if function.Labels != nil {
				labelMap = *function.Labels
			}

			labelArgumentMap, labelErr := parseMap(labelOpts, "label")
			if labelErr != nil {
				fmt.Printf("Error parsing labels: %v\n", labelErr)
				os.Exit(1)
			}

			allLabels := mergeMap(labelMap, labelArgumentMap)

			allEnvironment, envErr := compileEnvironment(envvarOpts, function.Environment, fileEnvironment)
			if envErr != nil {
				log.Fatalln(envErr)
			}

			// Get FProcess to use from the ./template/template.yml, if a template is being used
			if function.Language != "" && strings.ToLower(function.Language) != "dockerfile" {
				var fprocessErr error
				function.FProcess, fprocessErr = deriveFprocess(function)
				if fprocessErr != nil {
					log.Fatalln(fprocessErr)

				}
			}

			proxy.DeployFunction(function.FProcess, services.Provider.GatewayURL, function.Name, function.Image, function.Language, replace, allEnvironment, services.Provider.Network, functionConstraints, update, secrets, allLabels)
		}
	} else {
		if len(image) == 0 {
			fmt.Println("Please provide a --image to be deployed.")
			return
		}
		if len(functionName) == 0 {
			fmt.Println("Please provide a --name for your function as it will be deployed on FaaS")
			return
		}

		envvars, err := parseMap(envvarOpts, "env")
		if err != nil {
			fmt.Printf("Error parsing envvars: %v\n", err)
			os.Exit(1)
		}

		labelMap, labelErr := parseMap(labelOpts, "label")
		if labelErr != nil {
			fmt.Printf("Error parsing labels: %v\n", labelErr)
			os.Exit(1)
		}

		proxy.DeployFunction(fprocess, gateway, functionName, image, language, replace, envvars, network, constraints, update, secrets, labelMap)
	}
}

func buildLabelMap(labelOpts []string) map[string]string {
	labelMap := map[string]string{}
	for _, opt := range labelOpts {
		if !strings.Contains(opt, "=") {
			fmt.Println("Error - label option does not contain a value")
		} else {
			index := strings.Index(opt, "=")

			labelMap[opt[0:index]] = opt[index+1:]
		}
	}
	return labelMap
}

func readFiles(files []string) (map[string]string, error) {
	envs := make(map[string]string)

	for _, file := range files {
		bytesOut, readErr := ioutil.ReadFile(file)
		if readErr != nil {
			return nil, readErr
		}

		envFile := stack.EnvironmentFile{}
		unmarshalErr := yaml.Unmarshal(bytesOut, &envFile)
		if unmarshalErr != nil {
			return nil, unmarshalErr
		}
		for k, v := range envFile.Environment {
			envs[k] = v
		}
	}
	return envs, nil
}

func parseMap(envvars []string, keyName string) (map[string]string, error) {
	result := make(map[string]string)
	for _, envvar := range envvars {
		s := strings.SplitN(strings.TrimSpace(envvar), "=", 2)
		envvarName := s[0]
		envvarValue := s[1]

		if !(len(envvarName) > 0) {
			return nil, fmt.Errorf("Empty %s name: [%s]", keyName, envvar)
		}
		if !(len(envvarValue) > 0) {
			return nil, fmt.Errorf("Empty %s value: [%s]", keyName, envvar)
		}

		result[envvarName] = envvarValue
	}
	return result, nil
}

func mergeMap(i map[string]string, j map[string]string) map[string]string {
	merged := make(map[string]string)

	for k, v := range i {
		merged[k] = v
	}
	for k, v := range j {
		merged[k] = v
	}
	return merged
}

func getGatewayURL(argumentURL string, defaultURL string, yamlURL string) string {
	var gatewayURL string

	if len(argumentURL) > 0 && argumentURL != defaultURL {
		gatewayURL = argumentURL
	} else if len(yamlURL) > 0 {
		gatewayURL = yamlURL
	} else {
		gatewayURL = defaultURL
	}

	return gatewayURL
}

func compileEnvironment(envvarOpts []string, yamlEnvironment map[string]string, fileEnvironment map[string]string) (map[string]string, error) {
	envvarArguments, err := parseMap(envvarOpts, "env")
	if err != nil {
		return nil, fmt.Errorf("error parsing envvars: %v", err)
	}

	functionAndStack := mergeMap(yamlEnvironment, fileEnvironment)
	return mergeMap(functionAndStack, envvarArguments), nil
}

<<<<<<< HEAD
func deriveFprocess(function stack.Function) (string, error) {
	var fprocess string

	pathToTemplateYAML := "./template/" + function.Language + "/template.yml"
	if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) {
		return "", err
	}

	var langTemplate stack.LanguageTemplate
	parsedLangTemplate, err := stack.ParseYAMLForLanguageTemplate(pathToTemplateYAML)

	if err != nil {
		return "", err

	}

	if parsedLangTemplate != nil {
		langTemplate = *parsedLangTemplate
		fprocess = langTemplate.FProcess
	}

	return fprocess, nil
=======
func languageExistsNotDockerfile(language string) bool {
	return len(language) > 0 && strings.ToLower(language) != "dockerfile"
>>>>>>> template pull - address issues from thread #201
}
<commit_msg>Refactor and rebase<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package commands

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/openfaas/faas-cli/proxy"
	"github.com/openfaas/faas-cli/stack"
	"github.com/spf13/cobra"
)

// Flags that are to be added to commands.

var (
	envvarOpts  []string
	replace     bool
	update      bool
	constraints []string
	secrets     []string
	labelOpts   []string
)

func init() {
	// Setup flags that are used by multiple commands (variables defined in faas.go)
	deployCmd.Flags().StringVar(&fprocess, "fprocess", "", "Fprocess to be run by the watchdog")
	deployCmd.Flags().StringVarP(&gateway, "gateway", "g", defaultGateway, "Gateway URL starting with http(s)://")
	deployCmd.Flags().StringVar(&handler, "handler", "", "Directory with handler for function, e.g. handler.js")
	deployCmd.Flags().StringVar(&image, "image", "", "Docker image name to build")
	deployCmd.Flags().StringVar(&language, "lang", "", "Programming language template")
	deployCmd.Flags().StringVar(&functionName, "name", "", "Name of the deployed function")
	deployCmd.Flags().StringVar(&network, "network", defaultNetwork, "Name of the network")

	// Setup flags that are used only by this command (variables defined above)
	deployCmd.Flags().StringArrayVarP(&envvarOpts, "env", "e", []string{}, "Set one or more environment variables (ENVVAR=VALUE)")

	deployCmd.Flags().StringArrayVarP(&labelOpts, "label", "l", []string{}, "Set one or more label (LABEL=VALUE)")

	deployCmd.Flags().BoolVar(&replace, "replace", true, "Replace any existing function")
	deployCmd.Flags().BoolVar(&update, "update", false, "Update existing functions")

	deployCmd.Flags().StringArrayVar(&constraints, "constraint", []string{}, "Apply a constraint to the function")
	deployCmd.Flags().StringArrayVar(&secrets, "secret", []string{}, "Give the function access to a secure secret")

	// Set bash-completion.
	_ = deployCmd.Flags().SetAnnotation("handler", cobra.BashCompSubdirsInDir, []string{})

	faasCmd.AddCommand(deployCmd)
}

// deployCmd handles deploying OpenFaaS function containers
var deployCmd = &cobra.Command{
	Use: `deploy -f YAML_FILE [--replace=false]
  faas-cli deploy --image IMAGE_NAME
                  --name FUNCTION_NAME
                  [--lang <ruby|python|node|csharp>]
                  [--gateway GATEWAY_URL]
                  [--network NETWORK_NAME]
                  [--handler HANDLER_DIR]
                  [--fprocess PROCESS]
                  [--env ENVVAR=VALUE ...]
                  [--label LABEL=VALUE ...]
				  [--replace=false]
				  [--update=false]
                  [--constraint PLACEMENT_CONSTRAINT ...]
                  [--regex "REGEX"]
                  [--filter "WILDCARD"]
				  [--secret "SECRET_NAME"]`,

	Short: "Deploy OpenFaaS functions",
	Long: `Deploys OpenFaaS function containers either via the supplied YAML config using
the "--yaml" flag (which may contain multiple function definitions), or directly
via flags. Note: --replace and --update are mutually exclusive.`,
	Example: `  faas-cli deploy -f https://domain/path/myfunctions.yml
  faas-cli deploy -f ./samples.yml
  faas-cli deploy -f ./samples.yml --label canary=true
  faas-cli deploy -f ./samples.yml --filter "*gif*" --secret dockerhuborg
  faas-cli deploy -f ./samples.yml --regex "fn[0-9]_.*"
  faas-cli deploy -f ./samples.yml --replace=false
  faas-cli deploy -f ./samples.yml --update=true
  faas-cli deploy --image=alexellis/faas-url-ping --name=url-ping
  faas-cli deploy --image=my_image --name=my_fn --handler=/path/to/fn/
                  --gateway=http://remote-site.com:8080 --lang=python
                  --env=MYVAR=myval`,
	Run: runDeploy,
}

func runDeploy(cmd *cobra.Command, args []string) {

	if update && replace {
		fmt.Println(`Cannot specify --update and --replace at the same time.
  --replace    removes an existing deployment before re-creating it
  --update     provides a rolling update to a new function image or configuration`)
		return
	}

	var services stack.Services
	if len(yamlFile) > 0 {
		parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter)
		if err != nil {
			log.Fatalln(err.Error())
			return
		}

		parsedServices.Provider.GatewayURL = getGatewayURL(gateway, defaultGateway, parsedServices.Provider.GatewayURL)

		// Override network if passed
		if len(network) > 0 && network != defaultNetwork {
			parsedServices.Provider.Network = network
		}

		if parsedServices != nil {
			services = *parsedServices
		}
	}

	if len(services.Functions) > 0 {
		if len(services.Provider.Network) == 0 {
			services.Provider.Network = defaultNetwork
		}

		for k, function := range services.Functions {

			function.Name = k
			if update {
				fmt.Printf("Updating: %s.\n", function.Name)
			} else {
				fmt.Printf("Deploying: %s.\n", function.Name)
			}

			var functionConstraints []string
			if function.Constraints != nil {
				functionConstraints = *function.Constraints
			} else if len(constraints) > 0 {
				functionConstraints = constraints
			}

			fileEnvironment, err := readFiles(function.EnvironmentFile)
			if err != nil {
				log.Fatalln(err)
			}

			labelMap := map[string]string{}
			if function.Labels != nil {
				labelMap = *function.Labels
			}

			labelArgumentMap, labelErr := parseMap(labelOpts, "label")
			if labelErr != nil {
				fmt.Printf("Error parsing labels: %v\n", labelErr)
				os.Exit(1)
			}

			allLabels := mergeMap(labelMap, labelArgumentMap)

			allEnvironment, envErr := compileEnvironment(envvarOpts, function.Environment, fileEnvironment)
			if envErr != nil {
				log.Fatalln(envErr)
			}

			// Get FProcess to use from the ./template/template.yml, if a template is being used
			if languageExistsNotDockerfile(function.Language) {
				var fprocessErr error
				function.FProcess, fprocessErr = deriveFprocess(function)
				if fprocessErr != nil {
					log.Fatalln(fprocessErr)

				}
			}

			proxy.DeployFunction(function.FProcess, services.Provider.GatewayURL, function.Name, function.Image, function.Language, replace, allEnvironment, services.Provider.Network, functionConstraints, update, secrets, allLabels)
		}
	} else {
		if len(image) == 0 {
			fmt.Println("Please provide a --image to be deployed.")
			return
		}
		if len(functionName) == 0 {
			fmt.Println("Please provide a --name for your function as it will be deployed on FaaS")
			return
		}

		envvars, err := parseMap(envvarOpts, "env")
		if err != nil {
			fmt.Printf("Error parsing envvars: %v\n", err)
			os.Exit(1)
		}

		labelMap, labelErr := parseMap(labelOpts, "label")
		if labelErr != nil {
			fmt.Printf("Error parsing labels: %v\n", labelErr)
			os.Exit(1)
		}

		proxy.DeployFunction(fprocess, gateway, functionName, image, language, replace, envvars, network, constraints, update, secrets, labelMap)
	}
}

func buildLabelMap(labelOpts []string) map[string]string {
	labelMap := map[string]string{}
	for _, opt := range labelOpts {
		if !strings.Contains(opt, "=") {
			fmt.Println("Error - label option does not contain a value")
		} else {
			index := strings.Index(opt, "=")

			labelMap[opt[0:index]] = opt[index+1:]
		}
	}
	return labelMap
}

func readFiles(files []string) (map[string]string, error) {
	envs := make(map[string]string)

	for _, file := range files {
		bytesOut, readErr := ioutil.ReadFile(file)
		if readErr != nil {
			return nil, readErr
		}

		envFile := stack.EnvironmentFile{}
		unmarshalErr := yaml.Unmarshal(bytesOut, &envFile)
		if unmarshalErr != nil {
			return nil, unmarshalErr
		}
		for k, v := range envFile.Environment {
			envs[k] = v
		}
	}
	return envs, nil
}

func parseMap(envvars []string, keyName string) (map[string]string, error) {
	result := make(map[string]string)
	for _, envvar := range envvars {
		s := strings.SplitN(strings.TrimSpace(envvar), "=", 2)
		envvarName := s[0]
		envvarValue := s[1]

		if !(len(envvarName) > 0) {
			return nil, fmt.Errorf("Empty %s name: [%s]", keyName, envvar)
		}
		if !(len(envvarValue) > 0) {
			return nil, fmt.Errorf("Empty %s value: [%s]", keyName, envvar)
		}

		result[envvarName] = envvarValue
	}
	return result, nil
}

func mergeMap(i map[string]string, j map[string]string) map[string]string {
	merged := make(map[string]string)

	for k, v := range i {
		merged[k] = v
	}
	for k, v := range j {
		merged[k] = v
	}
	return merged
}

func getGatewayURL(argumentURL string, defaultURL string, yamlURL string) string {
	var gatewayURL string

	if len(argumentURL) > 0 && argumentURL != defaultURL {
		gatewayURL = argumentURL
	} else if len(yamlURL) > 0 {
		gatewayURL = yamlURL
	} else {
		gatewayURL = defaultURL
	}

	return gatewayURL
}

func compileEnvironment(envvarOpts []string, yamlEnvironment map[string]string, fileEnvironment map[string]string) (map[string]string, error) {
	envvarArguments, err := parseMap(envvarOpts, "env")
	if err != nil {
		return nil, fmt.Errorf("error parsing envvars: %v", err)
	}

	functionAndStack := mergeMap(yamlEnvironment, fileEnvironment)
	return mergeMap(functionAndStack, envvarArguments), nil
}

func deriveFprocess(function stack.Function) (string, error) {
	var fprocess string

	pathToTemplateYAML := "./template/" + function.Language + "/template.yml"
	if _, err := os.Stat(pathToTemplateYAML); os.IsNotExist(err) {
		return "", err
	}

	var langTemplate stack.LanguageTemplate
	parsedLangTemplate, err := stack.ParseYAMLForLanguageTemplate(pathToTemplateYAML)

	if err != nil {
		return "", err

	}

	if parsedLangTemplate != nil {
		langTemplate = *parsedLangTemplate
		fprocess = langTemplate.FProcess
	}

	return fprocess, nil
}

func languageExistsNotDockerfile(language string) bool {
	return len(language) > 0 && strings.ToLower(language) != "dockerfile"
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package commands

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/pagodabox/nanobox-cli/config"
	"github.com/pagodabox/nanobox-cli/ui"
	"github.com/pagodabox/nanobox-golang-stylish"
)

// DomainCommand satisfies the Command interface
type DomainCommand struct{}

// Help prints detailed help text for the app list command
func (c *DomainCommand) Help() {
	ui.CPrint("Intended for internal use only")
}

// Run halts the specified virtual machine
func (c *DomainCommand) Run(opts []string) {

	// flags
	flags := flag.NewFlagSet("flags", flag.ContinueOnError)
	flags.Usage = func() { c.Help() }

	var fWrite bool
	flags.BoolVar(&fWrite, "w", false, "")
	flags.BoolVar(&fWrite, "write", false, "")

	var fRemove bool
	flags.BoolVar(&fRemove, "x", false, "")
	flags.BoolVar(&fRemove, "remove", false, "")

	if err := flags.Parse(opts); err != nil {
		ui.LogFatal("[commands.domain] flags.Parse() failed", err)
	}

	f, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0644)
	defer f.Close()

	//
	if err != nil {
		ui.LogFatal("[commands.domain] os.OpenFile() failed", err)
	}

	//
	switch {

	// write the entry to the hosts file
	case fWrite:
		entry := fmt.Sprintf("\n%-15v   %s.%s # '%v' private network (added by nanobox)", config.Boxfile.IP, config.App, config.Boxfile.Domain, config.App)

		if _, err := f.WriteString(entry); err != nil {
			ui.LogFatal("[commands.domain] WriteString() failed", err)
		}

		fmt.Println(stylish.Bullet(config.App + ".nano.dev added to /etc/hosts"))

	//
	case fRemove:

		scanner := bufio.NewScanner(f)
		contents := ""

		// remove entry from /etc/hosts
		for scanner.Scan() {

			// if the line doesn't contain the entry add it back to what is going to be
			// re-written to the file
			if !strings.HasPrefix(scanner.Text(), config.Boxfile.IP) {
				contents += fmt.Sprintf("%s\n", scanner.Text())
			}
		}

		// write back the contents of the hosts file minus the removed entry
		if err := ioutil.WriteFile("/etc/hosts", []byte(contents), 0644); err != nil {
			ui.LogFatal("[commands.destroy] ioutil.WriteFile failed", err)
		}

		fmt.Println(stylish.Bullet(config.App + ".nano.dev removed from /etc/hosts"))
	}
}

// modifyHosts
func modifyHosts(mod string) {

	// attempt to open /etc/hosts file...
	f, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0644)
	defer f.Close()

	//
	if err != nil {

		fmt.Println("ERR??", err)

		// if nanobox doesn't have permission to modify the hosts file, it needs to
		// request it
		if perm := os.IsPermission(err); perm == true {

			entry := fmt.Sprintf("%-15v   %s # '%v' private network (added by nanobox)", config.Boxfile.IP, config.Boxfile.Domain, config.App)

			//
			switch mod {

			// add
			case "w":
				fmt.Printf(`
Nanobox needs your permission to add the following entry to your /etc/hosts file:
%v

`, entry)

			// delete
			case "x":
				fmt.Printf(`
Nanobox needs your permission to remove the following entry from your /etc/hosts file:
%v

`, entry)
			}

			//
			cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("sudo %v domain -%v", os.Args[0], mod))

			// connect standard in/outputs
			cmd.Stdin = os.Stdin
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			// run command
			if err := cmd.Run(); err != nil {
				ui.LogFatal("[commands.domain] cmd.Run() failed", err)
			}

			//
		} else {
			ui.LogFatal("[commands.domain] os.OpenFile() failed", err)
		}
	}
}
<commit_msg>removing some crufty debug output<commit_after>// Copyright (c) 2015 Pagoda Box Inc
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v.
// 2.0. If a copy of the MPL was not distributed with this file, You can obtain one
// at http://mozilla.org/MPL/2.0/.
//

package commands

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"

	"github.com/pagodabox/nanobox-cli/config"
	"github.com/pagodabox/nanobox-cli/ui"
	"github.com/pagodabox/nanobox-golang-stylish"
)

// DomainCommand satisfies the Command interface
type DomainCommand struct{}

// Help prints detailed help text for the app list command
func (c *DomainCommand) Help() {
	ui.CPrint("Intended for internal use only")
}

// Run halts the specified virtual machine
func (c *DomainCommand) Run(opts []string) {

	// flags
	flags := flag.NewFlagSet("flags", flag.ContinueOnError)
	flags.Usage = func() { c.Help() }

	var fWrite bool
	flags.BoolVar(&fWrite, "w", false, "")
	flags.BoolVar(&fWrite, "write", false, "")

	var fRemove bool
	flags.BoolVar(&fRemove, "x", false, "")
	flags.BoolVar(&fRemove, "remove", false, "")

	if err := flags.Parse(opts); err != nil {
		ui.LogFatal("[commands.domain] flags.Parse() failed", err)
	}

	f, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0644)
	defer f.Close()

	//
	if err != nil {
		ui.LogFatal("[commands.domain] os.OpenFile() failed", err)
	}

	//
	switch {

	// write the entry to the hosts file
	case fWrite:
		entry := fmt.Sprintf("\n%-15v   %s.%s # '%v' private network (added by nanobox)", config.Boxfile.IP, config.App, config.Boxfile.Domain, config.App)

		if _, err := f.WriteString(entry); err != nil {
			ui.LogFatal("[commands.domain] WriteString() failed", err)
		}

		fmt.Println(stylish.Bullet(config.App + ".nano.dev added to /etc/hosts"))

	//
	case fRemove:

		scanner := bufio.NewScanner(f)
		contents := ""

		// remove entry from /etc/hosts
		for scanner.Scan() {

			// if the line doesn't contain the entry add it back to what is going to be
			// re-written to the file
			if !strings.HasPrefix(scanner.Text(), config.Boxfile.IP) {
				contents += fmt.Sprintf("%s\n", scanner.Text())
			}
		}

		// write back the contents of the hosts file minus the removed entry
		if err := ioutil.WriteFile("/etc/hosts", []byte(contents), 0644); err != nil {
			ui.LogFatal("[commands.destroy] ioutil.WriteFile failed", err)
		}

		fmt.Println(stylish.Bullet(config.App + ".nano.dev removed from /etc/hosts"))
	}
}

// modifyHosts
func modifyHosts(mod string) {

	// attempt to open /etc/hosts file...
	f, err := os.OpenFile("/etc/hosts", os.O_RDWR|os.O_APPEND, 0644)
	defer f.Close()

	//
	if err != nil {

		// if nanobox doesn't have permission to modify the hosts file, it needs to
		// request it
		if perm := os.IsPermission(err); perm == true {

			entry := fmt.Sprintf("%-15v   %s # '%v' private network (added by nanobox)", config.Boxfile.IP, config.Boxfile.Domain, config.App)

			//
			switch mod {

			// add
			case "w":
				fmt.Printf(`
Nanobox needs your permission to add the following entry to your /etc/hosts file:
%v

`, entry)

			// delete
			case "x":
				fmt.Printf(`
Nanobox needs your permission to remove the following entry from your /etc/hosts file:
%v

`, entry)
			}

			//
			cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("sudo %v domain -%v", os.Args[0], mod))

			// connect standard in/outputs
			cmd.Stdin = os.Stdin
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			// run command
			if err := cmd.Run(); err != nil {
				ui.LogFatal("[commands.domain] cmd.Run() failed", err)
			}

			//
		} else {
			ui.LogFatal("[commands.domain] os.OpenFile() failed", err)
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"path/filepath"
	"os"
	"strings"
	"os/exec"

	"github.com/urfave/cli"
	"github.com/cbegin/graven/domain"
)

var TestCommand = cli.Command{
	Name: "test",
	Usage:       "Finds and runs tests in this project",
	Action: tester,
}

func tester(c *cli.Context) error {
	project, err := domain.FindProject()
	if err != nil {
		return err
	}

	return filepath.Walk(project.ProjectPath(), getTestWalkerFunc(project))
}

func getTestWalkerFunc(project *domain.Project) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			subDir := path[len(project.ProjectPath()):]
			subDirParts := strings.Split(subDir, string(filepath.Separator))
			matches, _ := filepath.Glob(filepath.Join(path, "*_test*"));
			if len(matches) > 0 && !contains(subDirParts, map[string]struct{}{
				"vendor":struct{}{},
				"target":struct{}{},
				".git":struct{}{}}) {
				_ = runTestCommand(subDir, project)
			}
		}
		return nil
	}
}

func contains(strings []string, exclusions map[string]struct{}) bool {
	for _, a := range strings {
		if _, ok := exclusions[a]; ok {
			return true
		}
	}
	return false
}

func runTestCommand(testPackage string, project *domain.Project) error {
	relativePath := "." + testPackage
	fmt.Println(relativePath)
	cmd := exec.Command("go", "test", "-p", "4", "-cover", "-parallel", "4", relativePath)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Dir = project.ProjectPath()

	environment := []string{}
	gopath, _ := os.LookupEnv("GOPATH")
	environment = append(environment, fmt.Sprintf("%s=%s", "GOPATH", gopath))
	cmd.Env = environment

	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("Build error. %v", err)
	}

	if !cmd.ProcessState.Success() {
		return fmt.Errorf("Build command exited in an error state. %v", cmd)
	}
	return err
}
<commit_msg>return nonzero exit code on test failure<commit_after>package commands

import (
	"fmt"
	"path/filepath"
	"os"
	"strings"
	"os/exec"

	"github.com/urfave/cli"
	"github.com/cbegin/graven/domain"
	"github.com/hashicorp/go-multierror"
)

var TestCommand = cli.Command{
	Name: "test",
	Usage:       "Finds and runs tests in this project",
	Action: tester,
}

func tester(c *cli.Context) error {
	project, err := domain.FindProject()
	if err != nil {
		return err
	}

	var merr error
	if err := filepath.Walk(project.ProjectPath(), getTestWalkerFunc(project, merr)); err != nil {
		merr = multierror.Append(merr, err)
	}
	return merr
}

func getTestWalkerFunc(project *domain.Project, merr error) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			subDir := path[len(project.ProjectPath()):]
			subDirParts := strings.Split(subDir, string(filepath.Separator))
			matches, _ := filepath.Glob(filepath.Join(path, "*_test*"));
			if len(matches) > 0 && !contains(subDirParts, map[string]struct{}{
				"vendor":struct{}{},
				"target":struct{}{},
				".git":struct{}{}}) {
				if err := runTestCommand(subDir, project); err != nil {
					merr = multierror.Append(merr, err)
				}
			}
		}
		return merr
	}
}

func contains(strings []string, exclusions map[string]struct{}) bool {
	for _, a := range strings {
		if _, ok := exclusions[a]; ok {
			return true
		}
	}
	return false
}

func runTestCommand(testPackage string, project *domain.Project) error {
	relativePath := "." + testPackage
	fmt.Println(relativePath)
	cmd := exec.Command("go", "test", "-p", "4", "-cover", "-parallel", "4", relativePath)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin
	cmd.Dir = project.ProjectPath()

	environment := []string{}
	gopath, _ := os.LookupEnv("GOPATH")
	environment = append(environment, fmt.Sprintf("%s=%s", "GOPATH", gopath))
	cmd.Env = environment

	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Error starting tests. %v", err)
	}

	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("Error state returned after tests. %v", err)
	}

	if !cmd.ProcessState.Success() {
		return fmt.Errorf("Build command exited in an error state. %v", cmd)
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/cbegin/graven/buildtool"
	"github.com/cbegin/graven/domain"
	"github.com/hashicorp/go-multierror"
	"github.com/urfave/cli"
)

var TestCommand = cli.Command{
	Name:   "test",
	Usage:  "Finds and runs tests in this project",
	Action: tester,
}

func tester(c *cli.Context) error {
	if err := clean(c); err != nil {
		return err
	}

	project, err := domain.FindProject()
	if err != nil {
		return err
	}

	err = os.MkdirAll(project.TargetPath("reports"), 0755)
	if err != nil {
		return fmt.Errorf("Could not create reports directory. %v", err)
	}

	waitGroup := &sync.WaitGroup{}
	var merr error
	if err := filepath.Walk(project.ProjectPath(), getTestWalkerFunc(project, waitGroup, &merr)); err != nil {
		merr = multierror.Append(merr, err)
	}
	waitGroup.Wait()
	return merr
}

func getTestWalkerFunc(project *domain.Project, waitGroup *sync.WaitGroup, merr *error) filepath.WalkFunc {
	// TODO - Make this configurable
	var buildTool builder.BuildTool = &builder.GoBuildTool{}
	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			subDir := path[len(project.ProjectPath()):]
			subDirParts := strings.Split(subDir, string(filepath.Separator))
			matches, _ := filepath.Glob(filepath.Join(path, "*_test.go"))
			if len(matches) > 0 && !contains(subDirParts, map[string]struct{}{
				"vendor": struct{}{},
				"target": struct{}{},
				".git":   struct{}{}}) {
				fmt.Printf("Testing %v\n", subDir)
				waitGroup.Add(1)
				go func() {
					if err := buildTool.Test(subDir, project); err != nil {
						*merr = multierror.Append(*merr, err)
					}
					waitGroup.Done()
				}()
			}
		}
		return nil
	}
}

func contains(strings []string, exclusions map[string]struct{}) bool {
	for _, a := range strings {
		if _, ok := exclusions[a]; ok {
			return true
		}
	}
	return false
}
<commit_msg>Serialized tests as running them parallel was crushing the CPU.<commit_after>package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/cbegin/graven/buildtool"
	"github.com/cbegin/graven/domain"
	"github.com/hashicorp/go-multierror"
	"github.com/urfave/cli"
)

var TestCommand = cli.Command{
	Name:   "test",
	Usage:  "Finds and runs tests in this project",
	Action: tester,
}

func tester(c *cli.Context) error {
	if err := clean(c); err != nil {
		return err
	}

	project, err := domain.FindProject()
	if err != nil {
		return err
	}

	err = os.MkdirAll(project.TargetPath("reports"), 0755)
	if err != nil {
		return fmt.Errorf("Could not create reports directory. %v", err)
	}

	var merr error
	if err := filepath.Walk(project.ProjectPath(), getTestWalkerFunc(project, &merr)); err != nil {
		merr = multierror.Append(merr, err)
	}
	return merr
}

func getTestWalkerFunc(project *domain.Project, merr *error) filepath.WalkFunc {
	// TODO - Make this configurable
	var buildTool builder.BuildTool = &builder.GoBuildTool{}
	return func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			subDir := path[len(project.ProjectPath()):]
			subDirParts := strings.Split(subDir, string(filepath.Separator))
			matches, _ := filepath.Glob(filepath.Join(path, "*_test.go"))
			if len(matches) > 0 && !contains(subDirParts, map[string]struct{}{
				"vendor": struct{}{},
				"target": struct{}{},
				".git":   struct{}{}}) {
				fmt.Printf("x Testing %v\n", subDir)
				if err := buildTool.Test(subDir, project); err != nil {
					*merr = multierror.Append(*merr, err)
				}
			}
		}
		return nil
	}
}

func contains(strings []string, exclusions map[string]struct{}) bool {
	for _, a := range strings {
		if _, ok := exclusions[a]; ok {
			return true
		}
	}
	return false
}
<|endoftext|>