<commit_before>package main

import (
	"fmt"
	"flag"
	"encoding/json"
	"os"
	"regexp"
	"sync/atomic"
	"log"
	"strings"
	"time"
	"path/filepath"
	"github.com/APTrust/bagman"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
)

// Global vars.
var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger
var taskCounter = int64(0)
var volume *bagman.Volume

// TODO: Move these out of global namespace.
// It's probably not even safe to have multiple
// go routines updated these vars without synchronization.
var succeeded = int64(0)
var failed = int64(0)
var bytesInS3 = int64(0)
var bytesProcessed = int64(0)

// test.go
//
// Downloads and tests tarred bag files from S3.
//
// Usage:
//
// $ go run test.go -config=<some config>
//
// Configuration options are defined in config.json,
// in this project's top-level directory. You can see
// a list of available configurations by running:
//
// $ go run test.go
//
// This program does the following:
//
// 1. Scans all the receiving buckets in S3.
// 2. Downloads all of the tar files in those buckets.
// 3. Untars the files.
// 4. Parses the untarred files, extracting tags and
//    verifying md5 sums.
// 5. Logs a summary for each tar file, which includes
//    information about whether the file was successfully
//    fetched, untarred, parsed and verified.
// 6. Deletes all of the downloaded and untarred files.
//
// You'll find the log in the directory specified in
// the configuration. See config.json for configuration
// options.
func main() {

	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bagman_cli")

	// Set up the volume to keep track of how much disk space is
	// available. We want to avoid downloading large files when
	// we know ahead of time that the volume containing the tar
	// directory doesn't have enough space to accomodate them.
	var err error
	volume, err = bagman.NewVolume(config.TarDirectory)
	if err != nil {
		panic(err.Error())
	}

	// Set up the channels. It's essential that that the fetchChannel
	// be limited to a relatively low number. If we are downloading
	// 1GB tar files, we need space to store the tar file AND the
	// untarred version. That's about 2 x 1GB. We do not want to pull
	// down 1000 files at once, or we'll run out of disk space!
	// If config sets fetchers to 10, we can pull down 10 files at a
	// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
	// have max 40 tar files + untarred directories on disk at once.
	// The number of workers should be close to the number of CPU
	// cores.
	fetcherBufferSize := config.Fetchers * 4
	workerBufferSize := config.Workers * 10

	fetchChannel := make(chan bagman.S3File, fetcherBufferSize)
	unpackChannel := make(chan bagman.ProcessResult, workerBufferSize)
	cleanUpChannel := make(chan bagman.ProcessResult, workerBufferSize)
	resultsChannel := make(chan bagman.ProcessResult, workerBufferSize)

	// Now fetch lists from S3 of what's in each bucket.
	messageLog.Println("[INFO]", "Checking S3 bucket lists")
	bucketSummaries, err := bagman.CheckAllBuckets(config.Buckets)
	if err != nil {
		messageLog.Println("[ERROR]", err)
		return
	}
	messageLog.Println("[INFO]", "Got info on", len(bucketSummaries), "buckets")

	// Set up our go routines. We do NOT want one go routine per
	// S3 file. If we do that, the system will run out of file handles,
	// as we'll have tens of thousands of open connections to S3
	// trying to write data into tens of thousands of local files.
	for i := 0; i < config.Fetchers; i++ {
		go doFetch(unpackChannel, resultsChannel, fetchChannel)
	}

	for i := 0; i < config.Workers; i++ {
		go doUnpack(resultsChannel, unpackChannel)
		go printResult(cleanUpChannel, resultsChannel)
		go doCleanUp(cleanUpChannel)
	}

	// Start adding S3 files into the fetch queue. Remember that this
	// queue blocks when it fills up, so we'll never be fetching more
	// than <queue size> files at once. That's what we want.
	for _, bucketSummary := range bucketSummaries {
		messageLog.Println("[INFO]", "Starting bucket", bucketSummary.BucketName,
			"which has", len(bucketSummary.Keys), "items")
		for _, key := range bucketSummary.Keys {
			if key.Size < config.MaxFileSize {
				if strings.HasSuffix(key.Key, ".tar") == false {
					messageLog.Println("[INFO]", "Ignoring non-tar file", key.Key,
						"in", bucketSummary.BucketName)
					continue
				}
				atomic.AddInt64(&taskCounter, 1)
				// TODO: Set attempt number correctly when queue is working.
				fetchChannel <- bagman.S3File{bucketSummary.BucketName, key}
				messageLog.Println("[INFO]", "Put", key.Key, "into fetch queue")
			}
		}
	}

	// Let the go routines run and wait for all tasks to complete.
	// sync.WaitGroup is the standard way of doing this, but it
	// seems to always fail on our longer-running jobs, blocking forever.
	// It works fine on shorting jobs. Perhaps we're using it incorrectly.
	// WaitGroup is really for counting go routines. We have only a handful
	// of go routines, and a huge number of tasks. We're counting the tasks.
	waitForAllTasks()
	printTotals()
}

// This function blocks until all tasks are complete.
func waitForAllTasks() {
	for {
		if atomic.LoadInt64(&taskCounter) == 0 {
			break
		}
		time.Sleep(5 * time.Second)
	}
}

// This runs as a go routine to fetch files from S3.
func doFetch(unpackChannel chan<- bagman.ProcessResult, resultsChannel chan<- bagman.ProcessResult, fetchChannel <-chan bagman.S3File) {
	for s3File := range fetchChannel {
		// Disk needs filesize * 2 disk space to accomodate tar file & untarred files
		err := volume.Reserve(uint64(s3File.Key.Size * 2))
		if err != nil {
			messageLog.Println("[WARNING]", "Requeueing", s3File.Key.Key, "- not enough disk space")
			resultsChannel <- bagman.ProcessResult{
				nil,
				nil,
				&s3File,
				err,
				nil,
				nil,
				nil,
				"",
				true}
		} else {
			messageLog.Println("[INFO]", "Fetching", s3File.Key.Key)
			fetchResult := Fetch(s3File.BucketName, s3File.Key)
			if fetchResult.Error != nil {
				resultsChannel <- bagman.ProcessResult{
					nil,
					nil,
					&s3File,
					fetchResult.Error,
					fetchResult,
					nil,
					nil,
					"",
					fetchResult.Retry}
			} else {
				unpackChannel <- bagman.ProcessResult{
					nil,
					nil,
					&s3File,
					nil,
					fetchResult,
					nil,
					nil,
					"",
					fetchResult.Retry}
			}
		}
	}
}

// This runs as a go routine to untar files downloaded from S3.
func doUnpack(resultsChannel chan<- bagman.ProcessResult, unpackChannel <-chan bagman.ProcessResult) {
	for result := range unpackChannel {
		if result.Error != nil {
			messageLog.Println("[INFO]", "Nothing to unpack for", result.S3File.Key.Key)
			resultsChannel <- result
		} else {
			messageLog.Println("[INFO]", "Unpacking", result.S3File.Key.Key)
			ProcessBagFile(&result)
			resultsChannel <- result
		}
	}
}

// This runs as a go routine to remove the files we downloaded
// and untarred.
func doCleanUp(cleanUpChannel <-chan bagman.ProcessResult) {
	for result := range cleanUpChannel {
		messageLog.Println("[INFO]", "Cleaning up", result.S3File.Key.Key)
		if result.FetchResult.LocalTarFile != "" {
			errors := CleanUp(result.FetchResult.LocalTarFile)
			if errors != nil && len(errors) > 0 {
				messageLog.Println("[WARNING]", "Errors cleaning up", result.FetchResult.LocalTarFile)
				for _, e := range errors {
					messageLog.Println("[ERROR]", e)
				}
			}
		}
		volume.Release(uint64(result.S3File.Key.Size * 2))
		atomic.AddInt64(&taskCounter, -1)
	}
}

// This prints to the log a summary of the total number of bags
// fetched and processed. This is printed at the end of the log.
func printTotals() {
	fmt.Println("-----------------------------------------------------------")
	fmt.Printf("Total Bags:       %d\n", succeeded + failed)
	fmt.Printf("Succeeded:        %d\n", succeeded)
	fmt.Printf("Failed:           %d\n", failed)
	fmt.Printf("Bytes in S3:      %d\n", bytesInS3)
	fmt.Printf("Bytes processed:  %d\n", bytesProcessed)
}

// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
func printResult(cleanUpChannel chan<- bagman.ProcessResult, resultsChannel <-chan bagman.ProcessResult) {
	for result := range resultsChannel {
		json, err := json.Marshal(result)
		if err != nil {
			messageLog.Println("[ERROR]", err)
		}
		jsonLog.Println(string(json))
		bytesInS3 += result.S3File.Key.Size
		if(result.Error != nil) {
			failed++
			messageLog.Println("[ERROR]",
				result.S3File.BucketName,
				result.S3File.Key.Key,
				"->", result.Error)
		} else {
			succeeded++
			bytesProcessed += result.S3File.Key.Size
			messageLog.Println("[INFO]", result.S3File.Key.Key, "-> OK")
		}
		cleanUpChannel <- result
	}
}

// This fetches a file from S3 and stores it locally.
func Fetch(bucketName string, key s3.Key) (result *bagman.FetchResult) {
	client, err := bagman.GetClient(aws.USEast)
	if err != nil {
		fetchResult := new(bagman.FetchResult)
		fetchResult.Error = err
		return fetchResult
	}
	// TODO: We fetched this bucket before. Do we need to fetch it again?
	bucket := client.Bucket(bucketName)
	tarFilePath := filepath.Join(config.TarDirectory, key.Key)
	return bagman.FetchToFile(bucket, key, tarFilePath)
}

// This deletes the tar file and all of the files that were
// unpacked from it. Param file is the path the tar file.
func CleanUp(file string) (errors []error) {
	errors = make([]error, 0)
	err := os.Remove(file)
	if err != nil {
		errors = append(errors, err)
	}
	// TODO: Don't assume the untarred dir name is the same as
	// the tar file. We have its actual name in the TarResult.
	re := regexp.MustCompile("\\.tar$")
	untarredDir := re.ReplaceAllString(file, "")
	err = os.RemoveAll(untarredDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error deleting dir %s: %s\n", untarredDir, err.Error())
		errors = append(errors, err)
	}
	return errors
}

// Runs tests on the bag file at path and returns information about
// whether it was successfully unpacked, valid and complete.
func ProcessBagFile(result *bagman.ProcessResult) {
	result.TarResult = bagman.Untar(result.FetchResult.LocalTarFile)
	if result.TarResult.Error != nil {
		result.Error = result.TarResult.Error
		// If we can't untar this, there's no reason to retry...
		// but we'll have to revisit this. There may be cases
		// where we do want to retry, such as if disk was full.
		result.Retry = false
	} else {
		result.BagReadResult = bagman.ReadBag(result.TarResult.OutputDir)
		if result.BagReadResult.Error != nil {
			result.Error = result.BagReadResult.Error
			// Something was wrong with this bag. Bad checksum,
			// missing file, etc. Don't reprocess it.
			result.Retry = false
		}
	}
}
<commit_msg>Brought cli up to speed with recent changes<commit_after>package main

import (
    "fmt"
    "flag"
    "encoding/json"
    "os"
    "regexp"
    "sync/atomic"
    "log"
    "strings"
    "time"
    "path/filepath"
    "github.com/APTrust/bagman"
    "launchpad.net/goamz/aws"
    "launchpad.net/goamz/s3"
)

// Global vars.
var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger
var taskCounter = int64(0)
var volume *bagman.Volume
var s3Client *bagman.S3Client

// TODO: Move these out of global namespace.
// It's probably not even safe to have multiple
// go routines updated these vars without synchronization.
var succeeded = int64(0)
var failed = int64(0)
var bytesInS3 = int64(0)
var bytesProcessed = int64(0)

// test.go
//
// Downloads and tests tarred bag files from S3.
//
// Usage:
//
// $ go run test.go -config=<some config>
//
// Configuration options are defined in config.json,
// in this project's top-level directory. You can see
// a list of available configurations by running:
//
// $ go run test.go
//
// This program does the following:
//
// 1. Scans all the receiving buckets in S3.
// 2. Downloads all of the tar files in those buckets.
// 3. Untars the files.
// 4. Parses the untarred files, extracting tags and
//    verifying md5 sums.
// 5. Logs a summary for each tar file, which includes
//    information about whether the file was successfully
//    fetched, untarred, parsed and verified.
// 6. Deletes all of the downloaded and untarred files.
//
// You'll find the log in the directory specified in
// the configuration. See config.json for configuration
// options.
func main() {

    // Load the config or die.
    requestedConfig := flag.String("config", "", "configuration to run")
    flag.Parse()
    config = bagman.LoadRequestedConfig(requestedConfig)
    jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bagman_cli")

    // Set up the volume to keep track of how much disk space is
    // available. We want to avoid downloading large files when
    // we know ahead of time that the volume containing the tar
    // directory doesn't have enough space to accomodate them.
    var err error
    volume, err = bagman.NewVolume(config.TarDirectory)
    if err != nil {
        panic(err.Error())
    }

    s3Client, err = bagman.NewS3Client(aws.USEast)
    if err != nil {
        panic(err.Error())
    }

    // Set up the channels. It's essential that that the fetchChannel
    // be limited to a relatively low number. If we are downloading
    // 1GB tar files, we need space to store the tar file AND the
    // untarred version. That's about 2 x 1GB. We do not want to pull
    // down 1000 files at once, or we'll run out of disk space!
    // If config sets fetchers to 10, we can pull down 10 files at a
    // time. The fetch queue could hold 10 * 4 = 40 items, so we'd
    // have max 40 tar files + untarred directories on disk at once.
    // The number of workers should be close to the number of CPU
    // cores.
    fetcherBufferSize := config.Fetchers * 4
    workerBufferSize := config.Workers * 10

    fetchChannel := make(chan bagman.S3File, fetcherBufferSize)
    unpackChannel := make(chan bagman.ProcessResult, workerBufferSize)
    cleanUpChannel := make(chan bagman.ProcessResult, workerBufferSize)
    resultsChannel := make(chan bagman.ProcessResult, workerBufferSize)

    // Now fetch lists from S3 of what's in each bucket.
    messageLog.Println("[INFO]", "Checking S3 bucket lists")
    bucketSummaries, err := s3Client.CheckAllBuckets(config.Buckets)
    if err != nil {
        messageLog.Println("[ERROR]", err)
        return
    }
    messageLog.Println("[INFO]", "Got info on", len(bucketSummaries), "buckets")

    // Set up our go routines. We do NOT want one go routine per
    // S3 file. If we do that, the system will run out of file handles,
    // as we'll have tens of thousands of open connections to S3
    // trying to write data into tens of thousands of local files.
    for i := 0; i < config.Fetchers; i++ {
        go doFetch(unpackChannel, resultsChannel, fetchChannel)
    }

    for i := 0; i < config.Workers; i++ {
        go doUnpack(resultsChannel, unpackChannel)
        go printResult(cleanUpChannel, resultsChannel)
        go doCleanUp(cleanUpChannel)
    }

    // Start adding S3 files into the fetch queue. Remember that this
    // queue blocks when it fills up, so we'll never be fetching more
    // than <queue size> files at once. That's what we want.
    for _, bucketSummary := range bucketSummaries {
        messageLog.Println("[INFO]", "Starting bucket", bucketSummary.BucketName,
            "which has", len(bucketSummary.Keys), "items")
        for _, key := range bucketSummary.Keys {
            if key.Size < config.MaxFileSize {
                if strings.HasSuffix(key.Key, ".tar") == false {
                    messageLog.Println("[INFO]", "Ignoring non-tar file", key.Key,
                        "in", bucketSummary.BucketName)
                    continue
                }
                atomic.AddInt64(&taskCounter, 1)
                // TODO: Set attempt number correctly when queue is working.
                fetchChannel <- bagman.S3File{bucketSummary.BucketName, key}
                messageLog.Println("[INFO]", "Put", key.Key, "into fetch queue")
            }
        }
    }

    // Let the go routines run and wait for all tasks to complete.
    // sync.WaitGroup is the standard way of doing this, but it
    // seems to always fail on our longer-running jobs, blocking forever.
    // It works fine on shorting jobs. Perhaps we're using it incorrectly.
    // WaitGroup is really for counting go routines. We have only a handful
    // of go routines, and a huge number of tasks. We're counting the tasks.
    waitForAllTasks()
    printTotals()
}

// This function blocks until all tasks are complete.
func waitForAllTasks() {
    for {
        if atomic.LoadInt64(&taskCounter) == 0 {
            break
        }
        time.Sleep(5 * time.Second)
    }
}

// This runs as a go routine to fetch files from S3.
func doFetch(unpackChannel chan<- bagman.ProcessResult, resultsChannel chan<- bagman.ProcessResult, fetchChannel <-chan bagman.S3File) {
    for s3File := range fetchChannel {
        // Disk needs filesize * 2 disk space to accomodate tar file & untarred files
        err := volume.Reserve(uint64(s3File.Key.Size * 2))
        if err != nil {
            messageLog.Println("[WARNING]", "Requeueing", s3File.Key.Key, "- not enough disk space")
            resultsChannel <- bagman.ProcessResult{
                nil,
                &s3File,
                err.Error(),
                nil,
                nil,
                nil,
                "",
                true}
        } else {
            messageLog.Println("[INFO]", "Fetching", s3File.Key.Key)
            fetchResult := Fetch(s3File.BucketName, s3File.Key)
            if fetchResult.ErrorMessage != "" {
                resultsChannel <- bagman.ProcessResult{
                    nil,
                    &s3File,
                    fetchResult.ErrorMessage,
                    fetchResult,
                    nil,
                    nil,
                    "",
                    fetchResult.Retry}
            } else {
                unpackChannel <- bagman.ProcessResult{
                    nil,
                    &s3File,
                    "",
                    fetchResult,
                    nil,
                    nil,
                    "",
                    fetchResult.Retry}
            }
        }
    }
}

// This runs as a go routine to untar files downloaded from S3.
func doUnpack(resultsChannel chan<- bagman.ProcessResult, unpackChannel <-chan bagman.ProcessResult) {
    for result := range unpackChannel {
        if result.ErrorMessage != "" {
            messageLog.Println("[INFO]", "Nothing to unpack for", result.S3File.Key.Key)
            resultsChannel <- result
        } else {
            messageLog.Println("[INFO]", "Unpacking", result.S3File.Key.Key)
            ProcessBagFile(&result)
            resultsChannel <- result
        }
    }
}

// This runs as a go routine to remove the files we downloaded
// and untarred.
func doCleanUp(cleanUpChannel <-chan bagman.ProcessResult) {
    for result := range cleanUpChannel {
        messageLog.Println("[INFO]", "Cleaning up", result.S3File.Key.Key)
        if result.FetchResult.LocalTarFile != "" {
            errors := CleanUp(result.FetchResult.LocalTarFile)
            if errors != nil && len(errors) > 0 {
                messageLog.Println("[WARNING]", "Errors cleaning up", result.FetchResult.LocalTarFile)
                for _, e := range errors {
                    messageLog.Println("[ERROR]", e)
                }
            }
        }
        volume.Release(uint64(result.S3File.Key.Size * 2))
        atomic.AddInt64(&taskCounter, -1)
    }
}

// This prints to the log a summary of the total number of bags
// fetched and processed. This is printed at the end of the log.
func printTotals() {
    fmt.Println("-----------------------------------------------------------")
    fmt.Printf("Total Bags:       %d\n", succeeded + failed)
    fmt.Printf("Succeeded:        %d\n", succeeded)
    fmt.Printf("Failed:           %d\n", failed)
    fmt.Printf("Bytes in S3:      %d\n", bytesInS3)
    fmt.Printf("Bytes processed:  %d\n", bytesProcessed)
}

// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
func printResult(cleanUpChannel chan<- bagman.ProcessResult, resultsChannel <-chan bagman.ProcessResult) {
    for result := range resultsChannel {
        json, err := json.Marshal(result)
        if err != nil {
            messageLog.Println("[ERROR]", err)
        }
        jsonLog.Println(string(json))
        bytesInS3 += result.S3File.Key.Size
        if(result.ErrorMessage != "") {
            failed++
            messageLog.Println("[ERROR]",
                result.S3File.BucketName,
                result.S3File.Key.Key,
                "->", result.ErrorMessage)
        } else {
            succeeded++
            bytesProcessed += result.S3File.Key.Size
            messageLog.Println("[INFO]", result.S3File.Key.Key, "-> OK")
        }
        cleanUpChannel <- result
    }
}

// This fetches a file from S3 and stores it locally.
func Fetch(bucketName string, key s3.Key) (result *bagman.FetchResult) {
    tarFilePath := filepath.Join(config.TarDirectory, key.Key)
    return s3Client.FetchToFile(bucketName, key, tarFilePath)
}

// This deletes the tar file and all of the files that were
// unpacked from it. Param file is the path the tar file.
func CleanUp(file string) (errors []error) {
    errors = make([]error, 0)
    err := os.Remove(file)
    if err != nil {
        errors = append(errors, err)
    }
    // TODO: Don't assume the untarred dir name is the same as
    // the tar file. We have its actual name in the TarResult.
    re := regexp.MustCompile("\\.tar$")
    untarredDir := re.ReplaceAllString(file, "")
    err = os.RemoveAll(untarredDir)
    if err != nil {
        fmt.Fprintf(os.Stderr, "Error deleting dir %s: %s\n", untarredDir, err.Error())
        errors = append(errors, err)
    }
    return errors
}

// Runs tests on the bag file at path and returns information about
// whether it was successfully unpacked, valid and complete.
func ProcessBagFile(result *bagman.ProcessResult) {
    result.TarResult = bagman.Untar(result.FetchResult.LocalTarFile)
    if result.TarResult.ErrorMessage != "" {
        result.ErrorMessage = result.TarResult.ErrorMessage
        // If we can't untar this, there's no reason to retry...
        // but we'll have to revisit this. There may be cases
        // where we do want to retry, such as if disk was full.
        result.Retry = false
    } else {
        result.BagReadResult = bagman.ReadBag(result.TarResult.OutputDir)
        if result.BagReadResult.ErrorMessage != "" {
            result.ErrorMessage = result.BagReadResult.ErrorMessage
            // Something was wrong with this bag. Bad checksum,
            // missing file, etc. Don't reprocess it.
            result.Retry = false
        }
    }
}
<|endoftext|>


<commit_before>package netutil

import (
	"bytes"

	"github.com/vmihailenco/msgpack"
	"github.com/xiaonanln/goworld/engine/gwlog"
)

// MessagePackMsgPacker packs and unpacks message in MessagePack format
type MessagePackMsgPacker struct{}

// PackMsg packs message to bytes in MessagePack format
func (mp MessagePackMsgPacker) PackMsg(msg interface{}, buf []byte) ([]byte, error) {
	buffer := bytes.NewBuffer(buf)

	encoder := msgpack.NewEncoder(buffer)
	err := encoder.Encode(msg)
	if err != nil {
		return buf, err
	}
	buf = buffer.Bytes()
	return buf, nil
}

// UnpackMsg unpacksbytes in MessagePack format to message
func (mp MessagePackMsgPacker) UnpackMsg(data []byte, msg interface{}) error {
	err := msgpack.Unmarshal(data, msg)
	if pv, ok := msg.(*interface{}); ok {
		*pv = mp.convertToStringKeys(*pv)
	} else if pv, ok := msg.(*[]interface{}); ok {
		*pv = mp.convertToStringKeys(*pv).([]interface{})
	} else if pv, ok := msg.(*map[string]interface{}); ok {
		*pv = mp.convertToStringKeys(*pv).(map[string]interface{})
	}

	return err
}

func (mp MessagePackMsgPacker) convertToStringKeys(v interface{}) interface{} {
	defer func() {
		err := recover()
		if err != nil {
			gwlog.Errorf("MessagePackMsgPacker.convertToStringKeys failed")
			panic(err)
		}
	}()

	if rv, ok := v.(map[interface{}]interface{}); ok {
		rrv := make(map[string]interface{}, len(rv))
		for k, _v := range rv {
			ks, ok := k.(string)
			if !ok {
				gwlog.Panicf("%v is not string, but %T", k, k)
			}

			rrv[ks] = mp.convertToStringKeys(_v)
		}
		return rrv
	}

	if rv, ok := v.(map[string]interface{}); ok {
		for k, _v := range rv {
			rv[k] = mp.convertToStringKeys(_v)
		}
		return rv
	}

	if rv, ok := v.([]interface{}); ok {
		for i, _v := range rv {
			rv[i] = mp.convertToStringKeys(_v)
		}
		return rv
	}

	return v
}
<commit_msg>use MessagePackMsgPacker for freeze<commit_after>package netutil

import (
	"bytes"

	"github.com/vmihailenco/msgpack"
	"github.com/xiaonanln/goworld/engine/gwlog"
)

// MessagePackMsgPacker packs and unpacks message in MessagePack format
type MessagePackMsgPacker struct{}

// PackMsg packs message to bytes in MessagePack format
func (mp MessagePackMsgPacker) PackMsg(msg interface{}, buf []byte) ([]byte, error) {
	buffer := bytes.NewBuffer(buf)

	encoder := msgpack.NewEncoder(buffer)
	err := encoder.Encode(msg)
	if err != nil {
		return buf, err
	}
	buf = buffer.Bytes()
	return buf, nil
}

// UnpackMsg unpacksbytes in MessagePack format to message
func (mp MessagePackMsgPacker) UnpackMsg(data []byte, msg interface{}) error {
	err := msgpack.Unmarshal(data, msg)
	if pv, ok := msg.(*interface{}); ok {
		*pv = mp.convertToStringKeys(*pv)
	} else if pv, ok := msg.(*[]interface{}); ok {
		*pv = mp.convertToStringKeys(*pv).([]interface{})
	} else if pv, ok := msg.(*map[string]interface{}); ok {
		*pv = mp.convertToStringKeys(*pv).(map[string]interface{})
	} else if pv, ok := msg.(*map[interface{}]interface{}); ok {
		*pv = mp.convertToStringKeys(*pv).(map[interface{}]interface{})
	}

	return err
}

func (mp MessagePackMsgPacker) convertToStringKeys(v interface{}) interface{} {
	defer func() {
		err := recover()
		if err != nil {
			gwlog.Errorf("MessagePackMsgPacker.convertToStringKeys failed")
			panic(err)
		}
	}()

	if rv, ok := v.(map[interface{}]interface{}); ok {
		rrv := make(map[string]interface{}, len(rv))
		for k, _v := range rv {
			ks, ok := k.(string)
			if !ok {
				gwlog.Panicf("%v is not string, but %T", k, k)
			}

			rrv[ks] = mp.convertToStringKeys(_v)
		}
		return rrv
	}

	if rv, ok := v.(map[string]interface{}); ok {
		for k, _v := range rv {
			rv[k] = mp.convertToStringKeys(_v)
		}
		return rv
	}

	if rv, ok := v.([]interface{}); ok {
		for i, _v := range rv {
			rv[i] = mp.convertToStringKeys(_v)
		}
		return rv
	}

	return v
}
<|endoftext|>


<commit_before>package cli

import (
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/envman/envman"
	"github.com/bitrise-io/envman/models"
	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/codegangsta/cli"
)

// CommandModel ...
type CommandModel struct {
	Command      string
	Argumentums  []string
	Environments []models.EnvironmentItemModel
}

func expandEnvsInString(inp string) string {
	return os.ExpandEnv(inp)
}

func commandEnvs(envs []models.EnvironmentItemModel) ([]string, error) {
	for _, env := range envs {
		key, value, err := env.GetKeyValuePair()
		if err != nil {
			return []string{}, err
		}

		opts, err := env.GetOptions()
		if err != nil {
			return []string{}, err
		}

		var valueStr string
		if *opts.IsExpand {
			valueStr = expandEnvsInString(value)
		} else {
			valueStr = value
		}

		if err := os.Setenv(key, valueStr); err != nil {
			return []string{}, err
		}
	}
	return os.Environ(), nil
}

func runCommandModel(cmdModel CommandModel) (int, error) {
	cmdEnvs, err := commandEnvs(cmdModel.Environments)
	if err != nil {
		return 1, err
	}

	return cmdex.RunCommandWithEnvsAndReturnExitCode(cmdEnvs, cmdModel.Command, cmdModel.Argumentums...)
}

func run(c *cli.Context) {
	log.Debugln("[ENVMAN] - Work path:", envman.CurrentEnvStoreFilePath)

	if len(c.Args()) > 0 {
		doCmdEnvs, err := envman.ReadEnvs(envman.CurrentEnvStoreFilePath)
		if err != nil {
			log.Fatal("[ENVMAN] - Failed to load EnvStore:", err)
		}

		doCommand := c.Args()[0]

		doArgs := []string{}
		if len(c.Args()) > 1 {
			doArgs = c.Args()[1:]
		}

		cmdToExecute := CommandModel{
			Command:      doCommand,
			Environments: doCmdEnvs,
			Argumentums:  doArgs,
		}

		log.Debugln("[ENVMAN] - Executing command:", cmdToExecute)

		if exit, err := runCommandModel(cmdToExecute); err != nil {
			os.Exit(exit)
		}

		log.Debugln("[ENVMAN] - Command executed")
	} else {
		log.Fatal("[ENVMAN] - No command specified")
	}
}
<commit_msg>exit code fix<commit_after>package cli

import (
	"os"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/envman/envman"
	"github.com/bitrise-io/envman/models"
	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/codegangsta/cli"
)

// CommandModel ...
type CommandModel struct {
	Command      string
	Argumentums  []string
	Environments []models.EnvironmentItemModel
}

func expandEnvsInString(inp string) string {
	return os.ExpandEnv(inp)
}

func commandEnvs(envs []models.EnvironmentItemModel) ([]string, error) {
	for _, env := range envs {
		key, value, err := env.GetKeyValuePair()
		if err != nil {
			return []string{}, err
		}

		opts, err := env.GetOptions()
		if err != nil {
			return []string{}, err
		}

		var valueStr string
		if *opts.IsExpand {
			valueStr = expandEnvsInString(value)
		} else {
			valueStr = value
		}

		if err := os.Setenv(key, valueStr); err != nil {
			return []string{}, err
		}
	}
	return os.Environ(), nil
}

func runCommandModel(cmdModel CommandModel) (int, error) {
	cmdEnvs, err := commandEnvs(cmdModel.Environments)
	if err != nil {
		return 1, err
	}

	return cmdex.RunCommandWithEnvsAndReturnExitCode(cmdEnvs, cmdModel.Command, cmdModel.Argumentums...)
}

func run(c *cli.Context) {
	log.Debugln("[ENVMAN] - Work path:", envman.CurrentEnvStoreFilePath)

	if len(c.Args()) > 0 {
		doCmdEnvs, err := envman.ReadEnvs(envman.CurrentEnvStoreFilePath)
		if err != nil {
			log.Fatal("[ENVMAN] - Failed to load EnvStore:", err)
		}

		doCommand := c.Args()[0]

		doArgs := []string{}
		if len(c.Args()) > 1 {
			doArgs = c.Args()[1:]
		}

		cmdToExecute := CommandModel{
			Command:      doCommand,
			Environments: doCmdEnvs,
			Argumentums:  doArgs,
		}

		log.Debugln("[ENVMAN] - Executing command:", cmdToExecute)

		if exit, err := runCommandModel(cmdToExecute); err != nil {
			if exit == 0 {
				exit = 1
			}
			os.Exit(exit)
		}

		log.Debugln("[ENVMAN] - Command executed")
	} else {
		log.Fatal("[ENVMAN] - No command specified")
	}
}
<|endoftext|>


<commit_before>package command

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	"github.com/coreos/etcd/store"
)

type set struct {
	key   string
	value string
	ttl   int64
}

func NewImportSnapCommand() cli.Command {
	return cli.Command{
		Name:  "import",
		Usage: "import a snapshot to a cluster",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "snap", Value: "", Usage: "Path to the vaild etcd 0.4.x snapshot."},
			cli.IntFlag{Name: "c", Value: 10, Usage: "Number of concurrent clients to import the data"},
		},
		Action: handleImportSnap,
	}
}

func handleImportSnap(c *cli.Context) {
	d, err := ioutil.ReadFile(c.String("snap"))
	if err != nil {
		if c.String("snap") == "" {
			fmt.Printf("no snapshot file provided (use --snap)")
		} else {
			fmt.Printf("cannot read snapshot file %s\n", c.String("snap"))
		}
		os.Exit(1)
	}

	st := store.New()
	err = st.Recovery(d)
	if err != nil {
		fmt.Printf("cannot recover the snapshot file: %v\n", err)
		os.Exit(1)
	}

	endpoints, err := getEndpoints(c)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}
	tr, err := getTransport(c)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}

	wg := &sync.WaitGroup{}
	setc := make(chan set)
	concurrent := c.Int("c")
	fmt.Printf("starting to import snapshot %s with %d clients\n", c.String("snap"), concurrent)
	for i := 0; i < concurrent; i++ {
		client := etcd.NewClient(endpoints)
		client.SetTransport(tr)

		if c.GlobalBool("debug") {
			go dumpCURL(client)
		}

		if ok := client.SyncCluster(); !ok {
			handleError(FailedToConnectToHost, errors.New("cannot sync with the cluster using endpoints "+strings.Join(endpoints, ", ")))
		}
		wg.Add(1)
		go runSet(client, setc, wg)
	}

	all, err := st.Get("/", true, true)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}
	n := copyKeys(all.Node, setc)
	close(setc)
	wg.Wait()
	fmt.Printf("finished importing %d keys\n", n)
}

func copyKeys(n *store.NodeExtern, setc chan set) int {
	num := 0
	if !n.Dir {
		setc <- set{n.Key, *n.Value, n.TTL}
		return 1
	}
	log.Println("entering dir:", n.Key)
	for _, nn := range n.Nodes {
		sub := copyKeys(nn, setc)
		num += sub
	}
	return num
}

func runSet(c *etcd.Client, setc chan set, wg *sync.WaitGroup) {
	for s := range setc {
		log.Println("copying key:", s.key)
		if s.ttl != 0 && s.ttl < 300 {
			log.Printf("extending key %s's ttl to 300 seconds", s.key)
			s.ttl = 5 * 60
		}
		_, err := c.Set(s.key, s.value, uint64(s.ttl))
		if err != nil {
			log.Fatalf("failed to copy key: %v\n", err)
		}
	}
	wg.Done()
}
<commit_msg>etcdctl: import hidden keys<commit_after>package command

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd"
	"github.com/coreos/etcd/store"
)

type set struct {
	key   string
	value string
	ttl   int64
}

func NewImportSnapCommand() cli.Command {
	return cli.Command{
		Name:  "import",
		Usage: "import a snapshot to a cluster",
		Flags: []cli.Flag{
			cli.StringFlag{Name: "snap", Value: "", Usage: "Path to the vaild etcd 0.4.x snapshot."},
			cli.StringSliceFlag{Name: "hidden", Value: nil, Usage: "Hidden key space to import from snapshot"},
			cli.IntFlag{Name: "c", Value: 10, Usage: "Number of concurrent clients to import the data"},
		},
		Action: handleImportSnap,
	}
}

func handleImportSnap(c *cli.Context) {
	d, err := ioutil.ReadFile(c.String("snap"))
	if err != nil {
		if c.String("snap") == "" {
			fmt.Printf("no snapshot file provided (use --snap)")
		} else {
			fmt.Printf("cannot read snapshot file %s\n", c.String("snap"))
		}
		os.Exit(1)
	}

	st := store.New()
	err = st.Recovery(d)
	if err != nil {
		fmt.Printf("cannot recover the snapshot file: %v\n", err)
		os.Exit(1)
	}

	endpoints, err := getEndpoints(c)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}
	tr, err := getTransport(c)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}

	wg := &sync.WaitGroup{}
	setc := make(chan set)
	concurrent := c.Int("c")
	fmt.Printf("starting to import snapshot %s with %d clients\n", c.String("snap"), concurrent)
	for i := 0; i < concurrent; i++ {
		client := etcd.NewClient(endpoints)
		client.SetTransport(tr)

		if c.GlobalBool("debug") {
			go dumpCURL(client)
		}

		if ok := client.SyncCluster(); !ok {
			handleError(FailedToConnectToHost, errors.New("cannot sync with the cluster using endpoints "+strings.Join(endpoints, ", ")))
		}
		wg.Add(1)
		go runSet(client, setc, wg)
	}

	all, err := st.Get("/", true, true)
	if err != nil {
		handleError(ErrorFromEtcd, err)
	}
	n := copyKeys(all.Node, setc)

	hiddens := c.StringSlice("hidden")
	for _, h := range hiddens {
		allh, err := st.Get(h, true, true)
		if err != nil {
			handleError(ErrorFromEtcd, err)
		}
		n += copyKeys(allh.Node, setc)
	}
	close(setc)
	wg.Wait()
	fmt.Printf("finished importing %d keys\n", n)
}

func copyKeys(n *store.NodeExtern, setc chan set) int {
	num := 0
	if !n.Dir {
		setc <- set{n.Key, *n.Value, n.TTL}
		return 1
	}
	log.Println("entering dir:", n.Key)
	for _, nn := range n.Nodes {
		sub := copyKeys(nn, setc)
		num += sub
	}
	return num
}

func runSet(c *etcd.Client, setc chan set, wg *sync.WaitGroup) {
	for s := range setc {
		log.Println("copying key:", s.key)
		if s.ttl != 0 && s.ttl < 300 {
			log.Printf("extending key %s's ttl to 300 seconds", s.key)
			s.ttl = 5 * 60
		}
		_, err := c.Set(s.key, s.value, uint64(s.ttl))
		if err != nil {
			log.Fatalf("failed to copy key: %v\n", err)
		}
	}
	wg.Done()
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-ego Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package ego

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"strings"
)

func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}

func CheckErr(err error) {
	if err != nil {
		panic(err)
	}
}

func ListFile(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func ListDir(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if !fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

// Get http
func Get(apiUrl string, params url.Values) (rs []byte, err error) {
	var Url *url.URL
	Url, err = url.Parse(apiUrl)
	if err != nil {
		fmt.Printf("analytic url error:\r\n%v", err)
		return nil, err
	}
	// URLEncode
	Url.RawQuery = params.Encode()
	resp, err := http.Get(Url.String())
	if err != nil {
		fmt.Println("err:", err)
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

// Post http, params is url.Values type
func Post(apiUrl string, params url.Values) (rs []byte, err error) {

	resp, err := http.PostForm(apiUrl, params)
	if err != nil {
		return nil, err
	}
	// fmt.Println("http:", resp)
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

// API
func API(httpUrl string, paramMap Map, method ...string) (rs []byte, err error) {
	param := url.Values{}
	for k, v := range paramMap {
		param.Set(k, v.(string))
	}

	amethod := "post"
	if len(method) > 0 {
		amethod = method[0]
	}

	// var rebody ioutil.ReadAll
	var rebody []byte
	var aerr error
	if amethod == "get" {
		rebody, aerr = Get(httpUrl, param)

	} else {
		rebody, aerr = Post(httpUrl, param)
	}

	return rebody, aerr
}

var Url url.Values = url.Values{}

// TestRest test restful and return json
func (router *Engine) TestRest(httpUrl string, param url.Values) {

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t_" + lastUrl

	router.GET(url, func(c *Context) {
		data, err := Post(httpUrl, param)

		if err != nil {
			fmt.Errorf("Request failed, error message:\r\n%v", err)
		} else {
			var netReturn map[string]interface{}

			json.Unmarshal(data, &netReturn)

			reContent := netReturn["content"]

			c.JSON(200, reContent)
		}
	})
}

// TestJson test restful and return json
func (router *Engine) TestJson(httpUrl string, param url.Values, args ...string) {
	var content string

	if len(args) > 0 {
		content = args[0]
	} else {
		// content = "content"
		content = "data"
	}

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t/" + lastUrl + "json"
	router.GET(url, func(c *Context) {
		data, err := Post(httpUrl, param)
		if err != nil {
			fmt.Errorf("Request failed, error message:\r\n%v", err)
		} else {
			var netReturn map[string]interface{}
			// ffjson.Unmarshal(data, &netReturn)
			json.Unmarshal(data, &netReturn)
			reContent := netReturn[content]

			c.JSON(200, reContent)
		}
	})
}

var (
	ajax int64
)

// TestHtml test restful and show pretty in the browser
func (router *Engine) TestHtml(httpUrl string, paramMap Map, args ...string) {
	if ajax != 1 {
		router.StaticFile("/t/ajax", "./views/js/ajax.js")
	}
	ajax = 1

	param := url.Values{}
	for k, v := range paramMap {
		param.Set(k, v.(string))
	}
	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t/" + lastUrl

	if len(args) > 0 {
		router.TestJson(httpUrl, param, args[0])
	} else {
		router.TestJson(httpUrl, param)
	}

	router.GET(url, func(c *Context) {
		c.HTML(200, "json.html", Map{
			"test": httpUrl,
		})
	})
}

// TestFile test restful and show pretty in the browser
func (router *Engine) TestFile(httpUrl string, paramMap Map, filename, upParam string) {
	if ajax != 1 {
		router.StaticFile("/t/ajax", "./views/js/ajax.js")
	}
	ajax = 1

	var (
		url string
		i   int64
	)

	for k, v := range paramMap {
		i++
		if i == 1 {
			url += k + "=" + v.(string)
		} else {
			url += "&" + k + "=" + v.(string)
		}
	}

	confUrl := httpUrl + "?" + url

	fmt.Println("confUrl-------", confUrl)
	// confUrl := url.Values{}

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	htmlurl := "/t/" + lastUrl
	jsonurl := htmlurl + "json"

	router.GET(jsonurl, func(c *Context) {
		resp, err := PostFile(filename, confUrl, upParam)
		if err != nil {
			fmt.Println("err--------", err)
		}
		fmt.Println("resp---------", resp)

		c.JSON(200, resp)
	})

	router.GET(htmlurl, func(c *Context) {
		c.HTML(200, "json.html", Map{
			"test": httpUrl,
		})
	})

}

// PostFile post file
func PostFile(filename, targetUrl, upParam string) (string, error) {
	bodyBuf := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(bodyBuf)

	// uploadfile
	fileWriter, err := bodyWriter.CreateFormFile(upParam, filename)
	if err != nil {
		fmt.Println("error writing to buffer")
		return "", err
	}

	// openfile
	fh, err := os.Open(filename)
	if err != nil {
		fmt.Println("error opening file")
		return "", err
	}

	// iocopy
	_, err = io.Copy(fileWriter, fh)
	if err != nil {
		return "", err
	}

	contentType := bodyWriter.FormDataContentType()
	bodyWriter.Close()

	resp, err := http.Post(targetUrl, contentType, bodyBuf)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(resp.Status)
	// fmt.Println(string(respBody))

	return string(respBody), nil
}
<commit_msg>Update http base<commit_after>// Copyright 2016 The go-ego Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package ego

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}

func CheckErr(err error) {
	if err != nil {
		panic(err)
	}
}

func ListFile(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

func ListDir(dirPth string, suffix string) (files []string, err error) {
	files = make([]string, 0, 10)
	dir, err := ioutil.ReadDir(dirPth)
	if err != nil {
		return nil, err
	}
	PthSep := string(os.PathSeparator)
	suffix = strings.ToUpper(suffix)
	for _, fi := range dir {
		if !fi.IsDir() {
			continue
		}
		if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {
			files = append(files, dirPth+PthSep+fi.Name())
		}
	}
	return files, nil
}

// Get http
func Get(apiUrl string, params url.Values) (rs []byte, err error) {
	var Url *url.URL
	Url, err = url.Parse(apiUrl)
	if err != nil {
		fmt.Printf("analytic url error:\r\n%v", err)
		return nil, err
	}
	// URLEncode
	Url.RawQuery = params.Encode()
	resp, err := http.Get(Url.String())
	if err != nil {
		fmt.Println("err:", err)
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

// Post http, params is url.Values type
func Post(apiUrl string, params url.Values, args ...int) (rs []byte, err error) {
	out := 1000
	if len(args) > 0 {
		out = args[0]
	}

	timeOut := time.Duration(out) * time.Millisecond

	c := &http.Client{
		Timeout: timeOut,
	}

	resp, err := c.PostForm(apiUrl, params)
	if err != nil {
		return nil, err
	}

	// fmt.Println("http:", resp)
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

// API
func API(httpUrl string, paramMap Map, method ...string) (rs []byte, err error) {
	param := url.Values{}
	for k, v := range paramMap {
		param.Set(k, v.(string))
	}

	amethod := "post"
	if len(method) > 0 {
		amethod = method[0]
	}

	// var rebody ioutil.ReadAll
	var rebody []byte
	var aerr error
	if amethod == "get" {
		rebody, aerr = Get(httpUrl, param)

	} else {
		rebody, aerr = Post(httpUrl, param)
	}

	return rebody, aerr
}

var Url url.Values = url.Values{}

// TestRest test restful and return json
func (router *Engine) TestRest(httpUrl string, param url.Values) {

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t_" + lastUrl

	router.GET(url, func(c *Context) {
		data, err := Post(httpUrl, param)

		if err != nil {
			fmt.Errorf("Request failed, error message:\r\n%v", err)
		} else {
			var netReturn map[string]interface{}

			json.Unmarshal(data, &netReturn)

			reContent := netReturn["content"]

			c.JSON(200, reContent)
		}
	})
}

// TestJson test restful and return json
func (router *Engine) TestJson(httpUrl string, param url.Values, args ...string) {
	var content string

	if len(args) > 0 {
		content = args[0]
	} else {
		// content = "content"
		content = "data"
	}

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t/" + lastUrl + "json"
	router.GET(url, func(c *Context) {
		data, err := Post(httpUrl, param)
		if err != nil {
			fmt.Errorf("Request failed, error message:\r\n%v", err)
		} else {
			var netReturn map[string]interface{}
			// ffjson.Unmarshal(data, &netReturn)
			json.Unmarshal(data, &netReturn)
			reContent := netReturn[content]

			c.JSON(200, reContent)
		}
	})
}

var (
	ajax int64
)

// TestHtml test restful and show pretty in the browser
func (router *Engine) TestHtml(httpUrl string, paramMap Map, args ...string) {
	if ajax != 1 {
		router.StaticFile("/t/ajax", "./views/js/ajax.js")
	}
	ajax = 1

	param := url.Values{}
	for k, v := range paramMap {
		param.Set(k, v.(string))
	}
	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	url := "/t/" + lastUrl

	if len(args) > 0 {
		router.TestJson(httpUrl, param, args[0])
	} else {
		router.TestJson(httpUrl, param)
	}

	router.GET(url, func(c *Context) {
		c.HTML(200, "json.html", Map{
			"test": httpUrl,
		})
	})
}

// TestFile test restful and show pretty in the browser
func (router *Engine) TestFile(httpUrl string, paramMap Map, filename, upParam string) {
	if ajax != 1 {
		router.StaticFile("/t/ajax", "./views/js/ajax.js")
	}
	ajax = 1

	var (
		url string
		i   int64
	)

	for k, v := range paramMap {
		i++
		if i == 1 {
			url += k + "=" + v.(string)
		} else {
			url += "&" + k + "=" + v.(string)
		}
	}

	confUrl := httpUrl + "?" + url

	fmt.Println("confUrl-------", confUrl)
	// confUrl := url.Values{}

	listUrl := strings.Split(httpUrl, "/")
	lastUrl := listUrl[len(listUrl)-1]

	htmlurl := "/t/" + lastUrl
	jsonurl := htmlurl + "json"

	router.GET(jsonurl, func(c *Context) {
		resp, err := PostFile(filename, confUrl, upParam)
		if err != nil {
			fmt.Println("err--------", err)
		}
		fmt.Println("resp---------", resp)

		c.JSON(200, resp)
	})

	router.GET(htmlurl, func(c *Context) {
		c.HTML(200, "json.html", Map{
			"test": httpUrl,
		})
	})

}

// PostFile post file
func PostFile(filename, targetUrl, upParam string) (string, error) {
	bodyBuf := &bytes.Buffer{}
	bodyWriter := multipart.NewWriter(bodyBuf)

	// uploadfile
	fileWriter, err := bodyWriter.CreateFormFile(upParam, filename)
	if err != nil {
		fmt.Println("error writing to buffer")
		return "", err
	}

	// openfile
	fh, err := os.Open(filename)
	if err != nil {
		fmt.Println("error opening file")
		return "", err
	}

	// iocopy
	_, err = io.Copy(fileWriter, fh)
	if err != nil {
		return "", err
	}

	contentType := bodyWriter.FormDataContentType()
	bodyWriter.Close()

	resp, err := http.Post(targetUrl, contentType, bodyBuf)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	fmt.Println(resp.Status)
	// fmt.Println(string(respBody))

	return string(respBody), nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"

	boshlog "github.com/cloudfoundry/bosh-agent/logger"
	boshsys "github.com/cloudfoundry/bosh-agent/system"

	slclient "github.com/maximilien/softlayer-go/client"

	bslcaction "github.com/maximilien/bosh-softlayer-cpi/action"
	bslcdisp "github.com/maximilien/bosh-softlayer-cpi/api/dispatcher"
	bslctrans "github.com/maximilien/bosh-softlayer-cpi/api/transport"
)

const mainLogTag = "main"

var (
	configPathOpt = flag.String("configPath", "", "Path to configuration file")
)

func main() {
	logger, fs, cmdRunner := basicDeps()

	defer logger.HandlePanic("Main")

	flag.Parse()

	config, err := NewConfigFromPath(*configPathOpt, fs)
	if err != nil {
		logger.Error(mainLogTag, "Loading config %s", err.Error())
		os.Exit(1)
	}

	dispatcher := buildDispatcher(config, logger, fs, cmdRunner)

	cli := bslctrans.NewCLI(os.Stdin, os.Stdout, dispatcher, logger)

	err = cli.ServeOnce()
	if err != nil {
		logger.Error(mainLogTag, "Serving once %s", err)
		os.Exit(1)
	}
}

func basicDeps() (boshlog.Logger, boshsys.FileSystem, boshsys.CmdRunner) {
	logger := boshlog.NewWriterLogger(boshlog.LevelDebug, os.Stderr, os.Stderr)

	fs := boshsys.NewOsFileSystem(logger)

	cmdRunner := boshsys.NewExecCmdRunner(logger)

	return logger, fs, cmdRunner
}

func buildDispatcher(
	config Config,
	logger boshlog.Logger,
	fs boshsys.FileSystem,
	cmdRunner boshsys.CmdRunner,
) bslcdisp.Dispatcher {

	softLayerClient := slclient.NewSoftLayerClient(config.SoftLayer.Username, config.SoftLayer.ApiKey)

	actionFactory := bslcaction.NewConcreteFactory(
		softLayerClient,
		config.Actions,
		logger,
	)

	caller := bslcdisp.NewJSONCaller()

	return bslcdisp.NewJSON(actionFactory, caller, logger)
}
<commit_msg>cosmetic<commit_after>package main

import (
	"flag"
	"os"

	boshlog "github.com/cloudfoundry/bosh-agent/logger"
	boshsys "github.com/cloudfoundry/bosh-agent/system"

	slclient "github.com/maximilien/softlayer-go/client"

	bslcaction "github.com/maximilien/bosh-softlayer-cpi/action"
	bslcdisp "github.com/maximilien/bosh-softlayer-cpi/api/dispatcher"
	bslctrans "github.com/maximilien/bosh-softlayer-cpi/api/transport"
)

const mainLogTag = "main"

var (
	configPathOpt = flag.String("configPath", "", "Path to configuration file")
)

func main() {
	logger, fs, cmdRunner := basicDeps()

	defer logger.HandlePanic("Main")

	flag.Parse()

	config, err := NewConfigFromPath(*configPathOpt, fs)
	if err != nil {
		logger.Error(mainLogTag, "Loading config %s", err.Error())
		os.Exit(1)
	}

	dispatcher := buildDispatcher(config, logger, fs, cmdRunner)

	cli := bslctrans.NewCLI(os.Stdin, os.Stdout, dispatcher, logger)

	err = cli.ServeOnce()
	if err != nil {
		logger.Error(mainLogTag, "Serving once %s", err)
		os.Exit(1)
	}
}

func basicDeps() (boshlog.Logger, boshsys.FileSystem, boshsys.CmdRunner) {
	logger := boshlog.NewWriterLogger(boshlog.LevelDebug, os.Stderr, os.Stderr)

	fs := boshsys.NewOsFileSystem(logger)

	cmdRunner := boshsys.NewExecCmdRunner(logger)

	return logger, fs, cmdRunner
}

func buildDispatcher(config Config, logger boshlog.Logger, fs boshsys.FileSystem, cmdRunner boshsys.CmdRunner) bslcdisp.Dispatcher {
	softLayerClient := slclient.NewSoftLayerClient(config.SoftLayer.Username, config.SoftLayer.ApiKey)

	actionFactory := bslcaction.NewConcreteFactory(
		softLayerClient,
		config.Actions,
		logger,
	)

	caller := bslcdisp.NewJSONCaller()

	return bslcdisp.NewJSON(actionFactory, caller, logger)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"
)

const (
	ACK        = 0x79
	NAK        = 0x1F
	GET_CMD    = 0x00
	GETID_CMD  = 0x02
	GO_CMD     = 0x21
	WRITE_CMD  = 0x31
	ERASE_CMD  = 0x43
	EXTERA_CMD = 0x44
	WRUNP_CMD  = 0x73
	RDUNP_CMD  = 0x92
)

var deviceMap = map[uint16]string{
	0x410: "STM32F1, performance, medium-density",
	0x411: "STM32F2",
	0x412: "STM32F1, performance, low-density",
	0x413: "STM32F4",
	0x414: "STM32F1, performance, high-density",
	0x416: "STM32L1, performance, medium-density",
	0x417: "STM32L0, low-power, category 3",
	0x418: "STM32F1, connectivity",
	0x420: "STM32F1, value, medium-density",
	0x425: "STM32L0, low-power, category 2",
	0x428: "STM32F1, value, high-density",
	0x430: "STM32F1, performance, XL-density",
	0x447: "STM32L0, low-power, category 5",
	0x457: "STM32L0, low-power, category 0",
}

var (
	checkSum uint8
	pending  []byte
	extended bool
)

func uploadSTM32(data []byte) {
	fmt.Print(" Synchronise: ")
	connectToTarget()
	fmt.Println(" OK")

	fmt.Printf(" Boot loader: %02x hex\n", getBootVersion())

	chip := getChipType()
	desc := ""
	if s, ok := deviceMap[chip]; ok {
		desc = " - " + s
	}
	fmt.Printf("   Chip type: %04x hex%s\n", chip, desc)

	fmt.Print("   Unprotect: ")
	sendCmd(RDUNP_CMD)
	wantAck()
	fmt.Println("OK")

	fmt.Print("      Resume: ")
	connectToTarget()
	fmt.Println(" OK")

	//fmt.Print(" WR-unprotect: ")
	//sendCmd(WRUNP_CMD)
	//wantAck()
	//fmt.Println("OK")

	//fmt.Print("      Resume: ")
	//connectToTarget()
	//fmt.Println(" OK")

	fmt.Print("  Mass erase: ")
	massErase(len(data))
	fmt.Println("OK")

	fmt.Print("   Uploading: ")
	writeFlash(data)
	fmt.Println(" OK")

	// this won't work if the uploaded coded expects to run in low-mem
	//fmt.Print("       Start: ")
	//sendGoCmd()
	//fmt.Println(" OK")
}

func getReply() uint8 {
	if len(pending) == 0 {
		pending = readWithTimeout()
	}
	if len(pending) == 0 {
		return 0
	}
	b := pending[0]
	if *verbose {
		fmt.Printf("<%02x", b)
	}
	pending = pending[1:]
	return b
}

func connectToTarget() {
	for {
		//conn.Flush()
		fmt.Print(".") // auto-baud greeting
		sendByte(0x7F)
		r := getReply()
		if r == ACK || r == NAK {
			if r == ACK {
				fmt.Print("+")
			}
			break
		}
		time.Sleep(time.Second)
	}
	// got a valid reply
}

func wantAck() {
	r := getReply()
	if r != ACK {
		fmt.Printf("\nFailed: %02x\n", r)
		os.Exit(1)
	}
	checkSum = 0
}

func wantSlowAck() {
	r := getReply()
	for r == 0 {
		r = getReply()
	}
	if r != ACK {
		fmt.Printf("\nFailed: %02x\n", r)
		os.Exit(1)
	}
	checkSum = 0
}

func sendByte(b uint8) {
	if *verbose {
		fmt.Printf(">%02x", b)
	}
	conn.Write([]byte{b})
	checkSum ^= b
}

func send2bytes(v int) {
	sendByte(uint8(v>>8))
	sendByte(uint8(v))
}

func send4bytes(v int) {
	send2bytes(v>>16)
	send2bytes(v)
}

func sendCmd(cmd uint8) {
	//getReply()  // get rid of pending data
	//conn.Flush()
	pending = nil
	sendByte(cmd)
	sendByte(^cmd)
	wantAck()
}

func getBootVersion() uint8 {
	sendCmd(GET_CMD)
	n := getReply()
	rev := getReply()
	for i := 0; i < int(n); i++ {
		if getReply() == EXTERA_CMD {
			extended = true
		}
	}
	wantAck()
	return rev
}

func getChipType() uint16 {
	sendCmd(GETID_CMD)
	getReply() // should be 1
	chipType := uint16(getReply()) << 8
	chipType |= uint16(getReply())
	wantAck()
	return chipType
}

func massErase(size int) {
	if extended {
		sendCmd(EXTERA_CMD)
		// for some reason, a "full" mass erase gets rejected with a NAK
		//send2bytes(0xFFFF)
		// ... so erase a list of segments instead, 1 more than needed
		n := (size+127)/128 + 1 // really? always 128 bytes per segment?
		n = 200
		send2bytes(n-1)
		for i := 0; i < n; i++ {
			send2bytes(i)
		}
	} else {
		sendCmd(ERASE_CMD)
		sendByte(0xFF)
	}
	sendByte(checkSum)
	wantSlowAck()
}

func writeFlash(data []byte) {
	for len(data)%256 != 0 {
		data = append(data, 0xFF)
	}
	for offset := 0; offset < len(data); offset += 256 {
		fmt.Print("+")
		sendCmd(WRITE_CMD)
		send4bytes(0x08000000 + offset)
		sendByte(checkSum)
		wantAck()
		sendByte(256 - 1)
		for i := 0; i < 256; i++ {
			sendByte(data[offset+i])
		}
		sendByte(checkSum)
		wantAck()
		*verbose = false // verbose mode off after one write, to reduce output
	}
}

func sendGoCmd() {
	sendCmd(GO_CMD)
	send4bytes(0x08000000)
	sendByte(checkSum)
	wantAck()
}
<commit_msg>go fmt cleanup<commit_after>package main

import (
	"fmt"
	"os"
	"time"
)

const (
	ACK        = 0x79
	NAK        = 0x1F
	GET_CMD    = 0x00
	GETID_CMD  = 0x02
	GO_CMD     = 0x21
	WRITE_CMD  = 0x31
	ERASE_CMD  = 0x43
	EXTERA_CMD = 0x44
	WRUNP_CMD  = 0x73
	RDUNP_CMD  = 0x92
)

var deviceMap = map[uint16]string{
	0x410: "STM32F1, performance, medium-density",
	0x411: "STM32F2",
	0x412: "STM32F1, performance, low-density",
	0x413: "STM32F4",
	0x414: "STM32F1, performance, high-density",
	0x416: "STM32L1, performance, medium-density",
	0x417: "STM32L0, low-power, category 3",
	0x418: "STM32F1, connectivity",
	0x420: "STM32F1, value, medium-density",
	0x425: "STM32L0, low-power, category 2",
	0x428: "STM32F1, value, high-density",
	0x430: "STM32F1, performance, XL-density",
	0x447: "STM32L0, low-power, category 5",
	0x457: "STM32L0, low-power, category 0",
}

var (
	checkSum uint8
	pending  []byte
	extended bool
)

func uploadSTM32(data []byte) {
	fmt.Print(" Synchronise: ")
	connectToTarget()
	fmt.Println(" OK")

	fmt.Printf(" Boot loader: %02x hex\n", getBootVersion())

	chip := getChipType()
	desc := ""
	if s, ok := deviceMap[chip]; ok {
		desc = " - " + s
	}
	fmt.Printf("   Chip type: %04x hex%s\n", chip, desc)

	fmt.Print("   Unprotect: ")
	sendCmd(RDUNP_CMD)
	wantAck()
	fmt.Println("OK")

	fmt.Print("      Resume: ")
	connectToTarget()
	fmt.Println(" OK")

	//fmt.Print(" WR-unprotect: ")
	//sendCmd(WRUNP_CMD)
	//wantAck()
	//fmt.Println("OK")

	//fmt.Print("      Resume: ")
	//connectToTarget()
	//fmt.Println(" OK")

	fmt.Print("  Mass erase: ")
	massErase(len(data))
	fmt.Println("OK")

	fmt.Print("   Uploading: ")
	writeFlash(data)
	fmt.Println(" OK")

	// this won't work if the uploaded coded expects to run in low-mem
	//fmt.Print("       Start: ")
	//sendGoCmd()
	//fmt.Println(" OK")
}

func getReply() uint8 {
	if len(pending) == 0 {
		pending = readWithTimeout()
	}
	if len(pending) == 0 {
		return 0
	}
	b := pending[0]
	if *verbose {
		fmt.Printf("<%02x", b)
	}
	pending = pending[1:]
	return b
}

func connectToTarget() {
	for {
		//conn.Flush()
		fmt.Print(".") // auto-baud greeting
		sendByte(0x7F)
		r := getReply()
		if r == ACK || r == NAK {
			if r == ACK {
				fmt.Print("+")
			}
			break
		}
		time.Sleep(time.Second)
	}
	// got a valid reply
}

func wantAck() {
	r := getReply()
	if r != ACK {
		fmt.Printf("\nFailed: %02x\n", r)
		os.Exit(1)
	}
	checkSum = 0
}

func wantSlowAck() {
	r := getReply()
	for r == 0 {
		r = getReply()
	}
	if r != ACK {
		fmt.Printf("\nFailed: %02x\n", r)
		os.Exit(1)
	}
	checkSum = 0
}

func sendByte(b uint8) {
	if *verbose {
		fmt.Printf(">%02x", b)
	}
	conn.Write([]byte{b})
	checkSum ^= b
}

func send2bytes(v int) {
	sendByte(uint8(v >> 8))
	sendByte(uint8(v))
}

func send4bytes(v int) {
	send2bytes(v >> 16)
	send2bytes(v)
}

func sendCmd(cmd uint8) {
	//getReply()  // get rid of pending data
	//conn.Flush()
	pending = nil
	sendByte(cmd)
	sendByte(^cmd)
	wantAck()
}

func getBootVersion() uint8 {
	sendCmd(GET_CMD)
	n := getReply()
	rev := getReply()
	for i := 0; i < int(n); i++ {
		if getReply() == EXTERA_CMD {
			extended = true
		}
	}
	wantAck()
	return rev
}

func getChipType() uint16 {
	sendCmd(GETID_CMD)
	getReply() // should be 1
	chipType := uint16(getReply()) << 8
	chipType |= uint16(getReply())
	wantAck()
	return chipType
}

func massErase(size int) {
	if extended {
		sendCmd(EXTERA_CMD)
		// for some reason, a "full" mass erase gets rejected with a NAK
		//send2bytes(0xFFFF)
		// ... so erase a list of segments instead, 1 more than needed
		n := (size+127)/128 + 1 // really? always 128 bytes per segment?
		n = 200
		send2bytes(n - 1)
		for i := 0; i < n; i++ {
			send2bytes(i)
		}
	} else {
		sendCmd(ERASE_CMD)
		sendByte(0xFF)
	}
	sendByte(checkSum)
	wantSlowAck()
}

func writeFlash(data []byte) {
	for len(data)%256 != 0 {
		data = append(data, 0xFF)
	}
	for offset := 0; offset < len(data); offset += 256 {
		fmt.Print("+")
		sendCmd(WRITE_CMD)
		send4bytes(0x08000000 + offset)
		sendByte(checkSum)
		wantAck()
		sendByte(256 - 1)
		for i := 0; i < 256; i++ {
			sendByte(data[offset+i])
		}
		sendByte(checkSum)
		wantAck()
		*verbose = false // verbose mode off after one write, to reduce output
	}
}

func sendGoCmd() {
	sendCmd(GO_CMD)
	send4bytes(0x08000000)
	sendByte(checkSum)
	wantAck()
}
<|endoftext|>


<commit_before>package dag

import (
	"fmt"
	"os"
	"strings"
	"fubsy/dsl"
	"fubsy/types"
)

type Action interface {
	String() string

	// Perform whatever task this action implies. Return nil on
	// success, error otherwise. Compound actions always fail on the
	// first error; they do not continue executing. (The global
	// "--keep-going" option is irrelevant at this level; the caller
	// of Execute() is responsible for respecting that option.)
	Execute(ns types.Namespace) error
}

type actionbase struct {
}

// an action that just consists of a list of other actions
type SequenceAction struct {
	actionbase
	subactions []Action
}

// an action that is a shell command to execute
type CommandAction struct {
	actionbase

	// as read from the build script, without variables expanded
	raw types.FuObject

	// with all variables expanded, ready to execute
	expanded types.FuObject
}

// an action that evaluates an expression and assigns the result to a
// local variable -- only affects the scope of one build rule
type AssignmentAction struct {
	actionbase
	assignment *dsl.ASTAssignment
}

// an action that calls a function with real-world side effects (e.g.
// remove(), copyfile()) -- a pure function would be useless here,
// since we do nothing with the return value!
type FunctionCallAction struct {
	actionbase
	fcall *dsl.ASTFunctionCall
}

func NewSequenceAction() *SequenceAction {
	result := new(SequenceAction)
	return result
}

func (self *SequenceAction) String() string {
	result := make([]string, len(self.subactions))
	for i, sub := range self.subactions {
		result[i] = sub.String()
	}
	var tail string
	if len(result) > 3 {
		result = result[0:3]
		tail = " && ..."
	}
	return strings.Join(result, " && ") + tail
}

func (self *SequenceAction) Execute(ns types.Namespace) error {
	var err error
	for _, sub := range self.subactions {
		err = sub.Execute(ns)
		if err != nil {
			return err
		}
	}
	return nil
}

func (self *SequenceAction) AddAction(action Action) {
	self.subactions = append(self.subactions, action)
}

func (self *SequenceAction) AddCommand(command *dsl.ASTString) {
	raw := types.FuString(command.Value())
	self.AddAction(&CommandAction{raw: raw})
}

func (self *SequenceAction) AddAssignment(assignment *dsl.ASTAssignment) {
	self.AddAction(&AssignmentAction{assignment: assignment})
}

func (self *SequenceAction) AddFunctionCall(fcall *dsl.ASTFunctionCall) {
	self.AddAction(&FunctionCallAction{fcall: fcall})
}

func (self *CommandAction) String() string {
	return self.raw.String()
}

func (self *CommandAction) Execute(ns types.Namespace) error {
	fmt.Println("raw command:", self.raw)
	fmt.Println("namespace for expansion:")
	ns.Dump(os.Stdout, "")

	command, err := self.raw.Expand(ns)
	if err != nil {
		return err
	}
	fmt.Println("command:", command)
	panic("command execution not implemented yet")

	// self.expanded = self.rule.Expand(self.raw)
	// fmt.Printf("execute:", self.expanded)
}

func (self *AssignmentAction) String() string {
	return self.assignment.Target() + " = ..."
	//return self.assignment.String()
}

func (self *AssignmentAction) Execute(ns types.Namespace) error {
	panic("assignment in build rule not implemented yet")
}

func (self *FunctionCallAction) String() string {
	return self.fcall.String() + "(...)"
}

func (self *FunctionCallAction) Execute(ns types.Namespace) error {
	panic("function call in build rule not implemeneted yet")
}
<commit_msg>dag: execute commands, i.e. flesh out CommandAction.Execute().<commit_after>package dag

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"fubsy/dsl"
	"fubsy/types"
)

type Action interface {
	String() string

	// Perform whatever task this action implies. Return nil on
	// success, error otherwise. Compound actions always fail on the
	// first error; they do not continue executing. (The global
	// "--keep-going" option is irrelevant at this level; the caller
	// of Execute() is responsible for respecting that option.)
	Execute(ns types.Namespace) error
}

type actionbase struct {
}

// an action that just consists of a list of other actions
type SequenceAction struct {
	actionbase
	subactions []Action
}

// an action that is a shell command to execute
type CommandAction struct {
	actionbase

	// as read from the build script, without variables expanded
	raw types.FuObject

	// with all variables expanded, ready to execute
	expanded types.FuObject
}

// an action that evaluates an expression and assigns the result to a
// local variable -- only affects the scope of one build rule
type AssignmentAction struct {
	actionbase
	assignment *dsl.ASTAssignment
}

// an action that calls a function with real-world side effects (e.g.
// remove(), copyfile()) -- a pure function would be useless here,
// since we do nothing with the return value!
type FunctionCallAction struct {
	actionbase
	fcall *dsl.ASTFunctionCall
}

func NewSequenceAction() *SequenceAction {
	result := new(SequenceAction)
	return result
}

func (self *SequenceAction) String() string {
	result := make([]string, len(self.subactions))
	for i, sub := range self.subactions {
		result[i] = sub.String()
	}
	var tail string
	if len(result) > 3 {
		result = result[0:3]
		tail = " && ..."
	}
	return strings.Join(result, " && ") + tail
}

func (self *SequenceAction) Execute(ns types.Namespace) error {
	var err error
	for _, sub := range self.subactions {
		err = sub.Execute(ns)
		if err != nil {
			return err
		}
	}
	return nil
}

func (self *SequenceAction) AddAction(action Action) {
	self.subactions = append(self.subactions, action)
}

func (self *SequenceAction) AddCommand(command *dsl.ASTString) {
	raw := types.FuString(command.Value())
	self.AddAction(&CommandAction{raw: raw})
}

func (self *SequenceAction) AddAssignment(assignment *dsl.ASTAssignment) {
	self.AddAction(&AssignmentAction{assignment: assignment})
}

func (self *SequenceAction) AddFunctionCall(fcall *dsl.ASTFunctionCall) {
	self.AddAction(&FunctionCallAction{fcall: fcall})
}

func (self *CommandAction) String() string {
	return self.raw.String()
}

func (self *CommandAction) Execute(ns types.Namespace) error {
	//fmt.Println(self.raw)

	var err error
	self.expanded, err = self.raw.Expand(ns)
	if err != nil {
		return err
	}
	fmt.Println(self.expanded)

	// Run commands with the shell because people expect redirection,
	// pipes, etc. to work from their build scripts. (And besides, all
	// we have is a string: Fubsy makes no effort to encourage
	// commands as lists. That just confuses people and causes excess
	// typing. And it's pointless on Windows, where command lists get
	// collapsed to a string and then parsed back into words by the
	// program being run.)
	// XXX can we mitigate security risks of using the shell?
	// XXX what about Windows?
	// XXX for parallel builds: gather stdout and stderr, accumulate
	// them in order but still distinguishing them, and dump them to
	// our stdout/stderr when the command finishes
	// XXX the error message doesn't say which command failed (and if
	// it did, it would probably say "/bin/sh", which is useless): can
	// we do better?
	cmd := exec.Command("/bin/sh", "-c", self.expanded.String())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err = cmd.Run()
	return err
}

func (self *AssignmentAction) String() string {
	return self.assignment.Target() + " = ..."
	//return self.assignment.String()
}

func (self *AssignmentAction) Execute(ns types.Namespace) error {
	panic("assignment in build rule not implemented yet")
}

func (self *FunctionCallAction) String() string {
	return self.fcall.String() + "(...)"
}

func (self *FunctionCallAction) Execute(ns types.Namespace) error {
	panic("function call in build rule not implemeneted yet")
}
<|endoftext|>


<commit_before>package alarm

import (
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/eliothedeman/bangarang/event"
)

func init() {
	log.SetFlags(log.Llongfile)
}

var (
	DEFAULT_GROUP_BY = map[string]string{
		"host":        "^(.*)$",
		"service":     "^(.*)$",
		"sub_service": "^(.*)$",
	}
)

type Policy struct {
	Match       map[string]string `json:"match"`
	NotMatch    map[string]string `json:"not_match"`
	GroupBy     map[string]string `json:"group_by"`
	Crit        *Condition        `json:"crit"`
	Warn        *Condition        `json:"warn"`
	Name        string            `json:"name"`
	r_match     map[string]*regexp.Regexp
	r_not_match map[string]*regexp.Regexp
	stop        chan struct{}
	in          chan *pack
	resolve     chan *event.Incident
}

// start the policy listening for events
func (p *Policy) start() {
	go func() {
		var in *pack
		for {
			select {
			case toResolve := <-p.resolve:
				var c *Condition

				// fetch the condition that created this incident
				if toResolve.Status == event.CRITICAL {
					c = p.Crit
				} else if toResolve.Status == event.WARNING {
					c = p.Warn
				}

				if c != nil {
					e := &event.Event{}
					e.Host = toResolve.Host
					e.Service = toResolve.Service
					e.SubService = toResolve.SubService
					t := c.getTracker(e)
					t.refresh()
				}
			case <-p.stop:
				logrus.Info("Stopping policy", p.Name)

				// cleanup the policy. Sometimes they hangaround.
				p.clean()

				// stop this policy from being stopped again
				p.stop = nil

				// catch resolve's that could be sent to this policy
				res := p.resolve
				p.resolve = nil

				go func() {
					for {
						select {
						case <-res:
							logrus.Info("Attempted to resolve an incident on a policy that no longer exists")

						case <-time.After(5 * time.Minute):
							return
						}
					}

				}()
				return
			case in = <-p.in:

				// process the event if it matches the policy
				if p.Matches(in.e) {
					// process the request

					// check critical
					if shouldAlert, status := p.ActionCrit(in.e); shouldAlert {
						incident := event.NewIncident(p.Name, p.Crit.Escalation, status, in.e)
						incident.SetResolve(p.resolve)

						in.n(incident)
						logrus.Info(incident.FormatDescription())

						// check warning
					} else if shouldAlert, status := p.ActionWarn(in.e); shouldAlert {
						incident := event.NewIncident(p.Name, p.Warn.Escalation, status, in.e)
						incident.SetResolve(p.resolve)
						in.n(incident)
						logrus.Info(incident.FormatDescription())
					}
				}
				in.e.WaitDec()
			}
		}
	}()
}

// remove all used memeory by this policy
func (p *Policy) clean() {
	p.Crit = nil
	p.Warn = nil
	p.r_match = nil
	p.r_not_match = nil
}

type pack struct {
	e *event.Event
	n func(*event.Incident)
}

// Process will send exicute the next function if the event satisfies the policy
func (p *Policy) Process(e *event.Event, next func(*event.Incident)) {
	p.in <- &pack{
		e: e,
		n: next,
	}
}

func (p *Policy) Stop() {
	p.stop <- struct{}{}
}

// check to see if an event satisfies the policy
func (p *Policy) Matches(e *event.Event) bool {
	return p.CheckMatch(e) && p.CheckNotMatch(e)
}

// compile the regex patterns for this policy
func (p *Policy) Compile() {
	logrus.Infof("Compiling regex maches for %s", p.Name)
	p.in = make(chan *pack, 10)
	p.stop = make(chan struct{})
	p.resolve = make(chan *event.Incident)
	p.start()

	if p.r_match == nil {
		p.r_match = make(map[string]*regexp.Regexp)
	}

	if p.r_not_match == nil {
		p.r_not_match = make(map[string]*regexp.Regexp)
	}

	// if we don't have at least three componants of the group by, establish them from the defaults
	if len(p.GroupBy) < 3 {

		if len(p.GroupBy) == 0 {
			p.GroupBy = DEFAULT_GROUP_BY

		} else {

			tmp := map[string]string{}
			for k, v := range DEFAULT_GROUP_BY {
				tmp[k] = v
			}

			for k, v := range p.GroupBy {
				tmp[k] = v
			}

			p.GroupBy = tmp
		}
	}

	for k, v := range p.Match {
		p.r_match[k] = regexp.MustCompile(v)
	}

	for k, v := range p.NotMatch {
		p.r_not_match[k] = regexp.MustCompile(v)
	}

	if p.Crit != nil {
		logrus.Infof("Initializing crit for %s", p.Name)
		p.Crit.init(p.GroupBy)
	}

	if p.Warn != nil {
		logrus.Infof("Initializing warn for %s", p.Name)
		p.Warn.init(p.GroupBy)
	}
}

func formatFileName(n string) string {
	s := strings.Split(n, "_")
	a := ""
	for _, k := range s {
		a = a + strings.Title(k)
	}
	return a
}

// return the action to take for a given event
func (p *Policy) ActionCrit(e *event.Event) (bool, int) {
	status := event.OK
	if p.Crit != nil {
		if p.Crit.TrackEvent(e) {
			status = event.CRITICAL
		} else {
			status = event.OK
		}

		return p.Crit.StateChanged(e), status
	}

	return false, status
}

func (p *Policy) ActionWarn(e *event.Event) (bool, int) {
	status := event.OK
	if p.Warn != nil {
		if p.Warn.TrackEvent(e) {
			status = event.WARNING
		} else {
			status = event.OK
		}
		return p.Warn.StateChanged(e), status
	}

	return false, status
}

func (p *Policy) CheckNotMatch(e *event.Event) bool {
	for k, m := range p.r_not_match {
		if m.MatchString(e.Get(k)) {
			return false
		}
	}
	return true
}

// check if any of p's matches are satisfied by the event
func (p *Policy) CheckMatch(e *event.Event) bool {
	for k, m := range p.r_match {
		// if the element does not match the regex pattern, the event does not fully match
		if !m.MatchString(e.Get(k)) {
			return false
		}
	}
	return true
}
<commit_msg>only need 1 minute to catch resolved events in process<commit_after>package alarm

import (
	"log"
	"regexp"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/eliothedeman/bangarang/event"
)

func init() {
	log.SetFlags(log.Llongfile)
}

var (
	DEFAULT_GROUP_BY = map[string]string{
		"host":        "^(.*)$",
		"service":     "^(.*)$",
		"sub_service": "^(.*)$",
	}
)

type Policy struct {
	Match       map[string]string `json:"match"`
	NotMatch    map[string]string `json:"not_match"`
	GroupBy     map[string]string `json:"group_by"`
	Crit        *Condition        `json:"crit"`
	Warn        *Condition        `json:"warn"`
	Name        string            `json:"name"`
	r_match     map[string]*regexp.Regexp
	r_not_match map[string]*regexp.Regexp
	stop        chan struct{}
	in          chan *pack
	resolve     chan *event.Incident
}

// start the policy listening for events
func (p *Policy) start() {
	go func() {
		var in *pack
		for {
			select {
			case toResolve := <-p.resolve:
				var c *Condition

				// fetch the condition that created this incident
				if toResolve.Status == event.CRITICAL {
					c = p.Crit
				} else if toResolve.Status == event.WARNING {
					c = p.Warn
				}

				if c != nil {
					e := &event.Event{}
					e.Host = toResolve.Host
					e.Service = toResolve.Service
					e.SubService = toResolve.SubService
					t := c.getTracker(e)
					t.refresh()
				}
			case <-p.stop:
				logrus.Info("Stopping policy", p.Name)

				// cleanup the policy. Sometimes they hangaround.
				p.clean()

				// stop this policy from being stopped again
				p.stop = nil

				// catch resolve's that could be sent to this policy
				res := p.resolve
				p.resolve = nil

				go func() {
					for {
						select {
						case <-res:
							logrus.Info("Attempted to resolve an incident on a policy that no longer exists")

						case <-time.After(1 * time.Minute):
							return
						}
					}

				}()
				return
			case in = <-p.in:

				// process the event if it matches the policy
				if p.Matches(in.e) {
					// process the request

					// check critical
					if shouldAlert, status := p.ActionCrit(in.e); shouldAlert {
						incident := event.NewIncident(p.Name, p.Crit.Escalation, status, in.e)
						incident.SetResolve(p.resolve)

						in.n(incident)
						logrus.Info(incident.FormatDescription())

						// check warning
					} else if shouldAlert, status := p.ActionWarn(in.e); shouldAlert {
						incident := event.NewIncident(p.Name, p.Warn.Escalation, status, in.e)
						incident.SetResolve(p.resolve)
						in.n(incident)
						logrus.Info(incident.FormatDescription())
					}
				}
				in.e.WaitDec()
			}
		}
	}()
}

// remove all used memeory by this policy
func (p *Policy) clean() {
	p.Crit = nil
	p.Warn = nil
	p.r_match = nil
	p.r_not_match = nil
}

type pack struct {
	e *event.Event
	n func(*event.Incident)
}

// Process will send exicute the next function if the event satisfies the policy
func (p *Policy) Process(e *event.Event, next func(*event.Incident)) {
	p.in <- &pack{
		e: e,
		n: next,
	}
}

func (p *Policy) Stop() {
	p.stop <- struct{}{}
}

// check to see if an event satisfies the policy
func (p *Policy) Matches(e *event.Event) bool {
	return p.CheckMatch(e) && p.CheckNotMatch(e)
}

// compile the regex patterns for this policy
func (p *Policy) Compile() {
	logrus.Infof("Compiling regex maches for %s", p.Name)
	p.in = make(chan *pack, 10)
	p.stop = make(chan struct{})
	p.resolve = make(chan *event.Incident)
	p.start()

	if p.r_match == nil {
		p.r_match = make(map[string]*regexp.Regexp)
	}

	if p.r_not_match == nil {
		p.r_not_match = make(map[string]*regexp.Regexp)
	}

	// if we don't have at least three componants of the group by, establish them from the defaults
	if len(p.GroupBy) < 3 {

		if len(p.GroupBy) == 0 {
			p.GroupBy = DEFAULT_GROUP_BY

		} else {

			tmp := map[string]string{}
			for k, v := range DEFAULT_GROUP_BY {
				tmp[k] = v
			}

			for k, v := range p.GroupBy {
				tmp[k] = v
			}

			p.GroupBy = tmp
		}
	}

	for k, v := range p.Match {
		p.r_match[k] = regexp.MustCompile(v)
	}

	for k, v := range p.NotMatch {
		p.r_not_match[k] = regexp.MustCompile(v)
	}

	if p.Crit != nil {
		logrus.Infof("Initializing crit for %s", p.Name)
		p.Crit.init(p.GroupBy)
	}

	if p.Warn != nil {
		logrus.Infof("Initializing warn for %s", p.Name)
		p.Warn.init(p.GroupBy)
	}
}

func formatFileName(n string) string {
	s := strings.Split(n, "_")
	a := ""
	for _, k := range s {
		a = a + strings.Title(k)
	}
	return a
}

// return the action to take for a given event
func (p *Policy) ActionCrit(e *event.Event) (bool, int) {
	status := event.OK
	if p.Crit != nil {
		if p.Crit.TrackEvent(e) {
			status = event.CRITICAL
		} else {
			status = event.OK
		}

		return p.Crit.StateChanged(e), status
	}

	return false, status
}

func (p *Policy) ActionWarn(e *event.Event) (bool, int) {
	status := event.OK
	if p.Warn != nil {
		if p.Warn.TrackEvent(e) {
			status = event.WARNING
		} else {
			status = event.OK
		}
		return p.Warn.StateChanged(e), status
	}

	return false, status
}

func (p *Policy) CheckNotMatch(e *event.Event) bool {
	for k, m := range p.r_not_match {
		if m.MatchString(e.Get(k)) {
			return false
		}
	}
	return true
}

// check if any of p's matches are satisfied by the event
func (p *Policy) CheckMatch(e *event.Event) bool {
	for k, m := range p.r_match {
		// if the element does not match the regex pattern, the event does not fully match
		if !m.MatchString(e.Get(k)) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/gophergala2016/goad"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/dustin/go-humanize"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/nsf/termbox-go"
	"github.com/gophergala2016/goad/queue"
)

var (
	url         string
	concurrency uint
	requests    uint
	timeout     uint
	regions     string
	method      string
)

const coldef = termbox.ColorDefault
const nano = 1000000000

func main() {
	flag.StringVar(&url, "u", "", "URL to load test (required)")
	flag.StringVar(&method, "m", "GET", "HTTP method")
	flag.UintVar(&concurrency, "c", 10, "number of concurrent requests")
	flag.UintVar(&requests, "n", 1000, "number of total requests to make")
	flag.UintVar(&timeout, "t", 15, "request timeout in seconds")
	flag.StringVar(&regions, "r", "us-east-1", "AWS regions to run in (comma separated, no spaces)")
	flag.Parse()

	if url == "" {
		flag.Usage()
		os.Exit(0)
	}

	test, testerr := goad.NewTest(&goad.TestConfig{
		URL:            url,
		Concurrency:    concurrency,
		TotalRequests:  requests,
		RequestTimeout: time.Duration(timeout) * time.Second,
		Regions:        strings.Split(regions, ","),
		Method:         method,
	})
	if testerr != nil {
		fmt.Println(testerr)
		os.Exit(1)
	}

	var finalResult queue.RegionsAggData
	defer printSummary(&finalResult)

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) // but interrupts from kbd are blocked by termbox

	start(test, &finalResult, sigChan)
}

func start(test *goad.Test, finalResult *queue.RegionsAggData, sigChan chan os.Signal) {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}

	defer termbox.Close()
	termbox.Sync()
	renderString(0, 0, "Launching on AWS...", coldef, coldef)
	renderLogo()
	termbox.Flush()

	resultChan := test.Start()

	_, h := termbox.Size()
	renderString(0, h-1, "Press ctrl-c to interrupt", coldef, coldef)
	termbox.Flush()

	go func() {
		for {
			event := termbox.PollEvent()
			if event.Key == 3 {
				sigChan <- syscall.SIGINT
			}
		}
	}()

	firstTime := true
outer:
	for {
		select {
		case result, ok := <-resultChan:
			if !ok {
				break outer
			}

			if firstTime {
				clearLogo()
				firstTime = false
			}

			// sort so that regions always appear in the same order
			var regions []string
			for key := range result.Regions {
				regions = append(regions, key)
			}
			sort.Strings(regions)
			y := 3
			totalReqs := 0
			for _, region := range regions {
				data := result.Regions[region]
				totalReqs += data.TotalReqs
				y = renderRegion(data, y)
				y++
			}

			y = 0
			percentDone := float64(totalReqs) / float64(result.TotalExpectedRequests)
			drawProgressBar(percentDone, y)

			termbox.Flush()
			finalResult.Regions = result.Regions

		case <-sigChan:
			break outer
		}
	}
}

func renderLogo() {
	s1 := `	  _____                 _`
	s2 := `  / ____|               | |`
	s3 := `	| |  __  ___   ____  __| |`
	s4 := `	| | |_ |/ _ \ / _  |/ _  |`
	s5 := `	| |__| | (_) | (_| | (_| |`
	s6 := `	 \_____|\___/ \__,_|\__,_|`
	s7 := " Global load testing with Go"
	arr := [...]string{s1, s2, s3, s4, s5, s6, s7}
	for i, str := range arr {
		renderString(0, i+1, str, coldef, coldef)
	}
}

// Also clears loading message
func clearLogo() {
	for i := 0; i < 8; i++ {
		renderString(0, i, "                            ", coldef, coldef)
	}
}

// renderRegion returns the y for the next empty line
func renderRegion(data queue.AggData, y int) int {
	x := 0
	renderString(x, y, "Region: ", termbox.ColorWhite, termbox.ColorBlue)
	x += 8
	regionStr := fmt.Sprintf("%-10s", data.Region)
	renderString(x, y, regionStr, termbox.ColorWhite|termbox.AttrBold, termbox.ColorBlue)
	x = 0
	y++
	headingStr := "   TotReqs   TotBytes    AvgTime   AvgReq/s  AvgKbps/s"
	renderString(x, y, headingStr, coldef|termbox.AttrBold, coldef)
	y++
	resultStr := fmt.Sprintf("%10d %10s   %7.3fs %10.2f %10.2f", data.TotalReqs, humanize.Bytes(uint64(data.TotBytesRead)), float64(data.AveTimeForReq)/nano, data.AveReqPerSec, data.AveKBytesPerSec)
	renderString(x, y, resultStr, coldef, coldef)
	y++
	headingStr = "   Slowest    Fastest   Timeouts  TotErrors"
	renderString(x, y, headingStr, coldef|termbox.AttrBold, coldef)
	y++
	resultStr = fmt.Sprintf("  %7.3fs   %7.3fs %10d %10d", float64(data.Slowest)/nano, float64(data.Fastest)/nano, data.TotalTimedOut, totErrors(&data))
	renderString(x, y, resultStr, coldef, coldef)
	y++

	return y
}

func totErrors(data *queue.AggData) int {
	var okReqs int
	for statusStr, value := range data.Statuses {
		status, _ := strconv.Atoi(statusStr)
		if status >= 200 && status <= 299 {
			okReqs += value
		}
	}
	return data.TotalReqs - okReqs
}

func drawProgressBar(percent float64, y int) {
	x := 0
	width := 52
	percentStr := fmt.Sprintf("%5.1f%%            ", percent*100)
	renderString(x, y, percentStr, coldef, coldef)
	y++
	hashes := int(percent * float64(width))
	if percent > 0.99 {
		hashes = width
	}
	renderString(x, y, "[", coldef, coldef)

	for x++; x <= hashes; x++ {
		renderString(x, y, "#", coldef, coldef)
	}
	renderString(width+1, y, "]", coldef, coldef)
}

func renderString(x int, y int, str string, f termbox.Attribute, b termbox.Attribute) {
	for i, c := range str {
		termbox.SetCell(x+i, y, c, f, b)
	}
}

func boldPrintln(msg string) {
	fmt.Printf("\033[1m%s\033[0m\n", msg)
}

func printData(data *queue.AggData) {
	boldPrintln("   TotReqs   TotBytes    AvgTime   AvgReq/s  AvgKbps/s")
	fmt.Printf("%10d %10s   %7.3fs %10.2f %10.2f\n", data.TotalReqs, humanize.Bytes(uint64(data.TotBytesRead)), float64(data.AveTimeForReq)/nano, data.AveReqPerSec, data.AveKBytesPerSec)
	boldPrintln("   Slowest    Fastest   Timeouts  TotErrors")
	fmt.Printf("  %7.3fs   %7.3fs %10d %10d", float64(data.Slowest)/nano, float64(data.Fastest)/nano, data.TotalTimedOut, totErrors(data))
	fmt.Println("")
}

func printSummary(result *queue.RegionsAggData) {
	if len(result.Regions) == 0 {
		boldPrintln("No results received")
		return
	}
	boldPrintln("Regional results")
	fmt.Println("")

	for region, data := range result.Regions {
		fmt.Println("Region: " + region)
		printData(&data)
	}

	overall := queue.SumRegionResults(result)

	fmt.Println("")
	boldPrintln("Overall")
	fmt.Println("")
	printData(overall)

	boldPrintln("HTTPStatus   Requests")
	for statusStr, value := range overall.Statuses {
		fmt.Printf("%10s %10d\n", statusStr, value)
	}
	fmt.Println("")
}
<commit_msg>Set default regions, layout improvements<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/gophergala2016/goad"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/dustin/go-humanize"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/nsf/termbox-go"
	"github.com/gophergala2016/goad/queue"
)

var (
	url         string
	concurrency uint
	requests    uint
	timeout     uint
	regions     string
	method      string
)

const coldef = termbox.ColorDefault
const nano = 1000000000

func main() {
	flag.StringVar(&url, "u", "", "URL to load test (required)")
	flag.StringVar(&method, "m", "GET", "HTTP method")
	flag.UintVar(&concurrency, "c", 10, "number of concurrent requests")
	flag.UintVar(&requests, "n", 1000, "number of total requests to make")
	flag.UintVar(&timeout, "t", 15, "request timeout in seconds")
	flag.StringVar(&regions, "r", "us-east-1,eu-west-1,ap-northeast-1", "AWS regions to run in (comma separated, no spaces)")
	flag.Parse()

	if url == "" {
		flag.Usage()
		os.Exit(0)
	}

	test, testerr := goad.NewTest(&goad.TestConfig{
		URL:            url,
		Concurrency:    concurrency,
		TotalRequests:  requests,
		RequestTimeout: time.Duration(timeout) * time.Second,
		Regions:        strings.Split(regions, ","),
		Method:         method,
	})
	if testerr != nil {
		fmt.Println(testerr)
		os.Exit(1)
	}

	var finalResult queue.RegionsAggData
	defer printSummary(&finalResult)

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) // but interrupts from kbd are blocked by termbox

	start(test, &finalResult, sigChan)
}

func start(test *goad.Test, finalResult *queue.RegionsAggData, sigChan chan os.Signal) {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}

	defer termbox.Close()
	termbox.Sync()
	renderString(0, 0, "Launching on AWS... (be patient)", coldef, coldef)
	renderLogo()
	termbox.Flush()

	resultChan := test.Start()

	_, h := termbox.Size()
	renderString(0, h-1, "Press ctrl-c to interrupt", coldef, coldef)
	termbox.Flush()

	go func() {
		for {
			event := termbox.PollEvent()
			if event.Key == 3 {
				sigChan <- syscall.SIGINT
			}
		}
	}()

	firstTime := true
outer:
	for {
		select {
		case result, ok := <-resultChan:
			if !ok {
				break outer
			}

			if firstTime {
				clearLogo()
				firstTime = false
			}

			// sort so that regions always appear in the same order
			var regions []string
			for key := range result.Regions {
				regions = append(regions, key)
			}
			sort.Strings(regions)
			y := 3
			totalReqs := 0
			for _, region := range regions {
				data := result.Regions[region]
				totalReqs += data.TotalReqs
				y = renderRegion(data, y)
				y++
			}

			y = 0
			percentDone := float64(totalReqs) / float64(result.TotalExpectedRequests)
			drawProgressBar(percentDone, y)

			termbox.Flush()
			finalResult.Regions = result.Regions

		case <-sigChan:
			break outer
		}
	}
}

func renderLogo() {
	s1 := `	  _____                 _`
	s2 := `  / ____|               | |`
	s3 := `	| |  __  ___   ____  __| |`
	s4 := `	| | |_ |/ _ \ / _  |/ _  |`
	s5 := `	| |__| | (_) | (_| | (_| |`
	s6 := `	 \_____|\___/ \__,_|\__,_|`
	s7 := " Global load testing with Go"
	arr := [...]string{s1, s2, s3, s4, s5, s6, s7}
	for i, str := range arr {
		renderString(0, i+1, str, coldef, coldef)
	}
}

// Also clears loading message
func clearLogo() {
	for i := 0; i < 8; i++ {
		renderString(0, i, "                                ", coldef, coldef)
	}
}

// renderRegion returns the y for the next empty line
func renderRegion(data queue.AggData, y int) int {
	x := 0
	renderString(x, y, "Region: ", termbox.ColorWhite, termbox.ColorBlue)
	x += 8
	regionStr := fmt.Sprintf("%s", data.Region)
	renderString(x, y, regionStr, termbox.ColorWhite|termbox.AttrBold, termbox.ColorBlue)
	x = 0
	y++
	headingStr := "   TotReqs   TotBytes    AvgTime   AvgReq/s  AvgKbps/s"
	renderString(x, y, headingStr, coldef|termbox.AttrBold, coldef)
	y++
	resultStr := fmt.Sprintf("%10d %10s   %7.3fs %10.2f %10.2f", data.TotalReqs, humanize.Bytes(uint64(data.TotBytesRead)), float64(data.AveTimeForReq)/nano, data.AveReqPerSec, data.AveKBytesPerSec)
	renderString(x, y, resultStr, coldef, coldef)
	y++
	headingStr = "   Slowest    Fastest   Timeouts  TotErrors"
	renderString(x, y, headingStr, coldef|termbox.AttrBold, coldef)
	y++
	resultStr = fmt.Sprintf("  %7.3fs   %7.3fs %10d %10d", float64(data.Slowest)/nano, float64(data.Fastest)/nano, data.TotalTimedOut, totErrors(&data))
	renderString(x, y, resultStr, coldef, coldef)
	y++

	return y
}

func totErrors(data *queue.AggData) int {
	var okReqs int
	for statusStr, value := range data.Statuses {
		status, _ := strconv.Atoi(statusStr)
		if status >= 200 && status <= 299 {
			okReqs += value
		}
	}
	return data.TotalReqs - okReqs
}

func drawProgressBar(percent float64, y int) {
	x := 0
	width := 52
	percentStr := fmt.Sprintf("%5.1f%%            ", percent*100)
	renderString(x, y, percentStr, coldef, coldef)
	y++
	hashes := int(percent * float64(width))
	if percent > 0.99 {
		hashes = width
	}
	renderString(x, y, "[", coldef, coldef)

	for x++; x <= hashes; x++ {
		renderString(x, y, "#", coldef, coldef)
	}
	renderString(width+1, y, "]", coldef, coldef)
}

func renderString(x int, y int, str string, f termbox.Attribute, b termbox.Attribute) {
	for i, c := range str {
		termbox.SetCell(x+i, y, c, f, b)
	}
}

func boldPrintln(msg string) {
	fmt.Printf("\033[1m%s\033[0m\n", msg)
}

func printData(data *queue.AggData) {
	boldPrintln("   TotReqs   TotBytes    AvgTime   AvgReq/s  AvgKbps/s")
	fmt.Printf("%10d %10s   %7.3fs %10.2f %10.2f\n", data.TotalReqs, humanize.Bytes(uint64(data.TotBytesRead)), float64(data.AveTimeForReq)/nano, data.AveReqPerSec, data.AveKBytesPerSec)
	boldPrintln("   Slowest    Fastest   Timeouts  TotErrors")
	fmt.Printf("  %7.3fs   %7.3fs %10d %10d", float64(data.Slowest)/nano, float64(data.Fastest)/nano, data.TotalTimedOut, totErrors(data))
	fmt.Println("")
}

func printSummary(result *queue.RegionsAggData) {
	if len(result.Regions) == 0 {
		boldPrintln("No results received")
		return
	}
	boldPrintln("Regional results")
	fmt.Println("")

	for region, data := range result.Regions {
		fmt.Println("Region: " + region)
		printData(&data)
	}

	overall := queue.SumRegionResults(result)

	fmt.Println("")
	boldPrintln("Overall")
	fmt.Println("")
	printData(overall)

	boldPrintln("HTTPStatus   Requests")
	for statusStr, value := range overall.Statuses {
		fmt.Printf("%10s %10d\n", statusStr, value)
	}
	fmt.Println("")
}
<|endoftext|>