<commit_before>package migrate

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/marcosnils/cmt/cmd"
	"github.com/marcosnils/cmt/validate"
)

var Command = cli.Command{
	Name:  "migrate",
	Usage: "Migrate running container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "src",
			Usage: "Source host where the container is running",
		},
		cli.StringFlag{
			Name:  "dst",
			Usage: "Target host to migrate the container",
		},
		cli.BoolFlag{
			Name:  "pre-dump",
			Usage: "Perform a pre-dump to minimize downtime",
		},
	},
	Action: func(c *cli.Context) {
		srcUrl := validate.ParseURL(c.String("src"))
		dstUrl := validate.ParseURL(c.String("dst"))

		log.Println("Performing validations")
		src, dst := validate.Validate(srcUrl, dstUrl)

		log.Println("Preparing everything to do a checkpoint")
		containerId := getContainerId(srcUrl.Path)
		var imagesPath string
		var restoreCmd cmd.Cmd
		var migrateStart time.Time
		var downtime time.Duration

		if c.Bool("pre-dump") {
			// Process pre-dump
			predumpPath := fmt.Sprintf("%s/images/0", srcUrl.Path)
			prepareDir(src, predumpPath)

			checkpoint(src, containerId, predumpPath, true)

			srcTarFile := fmt.Sprintf("%s/predump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, predumpPath)

			prepareDir(dst, fmt.Sprintf("%s/images/0", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/0", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/0/predump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/0", dstUrl.Path))

			// Process final image
			migrateStart = time.Now()

			imagesPath = fmt.Sprintf("%s/images/1", srcUrl.Path)
			log.Println("Performing the checkpoint")
			_, _, err = src.Run("sudo", "runc", "--id", containerId, "checkpoint", "--image-path", imagesPath, "--prev-images-dir", predumpPath)
			if err != nil {
				log.Fatal("Error performing checkpoint:", err)
			}

			srcTarFile = fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)
			prepareDir(dst, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err = cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/1", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile = fmt.Sprintf("%s/images/1/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Performing the restore")
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images/1", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		} else {
			imagesPath = fmt.Sprintf("%s/images", srcUrl.Path)
			prepareDir(src, imagesPath)

			migrateStart = time.Now()
			checkpoint(src, containerId, imagesPath, false)

			srcTarFile := fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)

			prepareDir(dst, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Copying checkpoint image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Performing the restore")
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		}

		var restoreSucceed bool
		var restoreError error
		var wg sync.WaitGroup
		wg.Add(1)

		go func() {
			restoreError = restoreCmd.Wait()
			wg.Done()
		}()

		go func() {
			log.Println("Waiting for container to start...")
			// We make a fast check so we don't wait for the first ticker internal
			if isRunning(containerId) {
				restoreSucceed = true
				wg.Done()
				return
			}
			ticker := time.NewTicker(200 * time.Millisecond)
			go func() {
				for _ = range ticker.C {
					if isRunning(containerId) {
						restoreSucceed = true
						break
					}

				}
				ticker.Stop()
				wg.Done()
			}()
		}()

		wg.Wait()

		downtime = time.Since(migrateStart)

		if restoreSucceed {
			log.Printf("Restore finished successfully, total downtime: %sms", downtime*time.Millisecond)
		} else {
			log.Println("Error performing restore:", restoreError)
			// Rollback
		}

	},
}

func isRunning(containerId string) bool {
	if _, err := os.Stat(fmt.Sprintf("/var/run/opencontainer/containers/%s", containerId)); err == nil {
		return true
	}
	return false
}

func unpackTar(cmd cmd.Cmd, tarFile, workDir string) {
	log.Println("Preparing image at destination host")
	_, _, err := cmd.Run("sudo", "tar", "-C", workDir, "-xvzf", tarFile)
	if err != nil {
		log.Fatal("Error uncompressing image in destination:", err)
	}
}

func prepareTar(cmd cmd.Cmd, tarFile, workDir string) {
	_, _, err := cmd.Run("sudo", "tar", "-czf", tarFile, "-C", fmt.Sprintf("%s/", workDir), ".")
	if err != nil {
		log.Fatal("Error compressing image in source:", err)
	}
}

func checkpoint(cmd cmd.Cmd, containerId, imagesPath string, predump bool) {
	log.Printf("Performing the checkpoint predump = %t\n", predump)
	args := []string{"runc", "--id", containerId, "checkpoint", "--image-path", imagesPath}
	if predump {
		args = append(args, "--pre-dump")
	}
	_, _, err := cmd.Run("sudo", args...)
	if err != nil {
		log.Fatal("Error performing checkpoint:", err)
	}
}

func prepareDir(cmd cmd.Cmd, path string) {
	_, _, err := cmd.Run("mkdir", "-p", path)
	if err != nil {
		log.Fatal("Error preparing pre-dump dir:", err)
	}
}

func getContainerId(path string) string {
	_, id := filepath.Split(path)
	return id
}
<commit_msg>Convert downtime to ms<commit_after>package migrate

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/marcosnils/cmt/cmd"
	"github.com/marcosnils/cmt/validate"
)

var Command = cli.Command{
	Name:  "migrate",
	Usage: "Migrate running container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "src",
			Usage: "Source host where the container is running",
		},
		cli.StringFlag{
			Name:  "dst",
			Usage: "Target host to migrate the container",
		},
		cli.BoolFlag{
			Name:  "pre-dump",
			Usage: "Perform a pre-dump to minimize downtime",
		},
	},
	Action: func(c *cli.Context) {
		srcUrl := validate.ParseURL(c.String("src"))
		dstUrl := validate.ParseURL(c.String("dst"))

		log.Println("Performing validations")
		src, dst := validate.Validate(srcUrl, dstUrl)

		log.Println("Preparing everything to do a checkpoint")
		containerId := getContainerId(srcUrl.Path)
		var imagesPath string
		var restoreCmd cmd.Cmd
		var migrateStart time.Time
		var downtime time.Duration

		if c.Bool("pre-dump") {
			// Process pre-dump
			predumpPath := fmt.Sprintf("%s/images/0", srcUrl.Path)
			prepareDir(src, predumpPath)

			checkpoint(src, containerId, predumpPath, true)

			srcTarFile := fmt.Sprintf("%s/predump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, predumpPath)

			prepareDir(dst, fmt.Sprintf("%s/images/0", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/0", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/0/predump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/0", dstUrl.Path))

			// Process final image
			migrateStart = time.Now()

			imagesPath = fmt.Sprintf("%s/images/1", srcUrl.Path)
			log.Println("Performing the checkpoint")
			_, _, err = src.Run("sudo", "runc", "--id", containerId, "checkpoint", "--image-path", imagesPath, "--prev-images-dir", predumpPath)
			if err != nil {
				log.Fatal("Error performing checkpoint:", err)
			}

			srcTarFile = fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)
			prepareDir(dst, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Copying predump image to dst")
			err = cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images/1", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying predump image files to dst", err)
			}

			dstTarFile = fmt.Sprintf("%s/images/1/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images/1", dstUrl.Path))

			log.Println("Performing the restore")
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images/1", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		} else {
			imagesPath = fmt.Sprintf("%s/images", srcUrl.Path)
			prepareDir(src, imagesPath)

			migrateStart = time.Now()
			checkpoint(src, containerId, imagesPath, false)

			srcTarFile := fmt.Sprintf("%s/dump.tar.gz", srcUrl.Path)
			prepareTar(src, srcTarFile, imagesPath)

			prepareDir(dst, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Copying checkpoint image to dst")
			err := cmd.Scp(src.URL(srcTarFile), dst.URL(fmt.Sprintf("%s/images", dstUrl.Path)))
			if err != nil {
				log.Fatal("Error copying image files to dst", err)
			}

			dstTarFile := fmt.Sprintf("%s/images/dump.tar.gz", dstUrl.Path)
			unpackTar(dst, dstTarFile, fmt.Sprintf("%s/images", dstUrl.Path))

			log.Println("Performing the restore")
			configFilePath := fmt.Sprintf("%s/config.json", dstUrl.Path)
			runtimeFilePath := fmt.Sprintf("%s/runtime.json", dstUrl.Path)
			dstImagesPath := fmt.Sprintf("%s/images", dstUrl.Path)
			restoreCmd, err = dst.Start("sudo", "runc", "--id", containerId, "restore", "--image-path", dstImagesPath, "--config-file", configFilePath, "--runtime-file", runtimeFilePath)
			if err != nil {
				log.Fatal("Error performing restore:", err)
			}

		}

		var restoreSucceed bool
		var restoreError error
		var wg sync.WaitGroup
		wg.Add(1)

		go func() {
			restoreError = restoreCmd.Wait()
			wg.Done()
		}()

		go func() {
			log.Println("Waiting for container to start...")
			// We make a fast check so we don't wait for the first ticker internal
			if isRunning(containerId) {
				restoreSucceed = true
				wg.Done()
				return
			}
			ticker := time.NewTicker(200 * time.Millisecond)
			go func() {
				for _ = range ticker.C {
					if isRunning(containerId) {
						restoreSucceed = true
						break
					}

				}
				ticker.Stop()
				wg.Done()
			}()
		}()

		wg.Wait()

		downtime = time.Since(migrateStart)

		if restoreSucceed {
			log.Printf("Restore finished successfully, total downtime: %dms", downtime/time.Millisecond)
		} else {
			log.Println("Error performing restore:", restoreError)
			// Rollback
		}

	},
}

func isRunning(containerId string) bool {
	if _, err := os.Stat(fmt.Sprintf("/var/run/opencontainer/containers/%s", containerId)); err == nil {
		return true
	}
	return false
}

func unpackTar(cmd cmd.Cmd, tarFile, workDir string) {
	log.Println("Preparing image at destination host")
	_, _, err := cmd.Run("sudo", "tar", "-C", workDir, "-xvzf", tarFile)
	if err != nil {
		log.Fatal("Error uncompressing image in destination:", err)
	}
}

func prepareTar(cmd cmd.Cmd, tarFile, workDir string) {
	_, _, err := cmd.Run("sudo", "tar", "-czf", tarFile, "-C", fmt.Sprintf("%s/", workDir), ".")
	if err != nil {
		log.Fatal("Error compressing image in source:", err)
	}
}

func checkpoint(cmd cmd.Cmd, containerId, imagesPath string, predump bool) {
	log.Printf("Performing the checkpoint predump = %t\n", predump)
	args := []string{"runc", "--id", containerId, "checkpoint", "--image-path", imagesPath}
	if predump {
		args = append(args, "--pre-dump")
	}
	_, _, err := cmd.Run("sudo", args...)
	if err != nil {
		log.Fatal("Error performing checkpoint:", err)
	}
}

func prepareDir(cmd cmd.Cmd, path string) {
	_, _, err := cmd.Run("mkdir", "-p", path)
	if err != nil {
		log.Fatal("Error preparing pre-dump dir:", err)
	}
}

func getContainerId(path string) string {
	_, id := filepath.Split(path)
	return id
}
<|endoftext|>


<commit_before>package build

import (
	"github.com/bmizerany/assert"
	"github.com/jingweno/gotask/task"
	"testing"
)

func TestManPageParse_Parse(t *testing.T) {
	doc := `NAME
    say-hello - Say hello to current user

DESCRIPTION
    Print out hello to current user
    one more line

OPTIONS
    -v, --verbose
        Run in verbose mode
    -n, --name=""
        Say hello to the given name
    -g, --greeting="Hello"
        Say hello with a custom type of greeting

`
	p := &manPageParser{doc}
	mp, err := p.Parse()

	assert.Equal(t, nil, err)
	assert.Equal(t, "say-hello", mp.Name)
	assert.Equal(t, "Say hello to current user", mp.Usage)
	assert.Equal(t, "Print out hello to current user\n   one more line", mp.Description)
	assert.Equal(t, 3, len(mp.Flags))

	firstFlag, ok := mp.Flags[0].(task.BoolFlag)
	assert.Tf(t, ok, "Can't convert flag to task.BoolFlag")
	assert.Equal(t, "v, verbose", firstFlag.Name)
	assert.Equal(t, "Run in verbose mode", firstFlag.Usage)

	stringFlag, ok := mp.Flags[1].(task.StringFlag)
	assert.Tf(t, ok, "Can't convert flag to task.StringFlag")
	assert.Equal(t, "n, name", stringFlag.Name)
	assert.Equal(t, "", stringFlag.Value)
	assert.Equal(t, "Say hello to the given name", stringFlag.Usage)

	stringFlag, ok = mp.Flags[2].(task.StringFlag)
	assert.Tf(t, ok, "Can't convert flag to task.StringFlag")
	assert.Equal(t, "g, greeting", stringFlag.Name)
	assert.Equal(t, "Hello", stringFlag.Value)
	assert.Equal(t, "Say hello with a custom type of greeting", stringFlag.Usage)

	doc = `Name
    say-hello - Say hello to current user

Description
    Print out hello to current user
`
	p = &manPageParser{doc}
	mp, err = p.Parse()

	assert.Equal(t, nil, err)
	assert.Equal(t, "", mp.Name)
	assert.Equal(t, "", mp.Usage)
	assert.Equal(t, "", mp.Description)
	assert.Equal(t, 0, len(mp.Flags))
}
<commit_msg>Swap test expectations to make sure order doesn't matter<commit_after>package build

import (
	"github.com/bmizerany/assert"
	"github.com/jingweno/gotask/task"
	"testing"
)

func TestManPageParser_Parse(t *testing.T) {
	doc := `NAME
    say-hello - Say hello to current user

DESCRIPTION
    Print out hello to current user
    one more line

OPTIONS
    -n, --name=""
        Say hello to the given name
    -v, --verbose
        Run in verbose mode
    -g, --greeting="Hello"
        Say hello with a custom type of greeting

`
	p := &manPageParser{doc}
	mp, err := p.Parse()

	assert.Equal(t, nil, err)
	assert.Equal(t, "say-hello", mp.Name)
	assert.Equal(t, "Say hello to current user", mp.Usage)
	assert.Equal(t, "Print out hello to current user\n   one more line", mp.Description)
	assert.Equal(t, 3, len(mp.Flags))

	stringFlag, ok := mp.Flags[0].(task.StringFlag)
	assert.Tf(t, ok, "Can't convert flag to task.StringFlag")
	assert.Equal(t, "n, name", stringFlag.Name)
	assert.Equal(t, "", stringFlag.Value)
	assert.Equal(t, "Say hello to the given name", stringFlag.Usage)

	boolFlag, ok := mp.Flags[1].(task.BoolFlag)
	assert.Tf(t, ok, "Can't convert flag to task.BoolFlag")
	assert.Equal(t, "v, verbose", boolFlag.Name)
	assert.Equal(t, "Run in verbose mode", boolFlag.Usage)

	stringFlag, ok = mp.Flags[2].(task.StringFlag)
	assert.Tf(t, ok, "Can't convert flag to task.StringFlag")
	assert.Equal(t, "g, greeting", stringFlag.Name)
	assert.Equal(t, "Hello", stringFlag.Value)
	assert.Equal(t, "Say hello with a custom type of greeting", stringFlag.Usage)

	doc = `Name
    say-hello - Say hello to current user

Description
    Print out hello to current user
`
	p = &manPageParser{doc}
	mp, err = p.Parse()

	assert.Equal(t, nil, err)
	assert.Equal(t, "", mp.Name)
	assert.Equal(t, "", mp.Usage)
	assert.Equal(t, "", mp.Description)
	assert.Equal(t, 0, len(mp.Flags))
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSnapshotCreateVolumePermission_Basic(t *testing.T) {
	var snapshotId, accountId string

	resource.ParallelTest(t, resource.TestCase{
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Scaffold everything
			{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(true),
				Check: resource.ComposeTestCheckFunc(
					testCheckResourceGetAttr("aws_ebs_snapshot.example_snapshot", "id", &snapshotId),
					testCheckResourceGetAttr("data.aws_caller_identity.current", "account_id", &accountId),
					testAccAWSSnapshotCreateVolumePermissionExists(&accountId, &snapshotId),
				),
			},
			// Drop just create volume permission to test destruction
			{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(false),
				Check: resource.ComposeTestCheckFunc(
					testAccAWSSnapshotCreateVolumePermissionDestroyed(&accountId, &snapshotId),
				),
			},
		},
	})
}

func testAccAWSSnapshotCreateVolumePermissionExists(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if !has {
			return fmt.Errorf("create volume permission does not exist for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionDestroyed(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if has {
			return fmt.Errorf("create volume permission still exists for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionConfig(includeCreateVolumePermission bool) string {
	base := `
data "aws_caller_identity" "current" {}

resource "aws_ebs_volume" "example" {
  availability_zone = "us-west-2a"
  size              = 40

  tags = {
    Name = "ebs_snap_perm"
  }
}

resource "aws_ebs_snapshot" "example_snapshot" {
  volume_id = "${aws_ebs_volume.example.id}"
}
`

	if !includeCreateVolumePermission {
		return base
	}

	return base + fmt.Sprintf(`
resource "aws_snapshot_create_volume_permission" "self-test" {
  snapshot_id = "${aws_ebs_snapshot.example_snapshot.id}"
  account_id  = "${data.aws_caller_identity.current.account_id}"
}
`)
}
<commit_msg>tests/resource/aws_snapshot_create_volume_permission: Use different account_id<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSnapshotCreateVolumePermission_Basic(t *testing.T) {
	var snapshotId string
	accountId := "111122223333"

	resource.ParallelTest(t, resource.TestCase{
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Scaffold everything
			{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(true, accountId),
				Check: resource.ComposeTestCheckFunc(
					testCheckResourceGetAttr("aws_ebs_snapshot.example_snapshot", "id", &snapshotId),
					testAccAWSSnapshotCreateVolumePermissionExists(&accountId, &snapshotId),
				),
			},
			// Drop just create volume permission to test destruction
			{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(false, accountId),
				Check: resource.ComposeTestCheckFunc(
					testAccAWSSnapshotCreateVolumePermissionDestroyed(&accountId, &snapshotId),
				),
			},
		},
	})
}

func testAccAWSSnapshotCreateVolumePermissionExists(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if !has {
			return fmt.Errorf("create volume permission does not exist for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionDestroyed(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if has {
			return fmt.Errorf("create volume permission still exists for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionConfig(includeCreateVolumePermission bool, accountID string) string {
	base := `
data "aws_availability_zones" "available" {}

resource "aws_ebs_volume" "example" {
  availability_zone = "${data.aws_availability_zones.available.names[0]}"
  size              = 1

  tags = {
    Name = "ebs_snap_perm"
  }
}

resource "aws_ebs_snapshot" "example_snapshot" {
  volume_id = "${aws_ebs_volume.example.id}"
}
`

	if !includeCreateVolumePermission {
		return base
	}

	return base + fmt.Sprintf(`
resource "aws_snapshot_create_volume_permission" "self-test" {
  snapshot_id = "${aws_ebs_snapshot.example_snapshot.id}"
  account_id  = %q
}
`, accountID)
}
<|endoftext|>


<commit_before><commit_msg>use daocloud<commit_after><|endoftext|>


<commit_before>package deployment

import (
	"errors"
	"fmt"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/inconshreveable/log15.v2"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/controller/worker/types"
	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/cluster"
)

type jobIDState struct {
	jobID string
	state ct.JobState
}

type DeployJob struct {
	*ct.Deployment
	client          *controller.Client
	deployEvents    chan<- ct.DeploymentEvent
	jobEvents       chan *ct.Job
	serviceNames    map[string]string
	serviceEvents   chan *discoverd.Event
	serviceMeta     *discoverd.ServiceMeta
	useJobEvents    map[string]struct{}
	logger          log15.Logger
	oldReleaseState map[string]int
	newReleaseState map[string]int
	knownJobStates  map[jobIDState]struct{}
	omni            map[string]struct{}
	hostCount       int
	stop            chan struct{}
}

func (d *DeployJob) isOmni(typ string) bool {
	_, ok := d.omni[typ]
	return ok
}

func (d *DeployJob) Perform() error {
	log := d.logger.New("fn", "Perform", "deployment_id", d.ID, "app_id", d.AppID)

	log.Info("validating deployment strategy")
	var deployFunc func() error
	switch d.Strategy {
	case "one-by-one":
		deployFunc = d.deployOneByOne
	case "all-at-once":
		deployFunc = d.deployAllAtOnce
	case "postgres":
		deployFunc = d.deployPostgres
	case "discoverd-meta":
		deployFunc = d.deployDiscoverdMeta
	default:
		err := UnknownStrategyError{d.Strategy}
		log.Error("error validating deployment strategy", "err", err)
		return err
	}

	log.Info("determining cluster size")
	hosts, err := cluster.NewClient().Hosts()
	if err != nil {
		log.Error("error listing cluster hosts", "err", err)
		return err
	}
	d.hostCount = len(hosts)

	log.Info("determining release services and deployment state")
	release, err := d.client.GetRelease(d.NewReleaseID)
	if err != nil {
		log.Error("error getting new release", "release_id", d.NewReleaseID, "err", err)
		return err
	}
	for typ, proc := range release.Processes {
		if proc.Omni {
			d.omni[typ] = struct{}{}
		}
		if proc.Service == "" {
			log.Info(fmt.Sprintf("using job events for %s process type, no service defined", typ))
			d.useJobEvents[typ] = struct{}{}
			continue
		}

		d.serviceNames[typ] = proc.Service

		log.Info(fmt.Sprintf("using service discovery for %s process type", typ), "service", proc.Service)
		events := make(chan *discoverd.Event)
		stream, err := discoverd.NewService(proc.Service).Watch(events)
		if err != nil {
			log.Error("error creating service discovery watcher", "service", proc.Service, "err", err)
			return err
		}
		defer stream.Close()

	outer:
		for {
			select {
			case <-d.stop:
				return worker.ErrStopped
			case event, ok := <-events:
				if !ok {
					log.Error("error creating service discovery watcher, channel closed", "service", proc.Service)
					return fmt.Errorf("deployer: could not create watcher for service: %s", proc.Service)
				}
				switch event.Kind {
				case discoverd.EventKindCurrent:
					break outer
				case discoverd.EventKindServiceMeta:
					d.serviceMeta = event.ServiceMeta
				case discoverd.EventKindUp:
					releaseID, ok := event.Instance.Meta["FLYNN_RELEASE_ID"]
					if !ok {
						continue
					}
					switch releaseID {
					case d.OldReleaseID:
						d.oldReleaseState[typ]++
					case d.NewReleaseID:
						d.newReleaseState[typ]++
					}
				}
			case <-time.After(5 * time.Second):
				log.Error("error creating service discovery watcher, timeout reached", "service", proc.Service)
				return fmt.Errorf("deployer: could not create watcher for service: %s", proc.Service)
			}
		}
		go func() {
			for {
				event, ok := <-events
				if !ok {
					// if this happens, it means defer cleanup is in progress

					// TODO: this could also happen if the stream connection
					// dropped. handle that case
					return
				}
				d.serviceEvents <- event
			}
		}()
	}

	log.Info("getting job event stream")
	d.jobEvents = make(chan *ct.Job)
	stream, err := d.client.StreamJobEvents(d.AppID, d.jobEvents)
	if err != nil {
		log.Error("error getting job event stream", "err", err)
		return err
	}
	defer stream.Close()

	log.Info("getting current jobs")
	jobs, err := d.client.JobList(d.AppID)
	if err != nil {
		log.Error("error getting current jobs", "err", err)
		return err
	}
	for _, job := range jobs {
		if job.State != ct.JobStateUp {
			continue
		}
		if _, ok := d.useJobEvents[job.Type]; !ok {
			continue
		}

		// track the jobs so we can drop any events received between
		// connecting the job stream and getting the list of jobs
		d.knownJobStates[jobIDState{job.ID, ct.JobStateUp}] = struct{}{}

		switch job.ReleaseID {
		case d.OldReleaseID:
			d.oldReleaseState[job.Type]++
		case d.NewReleaseID:
			d.newReleaseState[job.Type]++
		}
	}

	log.Info(
		"determined deployment state",
		"original", d.Processes,
		"old_release", d.oldReleaseState,
		"new_release", d.newReleaseState,
	)
	return deployFunc()
}

func (d *DeployJob) waitForJobEvents(releaseID string, expected ct.JobEvents, log log15.Logger) error {
	actual := make(ct.JobEvents)

	handleEvent := func(jobID, typ string, state ct.JobState) {
		// ignore pending events
		if state == ct.JobStatePending {
			return
		}

		// don't send duplicate events
		if _, ok := d.knownJobStates[jobIDState{jobID, state}]; ok {
			return
		}
		d.knownJobStates[jobIDState{jobID, state}] = struct{}{}

		if _, ok := actual[typ]; !ok {
			actual[typ] = make(map[ct.JobState]int)
		}
		actual[typ][state] += 1
		d.deployEvents <- ct.DeploymentEvent{
			ReleaseID: releaseID,
			JobState:  state,
			JobType:   typ,
		}
	}

	for {
		select {
		case <-d.stop:
			return worker.ErrStopped
		case event := <-d.serviceEvents:
			if event.Kind != discoverd.EventKindUp {
				continue
			}
			if id, ok := event.Instance.Meta["FLYNN_APP_ID"]; !ok || id != d.AppID {
				continue
			}
			if id, ok := event.Instance.Meta["FLYNN_RELEASE_ID"]; !ok || id != releaseID {
				continue
			}
			typ, ok := event.Instance.Meta["FLYNN_PROCESS_TYPE"]
			if !ok {
				continue
			}
			if _, ok := d.useJobEvents[typ]; ok {
				continue
			}
			jobID, ok := event.Instance.Meta["FLYNN_JOB_ID"]
			if !ok {
				continue
			}
			log.Info("got service event", "job.id", jobID, "job.type", typ, "job.state", event.Kind)
			handleEvent(jobID, typ, ct.JobStateUp)
			if expected.Equals(actual) {
				return nil
			}
		case event, ok := <-d.jobEvents:
			if !ok {
				return errors.New("unexpected close of job event stream")
			}
			if event.ReleaseID != releaseID {
				continue
			}

			// if service discovery is being used for the job's type, ignore up events and fail
			// the deployment if we get a down event when waiting for the job to come up.
			if _, ok := d.useJobEvents[event.Type]; !ok {
				if event.State == ct.JobStateUp {
					continue
				}
				if expected[event.Type][ct.JobStateUp] > 0 && event.IsDown() {
					handleEvent(event.ID, event.Type, ct.JobStateDown)
					return fmt.Errorf("%s process type failed to start, got %s job event", event.Type, event.State)
				}
			}

			log.Info("got job event", "job.id", event.ID, "job.type", event.Type, "job.state", event.State)
			if event.State == ct.JobStateStarting {
				continue
			}
			if _, ok := actual[event.Type]; !ok {
				actual[event.Type] = make(map[ct.JobState]int)
			}
			handleEvent(event.ID, event.Type, event.State)
			if event.HostError != nil {
				return fmt.Errorf("deployer: %s job failed to start: %s", event.Type, *event.HostError)
			}
			if expected.Equals(actual) {
				return nil
			}
		case <-time.After(60 * time.Second):
			return fmt.Errorf("timed out waiting for job events: %v", expected)
		}
	}
}
<commit_msg>deployer: Check service Update events as well as Up events<commit_after>package deployment

import (
	"errors"
	"fmt"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/inconshreveable/log15.v2"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/controller/worker/types"
	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/cluster"
)

type jobIDState struct {
	jobID string
	state ct.JobState
}

type DeployJob struct {
	*ct.Deployment
	client          *controller.Client
	deployEvents    chan<- ct.DeploymentEvent
	jobEvents       chan *ct.Job
	serviceNames    map[string]string
	serviceEvents   chan *discoverd.Event
	serviceMeta     *discoverd.ServiceMeta
	useJobEvents    map[string]struct{}
	logger          log15.Logger
	oldReleaseState map[string]int
	newReleaseState map[string]int
	knownJobStates  map[jobIDState]struct{}
	omni            map[string]struct{}
	hostCount       int
	stop            chan struct{}
}

func (d *DeployJob) isOmni(typ string) bool {
	_, ok := d.omni[typ]
	return ok
}

func (d *DeployJob) Perform() error {
	log := d.logger.New("fn", "Perform", "deployment_id", d.ID, "app_id", d.AppID)

	log.Info("validating deployment strategy")
	var deployFunc func() error
	switch d.Strategy {
	case "one-by-one":
		deployFunc = d.deployOneByOne
	case "all-at-once":
		deployFunc = d.deployAllAtOnce
	case "postgres":
		deployFunc = d.deployPostgres
	case "discoverd-meta":
		deployFunc = d.deployDiscoverdMeta
	default:
		err := UnknownStrategyError{d.Strategy}
		log.Error("error validating deployment strategy", "err", err)
		return err
	}

	log.Info("determining cluster size")
	hosts, err := cluster.NewClient().Hosts()
	if err != nil {
		log.Error("error listing cluster hosts", "err", err)
		return err
	}
	d.hostCount = len(hosts)

	log.Info("determining release services and deployment state")
	release, err := d.client.GetRelease(d.NewReleaseID)
	if err != nil {
		log.Error("error getting new release", "release_id", d.NewReleaseID, "err", err)
		return err
	}
	for typ, proc := range release.Processes {
		if proc.Omni {
			d.omni[typ] = struct{}{}
		}
		if proc.Service == "" {
			log.Info(fmt.Sprintf("using job events for %s process type, no service defined", typ))
			d.useJobEvents[typ] = struct{}{}
			continue
		}

		d.serviceNames[typ] = proc.Service

		log.Info(fmt.Sprintf("using service discovery for %s process type", typ), "service", proc.Service)
		events := make(chan *discoverd.Event)
		stream, err := discoverd.NewService(proc.Service).Watch(events)
		if err != nil {
			log.Error("error creating service discovery watcher", "service", proc.Service, "err", err)
			return err
		}
		defer stream.Close()

	outer:
		for {
			select {
			case <-d.stop:
				return worker.ErrStopped
			case event, ok := <-events:
				if !ok {
					log.Error("error creating service discovery watcher, channel closed", "service", proc.Service)
					return fmt.Errorf("deployer: could not create watcher for service: %s", proc.Service)
				}
				switch event.Kind {
				case discoverd.EventKindCurrent:
					break outer
				case discoverd.EventKindServiceMeta:
					d.serviceMeta = event.ServiceMeta
				case discoverd.EventKindUp:
					releaseID, ok := event.Instance.Meta["FLYNN_RELEASE_ID"]
					if !ok {
						continue
					}
					switch releaseID {
					case d.OldReleaseID:
						d.oldReleaseState[typ]++
					case d.NewReleaseID:
						d.newReleaseState[typ]++
					}
				}
			case <-time.After(5 * time.Second):
				log.Error("error creating service discovery watcher, timeout reached", "service", proc.Service)
				return fmt.Errorf("deployer: could not create watcher for service: %s", proc.Service)
			}
		}
		go func() {
			for {
				event, ok := <-events
				if !ok {
					// if this happens, it means defer cleanup is in progress

					// TODO: this could also happen if the stream connection
					// dropped. handle that case
					return
				}
				d.serviceEvents <- event
			}
		}()
	}

	log.Info("getting job event stream")
	d.jobEvents = make(chan *ct.Job)
	stream, err := d.client.StreamJobEvents(d.AppID, d.jobEvents)
	if err != nil {
		log.Error("error getting job event stream", "err", err)
		return err
	}
	defer stream.Close()

	log.Info("getting current jobs")
	jobs, err := d.client.JobList(d.AppID)
	if err != nil {
		log.Error("error getting current jobs", "err", err)
		return err
	}
	for _, job := range jobs {
		if job.State != ct.JobStateUp {
			continue
		}
		if _, ok := d.useJobEvents[job.Type]; !ok {
			continue
		}

		// track the jobs so we can drop any events received between
		// connecting the job stream and getting the list of jobs
		d.knownJobStates[jobIDState{job.ID, ct.JobStateUp}] = struct{}{}

		switch job.ReleaseID {
		case d.OldReleaseID:
			d.oldReleaseState[job.Type]++
		case d.NewReleaseID:
			d.newReleaseState[job.Type]++
		}
	}

	log.Info(
		"determined deployment state",
		"original", d.Processes,
		"old_release", d.oldReleaseState,
		"new_release", d.newReleaseState,
	)
	return deployFunc()
}

func (d *DeployJob) waitForJobEvents(releaseID string, expected ct.JobEvents, log log15.Logger) error {
	actual := make(ct.JobEvents)

	handleEvent := func(jobID, typ string, state ct.JobState) {
		// ignore pending events
		if state == ct.JobStatePending {
			return
		}

		// don't send duplicate events
		if _, ok := d.knownJobStates[jobIDState{jobID, state}]; ok {
			return
		}
		d.knownJobStates[jobIDState{jobID, state}] = struct{}{}

		if _, ok := actual[typ]; !ok {
			actual[typ] = make(map[ct.JobState]int)
		}
		actual[typ][state] += 1
		d.deployEvents <- ct.DeploymentEvent{
			ReleaseID: releaseID,
			JobState:  state,
			JobType:   typ,
		}
	}

	for {
		select {
		case <-d.stop:
			return worker.ErrStopped
		case event := <-d.serviceEvents:
			if !event.Kind.Any(discoverd.EventKindUp, discoverd.EventKindUpdate) {
				continue
			}
			if id, ok := event.Instance.Meta["FLYNN_APP_ID"]; !ok || id != d.AppID {
				continue
			}
			if id, ok := event.Instance.Meta["FLYNN_RELEASE_ID"]; !ok || id != releaseID {
				continue
			}
			typ, ok := event.Instance.Meta["FLYNN_PROCESS_TYPE"]
			if !ok {
				continue
			}
			if _, ok := d.useJobEvents[typ]; ok {
				continue
			}
			jobID, ok := event.Instance.Meta["FLYNN_JOB_ID"]
			if !ok {
				continue
			}
			log.Info("got service event", "job.id", jobID, "job.type", typ, "job.state", event.Kind)
			handleEvent(jobID, typ, ct.JobStateUp)
			if expected.Equals(actual) {
				return nil
			}
		case event, ok := <-d.jobEvents:
			if !ok {
				return errors.New("unexpected close of job event stream")
			}
			if event.ReleaseID != releaseID {
				continue
			}

			// if service discovery is being used for the job's type, ignore up events and fail
			// the deployment if we get a down event when waiting for the job to come up.
			if _, ok := d.useJobEvents[event.Type]; !ok {
				if event.State == ct.JobStateUp {
					continue
				}
				if expected[event.Type][ct.JobStateUp] > 0 && event.IsDown() {
					handleEvent(event.ID, event.Type, ct.JobStateDown)
					return fmt.Errorf("%s process type failed to start, got %s job event", event.Type, event.State)
				}
			}

			log.Info("got job event", "job.id", event.ID, "job.type", event.Type, "job.state", event.State)
			if event.State == ct.JobStateStarting {
				continue
			}
			if _, ok := actual[event.Type]; !ok {
				actual[event.Type] = make(map[ct.JobState]int)
			}
			handleEvent(event.ID, event.Type, event.State)
			if event.HostError != nil {
				return fmt.Errorf("deployer: %s job failed to start: %s", event.Type, *event.HostError)
			}
			if expected.Equals(actual) {
				return nil
			}
		case <-time.After(60 * time.Second):
			return fmt.Errorf("timed out waiting for job events: %v", expected)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>fix(api): get integration with clear password from services (#5149)<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"

	sci "github.com/samuell/scipipe"
)

func main() {
	foo := NewFooer()
	f2b := NewFoo2Barer()
	snk := sci.NewSink()

	f2b.InFoo = foo.OutFoo
	snk.In = f2b.OutBar

	pl := sci.NewPipeline()
	pl.AddProcs(foo, f2b, snk)
	pl.Run()
}

// ------------------------------------------------------------------------
// Components
// ------------------------------------------------------------------------

// Fooer

type Fooer struct {
	InnerProc *sci.ShellProcess
	OutFoo    chan *sci.FileTarget
}

func NewFooer() *Fooer {
	// Initiate task from a "shell like" pattern, though here we
	// just specify the out-port foo
	innerFoo := sci.Shell("{o:foo}")
	// Set the output formatter to a static string
	innerFoo.SetPathFormatterString("foo", "foo.txt")
	// Connect the ports of the outer task to the inner, generic one
	fooer := &Fooer{
		InnerProc: innerFoo,
		OutFoo:    innerFoo.OutPorts["foo"],
	}
	// Create the custom execute function, with pure Go code
	fooer.InnerProc.CustomExecute = func(task *sci.ShellTask) {
		task.OutTargets["foo"].WriteTempFile([]byte("foo\n"))
	}
	return fooer
}

func (p *Fooer) Run() {
	// Connect inner ports to outer ones again, in order to update
	// connectivity after the workflow wiring has taken place.
	p.InnerProc.OutPorts["foo"] = p.OutFoo
	// Run the inner process
	p.InnerProc.Run()
}

// Foo2Barer

type Foo2Barer struct {
	InnerProc *sci.ShellProcess
	InFoo     chan *sci.FileTarget
	OutBar    chan *sci.FileTarget
}

func NewFoo2Barer() *Foo2Barer {
	// Initiate task from a "shell like" pattern, though here we
	// just specify the in-port foo and the out-port bar
	innerFoo2Bar := sci.Shell("{i:foo}{o:bar}")
	// Set the output formatter to extend the path on the "bar"" in-port
	innerFoo2Bar.SetPathFormatterExtend("bar", "foo", ".bar.txt")
	// Connect the ports of the outer task to the inner, generic one
	foo2bar := &Foo2Barer{
		InnerProc: innerFoo2Bar,
		InFoo:     innerFoo2Bar.InPorts["foo"],
		OutBar:    innerFoo2Bar.OutPorts["bar"],
	}
	// Create the custom execute function, with pure Go code
	foo2bar.InnerProc.CustomExecute = func(task *sci.ShellTask) {
		task.OutTargets["bar"].WriteTempFile(bytes.Replace(task.InTargets["foo"].Read(), []byte("foo"), []byte("bar"), 1))
	}
	return foo2bar
}

func (p *Foo2Barer) Run() {
	// Connect inner ports to outer ones again, in order to update
	// connectivity after the workflow wiring has taken place.
	p.InnerProc.InPorts["foo"] = p.InFoo
	p.InnerProc.OutPorts["bar"] = p.OutBar
	// Run the inner process
	p.InnerProc.Run()
}
<commit_msg>Update in example 15<commit_after>package main

import (
	"bytes"

	sci "github.com/samuell/scipipe"
)

func main() {
	sci.InitLogInfo()

	foo := NewFooer()
	f2b := NewFoo2Barer()
	snk := sci.NewSink()

	f2b.InFoo = foo.OutFoo
	snk.In = f2b.OutBar

	pl := sci.NewPipeline()
	pl.AddProcs(foo, f2b, snk)
	pl.Run()
}

// ------------------------------------------------------------------------
// Components
// ------------------------------------------------------------------------

// Fooer

type Fooer struct {
	InnerProc *sci.ShellProcess
	OutFoo    chan *sci.FileTarget
}

func NewFooer() *Fooer {
	// Initiate task from a "shell like" pattern, though here we
	// just specify the out-port foo
	innerFoo := sci.Shell("{o:foo}")
	// Set the output formatter to a static string
	innerFoo.SetPathFormatterString("foo", "foo.txt")
	// Create the custom execute function, with pure Go code
	innerFoo.CustomExecute = func(task *sci.ShellTask) {
		task.OutTargets["foo"].WriteTempFile([]byte("foo\n"))
	}
	// Connect the ports of the outer task to the inner, generic one
	fooer := &Fooer{
		InnerProc: innerFoo,
		OutFoo:    innerFoo.OutPorts["foo"],
	}
	return fooer
}

func (p *Fooer) Run() {
	// Connect inner ports to outer ones again, in order to update
	// connectivity after the workflow wiring has taken place.
	p.InnerProc.OutPorts["foo"] = p.OutFoo
	// Run the inner process
	p.InnerProc.Run()
}

// Foo2Barer

type Foo2Barer struct {
	InnerProc *sci.ShellProcess
	InFoo     chan *sci.FileTarget
	OutBar    chan *sci.FileTarget
}

func NewFoo2Barer() *Foo2Barer {
	// Initiate task from a "shell like" pattern, though here we
	// just specify the in-port foo and the out-port bar
	innerFoo2Bar := sci.Shell("{i:foo}{o:bar}")
	// Set the output formatter to extend the path on the "bar"" in-port
	innerFoo2Bar.SetPathFormatterExtend("bar", "foo", ".bar.txt")
	// Connect the ports of the outer task to the inner, generic one
	foo2bar := &Foo2Barer{
		InnerProc: innerFoo2Bar,
		InFoo:     innerFoo2Bar.InPorts["foo"],
		OutBar:    innerFoo2Bar.OutPorts["bar"],
	}
	// Create the custom execute function, with pure Go code
	foo2bar.InnerProc.CustomExecute = func(task *sci.ShellTask) {
		task.OutTargets["bar"].WriteTempFile(bytes.Replace(task.InTargets["foo"].Read(), []byte("foo"), []byte("bar"), 1))
	}
	return foo2bar
}

func (p *Foo2Barer) Run() {
	// Connect inner ports to outer ones again, in order to update
	// connectivity after the workflow wiring has taken place.
	p.InnerProc.InPorts["foo"] = p.InFoo
	p.InnerProc.OutPorts["bar"] = p.OutBar
	// Run the inner process
	p.InnerProc.Run()
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"log"
	"net"
	"runtime"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/mitchellh/go-vnc"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
)

const KeyLeftShift uint32 = 0xFFE1

type bootCommandTemplateData struct {
	HTTPIP   string
	HTTPPort uint
	Name     string
}

// This step "types" the boot command into the VM over VNC.
//
// Uses:
//   http_port int
//   ui     packer.Ui
//   vnc_port uint
//
// Produces:
//   <nothing>
type StepTypeBootCommand struct {
	BootCommand []string
	VMName      string
	Tpl         *packer.ConfigTemplate
}

func (s *StepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	httpPort := state.Get("http_port").(uint)
	ui := state.Get("ui").(packer.Ui)
	vncIp := state.Get("vnc_ip").(string)
	vncPort := state.Get("vnc_port").(uint)

	// Connect to VNC
	ui.Say("Connecting to VM via VNC")
	nc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", vncIp, vncPort))
	if err != nil {
		err := fmt.Errorf("Error connecting to VNC: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer nc.Close()

	c, err := vnc.Client(nc, &vnc.ClientConfig{Exclusive: true})
	if err != nil {
		err := fmt.Errorf("Error handshaking with VNC: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer c.Close()

	log.Printf("Connected to VNC desktop: %s", c.DesktopName)

	// Determine the host IP
	var ipFinder HostIPFinder
	if finder, ok := driver.(HostIPFinder); ok {
		ipFinder = finder
	} else if runtime.GOOS == "windows" {
		ipFinder = new(VMnetNatConfIPFinder)
	} else {
		ipFinder = &IfconfigIPFinder{Device: "vmnet8"}
	}

	hostIp, err := ipFinder.HostIP()
	if err != nil {
		err := fmt.Errorf("Error detecting host IP: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	log.Printf("Host IP for the VMware machine: %s", hostIp)

	tplData := &bootCommandTemplateData{
		hostIp,
		httpPort,
		s.VMName,
	}

	ui.Say("Typing the boot command over VNC...")
	for _, command := range s.BootCommand {
		command, err := s.Tpl.Process(command, tplData)
		if err != nil {
			err := fmt.Errorf("Error preparing boot command: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		// Check for interrupts between typing things so we can cancel
		// since this isn't the fastest thing.
		if _, ok := state.GetOk(multistep.StateCancelled); ok {
			return multistep.ActionHalt
		}

		vncSendString(c, command)
	}

	return multistep.ActionContinue
}

func (*StepTypeBootCommand) Cleanup(multistep.StateBag) {}

func vncSendString(c *vnc.ClientConn, original string) {
	// Scancodes reference: https://github.com/qemu/qemu/blob/master/ui/vnc_keysym.h
	special := make(map[string]uint32)
	special["<bs>"] = 0xFF08
	special["<del>"] = 0xFFFF
	special["<enter>"] = 0xFF0D
	special["<esc>"] = 0xFF1B
	special["<f1>"] = 0xFFBE
	special["<f2>"] = 0xFFBF
	special["<f3>"] = 0xFFC0
	special["<f4>"] = 0xFFC1
	special["<f5>"] = 0xFFC2
	special["<f6>"] = 0xFFC3
	special["<f7>"] = 0xFFC4
	special["<f8>"] = 0xFFC5
	special["<f9>"] = 0xFFC6
	special["<f10>"] = 0xFFC7
	special["<f11>"] = 0xFFC8
	special["<f12>"] = 0xFFC9
	special["<return>"] = 0xFF0D
	special["<tab>"] = 0xFF09
	special["<up>"] = 0xFF52
	special["<down>"] = 0xFF54
	special["<left>"] = 0xFF51
	special["<right>"] = 0xFF53
	special["<spacebar>"] = 0x020
	special["<insert>"] = 0xFF63
	special["<home>"] = 0xFF50
	special["<end>"] = 0xFF57
	special["<pageUp>"] = 0xFF55
	special["<pageDown>"] = 0xFF56

	shiftedChars := "~!@#$%^&*()_+{}|:\"<>?"

	// TODO(mitchellh): Ripe for optimizations of some point, perhaps.
	for len(original) > 0 {
		var keyCode uint32
		keyShift := false

		if strings.HasPrefix(original, "<wait>") {
			log.Printf("Special code '<wait>' found, sleeping one second")
			time.Sleep(1 * time.Second)
			original = original[len("<wait>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait5>") {
			log.Printf("Special code '<wait5>' found, sleeping 5 seconds")
			time.Sleep(5 * time.Second)
			original = original[len("<wait5>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait10>") {
			log.Printf("Special code '<wait10>' found, sleeping 10 seconds")
			time.Sleep(10 * time.Second)
			original = original[len("<wait10>"):]
			continue
		}

		for specialCode, specialValue := range special {
			if strings.HasPrefix(original, specialCode) {
				log.Printf("Special code '%s' found, replacing with: %d", specialCode, specialValue)
				keyCode = specialValue
				original = original[len(specialCode):]
				break
			}
		}

		if keyCode == 0 {
			r, size := utf8.DecodeRuneInString(original)
			original = original[size:]
			keyCode = uint32(r)
			keyShift = unicode.IsUpper(r) || strings.ContainsRune(shiftedChars, r)

			log.Printf("Sending char '%c', code %d, shift %v", r, keyCode, keyShift)
		}

		if keyShift {
			c.KeyEvent(KeyLeftShift, true)
		}

		c.KeyEvent(keyCode, true)
		c.KeyEvent(keyCode, false)

		if keyShift {
			c.KeyEvent(KeyLeftShift, false)
		}
	}
}
<commit_msg>Add 1/10th second delay between key events to VNC<commit_after>package common

import (
	"fmt"
	"log"
	"net"
	"runtime"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/mitchellh/go-vnc"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
)

const KeyLeftShift uint32 = 0xFFE1

type bootCommandTemplateData struct {
	HTTPIP   string
	HTTPPort uint
	Name     string
}

// This step "types" the boot command into the VM over VNC.
//
// Uses:
//   http_port int
//   ui     packer.Ui
//   vnc_port uint
//
// Produces:
//   <nothing>
type StepTypeBootCommand struct {
	BootCommand []string
	VMName      string
	Tpl         *packer.ConfigTemplate
}

func (s *StepTypeBootCommand) Run(state multistep.StateBag) multistep.StepAction {
	driver := state.Get("driver").(Driver)
	httpPort := state.Get("http_port").(uint)
	ui := state.Get("ui").(packer.Ui)
	vncIp := state.Get("vnc_ip").(string)
	vncPort := state.Get("vnc_port").(uint)

	// Connect to VNC
	ui.Say("Connecting to VM via VNC")
	nc, err := net.Dial("tcp", fmt.Sprintf("%s:%d", vncIp, vncPort))
	if err != nil {
		err := fmt.Errorf("Error connecting to VNC: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer nc.Close()

	c, err := vnc.Client(nc, &vnc.ClientConfig{Exclusive: true})
	if err != nil {
		err := fmt.Errorf("Error handshaking with VNC: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}
	defer c.Close()

	log.Printf("Connected to VNC desktop: %s", c.DesktopName)

	// Determine the host IP
	var ipFinder HostIPFinder
	if finder, ok := driver.(HostIPFinder); ok {
		ipFinder = finder
	} else if runtime.GOOS == "windows" {
		ipFinder = new(VMnetNatConfIPFinder)
	} else {
		ipFinder = &IfconfigIPFinder{Device: "vmnet8"}
	}

	hostIp, err := ipFinder.HostIP()
	if err != nil {
		err := fmt.Errorf("Error detecting host IP: %s", err)
		state.Put("error", err)
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	log.Printf("Host IP for the VMware machine: %s", hostIp)

	tplData := &bootCommandTemplateData{
		hostIp,
		httpPort,
		s.VMName,
	}

	ui.Say("Typing the boot command over VNC...")
	for _, command := range s.BootCommand {
		command, err := s.Tpl.Process(command, tplData)
		if err != nil {
			err := fmt.Errorf("Error preparing boot command: %s", err)
			state.Put("error", err)
			ui.Error(err.Error())
			return multistep.ActionHalt
		}

		// Check for interrupts between typing things so we can cancel
		// since this isn't the fastest thing.
		if _, ok := state.GetOk(multistep.StateCancelled); ok {
			return multistep.ActionHalt
		}

		vncSendString(c, command)
	}

	return multistep.ActionContinue
}

func (*StepTypeBootCommand) Cleanup(multistep.StateBag) {}

func vncSendString(c *vnc.ClientConn, original string) {
	// Scancodes reference: https://github.com/qemu/qemu/blob/master/ui/vnc_keysym.h
	special := make(map[string]uint32)
	special["<bs>"] = 0xFF08
	special["<del>"] = 0xFFFF
	special["<enter>"] = 0xFF0D
	special["<esc>"] = 0xFF1B
	special["<f1>"] = 0xFFBE
	special["<f2>"] = 0xFFBF
	special["<f3>"] = 0xFFC0
	special["<f4>"] = 0xFFC1
	special["<f5>"] = 0xFFC2
	special["<f6>"] = 0xFFC3
	special["<f7>"] = 0xFFC4
	special["<f8>"] = 0xFFC5
	special["<f9>"] = 0xFFC6
	special["<f10>"] = 0xFFC7
	special["<f11>"] = 0xFFC8
	special["<f12>"] = 0xFFC9
	special["<return>"] = 0xFF0D
	special["<tab>"] = 0xFF09
	special["<up>"] = 0xFF52
	special["<down>"] = 0xFF54
	special["<left>"] = 0xFF51
	special["<right>"] = 0xFF53
	special["<spacebar>"] = 0x020
	special["<insert>"] = 0xFF63
	special["<home>"] = 0xFF50
	special["<end>"] = 0xFF57
	special["<pageUp>"] = 0xFF55
	special["<pageDown>"] = 0xFF56

	shiftedChars := "~!@#$%^&*()_+{}|:\"<>?"

	// TODO(mitchellh): Ripe for optimizations of some point, perhaps.
	for len(original) > 0 {
		var keyCode uint32
		keyShift := false

		if strings.HasPrefix(original, "<wait>") {
			log.Printf("Special code '<wait>' found, sleeping one second")
			time.Sleep(1 * time.Second)
			original = original[len("<wait>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait5>") {
			log.Printf("Special code '<wait5>' found, sleeping 5 seconds")
			time.Sleep(5 * time.Second)
			original = original[len("<wait5>"):]
			continue
		}

		if strings.HasPrefix(original, "<wait10>") {
			log.Printf("Special code '<wait10>' found, sleeping 10 seconds")
			time.Sleep(10 * time.Second)
			original = original[len("<wait10>"):]
			continue
		}

		for specialCode, specialValue := range special {
			if strings.HasPrefix(original, specialCode) {
				log.Printf("Special code '%s' found, replacing with: %d", specialCode, specialValue)
				keyCode = specialValue
				original = original[len(specialCode):]
				break
			}
		}

		if keyCode == 0 {
			r, size := utf8.DecodeRuneInString(original)
			original = original[size:]
			keyCode = uint32(r)
			keyShift = unicode.IsUpper(r) || strings.ContainsRune(shiftedChars, r)

			log.Printf("Sending char '%c', code %d, shift %v", r, keyCode, keyShift)
		}

		if keyShift {
			c.KeyEvent(KeyLeftShift, true)
		}

		c.KeyEvent(keyCode, true)
                time.Sleep(time.Second/10)
		c.KeyEvent(keyCode, false)
                time.Sleep(time.Second/10)

		if keyShift {
			c.KeyEvent(KeyLeftShift, false)
		}
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package builds

import (
	"encoding/json"
	"net/http"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Review : Resolves a build that is queued pending approval
func Review(au models.User, env string, action *models.Action) (int, []byte) {
	var e models.Env

	err := e.FindByName(env)
	if err != nil {
		h.L.Error(err.Error())
		return 404, []byte("Environment not found")
	}

	if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, e.GetType(), e.Name); st != 200 {
		return st, res
	}

	id, err := e.RequestReview(au, action.Options.Resolution)
	if err != nil {
		return 500, []byte(err.Error())
	}

	action.Status = "done"

	if id != "" {
		action.ResourceType = "build"
		action.ResourceID = id
		action.Status = "in_progress"
	}

	data, err := json.Marshal(action)
	if err != nil {
		return 500, []byte("could not process sync request")
	}

	return http.StatusOK, data
}
<commit_msg>returning error<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package builds

import (
	"encoding/json"
	"net/http"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
)

// Review : Resolves a build that is queued pending approval
func Review(au models.User, env string, action *models.Action) (int, []byte) {
	var e models.Env

	err := e.FindByName(env)
	if err != nil {
		h.L.Error(err.Error())
		return 404, []byte("Environment not found")
	}

	if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, e.GetType(), e.Name); st != 200 {
		return st, res
	}

	id, err := e.RequestReview(au, action.Options.Resolution)
	if err != nil {
		return 500, []byte(`{"error":"` + err.Error() + `"}`)
	}

	action.Status = "done"

	if id != "" {
		action.ResourceType = "build"
		action.ResourceID = id
		action.Status = "in_progress"
	}

	data, err := json.Marshal(action)
	if err != nil {
		return 500, []byte("could not process sync request")
	}

	return http.StatusOK, data
}
<|endoftext|>


<commit_before>package coordinator

import (
	"bytes"
	"encoding/binary"
	"io"
	"net"
	"testing"
	"time"

	log "code.google.com/p/log4go"
	"github.com/influxdb/influxdb/cluster"
	"github.com/influxdb/influxdb/protocol"
)

type PingResponseServer struct {
	Listener net.Listener
}

func (prs *PingResponseServer) Start() {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	prs.Listener = l
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				break
			}

			go prs.handleConnection(conn)
		}
	}()
}

func (prs *PingResponseServer) handleConnection(conn net.Conn) {
	message := make([]byte, 0, MAX_REQUEST_SIZE)
	buff := bytes.NewBuffer(message)
	var messageSizeU uint32

	for {
		buff.Reset()
		err := binary.Read(conn, binary.LittleEndian, &messageSizeU)
		if err != nil {
			log.Error("Error reading from connection (%s): %s", conn.RemoteAddr().String(), err)
			return
		}

		_, err = io.CopyN(buff, conn, int64(messageSizeU))

		if err != nil {
			break
		}

		request, err := protocol.DecodeRequest(buff)
		if err != nil {
			break
		}

		switch *request.Type {
		case protocol.Request_HEARTBEAT:
			response := &protocol.Response{RequestId: request.Id, Type: protocol.Response_HEARTBEAT.Enum()}

			data, err := response.Encode()
			if err != nil {
				panic(err)
			}
			binary.Write(conn, binary.LittleEndian, uint32(len(data)))
			_, err = conn.Write(data)
		default:
			panic("Not a heartbeat request")

		}
	}
	conn.Close()
}

func FakeHeartbeatServer() *PingResponseServer {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	prs := PingResponseServer{Listener: l}
	prs.Start()
	return &prs
}

func BenchmarkSingle(b *testing.B) {
	var HEARTBEAT_TYPE = protocol.Request_HEARTBEAT
	prs := FakeHeartbeatServer()
	client := NewProtobufClient(prs.Listener.Addr().String(), time.Second)
	client.Connect()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		responseChan := make(chan *protocol.Response, 1)
		heartbeatRequest := &protocol.Request{
			Type:     &HEARTBEAT_TYPE,
			Database: protocol.String(""),
		}
		rcw := cluster.NewResponseChannelWrapper(responseChan)
		client.MakeRequest(heartbeatRequest, rcw)
		<-responseChan
	}
}
<commit_msg>use gocheck in the coordinator benchmark<commit_after>package coordinator

import (
	"bytes"
	"encoding/binary"
	"io"
	"net"
	"time"

	"launchpad.net/gocheck"

	log "code.google.com/p/log4go"
	"github.com/influxdb/influxdb/cluster"
	"github.com/influxdb/influxdb/protocol"
)

type PingResponseServer struct {
	Listener net.Listener
}

func (prs *PingResponseServer) Start() {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	prs.Listener = l
	go func() {
		for {
			conn, err := l.Accept()
			if err != nil {
				break
			}

			go prs.handleConnection(conn)
		}
	}()
}

func (prs *PingResponseServer) handleConnection(conn net.Conn) {
	message := make([]byte, 0, MAX_REQUEST_SIZE)
	buff := bytes.NewBuffer(message)
	var messageSizeU uint32

	for {
		buff.Reset()
		err := binary.Read(conn, binary.LittleEndian, &messageSizeU)
		if err != nil {
			log.Error("Error reading from connection (%s): %s", conn.RemoteAddr().String(), err)
			return
		}

		_, err = io.CopyN(buff, conn, int64(messageSizeU))

		if err != nil {
			break
		}

		request, err := protocol.DecodeRequest(buff)
		if err != nil {
			break
		}

		switch *request.Type {
		case protocol.Request_HEARTBEAT:
			response := &protocol.Response{RequestId: request.Id, Type: protocol.Response_HEARTBEAT.Enum()}

			data, err := response.Encode()
			if err != nil {
				panic(err)
			}
			binary.Write(conn, binary.LittleEndian, uint32(len(data)))
			_, err = conn.Write(data)
		default:
			panic("Not a heartbeat request")

		}
	}
	conn.Close()
}

func FakeHeartbeatServer() *PingResponseServer {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		panic(err)
	}
	prs := PingResponseServer{Listener: l}
	prs.Start()
	return &prs
}

type ProtobufClientSuite struct{}

var _ = gocheck.Suite(&ProtobufClient{})

func (self *ProtobufClientSuite) BenchmarkSingle(c *gocheck.C) {
	var HEARTBEAT_TYPE = protocol.Request_HEARTBEAT
	prs := FakeHeartbeatServer()
	client := NewProtobufClient(prs.Listener.Addr().String(), time.Second)
	client.Connect()
	c.ResetTimer()
	for i := 0; i < c.N; i++ {
		responseChan := make(chan *protocol.Response, 1)
		heartbeatRequest := &protocol.Request{
			Type:     &HEARTBEAT_TYPE,
			Database: protocol.String(""),
		}
		rcw := cluster.NewResponseChannelWrapper(responseChan)
		client.MakeRequest(heartbeatRequest, rcw)
		<-responseChan
	}
}
<|endoftext|>


<commit_before>package model

import (
	"testing"

	"github.com/google/go-github/github"
	"github.com/stretchr/testify/assert"
)

func TestNewStarFromGithubShouldCopyFields(t *testing.T) {
	id := 33
	name := "larry-bird"
	fullName := "celtics/larry-bird"
	description := "larry legend"
	homepage := "http ://www.nba.com/celtics/"
	url := "http ://www.nba.com/pacers/"
	language := "hoosier"
	stargazersCount := 10000

	github := github.Repository{
		ID:              &id,
		Name:            &name,
		FullName:        &fullName,
		Description:     &description,
		Homepage:        &homepage,
		CloneURL:        &url,
		Language:        &language,
		StargazersCount: &stargazersCount,
	}

	star, err := NewStarFromGithub(github)
	assert.Nil(t, err)
	assert.Equal(t, "33", star.RemoteID)
	assert.Equal(t, name, *star.Name)
	assert.Equal(t, fullName, *star.FullName)
	assert.Equal(t, description, *star.Description)
	assert.Equal(t, homepage, *star.Homepage)
	assert.Equal(t, url, *star.URL)
	assert.Equal(t, language, *star.Language)
	assert.Equal(t, stargazersCount, star.Stargazers)
}

func TestNewStarFromGithubShouldHandleEmpty(t *testing.T) {
	star, err := NewStarFromGithub(github.Repository{})
	assert.NotNil(t, err)
	assert.Equal(t, "ID from GitHub is required", err.Error())
	assert.Nil(t, star)
}

func TestNewStarFromGithubShouldHandleOnlyID(t *testing.T) {
	id := 33
	star, err := NewStarFromGithub(github.Repository{
		ID: &id,
	})
	assert.Nil(t, err)
	assert.NotNil(t, star)
	assert.Equal(t, "33", star.RemoteID)
}
<commit_msg>Add tests for star fuzzy find<commit_after>package model

import (
	"testing"

	"github.com/google/go-github/github"
	"github.com/stretchr/testify/assert"
)

func TestNewStarFromGithubShouldCopyFields(t *testing.T) {
	id := 33
	name := "larry-bird"
	fullName := "celtics/larry-bird"
	description := "larry legend"
	homepage := "http ://www.nba.com/celtics/"
	url := "http ://www.nba.com/pacers/"
	language := "hoosier"
	stargazersCount := 10000

	github := github.Repository{
		ID:              &id,
		Name:            &name,
		FullName:        &fullName,
		Description:     &description,
		Homepage:        &homepage,
		CloneURL:        &url,
		Language:        &language,
		StargazersCount: &stargazersCount,
	}

	star, err := NewStarFromGithub(github)
	assert.Nil(t, err)
	assert.Equal(t, "33", star.RemoteID)
	assert.Equal(t, name, *star.Name)
	assert.Equal(t, fullName, *star.FullName)
	assert.Equal(t, description, *star.Description)
	assert.Equal(t, homepage, *star.Homepage)
	assert.Equal(t, url, *star.URL)
	assert.Equal(t, language, *star.Language)
	assert.Equal(t, stargazersCount, star.Stargazers)
}

func TestNewStarFromGithubShouldHandleEmpty(t *testing.T) {
	star, err := NewStarFromGithub(github.Repository{})
	assert.NotNil(t, err)
	assert.Equal(t, "ID from GitHub is required", err.Error())
	assert.Nil(t, star)
}

func TestNewStarFromGithubShouldHandleOnlyID(t *testing.T) {
	id := 33
	star, err := NewStarFromGithub(github.Repository{
		ID: &id,
	})
	assert.Nil(t, err)
	assert.NotNil(t, star)
	assert.Equal(t, "33", star.RemoteID)
}

func TestFuzzyFindStarsWithNameShouldFuzzyFind(t *testing.T) {
	fullName := "Apple/Baker"
	name := "Charlie"

	star := Star{
		FullName: &fullName,
		Name:     &name,
	}
	assert.Nil(t, db.Create(&star).Error)

	stars, err := FuzzyFindStarsWithName(db, "Apple/Baker")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "Charlie")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "apple/baker")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "charlie")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "apple")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "harl")
	assert.Nil(t, err)
	assert.Equal(t, 1, len(stars))
	assert.Equal(t, fullName, *stars[0].FullName)
	assert.Equal(t, name, *stars[0].Name)

	stars, err = FuzzyFindStarsWithName(db, "boogers")
	assert.Nil(t, err)
	assert.Equal(t, 0, len(stars))
}
<|endoftext|>


<commit_before>package merkledag

import (
	"fmt"
	"time"

	"code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return fmt.Errorf("merkledag: %s not found", name)
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debug("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}
<commit_msg>fix(merkledag) return static error<commit_after>package merkledag

import (
	"fmt"
	"time"

	"code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debug("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}
<|endoftext|>


<commit_before>package message

import (
	"errors"
	"log"
	"reflect"
	"unicode"
	"unicode/utf8"
)

type HandlerMap map[reflect.Type]*reflect.Method

func NewHandlerMap() HandlerMap {
	return make(map[reflect.Type]*reflect.Method)
}

func (handlerMap HandlerMap) Handle(contex interface{}, message interface{}) error {
	handler := handlerMap[reflect.TypeOf(message)]
	if handler == nil {
		return errors.New("Message handler not found:" + reflect.TypeOf(message).String() + "\n")
	}
	err := handler.Func.Call([]reflect.Value{reflect.ValueOf(contex), reflect.ValueOf(message)})
	if err[0].IsNil() {
		return nil
	} else {
		return err[0].Interface().(error)
	}
}

func (handlerMap HandlerMap) Register(contex interface{}) {
	handlerMap.register(contex, true)
}

func (handlerMap HandlerMap) RegisterWithoutLog(contex interface{}) {
	handlerMap.register(contex, false)
}

func (handlerMap HandlerMap) register(contex interface{}, printLog bool) {
	t := reflect.TypeOf(contex)
	num := 0
	for i := 0; i < t.NumMethod(); i++ {
		method := t.Method(i)
		if isSuitableMethod(&method, printLog) {
			handlerMap.registerHandler(method.Type.In(1), &method)
			num++
		}
	}
	if printLog {
		log.Printf("%v:%v handler(s) registered.\n", t.Name(), num)
	}
}

func (handlerMap HandlerMap) registerHandler(typ reflect.Type, method *reflect.Method) {
	handlerMap[typ] = method
}

var typeOfError = reflect.TypeOf((*error)(nil)).Elem()

func isExported(name string) bool {
	rune, _ := utf8.DecodeRuneInString(name)
	return unicode.IsUpper(rune)
}

// Is this type exported or a builtin?
func isExportedOrBuiltinType(t reflect.Type) bool {
	for t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	// so we need to check the type name as well.
	return isExported(t.Name()) || t.PkgPath() == ""
}

func isSuitableMethod(method *reflect.Method, printLog bool) bool {
	if method.Type.PkgPath() != "" {
		if printLog {
			log.Printf("%v is not exported.", method.Name)
		}
		return false
	}
	if method.Type.NumIn() != 2 {
		if printLog {
			log.Printf("%v number of args not matched.", method.Name)
		}
		return false
	}
	argType := method.Type.In(1)
	if argType.Kind() != reflect.Ptr {
		if printLog {
			log.Printf("%v the argument must be a pointer.", method.Name)
		}
		return false
	}
	if !isExportedOrBuiltinType(argType) {
		if printLog {
			log.Printf("%v the argument must be exported.", method.Name)
		}
		return false
	}
	if method.Type.NumOut() != 1 {
		if printLog {
			log.Printf("%v number of return value must be 1.", method.Name)
		}
		return false
	}
	if returnType := method.Type.Out(0); returnType != typeOfError {
		if printLog {
			log.Printf("%v the type of return value must be error.", method.Name)
		}
		return false
	}
	return true
}
<commit_msg>Add log info of handler_map.go<commit_after>package message

import (
	"errors"
	"log"
	"reflect"
	"unicode"
	"unicode/utf8"
)

type HandlerMap map[reflect.Type]*reflect.Method

func NewHandlerMap() HandlerMap {
	return make(map[reflect.Type]*reflect.Method)
}

func (handlerMap HandlerMap) Handle(contex interface{}, message interface{}) error {
	handler := handlerMap[reflect.TypeOf(message)]
	if handler == nil {
		return errors.New("Message handler not found:" + reflect.TypeOf(message).String() + "\n")
	}
	err := handler.Func.Call([]reflect.Value{reflect.ValueOf(contex), reflect.ValueOf(message)})
	if err[0].IsNil() {
		return nil
	} else {
		return err[0].Interface().(error)
	}
}

func (handlerMap HandlerMap) Register(contex interface{}) {
	handlerMap.register(contex, true)
}

func (handlerMap HandlerMap) RegisterWithoutLog(contex interface{}) {
	handlerMap.register(contex, false)
}

func (handlerMap HandlerMap) register(contex interface{}, printLog bool) {
	t := reflect.TypeOf(contex)
	num := 0
	for i := 0; i < t.NumMethod(); i++ {
		method := t.Method(i)
		if isSuitableMethod(&method, printLog) {
			key := method.Type.In(1)
			if _, ok := handlerMap[key]; ok {
				if printLog {
					log.Printf("%v:duplicate handler of %v, override.\n", t.Name(), key.Name())
				}
			}
			handlerMap[method.Type.In(1)] = &method
			num++
		}
	}
	if printLog {
		log.Printf("%v:%v handler(s) registered.\n", t.Name(), num)
	}
}

var typeOfError = reflect.TypeOf((*error)(nil)).Elem()

func isExported(name string) bool {
	rune, _ := utf8.DecodeRuneInString(name)
	return unicode.IsUpper(rune)
}

// Is this type exported or a builtin?
func isExportedOrBuiltinType(t reflect.Type) bool {
	for t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	// so we need to check the type name as well.
	return isExported(t.Name()) || t.PkgPath() == ""
}

func isSuitableMethod(method *reflect.Method, printLog bool) bool {
	if method.Type.PkgPath() != "" {
		if printLog {
			log.Printf("%v is not exported.", method.Name)
		}
		return false
	}
	if method.Type.NumIn() != 2 {
		if printLog {
			log.Printf("%v number of args not matched.", method.Name)
		}
		return false
	}
	argType := method.Type.In(1)
	if argType.Kind() != reflect.Ptr {
		if printLog {
			log.Printf("%v the argument must be a pointer.", method.Name)
		}
		return false
	}
	if !isExportedOrBuiltinType(argType) {
		if printLog {
			log.Printf("%v the argument must be exported.", method.Name)
		}
		return false
	}
	if method.Type.NumOut() != 1 {
		if printLog {
			log.Printf("%v number of return value must be 1.", method.Name)
		}
		return false
	}
	if returnType := method.Type.Out(0); returnType != typeOfError {
		if printLog {
			log.Printf("%v the type of return value must be error.", method.Name)
		}
		return false
	}
	return true
}
<|endoftext|>


<commit_before>// bulk_query_gen generates queries for various use cases. Its output will
// typically be consume by query_benchmarker.
//
// Output formats: InfluxDB, ElasticSearch.
//
// Query style use cases: Devops.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"
)

// query generator choices {use-case, query-type, format}
var useCaseMatrix = map[string]map[string]map[string]QueryGeneratorMaker {
	"devops": {
		"single-host": {
			"influx-http": NewInfluxDevopsSingleHost,
			"es-http": NewElasticSearchDevopsSingleHost,
		},
	},
}

// Program option vars:
var (
	useCase   string
	queryType string
	format    string

	scaleVar   int
	queryCount int

	dbName string // TODO(rw): make this a map[string]string -> DatabaseConfig

	timestampStartStr string
	timestampEndStr   string

	timestampStart time.Time
	timestampEnd   time.Time

	seed  int64
	debug int
)

// Parse args:
func init() {
	oldUsage := flag.Usage
	flag.Usage = func() {
		oldUsage()
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "The use case matrix of choices is:\n")
		for uc, queryTypes := range useCaseMatrix {
			for qt, formats := range queryTypes {
				for ff := range formats {
					fmt.Fprintf(os.Stderr, "  use case: %s, query type: %s, format: %s\n", uc, qt, ff)
				}
			}
		}

	}
	flag.StringVar(&format, "format", "influx-http", "Format to emit. (Choices are in the use case matrix.)")
	flag.StringVar(&useCase, "use-case", "devops", "Use case to model. (Choices are in the use case matrix.)")
	flag.StringVar(&queryType, "query-type", "", "Query type. (Choices are in the use case matrix.)")
	flag.IntVar(&scaleVar, "scale-var", 1, "Scaling variable (must be the equal to the scalevar used for data generation).")
	flag.IntVar(&queryCount, "queries", 1000, "Number of queries to generate.")

	flag.StringVar(&dbName, "db", "benchmark_db", "Database for influx to use (ignored for elastic)")

	flag.StringVar(&timestampStartStr, "timestamp-start", "2016-01-01T00:00:00Z", "Beginning timestamp (RFC3339).")
	flag.StringVar(&timestampEndStr, "timestamp-end", "2016-01-01T06:00:00Z", "Ending timestamp (RFC3339).")

	flag.Int64Var(&seed, "seed", 0, "PRNG seed (default, or 0, uses the current timestamp).")
	flag.IntVar(&debug, "debug", 0, "Debug printing (choices: 0, 1) (default 0).")

	flag.Parse()

	if _, ok := useCaseMatrix[useCase]; !ok {
		log.Fatal("invalid use case specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType]; !ok {
		log.Fatal("invalid query type specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType][format]; !ok {
		log.Fatal("invalid format specifier")
	}

	// the default seed is the current timestamp:
	if seed == 0 {
		seed = int64(time.Now().Nanosecond())
	}
	fmt.Fprintf(os.Stderr, "using random seed %d\n", seed)

	// Parse timestamps:
	var err error
	timestampStart, err = time.Parse(time.RFC3339, timestampStartStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampStart = timestampStart.UTC()
	timestampEnd, err = time.Parse(time.RFC3339, timestampEndStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampEnd = timestampEnd.UTC()
}

func main() {
	rand.Seed(seed)

	dbConfig := DatabaseConfig{
		"database-name": dbName,
	}

	qgMaker := useCaseMatrix[useCase][queryType][format]
	var generator QueryGenerator = qgMaker(dbConfig, timestampStart, timestampEnd)

	stats := make(map[string]int64)

	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()

	// create request instances, serializing them to stdout and collecting
	// counts for each kind:
	enc := gob.NewEncoder(out)
	q := &Query{}
	for i := 0; i < queryCount; i++ {
		generator.Dispatch(i, q, scaleVar)
		err := enc.Encode(q)
		if err != nil {
			log.Fatal(err)
		}
		stats[string(q.HumanLabel)]++

		if debug == 1 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanLabel)
			if err != nil {
				log.Fatal(err)
			}
		} else if debug == 2 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanDescription)
			if err != nil {
				log.Fatal(err)
			}
		} else if debug >= 3 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.String())
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// print stats:
	keys := []string{}
	for k, _ := range stats {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		_, err := fmt.Fprintf(os.Stderr, "%s: %d points\n", k, stats[k])
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Tidying up<commit_after>// bulk_query_gen generates queries for various use cases. Its output will
// be consumed by query_benchmarker.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"time"
)

// query generator choices {use-case, query-type, format}
// (This object is shown to the user when flag.Usage is called.)
var useCaseMatrix = map[string]map[string]map[string]QueryGeneratorMaker {
	"devops": {
		"single-host": {
			"influx-http": NewInfluxDevopsSingleHost,
			"es-http": NewElasticSearchDevopsSingleHost,
		},
	},
}

// Program option vars:
var (
	useCase   string
	queryType string
	format    string

	scaleVar   int
	queryCount int

	dbName string // TODO(rw): make this a map[string]string -> DatabaseConfig

	timestampStartStr string
	timestampEndStr   string

	timestampStart time.Time
	timestampEnd   time.Time

	seed  int64
	debug int
)

// Parse args:
func init() {
	// Change the Usage function to print the use case matrix of choices:
	oldUsage := flag.Usage
	flag.Usage = func() {
		oldUsage()

		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "The use case matrix of choices is:\n")
		for uc, queryTypes := range useCaseMatrix {
			for qt, formats := range queryTypes {
				for f := range formats {
					fmt.Fprintf(os.Stderr, "  use case: %s, query type: %s, format: %s\n", uc, qt, f)
				}
			}
		}

	}

	flag.StringVar(&format, "format", "influx-http", "Format to emit. (Choices are in the use case matrix.)")
	flag.StringVar(&useCase, "use-case", "devops", "Use case to model. (Choices are in the use case matrix.)")
	flag.StringVar(&queryType, "query-type", "", "Query type. (Choices are in the use case matrix.)")

	flag.IntVar(&scaleVar, "scale-var", 1, "Scaling variable (must be the equal to the scalevar used for data generation).")
	flag.IntVar(&queryCount, "queries", 1000, "Number of queries to generate.")

	flag.StringVar(&dbName, "db", "benchmark_db", "Database for influx to use (ignored for ElasticSearch).")

	flag.StringVar(&timestampStartStr, "timestamp-start", "2016-01-01T00:00:00Z", "Beginning timestamp (RFC3339).")
	flag.StringVar(&timestampEndStr, "timestamp-end", "2016-01-01T06:00:00Z", "Ending timestamp (RFC3339).")

	flag.Int64Var(&seed, "seed", 0, "PRNG seed (default, or 0, uses the current timestamp).")
	flag.IntVar(&debug, "debug", 0, "Debug printing (choices: 0, 1) (default 0).")

	flag.Parse()

	if _, ok := useCaseMatrix[useCase]; !ok {
		log.Fatal("invalid use case specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType]; !ok {
		log.Fatal("invalid query type specifier")
	}

	if _, ok := useCaseMatrix[useCase][queryType][format]; !ok {
		log.Fatal("invalid format specifier")
	}

	// the default seed is the current timestamp:
	if seed == 0 {
		seed = int64(time.Now().Nanosecond())
	}
	fmt.Fprintf(os.Stderr, "using random seed %d\n", seed)

	// Parse timestamps:
	var err error
	timestampStart, err = time.Parse(time.RFC3339, timestampStartStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampStart = timestampStart.UTC()
	timestampEnd, err = time.Parse(time.RFC3339, timestampEndStr)
	if err != nil {
		log.Fatal(err)
	}
	timestampEnd = timestampEnd.UTC()
}

func main() {
	rand.Seed(seed)

	// TODO(rw): Parse this from the CLI (maybe).
	dbConfig := DatabaseConfig{
		"database-name": dbName,
	}

	// Make the query generator:
	maker := useCaseMatrix[useCase][queryType][format]
	var generator QueryGenerator = maker(dbConfig, timestampStart, timestampEnd)

	// Set up bookkeeping:
	stats := make(map[string]int64)

	// Set up output buffering:
	out := bufio.NewWriter(os.Stdout)
	defer out.Flush()

	// Create request instances, serializing them to stdout and collecting
	// counts for each kind:
	enc := gob.NewEncoder(out)
	q := &Query{}
	for i := 0; i < queryCount; i++ {
		generator.Dispatch(i, q, scaleVar)
		err := enc.Encode(q)
		if err != nil {
			log.Fatal(err)
		}
		stats[string(q.HumanLabel)]++

		if debug == 1 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanLabel)
			if err != nil {
				log.Fatal(err)
			}
		} else if debug == 2 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.HumanDescription)
			if err != nil {
				log.Fatal(err)
			}
		} else if debug >= 3 {
			_, err := fmt.Fprintf(os.Stderr, "%s\n", q.String())
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	// Print stats:
	keys := []string{}
	for k, _ := range stats {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		_, err := fmt.Fprintf(os.Stderr, "%s: %d points\n", k, stats[k])
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hillu/go-ntdll"

	"flag"
	"fmt"
	"os"
	"unsafe"
)

func query_vm(ph ntdll.Handle, base uintptr) (*ntdll.MemoryBasicInformationT, ntdll.NtStatus) {
	var mbi ntdll.MemoryBasicInformationT
	s := ntdll.NtQueryVirtualMemory(
		ph,
		(*uint8)(unsafe.Pointer(base)),
		ntdll.MemoryBasicInformation,
		(*uint8)(unsafe.Pointer(&mbi)),
		unsafe.Sizeof(mbi),
		nil,
	)
	if s.IsError() {
		return nil, s
	}
	return &mbi, s
}

func query_name(ph ntdll.Handle, base uintptr) (string, ntdll.NtStatus) {
	buf := [1024]byte{}
	s := ntdll.NtQueryVirtualMemory(
		ph,
		(*uint8)(unsafe.Pointer(base)),
		ntdll.MemoryMappedFilenameInformation,
		&buf[0],
		1024,
		nil,
	)
	if s.IsError() {
		return "", s
	}
	return (*ntdll.UnicodeString)(unsafe.Pointer(&buf[0])).String(), s
}

type memProtect uint32

func (p memProtect) String() string {
	r := []byte("----")
	if p&ntdll.PAGE_NOACCESS != 0 {
		r[3] = '!'
	}
	if p&ntdll.PAGE_READONLY != 0 {
		r[0] = 'r'
	}
	if p&ntdll.PAGE_READWRITE != 0 {
		r[0] = 'r'
		r[1] = 'w'
	}
	if p&ntdll.PAGE_WRITECOPY != 0 {
		r[1] = 'w'
		r[3] = 'c'
	}
	if p&ntdll.PAGE_EXECUTE != 0 {
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_READ != 0 {
		r[0] = 'r'
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_READWRITE != 0 {
		r[0] = 'r'
		r[1] = 'w'
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_WRITECOPY != 0 {
		r[1] = 'x'
		r[2] = 'x'
		r[3] = 'c'
	}
	if p&ntdll.PAGE_GUARD != 0 {
		r[3] = 'G'
	}
	if p&ntdll.PAGE_NOCACHE != 0 {
		r[3] = 'C'
	}
	return string(r)
}

type memState uint32

func (s memState) String() string {
	r := []byte("---")
	if s&ntdll.MEM_COMMIT != 0 {
		r[0] = 'c'
	}
	if s&ntdll.MEM_RESERVE != 0 {
		r[1] = 'r'
	}
	if s&ntdll.MEM_FREE != 0 {
		r[2] = 'f'
	}
	return string(r)
}

type memType uint32

func (s memType) String() string {
	r := []byte("---")
	if s&ntdll.MEM_MAPPED != 0 {
		r[0] = 'm'
	}
	if s&ntdll.MEM_PRIVATE != 0 {
		r[1] = 'p'
	}
	if s&ntdll.MEM_IMAGE != 0 {
		r[2] = 'i'
	}
	return string(r)
}

func main() {
	var pid uint
	flag.UintVar(&pid, "pid", 0, "target process id")
	flag.Parse()
	if pid == 0 {
		fmt.Printf("usage: %s -pid <pid>\n", os.Args[0])
		os.Exit(1)
	}

	var ph ntdll.Handle
	oa := ntdll.NewObjectAttributes("", 0, 0, nil)
	client := ntdll.ClientId{ntdll.Handle(pid), 0}
	s := ntdll.NtOpenProcess(&ph, ntdll.PROCESS_QUERY_INFORMATION, oa, &client)
	if s.IsError() {
		fmt.Printf("NtOpenProcess: %s", s)
		os.Exit(1)
	}

	addr := uintptr(0)
	for {
		mbi, s := query_vm(ph, addr)
		if s.IsError() {
			if s != ntdll.STATUS_INVALID_PARAMETER {
				fmt.Printf("NtQueryVirtualMemory: %08x: %s", addr, s.Error())
			}
			break
		}
		name, _ := query_name(ph, addr)
		if mbi.AllocationBase != nil {
			fmt.Printf("%12x %12x + %12x ty:%s st:%s pr:=%s %s\n",
				uintptr(unsafe.Pointer(mbi.AllocationBase)),
				uintptr(unsafe.Pointer(mbi.BaseAddress)),
				mbi.RegionSize,
				memType(mbi.Type),
				memState(mbi.State),
				memProtect(mbi.Protect),
				name,
			)
		}
		addr = uintptr(unsafe.Pointer(mbi.BaseAddress)) + mbi.RegionSize
	}
}
<commit_msg>Fix typo<commit_after>package main

import (
	"github.com/hillu/go-ntdll"

	"flag"
	"fmt"
	"os"
	"unsafe"
)

func query_vm(ph ntdll.Handle, base uintptr) (*ntdll.MemoryBasicInformationT, ntdll.NtStatus) {
	var mbi ntdll.MemoryBasicInformationT
	s := ntdll.NtQueryVirtualMemory(
		ph,
		(*uint8)(unsafe.Pointer(base)),
		ntdll.MemoryBasicInformation,
		(*uint8)(unsafe.Pointer(&mbi)),
		unsafe.Sizeof(mbi),
		nil,
	)
	if s.IsError() {
		return nil, s
	}
	return &mbi, s
}

func query_name(ph ntdll.Handle, base uintptr) (string, ntdll.NtStatus) {
	buf := [1024]byte{}
	s := ntdll.NtQueryVirtualMemory(
		ph,
		(*uint8)(unsafe.Pointer(base)),
		ntdll.MemoryMappedFilenameInformation,
		&buf[0],
		1024,
		nil,
	)
	if s.IsError() {
		return "", s
	}
	return (*ntdll.UnicodeString)(unsafe.Pointer(&buf[0])).String(), s
}

type memProtect uint32

func (p memProtect) String() string {
	r := []byte("----")
	if p&ntdll.PAGE_NOACCESS != 0 {
		r[3] = '!'
	}
	if p&ntdll.PAGE_READONLY != 0 {
		r[0] = 'r'
	}
	if p&ntdll.PAGE_READWRITE != 0 {
		r[0] = 'r'
		r[1] = 'w'
	}
	if p&ntdll.PAGE_WRITECOPY != 0 {
		r[1] = 'w'
		r[3] = 'c'
	}
	if p&ntdll.PAGE_EXECUTE != 0 {
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_READ != 0 {
		r[0] = 'r'
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_READWRITE != 0 {
		r[0] = 'r'
		r[1] = 'w'
		r[2] = 'x'
	}
	if p&ntdll.PAGE_EXECUTE_WRITECOPY != 0 {
		r[1] = 'x'
		r[2] = 'x'
		r[3] = 'c'
	}
	if p&ntdll.PAGE_GUARD != 0 {
		r[3] = 'G'
	}
	if p&ntdll.PAGE_NOCACHE != 0 {
		r[3] = 'C'
	}
	return string(r)
}

type memState uint32

func (s memState) String() string {
	r := []byte("---")
	if s&ntdll.MEM_COMMIT != 0 {
		r[0] = 'c'
	}
	if s&ntdll.MEM_RESERVE != 0 {
		r[1] = 'r'
	}
	if s&ntdll.MEM_FREE != 0 {
		r[2] = 'f'
	}
	return string(r)
}

type memType uint32

func (s memType) String() string {
	r := []byte("---")
	if s&ntdll.MEM_MAPPED != 0 {
		r[0] = 'm'
	}
	if s&ntdll.MEM_PRIVATE != 0 {
		r[1] = 'p'
	}
	if s&ntdll.MEM_IMAGE != 0 {
		r[2] = 'i'
	}
	return string(r)
}

func main() {
	var pid uint
	flag.UintVar(&pid, "pid", 0, "target process id")
	flag.Parse()
	if pid == 0 {
		fmt.Printf("usage: %s -pid <pid>\n", os.Args[0])
		os.Exit(1)
	}

	var ph ntdll.Handle
	oa := ntdll.NewObjectAttributes("", 0, 0, nil)
	client := ntdll.ClientId{ntdll.Handle(pid), 0}
	s := ntdll.NtOpenProcess(&ph, ntdll.PROCESS_QUERY_INFORMATION, oa, &client)
	if s.IsError() {
		fmt.Printf("NtOpenProcess: %s", s)
		os.Exit(1)
	}

	addr := uintptr(0)
	for {
		mbi, s := query_vm(ph, addr)
		if s.IsError() {
			if s != ntdll.STATUS_INVALID_PARAMETER {
				fmt.Printf("NtQueryVirtualMemory: %08x: %s", addr, s.Error())
			}
			break
		}
		name, _ := query_name(ph, addr)
		if mbi.AllocationBase != nil {
			fmt.Printf("%12x %12x + %12x ty:%s st:%s pr:%s %s\n",
				uintptr(unsafe.Pointer(mbi.AllocationBase)),
				uintptr(unsafe.Pointer(mbi.BaseAddress)),
				mbi.RegionSize,
				memType(mbi.Type),
				memState(mbi.State),
				memProtect(mbi.Protect),
				name,
			)
		}
		addr = uintptr(unsafe.Pointer(mbi.BaseAddress)) + mbi.RegionSize
	}
}
<|endoftext|>


<commit_before>/* This example show a basic menu similar to that found in the ncurses
 * examples from TLDP */

package main

import . "goncurses.googlecode.com/hg/goncurses"

func main() {
    stdscr, _ := Init();
    defer End()

    StartColor()
    Raw(true)
    Echo(false)
    Cursor(0)
    stdscr.Keypad(true)

    // build the menu items
    menu_items := []string{
        "Choice 1",
        "Choice 2",
        "Choice 3",
        "Choice 4",
        "Choice 5",
        "Choice 6",
        "Choice 7",
        "Exit"}
    items := make([]*MenuItem, len(menu_items))
    for i, val := range menu_items {
        items[i], _ = NewItem(val, "")
        defer items[i].Free()
    }

    // create the menu
    menu, _ := NewMenu(items)
    defer menu.Free()

    menu.Option(O_ONEVALUE, false)

    y, _ := stdscr.Maxyx()
    stdscr.Print(y-3, "Use up/down arrows to move, spacebar to toggle and " +
        "enter to print. 'q' to exit")
    stdscr.Refresh()

    menu.Post()
    defer menu.UnPost()

    for {
        Update()
        ch := stdscr.GetChar()

        switch ch {
        case 'q':
            return
        case ' ':
            menu.Driver(REQ_TOGGLE)
        case KEY_RETURN:
            var list string
            for _, item := range menu.Items(nil) {
                if item.Value() {
                    list += "\"" + item.Name() + "\" "
                }
            }
            stdscr.Move(20, 0)
            stdscr.ClearToEOL()
            stdscr.Print(20, list)
            stdscr.Refresh()
        default:
            menu.Driver(DriverActions[ch])
        }
    }
}
<commit_msg>Update menutoggle example<commit_after>/* This example show a basic menu similar to that found in the ncurses
 * examples from TLDP */

package main

import gc "code.google.com/p/goncurses"

func main() {
	stdscr, _ := gc.Init()
	defer gc.End()

	gc.StartColor()
	gc.Raw(true)
	gc.Echo(false)
	gc.Cursor(0)
	stdscr.Keypad(true)

	// build the menu items
	menu_items := []string{
		"Choice 1",
		"Choice 2",
		"Choice 3",
		"Choice 4",
		"Choice 5",
		"Choice 6",
		"Choice 7",
		"Exit"}
	items := make([]*gc.MenuItem, len(menu_items))
	for i, val := range menu_items {
		items[i], _ = gc.NewItem(val, "")
		defer items[i].Free()
	}

	// create the menu
	menu, _ := gc.NewMenu(items)
	defer menu.Free()

	menu.Option(gc.O_ONEVALUE, false)

	y, _ := stdscr.Maxyx()
	stdscr.Print(y-3, "Use up/down arrows to move, spacebar to toggle and "+
		"enter to print. 'q' to exit")
	stdscr.Refresh()

	menu.Post()
	defer menu.UnPost()

	for {
		gc.Update()
		ch := stdscr.GetChar()

		switch ch {
		case 'q':
			return
		case ' ':
			menu.Driver(gc.REQ_TOGGLE)
		case gc.KEY_RETURN:
			var list string
			for _, item := range menu.Items() {
				if item.Value() {
					list += "\"" + item.Name() + "\" "
				}
			}
			stdscr.Move(20, 0)
			stdscr.ClearToEOL()
			stdscr.Print(20, list)
			stdscr.Refresh()
		default:
			menu.Driver(gc.DriverActions[ch])
		}
	}
}
<|endoftext|>


<commit_before>package models

type Customer struct {
	OrganizationId int
	Id             int
	DisplayName    string
	FirstName      string
	LastName       string
	CompanyName    string
	Addresses      []CustomerAddress
	Email          string
	PhoneNumber    string
	VATNumber      string
}

type CustomerAddress struct {
	Id         int64
	CustomerId int64
	Address
}
<commit_msg>Omit empty addresses<commit_after>package models

type Customer struct {
	OrganizationId int
	Id             int
	DisplayName    string
	FirstName      string
	LastName       string
	CompanyName    string
	Addresses      []CustomerAddress `json:",omitempty"`
	Email          string
	PhoneNumber    string
	VATNumber      string
}

type CustomerAddress struct {
	Id         int64
	CustomerId int64
	Address
}
<|endoftext|>