<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A small helper for using gcsfuse with mount(8).
//
// Can be invoked using a command-line of the form expected for mount helpers.
// Calls the gcsfuse binary, which must be in $PATH, and waits for it to
// complete. The device is passed as --bucket, and other known options are
// converted to appropriate flags.
//
// This binary does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

// Example invocation on OS X:
//
//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
//
// becomes the following arguments:
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "-o"
//     Arg 2: "key_file=/some file.json"
//     Arg 3: "-o"
//     Arg 4: "ro"
//     Arg 5: "-o"
//     Arg 6: "blah"
//     Arg 7: "bucket"
//     Arg 8: "/path/to/mp"
//
// On Linux, the fstab entry
//
//     bucket /path/to/mp porp user,key_file=/some\040file.json
//
// becomes
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "bucket"
//     Arg 2: "/path/to/mp"
//     Arg 3: "-o"
//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
//

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/googlecloudplatform/gcsfuse/mount"
)

var fOptions = make(map[string]string)

func init() {
	flag.Var(
		mount.OptionValue(fOptions),
		"o",
		"Mount options. May be repeated.")
}

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(
	device string,
	mountPoint string,
	opts map[string]string) (args []string, err error) {
	// Deal with options.
	for name, value := range opts {
		switch name {
		case "key_file":
			args = append(args, "--key_file="+value)

		case "fuse_debug":
			args = append(args, "--fuse.debug")

		case "gcs_debug":
			args = append(args, "--gcs.debug")

		case "uid":
			args = append(args, "--uid="+value)

		case "gid":
			args = append(args, "--gid="+value)

		case "file_mode":
			args = append(args, "--file_mode="+value)

		case "dir_mode":
			args = append(args, "--dir_mode="+value)

		// Pass through everything else.
		default:
			var formatted string
			if value == "" {
				formatted = name
			} else {
				formatted = fmt.Sprintf("%s=%s", name, value)
			}

			args = append(args, "-o", formatted)
		}
	}

	// Set the bucket.
	args = append(args, "--bucket="+device)

	// Set the mount point.
	args = append(args, "--mount_point="+mountPoint)

	return
}

// Parse the supplied command-line arguments from a mount(8) invocation on OS X
// or Linux.
func parseArgs(
	args []string) (
	device string,
	mountPoint string,
	opts map[string]string,
	err error) {
	opts = make(map[string]string)

	// Process each argument in turn.
	for i, s := range args {
		switch {
		// Is this an options string following a "-o"?
		case i > 0 && args[i-1] == "-o":
			err = mount.ParseOptions(opts, s)
			if err != nil {
				err = fmt.Errorf("ParseOptions(%q): %v", s, err)
				return
			}

		default:
			err = fmt.Errorf("Unexpected arg: %q", s)
			return
		}
	}
}

func main() {
	flag.Parse()

	// Print out each argument.
	args := flag.Args()
	for i, arg := range args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, err := parseArgs(args)
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for name, value := range fOptions {
		log.Printf("Option %q: %q", name, value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(device, mountPoint, fOptions)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<commit_msg>parseArgs<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A small helper for using gcsfuse with mount(8).
//
// Can be invoked using a command-line of the form expected for mount helpers.
// Calls the gcsfuse binary, which must be in $PATH, and waits for it to
// complete. The device is passed as --bucket, and other known options are
// converted to appropriate flags.
//
// This binary does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

// Example invocation on OS X:
//
//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
//
// becomes the following arguments:
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "-o"
//     Arg 2: "key_file=/some file.json"
//     Arg 3: "-o"
//     Arg 4: "ro"
//     Arg 5: "-o"
//     Arg 6: "blah"
//     Arg 7: "bucket"
//     Arg 8: "/path/to/mp"
//
// On Linux, the fstab entry
//
//     bucket /path/to/mp porp user,key_file=/some\040file.json
//
// becomes
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "bucket"
//     Arg 2: "/path/to/mp"
//     Arg 3: "-o"
//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
//

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/googlecloudplatform/gcsfuse/mount"
)

var fOptions = make(map[string]string)

func init() {
	flag.Var(
		mount.OptionValue(fOptions),
		"o",
		"Mount options. May be repeated.")
}

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(
	device string,
	mountPoint string,
	opts map[string]string) (args []string, err error) {
	// Deal with options.
	for name, value := range opts {
		switch name {
		case "key_file":
			args = append(args, "--key_file="+value)

		case "fuse_debug":
			args = append(args, "--fuse.debug")

		case "gcs_debug":
			args = append(args, "--gcs.debug")

		case "uid":
			args = append(args, "--uid="+value)

		case "gid":
			args = append(args, "--gid="+value)

		case "file_mode":
			args = append(args, "--file_mode="+value)

		case "dir_mode":
			args = append(args, "--dir_mode="+value)

		// Pass through everything else.
		default:
			var formatted string
			if value == "" {
				formatted = name
			} else {
				formatted = fmt.Sprintf("%s=%s", name, value)
			}

			args = append(args, "-o", formatted)
		}
	}

	// Set the bucket.
	args = append(args, "--bucket="+device)

	// Set the mount point.
	args = append(args, "--mount_point="+mountPoint)

	return
}

// Parse the supplied command-line arguments from a mount(8) invocation on OS X
// or Linux.
func parseArgs(
	args []string) (
	device string,
	mountPoint string,
	opts map[string]string,
	err error) {
	opts = make(map[string]string)

	// Process each argument in turn.
	positionalCount := 0
	for i, s := range args {
		switch {
		// "-o" is illegal only when at the end. We handle its argument in the case
		// below.
		case s == "-o" && i == len(args)-1:
			err = fmt.Errorf("Unexpected -o at end of args.")
			return

		// Is this an options string following a "-o"?
		case i > 0 && args[i-1] == "-o":
			err = mount.ParseOptions(opts, s)
			if err != nil {
				err = fmt.Errorf("ParseOptions(%q): %v", s, err)
				return
			}

		// Is this the device?
		case positionalCount == 0:
			device = s
			positionalCount++

		// Is this the mount point?
		case positionalCount == 1:
			mountPoint = s
			positionalCount++

		default:
			err = fmt.Errorf("Unexpected arg: %q", s)
			return
		}
	}

	return
}

func main() {
	flag.Parse()

	// Print out each argument.
	args := flag.Args()
	for i, arg := range args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, err := parseArgs(args)
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for name, value := range fOptions {
		log.Printf("Option %q: %q", name, value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(device, mountPoint, fOptions)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"math"
	"strconv"
	"strings"

	"github.com/chzyer/readline"

	"encoding/json"
	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/remote"
	proto "github.com/gogo/protobuf/proto"
)

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

// Function constructor - constructs new function for listing given directory
var completer = readline.NewPrefixCompleter(
	readline.PcItem("tell"),
	readline.PcItem("exit"),
)

func filterInput(r rune) (rune, bool) {
	switch r {
	// block CtrlZ feature
	case readline.CharCtrlZ:
		return r, false
	}
	return r, true
}

func main() {
	logo := `
     ___         _         ___ _    ___
    | _ \_ _ ___| |_ ___  / __| |  |_ _|
    |  _/ '_/ _ \  _/ _ \| (__| |__ | |
    |_| |_| \___/\__\___(_)___|____|___|
`
	fmt.Println(logo)

	remote.DefaultSerializerID = 1
	remote.Start("127.0.0.1:0")
	actor.SpawnNamed(actor.FromFunc(func(ctx actor.Context) {
		fmt.Printf("ECHO: %+v\n", ctx.Message())
	}), "echo")

	vars := make(map[string]string)
	vars["%address%"] = actor.ProcessRegistry.Address
	vars["%echo%"] = fmt.Sprintf(`{"Address":"%v", "Id":"echo"}`, actor.ProcessRegistry.Address)

	l, err := readline.NewEx(&readline.Config{
		Prompt:          "\033[31m»\033[0m ",
		HistoryFile:     "/tmp/readline.tmp",
		AutoComplete:    completer,
		InterruptPrompt: "^C",
		EOFPrompt:       "exit",

		HistorySearchFold:   true,
		FuncFilterInputRune: filterInput,
	})

	if err != nil {
		panic(err)
	}
	defer l.Close()

	log.SetOutput(l.Stderr())
	for {
		line, err := l.Readline()
		if err == readline.ErrInterrupt {
			if len(line) == 0 {
				break
			} else {
				continue
			}
		} else if err == io.EOF {
			break
		}

		line = strings.TrimSpace(line)
		for k, v := range vars {
			line = strings.Replace(line, k, v, 1000)
		}
		switch {

		case strings.HasPrefix(line, "tell "):
			tell(line)

		case line == "exit":
			goto exit
		case line == "":
		default:
			log.Println("Unknown command :", strconv.Quote(line))
		}
	}
exit:
}
func tell(line string) {
	parts := strings.SplitN(line, " ", 4)

	if len(parts) != 4 {
		fmt.Printf("Wrong number of arguments for `tell`. expected: pid type-name json\n")
	} else {

		pidStr := parts[1]
		typeNameStr := parts[2]
		jsonStr := parts[3]

		x := strings.SplitN(pidStr, "/", 2)
		address := x[0]
		id := x[1]

		err := parseJson(jsonStr)
		if err == nil {
			m := &remote.JsonMessage{
				Json:     jsonStr,
				TypeName: typeNameStr,
			}
			pid := actor.NewPID(address, id)
			remote.SendMessage(pid, m, nil, 1)
		} else {
			fmt.Printf("Invalid JSON payload: %v\n", err)
		}
	}
}

func parseJson(s string) error {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js)
}
<commit_msg>remote watch<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"math"
	"strconv"
	"strings"

	"github.com/chzyer/readline"

	"encoding/json"
	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/remote"
	proto "github.com/gogo/protobuf/proto"
)

// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf

// Function constructor - constructs new function for listing given directory
var completer = readline.NewPrefixCompleter(
	readline.PcItem("tell"),
	readline.PcItem("watch"),
	readline.PcItem("exit"),
)

func filterInput(r rune) (rune, bool) {
	switch r {
	// block CtrlZ feature
	case readline.CharCtrlZ:
		return r, false
	}
	return r, true
}

var echoPID *actor.PID

func main() {
	logo := `
     ___         _         ___ _    ___
    | _ \_ _ ___| |_ ___  / __| |  |_ _|
    |  _/ '_/ _ \  _/ _ \| (__| |__ | |
    |_| |_| \___/\__\___(_)___|____|___|
`
	fmt.Println(logo)

	remote.DefaultSerializerID = 1
	remote.Start("127.0.0.1:0")
	spawnEcho()

	vars := make(map[string]string)
	vars["%address%"] = actor.ProcessRegistry.Address
	vars["%echo%"] = fmt.Sprintf(`{"Address":"%v", "Id":"echo"}`, actor.ProcessRegistry.Address)

	l, err := readline.NewEx(&readline.Config{
		Prompt:          "\033[31m»\033[0m ",
		HistoryFile:     "/tmp/readline.tmp",
		AutoComplete:    completer,
		InterruptPrompt: "^C",
		EOFPrompt:       "exit",

		HistorySearchFold:   true,
		FuncFilterInputRune: filterInput,
	})

	if err != nil {
		panic(err)
	}
	defer l.Close()

	log.SetOutput(l.Stderr())
	for {
		line, err := l.Readline()
		if err == readline.ErrInterrupt {
			if len(line) == 0 {
				break
			} else {
				continue
			}
		} else if err == io.EOF {
			break
		}

		line = strings.TrimSpace(line)
		for k, v := range vars {
			line = strings.Replace(line, k, v, 1000)
		}
		switch {

		case strings.HasPrefix(line, "tell "):
			tell(line)
		case strings.HasPrefix(line, "watch "):
			watch(line)
		case line == "exit":
			goto exit
		case line == "":
		default:
			log.Println("Unknown command :", strconv.Quote(line))
		}
	}
exit:
}
func spawnEcho() {
	echoPID, _ = actor.SpawnNamed(actor.FromFunc(func(ctx actor.Context) {
		switch msg := ctx.Message().(type) {
		case *actor.Started:
			fmt.Println("ECHO: Started")
		case *watchRequest:
			fmt.Printf("ECHO: Watching %v\n", msg.target.String())
			ctx.Watch(msg.target)
		case *actor.Terminated:
			fmt.Printf("ECHO:Actor %v terminated \n", msg.Who.String())
		default:
			fmt.Printf("ECHO: %+v\n", msg)
		}

	}), "echo")
}
func watch(line string) {
	parts := strings.SplitN(line, " ", 2)

	if len(parts) != 2 {
		fmt.Printf("Wrong number of arguments for `watch`. expected: pid\n")
	} else {

		pidStr := parts[1]
		x := strings.SplitN(pidStr, "/", 2)
		address := x[0]
		id := x[1]
		pid := actor.NewPID(address, id)
		echoPID.Tell(&watchRequest{
			target: pid,
		})
	}
}
func tell(line string) {
	parts := strings.SplitN(line, " ", 4)

	if len(parts) != 4 {
		fmt.Printf("Wrong number of arguments for `tell`. expected: pid type-name json\n")
	} else {

		pidStr := parts[1]
		typeNameStr := parts[2]
		jsonStr := parts[3]

		x := strings.SplitN(pidStr, "/", 2)
		address := x[0]
		id := x[1]

		err := parseJson(jsonStr)
		if err == nil {
			m := &remote.JsonMessage{
				Json:     jsonStr,
				TypeName: typeNameStr,
			}
			pid := actor.NewPID(address, id)
			remote.SendMessage(pid, m, nil, 1)
		} else {
			fmt.Printf("Invalid JSON payload: %v\n", err)
		}
	}
}

type watchRequest struct {
	target *actor.PID
}

func parseJson(s string) error {
	var js map[string]interface{}
	return json.Unmarshal([]byte(s), &js)
}
<|endoftext|>


<commit_before>package enaml

type Deployment interface {
	GetDeployment() DeploymentManifest
}

type CloudConfig interface {
	GetManifest() CloudConfigManifest
}

type DeploymentManifest struct {
	Name           string              `yaml:"name"`
	DirectorUUID   string              `yaml:"director_uuid,omitempty"`
	Releases       []Release           `yaml:"releases,omitempty"`
	Stemcells      []Stemcell          `yaml:"stemcells,omitempty"`
	InstanceGroups []*InstanceGroup    `yaml:"instance_groups,omitempty"`
	Networks       []DeploymentNetwork `yaml:"networks,omitempty"`
	ResourcePools  []ResourcePool      `yaml:"resource_pools,omitempty"`
	DiskPools      []DiskPool          `yaml:"disk_pools,omitempty"`
	Compilation    *Compilation        `yaml:"compilation,omitempty"`
	Update         Update              `yaml:"update,omitempty"`
	Jobs           []Job               `yaml:"jobs,omitempty"`
	Properties     Properties          `yaml:"properties,omitempty"`
	CloudProvider  *CloudProvider      `yaml:"cloud_provider,omitempty"`
}

type InstanceGroup struct {
	Name               string        `yaml:"name"`
	ResourcePool       string        `yaml:"resource_pool,omitempty"`
	PersistentDisk     int           `yaml:"persistent_disk,omitempty"`
	PersistentDiskType string        `yaml:"persistent_disk_type,omitempty"`
	Instances          int           `yaml:"instances"`
	VMType             string        `yaml:"vm_type,omitempty"`
	Stemcell           string        `yaml:"stemcell,omitempty"`
	AZs                []string      `yaml:"azs,flow,omitempty"`
	Networks           []Network     `yaml:"networks,flow"`
	Jobs               []InstanceJob `yaml:"jobs"`
	Update             Update        `yaml:"update,omitempty"`
}

type InstanceJob struct {
	Name       string      `yaml:"name"`
	Release    string      `yaml:"release"`
	Properties interface{} `yaml:"properties"`
}

type DeploymentNetwork interface {
}

type Release struct {
	Name    string `yaml:"name"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

func NewVIPNetwork(name string) VIPNetwork {
	return VIPNetwork{
		Name: name,
		Type: "vip",
	}
}

type VIPNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	CloudProperties CloudProperties `yaml:"cloud_properties,omitempty"`
}

type DynamicNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	DNS             []string        `yaml:"dns"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s DynamicNetwork) GetName() (name string) {
	return s.Name
}

func NewManualNetwork(name string) ManualNetwork {
	return ManualNetwork{
		Name: name,
		Type: "manual",
	}
}

type ManualNetwork struct {
	Name    string   `yaml:"name"`
	Type    string   `yaml:"type"`
	Subnets []Subnet `yaml:"subnets"`
}

func (s *ManualNetwork) AddSubnet(subnet Subnet) (err error) {
	s.Subnets = append(s.Subnets, subnet)
	return
}

func NewSubnet(cidr, gateway, azName string) Subnet {
	return Subnet{
		Range:   cidr,
		Gateway: gateway,
		AZ:      azName,
	}
}

type Subnet struct {
	Range           string          `yaml:"range,omitempty"`
	Gateway         string          `yaml:"gateway,omitempty"`
	DNS             []string        `yaml:"dns,omitempty"`
	Reserved        []string        `yaml:"reserved,omitempty"`
	Static          []string        `yaml:"static,omitempty"`
	AZ              string          `yaml:"az,omitempty"`
	AZs             []string        `yaml:"azs,omitempty"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s *Subnet) AddDNS(dns string) (err error) {
	s.DNS = append(s.DNS, dns)
	return
}

func (s *Subnet) AddReserved(rsv string) (err error) {
	s.Reserved = append(s.Reserved, rsv)
	return
}

type ResourcePool struct {
	Name            string                 `yaml:"name"`
	Network         string                 `yaml:"network"`
	Size            int                    `yaml:"size,omitempty"`
	Stemcell        Stemcell               `yaml:"stemcell"`
	CloudProperties CloudProperties        `yaml:"cloud_properties"`
	Env             map[string]interface{} `yaml:"env,omitempty"`
}

func (s *ResourcePool) SetStemcell(sc Stemcell) (err error) {
	s.Stemcell = sc
	return
}

type Stemcell struct {
	Alias   string `yaml:"alias,omitempty"`
	OS      string `yaml:"os,omitempty"`
	Name    string `yaml:"name,omitempty"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

type DiskType DiskPool
type DiskPool struct {
	Name            string          `yaml:"name"`
	DiskSize        int             `yaml:"disk_size"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

type Compilation struct {
	Workers             int             `yaml:"workers"`
	Network             string          `yaml:"network"`
	ReuseCompilationVMs bool            `yaml:"reuse_compilation_vms"`
	CloudProperties     CloudProperties `yaml:"cloud_properties,omitempty"`
	VMType              string          `yaml:"vm_type,omitempty"`
	AZ                  string          `yaml:"az,omitempty"`
}

type Update struct {
	Canaries        int    `yaml:"canaries,omitempty"`
	MaxInFlight     int    `yaml:"max_in_flight,omitempty"`
	CanaryWatchTime string `yaml:"canary_watch_time,omitempty"`
	UpdateWatchTime string `yaml:"update_watch_time,omitempty"`
	Serial          bool   `yaml:"serial,omitempty"`
}

type Job struct {
	Name               string     `yaml:"name"`
	Templates          []Template `yaml:"templates,flow"`
	Lifecycle          string     `yaml:"lifeycle,omitempty"`
	PersistentDisk     string     `yaml:"persistent_disk,omitempty"`
	PersistentDiskPool string     `yaml:"persistent_disk_pool,omitempty"`
	Properties         Properties `yaml:"properties,omitempty"`
	ResourcePool       string     `yaml:"resource_pool"`
	Update             Update     `yaml:"update,omitempty"`
	Instances          int        `yaml:"instances"`
	Networks           []Network  `yaml:"networks"`
}

func (s *Job) AddTemplate(t Template) (err error) {
	s.Templates = append(s.Templates, t)
	return
}

func (s *Job) AddNetwork(n Network) (err error) {
	s.Networks = append(s.Networks, n)
	return
}

func (s *Job) AddProperty(name string, property interface{}) (err error) {
	if s.Properties == nil {
		s.Properties = make(map[string]interface{})
	}
	s.Properties[name] = property
	return
}

type Network struct {
	Name      string        `yaml:"name"`
	StaticIPs []string      `yaml:"static_ips,omitempty"`
	Default   []interface{} `yaml:"default,omitempty"`
}

type Template struct {
	Name    string `yaml:"name"`
	Release string `yaml:"release"`
}

type CloudProvider struct {
	Template   Template                `yaml:"template,flow"`
	MBus       string                  `yaml:"mbus"`
	Properties CloudProviderProperties `yaml:"properties"`
	SSHTunnel  SSHTunnel               `yaml:"ssh_tunnel"`
}

type SSHTunnel struct {
	Host           string `yaml:"host"`
	Port           int    `yaml:"port"`
	User           string `yaml:"user"`
	PrivateKeyPath string `yaml:"private_key"`
}
type CloudProviderProperties interface{}
type Properties map[string]interface{}
type CloudProperties interface{}
<commit_msg>adding lifecyle attribute to instance group to allow specification of errand vs service<commit_after>package enaml

type Deployment interface {
	GetDeployment() DeploymentManifest
}

type CloudConfig interface {
	GetManifest() CloudConfigManifest
}

type DeploymentManifest struct {
	Name           string              `yaml:"name"`
	DirectorUUID   string              `yaml:"director_uuid,omitempty"`
	Releases       []Release           `yaml:"releases,omitempty"`
	Stemcells      []Stemcell          `yaml:"stemcells,omitempty"`
	InstanceGroups []*InstanceGroup    `yaml:"instance_groups,omitempty"`
	Networks       []DeploymentNetwork `yaml:"networks,omitempty"`
	ResourcePools  []ResourcePool      `yaml:"resource_pools,omitempty"`
	DiskPools      []DiskPool          `yaml:"disk_pools,omitempty"`
	Compilation    *Compilation        `yaml:"compilation,omitempty"`
	Update         Update              `yaml:"update,omitempty"`
	Jobs           []Job               `yaml:"jobs,omitempty"`
	Properties     Properties          `yaml:"properties,omitempty"`
	CloudProvider  *CloudProvider      `yaml:"cloud_provider,omitempty"`
}

type InstanceGroup struct {
	Name               string        `yaml:"name"`
	ResourcePool       string        `yaml:"resource_pool,omitempty"`
	PersistentDisk     int           `yaml:"persistent_disk,omitempty"`
	PersistentDiskType string        `yaml:"persistent_disk_type,omitempty"`
	Instances          int           `yaml:"instances"`
	VMType             string        `yaml:"vm_type,omitempty"`
	Stemcell           string        `yaml:"stemcell,omitempty"`
	AZs                []string      `yaml:"azs,flow,omitempty"`
	Networks           []Network     `yaml:"networks,flow"`
	Jobs               []InstanceJob `yaml:"jobs"`
	Update             Update        `yaml:"update,omitempty"`
	Lifecycle          string        `yaml:"lifecycle,omitempty"`
}

type InstanceJob struct {
	Name       string      `yaml:"name"`
	Release    string      `yaml:"release"`
	Properties interface{} `yaml:"properties"`
}

type DeploymentNetwork interface {
}

type Release struct {
	Name    string `yaml:"name"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

func NewVIPNetwork(name string) VIPNetwork {
	return VIPNetwork{
		Name: name,
		Type: "vip",
	}
}

type VIPNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	CloudProperties CloudProperties `yaml:"cloud_properties,omitempty"`
}

type DynamicNetwork struct {
	Name            string          `yaml:"name"`
	Type            string          `yaml:"type"`
	DNS             []string        `yaml:"dns"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s DynamicNetwork) GetName() (name string) {
	return s.Name
}

func NewManualNetwork(name string) ManualNetwork {
	return ManualNetwork{
		Name: name,
		Type: "manual",
	}
}

type ManualNetwork struct {
	Name    string   `yaml:"name"`
	Type    string   `yaml:"type"`
	Subnets []Subnet `yaml:"subnets"`
}

func (s *ManualNetwork) AddSubnet(subnet Subnet) (err error) {
	s.Subnets = append(s.Subnets, subnet)
	return
}

func NewSubnet(cidr, gateway, azName string) Subnet {
	return Subnet{
		Range:   cidr,
		Gateway: gateway,
		AZ:      azName,
	}
}

type Subnet struct {
	Range           string          `yaml:"range,omitempty"`
	Gateway         string          `yaml:"gateway,omitempty"`
	DNS             []string        `yaml:"dns,omitempty"`
	Reserved        []string        `yaml:"reserved,omitempty"`
	Static          []string        `yaml:"static,omitempty"`
	AZ              string          `yaml:"az,omitempty"`
	AZs             []string        `yaml:"azs,omitempty"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

func (s *Subnet) AddDNS(dns string) (err error) {
	s.DNS = append(s.DNS, dns)
	return
}

func (s *Subnet) AddReserved(rsv string) (err error) {
	s.Reserved = append(s.Reserved, rsv)
	return
}

type ResourcePool struct {
	Name            string                 `yaml:"name"`
	Network         string                 `yaml:"network"`
	Size            int                    `yaml:"size,omitempty"`
	Stemcell        Stemcell               `yaml:"stemcell"`
	CloudProperties CloudProperties        `yaml:"cloud_properties"`
	Env             map[string]interface{} `yaml:"env,omitempty"`
}

func (s *ResourcePool) SetStemcell(sc Stemcell) (err error) {
	s.Stemcell = sc
	return
}

type Stemcell struct {
	Alias   string `yaml:"alias,omitempty"`
	OS      string `yaml:"os,omitempty"`
	Name    string `yaml:"name,omitempty"`
	Version string `yaml:"version,omitempty"`
	URL     string `yaml:"url,omitempty"`
	SHA1    string `yaml:"sha1,omitempty"`
}

type DiskType DiskPool
type DiskPool struct {
	Name            string          `yaml:"name"`
	DiskSize        int             `yaml:"disk_size"`
	CloudProperties CloudProperties `yaml:"cloud_properties"`
}

type Compilation struct {
	Workers             int             `yaml:"workers"`
	Network             string          `yaml:"network"`
	ReuseCompilationVMs bool            `yaml:"reuse_compilation_vms"`
	CloudProperties     CloudProperties `yaml:"cloud_properties,omitempty"`
	VMType              string          `yaml:"vm_type,omitempty"`
	AZ                  string          `yaml:"az,omitempty"`
}

type Update struct {
	Canaries        int    `yaml:"canaries,omitempty"`
	MaxInFlight     int    `yaml:"max_in_flight,omitempty"`
	CanaryWatchTime string `yaml:"canary_watch_time,omitempty"`
	UpdateWatchTime string `yaml:"update_watch_time,omitempty"`
	Serial          bool   `yaml:"serial,omitempty"`
}

type Job struct {
	Name               string     `yaml:"name"`
	Templates          []Template `yaml:"templates,flow"`
	Lifecycle          string     `yaml:"lifeycle,omitempty"`
	PersistentDisk     string     `yaml:"persistent_disk,omitempty"`
	PersistentDiskPool string     `yaml:"persistent_disk_pool,omitempty"`
	Properties         Properties `yaml:"properties,omitempty"`
	ResourcePool       string     `yaml:"resource_pool"`
	Update             Update     `yaml:"update,omitempty"`
	Instances          int        `yaml:"instances"`
	Networks           []Network  `yaml:"networks"`
}

func (s *Job) AddTemplate(t Template) (err error) {
	s.Templates = append(s.Templates, t)
	return
}

func (s *Job) AddNetwork(n Network) (err error) {
	s.Networks = append(s.Networks, n)
	return
}

func (s *Job) AddProperty(name string, property interface{}) (err error) {
	if s.Properties == nil {
		s.Properties = make(map[string]interface{})
	}
	s.Properties[name] = property
	return
}

type Network struct {
	Name      string        `yaml:"name"`
	StaticIPs []string      `yaml:"static_ips,omitempty"`
	Default   []interface{} `yaml:"default,omitempty"`
}

type Template struct {
	Name    string `yaml:"name"`
	Release string `yaml:"release"`
}

type CloudProvider struct {
	Template   Template                `yaml:"template,flow"`
	MBus       string                  `yaml:"mbus"`
	Properties CloudProviderProperties `yaml:"properties"`
	SSHTunnel  SSHTunnel               `yaml:"ssh_tunnel"`
}

type SSHTunnel struct {
	Host           string `yaml:"host"`
	Port           int    `yaml:"port"`
	User           string `yaml:"user"`
	PrivateKeyPath string `yaml:"private_key"`
}
type CloudProviderProperties interface{}
type Properties map[string]interface{}
type CloudProperties interface{}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// mount_gcsfuse is a small helper for using gcsfuse with mount(8).
//
// mount_gcsfuse can be invoked using a command-line of the form expected for
// mount helpers. It calls the gcsfuse binary, which must be in $PATH, and
// waits for it to complete. The device is passed as --bucket, and other known
// options are converted to appropriate flags.
//
// mount_gcsfuse does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

// A 'name=value' mount option. If '=value' is not present, only the name will
// be filled in.
type Option struct {
	Name  string
	Value string
}

// Parse a single comma-separated list of mount options.
func parseOpts(s string) (opts []Option, err error) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var opt Option

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			opt.Name = p[:equalsIndex]
			opt.Value = p[equalsIndex+1:]
		} else {
			opt.Name = p
		}

		opts = append(opts, opt)
	}

	return
}

// Attempt to parse the terrible undocumented format that mount(8) gives us.
// Return the 'device' (aka 'special' on OS X), the mount point, and a list of
// mount options encountered.
func parseArgs() (device string, mountPoint string, opts []Option, err error) {
	// Example invocation on OS X:
	//
	//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
	//
	// becomes the following arguments:
	//
	//     Arg 0: "/path/to/mount_gcsfuse"
	//     Arg 1: "-o"
	//     Arg 2: "key_file=/some file.json"
	//     Arg 3: "-o"
	//     Arg 4: "ro"
	//     Arg 5: "-o"
	//     Arg 6: "blah"
	//     Arg 7: "bucket"
	//     Arg 8: "/path/to/mp"
	//
	// On Linux, the fstab entry
	//
	//     bucket /path/to/mp porp user,key_file=/some\040file.json
	//
	// becomes
	//
	//     Arg 0: "/path/to/mount_gcsfuse"
	//     Arg 1: "bucket"
	//     Arg 2: "/path/to/mp"
	//     Arg 3: "-o"
	//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
	//

	// Linux and OS X differ on the position of the options. So scan all
	// arguments (aside from the name of the binary), and:
	//
	//  *  Treat the first argument not following "-o" as the device name.
	//  *  Treat the second argument not following "-o" as the mount point.
	//  *  Treat the third argument not following "-o" as an error.
	//  *  Treat all arguments following "-o" as comma-separated options lists.
	//
	rawArgs := 0
	for i, arg := range os.Args {
		// Skip the binary name.
		if i == 0 {
			continue
		}

		// Skip "-o"; we will look back on the next iteration.
		if arg == "-o" {
			continue
		}

		// If the previous argument was "-o", this is a list of options.
		if os.Args[i-1] == "-o" {
			var tmp []Option
			tmp, err = parseOpts(arg)
			if err != nil {
				err = fmt.Errorf("parseOpts(%q): %v", arg, err)
				return
			}

			opts = append(opts, tmp...)
			continue
		}

		// Otherwise this is a non-option argument.
		switch rawArgs {
		case 0:
			device = arg

		case 1:
			mountPoint = arg

		default:
			err = fmt.Errorf(
				"Too many non-option arguments. The straw that broke the "+
					"camel's back: %q",
				arg)
			return
		}

		rawArgs++
	}

	// Did we see all of the raw arguments we expected?
	if rawArgs != 2 {
		err = fmt.Errorf("Expected 2 non-option arguments; got %d", rawArgs)
	}

	return
}

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(opts []Option) (args []string, err error) {
	for _, opt := range opts {
		switch opt.Name {
		case "key_file":
			args = append(args, "--key_file="+opt.Value)

		case "ro":
			args = append(args, "--read_only")

		default:
			err = fmt.Errorf(
				"Unrecognized mount option: %q (value %q)",
				opt.Name,
				opt.Value)

			return
		}
	}

	return
}

func main() {
	// Print out each argument.
	//
	// TODO(jacobsa): Get rid of some or all of the debug logging.
	for i, arg := range os.Args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, opts, err := parseArgs()
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for _, opt := range opts {
		log.Printf("Option %q: %q", opt.Name, opt.Value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(opts)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<commit_msg>Include mount point and bucket.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// mount_gcsfuse is a small helper for using gcsfuse with mount(8).
//
// mount_gcsfuse can be invoked using a command-line of the form expected for
// mount helpers. It calls the gcsfuse binary, which must be in $PATH, and
// waits for it to complete. The device is passed as --bucket, and other known
// options are converted to appropriate flags.
//
// mount_gcsfuse does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

// A 'name=value' mount option. If '=value' is not present, only the name will
// be filled in.
type Option struct {
	Name  string
	Value string
}

// Parse a single comma-separated list of mount options.
func parseOpts(s string) (opts []Option, err error) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var opt Option

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			opt.Name = p[:equalsIndex]
			opt.Value = p[equalsIndex+1:]
		} else {
			opt.Name = p
		}

		opts = append(opts, opt)
	}

	return
}

// Attempt to parse the terrible undocumented format that mount(8) gives us.
// Return the 'device' (aka 'special' on OS X), the mount point, and a list of
// mount options encountered.
func parseArgs() (device string, mountPoint string, opts []Option, err error) {
	// Example invocation on OS X:
	//
	//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
	//
	// becomes the following arguments:
	//
	//     Arg 0: "/path/to/mount_gcsfuse"
	//     Arg 1: "-o"
	//     Arg 2: "key_file=/some file.json"
	//     Arg 3: "-o"
	//     Arg 4: "ro"
	//     Arg 5: "-o"
	//     Arg 6: "blah"
	//     Arg 7: "bucket"
	//     Arg 8: "/path/to/mp"
	//
	// On Linux, the fstab entry
	//
	//     bucket /path/to/mp porp user,key_file=/some\040file.json
	//
	// becomes
	//
	//     Arg 0: "/path/to/mount_gcsfuse"
	//     Arg 1: "bucket"
	//     Arg 2: "/path/to/mp"
	//     Arg 3: "-o"
	//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
	//

	// Linux and OS X differ on the position of the options. So scan all
	// arguments (aside from the name of the binary), and:
	//
	//  *  Treat the first argument not following "-o" as the device name.
	//  *  Treat the second argument not following "-o" as the mount point.
	//  *  Treat the third argument not following "-o" as an error.
	//  *  Treat all arguments following "-o" as comma-separated options lists.
	//
	rawArgs := 0
	for i, arg := range os.Args {
		// Skip the binary name.
		if i == 0 {
			continue
		}

		// Skip "-o"; we will look back on the next iteration.
		if arg == "-o" {
			continue
		}

		// If the previous argument was "-o", this is a list of options.
		if os.Args[i-1] == "-o" {
			var tmp []Option
			tmp, err = parseOpts(arg)
			if err != nil {
				err = fmt.Errorf("parseOpts(%q): %v", arg, err)
				return
			}

			opts = append(opts, tmp...)
			continue
		}

		// Otherwise this is a non-option argument.
		switch rawArgs {
		case 0:
			device = arg

		case 1:
			mountPoint = arg

		default:
			err = fmt.Errorf(
				"Too many non-option arguments. The straw that broke the "+
					"camel's back: %q",
				arg)
			return
		}

		rawArgs++
	}

	// Did we see all of the raw arguments we expected?
	if rawArgs != 2 {
		err = fmt.Errorf("Expected 2 non-option arguments; got %d", rawArgs)
	}

	return
}

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(
	device string,
	mountPoint string,
	opts []Option) (args []string, err error) {
	// Deal with options.
	for _, opt := range opts {
		switch opt.Name {
		case "key_file":
			args = append(args, "--key_file="+opt.Value)

		case "ro":
			args = append(args, "--read_only")

		default:
			err = fmt.Errorf(
				"Unrecognized mount option: %q (value %q)",
				opt.Name,
				opt.Value)

			return
		}
	}

	// Set up the bucket.
	args = append(args, "--bucket="+device)

	// Include the mount point last.
	args = append(args, mountPoint)

	return
}

func main() {
	// Print out each argument.
	//
	// TODO(jacobsa): Get rid of some or all of the debug logging.
	for i, arg := range os.Args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, opts, err := parseArgs()
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for _, opt := range opts {
		log.Printf("Option %q: %q", opt.Name, opt.Value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(device, mountPoint, opts)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<|endoftext|>


<commit_before>// Packge gdbm implements a wrapper around libgdbm, the GNU DataBase Manager
// library, for Go.
package gdbm

// #cgo CFLAGS: -std=gnu99
// #cgo LDFLAGS: -lgdbm
// #include <stdlib.h>
// #include <gdbm.h>
// #include <string.h>
// inline datum mk_datum(char * s) {
//     datum d;
//     d.dptr = s;
//     d.dsize = strlen(s);
//     return d;
// }
import "C"

import (
	"errors"
	"unsafe"
)

// GDBM database "connection" type.
type Database struct {
	dbf  C.GDBM_FILE
	mode int
}

// GDBM database configuration type that can be used to specify how the
// database is created and treated. The `Mode` determines what the user of the
// `Database` object can do with it. The field can be a string, either "r" for
// Read-only, "w" for Read-Write, "c" for Read-Write and create if it doesn't
// exist, and "n" for Read-Write and always recreate database, even if it
// exists. TODO: write descriptions for other params... too lazy right now
type DatabaseCfg struct {
	Mode        string
	BlockSize   int
	Permissions int
}

func lastError() error {
	return errors.New(C.GoString(C.gdbm_strerror(C.gdbm_errno)))
}

// Simple function to open a database file with default parameters (block size
// is default for the filesystem and file permissions are set to 0666).
func Open(filename string, mode string) (db *Database, err error) {
	return OpenWithCfg(filename, DatabaseCfg{mode, 0, 0666})
}

// More complex database initialization function that takes in a `DatabaseCfg`
// struct to allow more fine-grained control over database settings.
func OpenWithCfg(filename string, cfg DatabaseCfg) (db *Database, err error) {
	db = new(Database)

	// Convert a human-readable mode string into a libgdbm-usable constant.
	switch cfg.Mode {
	case "r":
		db.mode = C.GDBM_READER
	case "w":
		db.mode = C.GDBM_WRITER
	case "c":
		db.mode = C.GDBM_WRCREAT
	case "n":
		db.mode = C.GDBM_NEWDB
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.dbf = C.gdbm_open(cs, C.int(cfg.BlockSize), C.int(db.mode), C.int(cfg.Permissions), nil)
	if db.dbf == nil {
		err = lastError()
	}
	return db, err
}

// Closes a database's internal file pointer.
func (db *Database) Close() {
	C.gdbm_close(db.dbf)
}

// Internal helper method to hide the two constants GDBM_INSERT and
// GDBM_REPLACE from the user.
func (db *Database) update(key string, value string, flag C.int) (err error) {
	// Convert key and value into libgdbm's `datum` data structure. See the
	// C definition at the top for the implementation of C.mk_datum(string).
	kcs := C.CString(key)
	vcs := C.CString(value)
	k := C.mk_datum(kcs)
	v := C.mk_datum(vcs)
	defer C.free(unsafe.Pointer(kcs))
	defer C.free(unsafe.Pointer(vcs))

	retv := C.gdbm_store(db.dbf, k, v, flag)
	if retv != 0 {
		err = lastError()
	}
	return err
}

// Inserts a key-value pair into the database. If the database is opened
// in "r" mode, this will return an error. Also, if the key already exists in
// the database, and error will be returned.
func (db *Database) Insert(key string, value string) (err error) {
	return db.update(key, value, C.GDBM_INSERT)
}

// Updates a key-value pair to use a new value, specified by the `value` string
// parameter. An error will be returned if the database is opened in "r" mode.
func (db *Database) Replace(key string, value string) (err error) {
	return db.update(key, value, C.GDBM_REPLACE)
}

// Returns true or false, depending on whether the specified key exists in the
// database.
func (db *Database) Exists(key string) bool {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	e := C.gdbm_exists(db.dbf, k)
	if e == 1 {
		return true
	}
	return false
}

// Returns the firstkey in this gdbm.Database. If there is not a key, an
// error will be returned in err.
func (db *Database) FirstKey() (value string, err error) {
	vdatum := C.gdbm_firstkey(db.dbf)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// Returns the nextkey after `key`. If there is not a next key, an
// error will be returned in err.
func (db *Database) NextKey(key string) (value string, err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	vdatum := C.gdbm_nextkey(db.dbf, k)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// return a map of [key]value, otherwise `err`
func (db *Database) ToMap() (db_map map[string]string, err error) {
	var (
		curr_k string
		curr_v string
		next_k string
		next_v string
	)
	db_map = make(map[string]string)

	curr_k, err = db.FirstKey()
	if err != nil {
		return
	}

	curr_v, err = db.Fetch(curr_k)
	if err != nil {
		return
	}

	db_map[curr_k] = curr_v

	for {
		next_k, err = db.NextKey(curr_k)
		if err != nil {
			break
		}

		next_v, err = db.Fetch(next_k)
		if err != nil {
			break
		}
		db_map[next_k] = next_v

		curr_k = next_k
	}
	return
}

// Fetches the value of the given key. If the key is not in the database, an
// error will be returned in err. Otherwise, value will be the value string
// that is keyed by `key`.
func (db *Database) Fetch(key string) (value string, err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	vdatum := C.gdbm_fetch(db.dbf, k)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// Removes a key-value pair from the database. If the database is opened in "r"
// mode, an error is returned
func (db *Database) Delete(key string) (err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	retv := C.gdbm_delete(db.dbf, k)
	if retv == -1 && db.mode == C.GDBM_READER {
		err = lastError()
	}
	return err
}

// Reorganizes the database for more efficient use of disk space. This method
// can be used if Delete(k) is called many times.
func (db *Database) Reorganize() {
	C.gdbm_reorganize(db.dbf)
}

// Synchronizes all pending database changes to the disk. TODO: note this is
// only needed in FAST mode, and FAST mode needs implemented!
func (db *Database) Sync() {
	C.gdbm_sync(db.dbf)
}
<commit_msg>be explicit, since the err that breaks the loop, should not trickle up<commit_after>// Packge gdbm implements a wrapper around libgdbm, the GNU DataBase Manager
// library, for Go.
package gdbm

// #cgo CFLAGS: -std=gnu99
// #cgo LDFLAGS: -lgdbm
// #include <stdlib.h>
// #include <gdbm.h>
// #include <string.h>
// inline datum mk_datum(char * s) {
//     datum d;
//     d.dptr = s;
//     d.dsize = strlen(s);
//     return d;
// }
import "C"

import (
	"errors"
	"unsafe"
)

// GDBM database "connection" type.
type Database struct {
	dbf  C.GDBM_FILE
	mode int
}

// GDBM database configuration type that can be used to specify how the
// database is created and treated. The `Mode` determines what the user of the
// `Database` object can do with it. The field can be a string, either "r" for
// Read-only, "w" for Read-Write, "c" for Read-Write and create if it doesn't
// exist, and "n" for Read-Write and always recreate database, even if it
// exists. TODO: write descriptions for other params... too lazy right now
type DatabaseCfg struct {
	Mode        string
	BlockSize   int
	Permissions int
}

func lastError() error {
	return errors.New(C.GoString(C.gdbm_strerror(C.gdbm_errno)))
}

// Simple function to open a database file with default parameters (block size
// is default for the filesystem and file permissions are set to 0666).
func Open(filename string, mode string) (db *Database, err error) {
	return OpenWithCfg(filename, DatabaseCfg{mode, 0, 0666})
}

// More complex database initialization function that takes in a `DatabaseCfg`
// struct to allow more fine-grained control over database settings.
func OpenWithCfg(filename string, cfg DatabaseCfg) (db *Database, err error) {
	db = new(Database)

	// Convert a human-readable mode string into a libgdbm-usable constant.
	switch cfg.Mode {
	case "r":
		db.mode = C.GDBM_READER
	case "w":
		db.mode = C.GDBM_WRITER
	case "c":
		db.mode = C.GDBM_WRCREAT
	case "n":
		db.mode = C.GDBM_NEWDB
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.dbf = C.gdbm_open(cs, C.int(cfg.BlockSize), C.int(db.mode), C.int(cfg.Permissions), nil)
	if db.dbf == nil {
		err = lastError()
	}
	return db, err
}

// Closes a database's internal file pointer.
func (db *Database) Close() {
	C.gdbm_close(db.dbf)
}

// Internal helper method to hide the two constants GDBM_INSERT and
// GDBM_REPLACE from the user.
func (db *Database) update(key string, value string, flag C.int) (err error) {
	// Convert key and value into libgdbm's `datum` data structure. See the
	// C definition at the top for the implementation of C.mk_datum(string).
	kcs := C.CString(key)
	vcs := C.CString(value)
	k := C.mk_datum(kcs)
	v := C.mk_datum(vcs)
	defer C.free(unsafe.Pointer(kcs))
	defer C.free(unsafe.Pointer(vcs))

	retv := C.gdbm_store(db.dbf, k, v, flag)
	if retv != 0 {
		err = lastError()
	}
	return err
}

// Inserts a key-value pair into the database. If the database is opened
// in "r" mode, this will return an error. Also, if the key already exists in
// the database, and error will be returned.
func (db *Database) Insert(key string, value string) (err error) {
	return db.update(key, value, C.GDBM_INSERT)
}

// Updates a key-value pair to use a new value, specified by the `value` string
// parameter. An error will be returned if the database is opened in "r" mode.
func (db *Database) Replace(key string, value string) (err error) {
	return db.update(key, value, C.GDBM_REPLACE)
}

// Returns true or false, depending on whether the specified key exists in the
// database.
func (db *Database) Exists(key string) bool {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	e := C.gdbm_exists(db.dbf, k)
	if e == 1 {
		return true
	}
	return false
}

// Returns the firstkey in this gdbm.Database. If there is not a key, an
// error will be returned in err.
func (db *Database) FirstKey() (value string, err error) {
	vdatum := C.gdbm_firstkey(db.dbf)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// Returns the nextkey after `key`. If there is not a next key, an
// error will be returned in err.
func (db *Database) NextKey(key string) (value string, err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	vdatum := C.gdbm_nextkey(db.dbf, k)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// return a map of [key]value, otherwise `err`
func (db *Database) ToMap() (db_map map[string]string, err error) {
	var (
		curr_k string
		curr_v string
		next_k string
		next_v string
	)
	db_map = make(map[string]string)

	curr_k, err = db.FirstKey()
	if err != nil {
		return
	}

	curr_v, err = db.Fetch(curr_k)
	if err != nil {
		return
	}

	db_map[curr_k] = curr_v

	for {
		next_k, err = db.NextKey(curr_k)
		if err != nil {
			break
		}

		next_v, err = db.Fetch(next_k)
		if err != nil {
			break
		}
		db_map[next_k] = next_v

		curr_k = next_k
	}
	return db_map, nil
}

// Fetches the value of the given key. If the key is not in the database, an
// error will be returned in err. Otherwise, value will be the value string
// that is keyed by `key`.
func (db *Database) Fetch(key string) (value string, err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	vdatum := C.gdbm_fetch(db.dbf, k)
	if vdatum.dptr == nil {
		return "", lastError()
	}

	value = C.GoString(vdatum.dptr)
	defer C.free(unsafe.Pointer(vdatum.dptr))
	return value, nil
}

// Removes a key-value pair from the database. If the database is opened in "r"
// mode, an error is returned
func (db *Database) Delete(key string) (err error) {
	kcs := C.CString(key)
	k := C.mk_datum(kcs)
	defer C.free(unsafe.Pointer(kcs))

	retv := C.gdbm_delete(db.dbf, k)
	if retv == -1 && db.mode == C.GDBM_READER {
		err = lastError()
	}
	return err
}

// Reorganizes the database for more efficient use of disk space. This method
// can be used if Delete(k) is called many times.
func (db *Database) Reorganize() {
	C.gdbm_reorganize(db.dbf)
}

// Synchronizes all pending database changes to the disk. TODO: note this is
// only needed in FAST mode, and FAST mode needs implemented!
func (db *Database) Sync() {
	C.gdbm_sync(db.dbf)
}
<|endoftext|>


<commit_before>package NEAT

import (
  "math"
  "math/rand"
)

type Gene struct{
  from int
  to int
  weight float64
  enabled bool
  innovation int
}

func NewGene() *Gene{
  return &Gene{0,0,0,true,0}
}

func CopyGene(g *Gene) *Gene{
  return &Gene{g.from,g.to,g.weight,g.enabled,g.innovation}
}

func containsLink(genes []Gene, link Gene) bool{
  for _,x := range(genes){
    if x.from == link.from && x.to == link.to {
      return true
    }
  }
  return false
}

func randomNeuron(g *Genome, notInput bool, c ConstContainer) int {
  if notInput {
     return rand.Intn(len(g.network)-c.nbInputs) + c.nbInputs
  }
  return rand.Intn(len(g.network))
}

// Returns the number of genes who are in one slice, not the other, and have an innovation
// number inferior to the other's (disjoint)

// Also returnes the number of genes who are in one slice, not the other, and
// are not disjoint (excess)

// Both are normalized by the number of genes in the larger genome

// TODO : check if the genes are guaranteed to be always sorted (avoids a test on gene)
func disjointExcess(g1, g2 []Gene) (disjoint, excess float64){
  disjoint = float64(len(g1) + len(g2))
  excess = 0.0
  innos := make(map[int]bool)

  maxinno1 := 0
  for _,g := range g1 {
    innos[g.innovation] = true
    if maxinno1 < g.innovation {
      maxinno1 = g.innovation
    }
  }

  maxinno2 := 0
  for _,g := range g2 {
    if g.innovation > maxinno1 {
      excess++
    } else if _,ok := innos[g.innovation]; ok {
      disjoint -= 2 // Two genes match
    }
    if maxinno2 < g.innovation {
       maxinno2 = g.innovation
    }
  }

  if maxinno2 < maxinno1 {
    for k,_ := range innos {
      if k > maxinno2 {
        disjoint-- // this is an excess, not a disjoint
        excess++
      }
    }
  }

  maxGene := math.Max(float64(len(g1)),float64(len(g2)))
  disjoint /= maxGene
  excess /= maxGene

  return
}

// Implementation to sort a slice of Genes
// Go has the bad habit of naming a slice type ._. No generecism sucks

type GeneSlice []Gene

func (a GeneSlice) Len() int {return len(a)}
func (a GeneSlice) Swap(i,j int) {a[i],a[j] = a[j],a[i]}
func (a GeneSlice) Less(i,j int) bool {return a[i].to < a[j].to}
<commit_msg>Add weight differential calculation<commit_after>package NEAT

import (
  "math"
  "math/rand"
)

type Gene struct{
  from int
  to int
  weight float64
  enabled bool
  innovation int
}

func NewGene() *Gene{
  return &Gene{0,0,0,true,0}
}

func CopyGene(g *Gene) *Gene{
  return &Gene{g.from,g.to,g.weight,g.enabled,g.innovation}
}

func containsLink(genes []Gene, link Gene) bool{
  for _,x := range(genes){
    if x.from == link.from && x.to == link.to {
      return true
    }
  }
  return false
}

func randomNeuron(g *Genome, notInput bool, c ConstContainer) int {
  if notInput {
     return rand.Intn(len(g.network)-c.nbInputs) + c.nbInputs
  }
  return rand.Intn(len(g.network))
}

// Returns the number of genes who are in one slice, not the other, and have an innovation
// number inferior to the other's (disjoint)

// Also returnes the number of genes who are in one slice, not the other, and
// are not disjoint (excess)

// Both are normalized by the number of genes in the larger genome

// TODO : check if the genes are guaranteed to be always sorted (avoids a test on gene)
func disjointExcess(g1, g2 []Gene) (disjoint, excess float64){
  disjoint = float64(len(g1) + len(g2))
  excess = 0.0
  innos := make(map[int]bool)

  maxinno1 := 0
  for _,g := range g1 {
    innos[g.innovation] = true
    if maxinno1 < g.innovation {
      maxinno1 = g.innovation
    }
  }

  maxinno2 := 0
  for _,g := range g2 {
    if g.innovation > maxinno1 {
      excess++
    } else if _,ok := innos[g.innovation]; ok {
      disjoint -= 2 // Two genes match
    }
    if maxinno2 < g.innovation {
       maxinno2 = g.innovation
    }
  }

  if maxinno2 < maxinno1 {
    for k,_ := range innos {
      if k > maxinno2 {
        disjoint-- // this is an excess, not a disjoint
        excess++
      }
    }
  }

  maxGene := math.Max(float64(len(g1)),float64(len(g2)))
  disjoint /= maxGene
  excess /= maxGene

  return
}

func weights(g1,g2 []Gene) float64{
  inno := make(map[int]Gene)

  for _,g := range g1 {
    inno[g.innovation] = g
  }

  result := 0.0
  matches := 0

  for _, g := range g2 {
    if v,ok := inno[g.innovation]; ok {
      result += math.Abs(g.weight-v.weight)
      matches++
    }
  }

  return result / float64(matches)
}

// Implementation to sort a slice of Genes
// Go has the bad habit of naming a slice type ._. No generecism sucks

type GeneSlice []Gene

func (a GeneSlice) Len() int {return len(a)}
func (a GeneSlice) Swap(i,j int) {a[i],a[j] = a[j],a[i]}
func (a GeneSlice) Less(i,j int) bool {return a[i].to < a[j].to}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/exitcodes"
	"github.com/rfjakob/gocryptfs/internal/prefer_openssl"
	"github.com/rfjakob/gocryptfs/internal/stupidgcm"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// argContainer stores the parsed CLI options and arguments
type argContainer struct {
	debug, init, zerokey, fusedebug, openssl, passwd, fg, version,
	plaintextnames, quiet, nosyslog, wpanic,
	longnames, allow_other, reverse, aessiv, nonempty, raw64,
	noprealloc, speed, hkdf, serialize_reads, forcedecode, hh, info,
	sharedstorage, devrandom, fsck bool
	// Mount options with opposites
	dev, nodev, suid, nosuid, exec, noexec, rw, ro bool
	masterkey, mountpoint, cipherdir, cpuprofile, extpass,
	memprofile, ko, passfile, ctlsock, fsname, force_owner, trace string
	// Configuration file name override
	config             string
	notifypid, scryptn int
	// Helper variables that are NOT cli options all start with an underscore
	// _configCustom is true when the user sets a custom config file name.
	_configCustom bool
	// _ctlsockFd stores the control socket file descriptor (ctlsock stores the path)
	_ctlsockFd net.Listener
	// _forceOwner is, if non-nil, a parsed, validated Owner (as opposed to the string above)
	_forceOwner *fuse.Owner
}

var flagSet *flag.FlagSet

// prefixOArgs transform options passed via "-o foo,bar" into regular options
// like "-foo -bar" and prefixes them to the command line.
// Testcases in TestPrefixOArgs().
func prefixOArgs(osArgs []string) ([]string, error) {
	// Need at least 3, example: gocryptfs -o    foo,bar
	//                               ^ 0    ^ 1    ^ 2
	if len(osArgs) < 3 {
		return osArgs, nil
	}
	// Passing "--" disables "-o" parsing. Ignore element 0 (program name).
	for _, v := range osArgs[1:] {
		if v == "--" {
			return osArgs, nil
		}
	}
	// Find and extract "-o foo,bar"
	var otherArgs, oOpts []string
	for i := 1; i < len(osArgs); i++ {
		if osArgs[i] == "-o" {
			// Last argument?
			if i+1 >= len(osArgs) {
				return nil, fmt.Errorf("The \"-o\" option requires an argument")
			}
			oOpts = strings.Split(osArgs[i+1], ",")
			// Skip over the arguments to "-o"
			i++
		} else if strings.HasPrefix(osArgs[i], "-o=") {
			oOpts = strings.Split(osArgs[i][3:], ",")
		} else {
			otherArgs = append(otherArgs, osArgs[i])
		}
	}
	// Start with program name
	newArgs := []string{osArgs[0]}
	// Add options from "-o"
	for _, o := range oOpts {
		if o == "" {
			continue
		}
		if o == "o" || o == "-o" {
			tlog.Fatal.Printf("You can't pass \"-o\" to \"-o\"")
			os.Exit(exitcodes.Usage)
		}
		newArgs = append(newArgs, "-"+o)
	}
	// Add other arguments
	newArgs = append(newArgs, otherArgs...)
	return newArgs, nil
}

// parseCliOpts - parse command line options (i.e. arguments that start with "-")
func parseCliOpts() (args argContainer) {
	var err error
	var opensslAuto string

	os.Args, err = prefixOArgs(os.Args)
	if err != nil {
		tlog.Fatal.Println(err)
		os.Exit(exitcodes.Usage)
	}

	flagSet = flag.NewFlagSet(tlog.ProgramName, flag.ContinueOnError)
	flagSet.Usage = helpShort
	flagSet.BoolVar(&args.debug, "d", false, "")
	flagSet.BoolVar(&args.debug, "debug", false, "Enable debug output")
	flagSet.BoolVar(&args.fusedebug, "fusedebug", false, "Enable fuse library debug output")
	flagSet.BoolVar(&args.init, "init", false, "Initialize encrypted directory")
	flagSet.BoolVar(&args.zerokey, "zerokey", false, "Use all-zero dummy master key")
	// Tri-state true/false/auto
	flagSet.StringVar(&opensslAuto, "openssl", "auto", "Use OpenSSL instead of built-in Go crypto")
	flagSet.BoolVar(&args.passwd, "passwd", false, "Change password")
	flagSet.BoolVar(&args.fg, "f", false, "")
	flagSet.BoolVar(&args.fg, "fg", false, "Stay in the foreground")
	flagSet.BoolVar(&args.version, "version", false, "Print version and exit")
	flagSet.BoolVar(&args.plaintextnames, "plaintextnames", false, "Do not encrypt file names")
	flagSet.BoolVar(&args.quiet, "q", false, "")
	flagSet.BoolVar(&args.quiet, "quiet", false, "Quiet - silence informational messages")
	flagSet.BoolVar(&args.nosyslog, "nosyslog", false, "Do not redirect output to syslog when running in the background")
	flagSet.BoolVar(&args.wpanic, "wpanic", false, "When encountering a warning, panic and exit immediately")
	flagSet.BoolVar(&args.longnames, "longnames", true, "Store names longer than 176 bytes in extra files")
	flagSet.BoolVar(&args.allow_other, "allow_other", false, "Allow other users to access the filesystem. "+
		"Only works if user_allow_other is set in /etc/fuse.conf.")
	flagSet.BoolVar(&args.reverse, "reverse", false, "Reverse mode")
	flagSet.BoolVar(&args.aessiv, "aessiv", false, "AES-SIV encryption")
	flagSet.BoolVar(&args.nonempty, "nonempty", false, "Allow mounting over non-empty directories")
	flagSet.BoolVar(&args.raw64, "raw64", true, "Use unpadded base64 for file names")
	flagSet.BoolVar(&args.noprealloc, "noprealloc", false, "Disable preallocation before writing")
	flagSet.BoolVar(&args.speed, "speed", false, "Run crypto speed test")
	flagSet.BoolVar(&args.hkdf, "hkdf", true, "Use HKDF as an additional key derivation step")
	flagSet.BoolVar(&args.serialize_reads, "serialize_reads", false, "Try to serialize read operations")
	flagSet.BoolVar(&args.forcedecode, "forcedecode", false, "Force decode of files even if integrity check fails."+
		" Requires gocryptfs to be compiled with openssl support and implies -openssl true")
	flagSet.BoolVar(&args.hh, "hh", false, "Show this long help text")
	flagSet.BoolVar(&args.info, "info", false, "Display information about CIPHERDIR")
	flagSet.BoolVar(&args.sharedstorage, "sharedstorage", false, "Make concurrent access to a shared CIPHERDIR safer")
	flagSet.BoolVar(&args.devrandom, "devrandom", false, "Use /dev/random for generating master key")
	flagSet.BoolVar(&args.fsck, "fsck", false, "Run a filesystem check on CIPHERDIR")

	// Mount options with opposites
	flagSet.BoolVar(&args.dev, "dev", false, "Allow device files")
	flagSet.BoolVar(&args.nodev, "nodev", false, "Deny device files")
	flagSet.BoolVar(&args.suid, "suid", false, "Allow suid binaries")
	flagSet.BoolVar(&args.nosuid, "nosuid", false, "Deny suid binaries")
	flagSet.BoolVar(&args.exec, "exec", false, "Allow executables")
	flagSet.BoolVar(&args.noexec, "noexec", false, "Deny executables")
	flagSet.BoolVar(&args.rw, "rw", false, "Mount the filesystem read-write")
	flagSet.BoolVar(&args.ro, "ro", false, "Mount the filesystem read-only")

	flagSet.StringVar(&args.masterkey, "masterkey", "", "Mount with explicit master key")
	flagSet.StringVar(&args.cpuprofile, "cpuprofile", "", "Write cpu profile to specified file")
	flagSet.StringVar(&args.memprofile, "memprofile", "", "Write memory profile to specified file")
	flagSet.StringVar(&args.config, "config", "", "Use specified config file instead of CIPHERDIR/gocryptfs.conf")
	flagSet.StringVar(&args.extpass, "extpass", "", "Use external program for the password prompt")
	flagSet.StringVar(&args.passfile, "passfile", "", "Read password from file")
	flagSet.StringVar(&args.ko, "ko", "", "Pass additional options directly to the kernel, comma-separated list")
	flagSet.StringVar(&args.ctlsock, "ctlsock", "", "Create control socket at specified path")
	flagSet.StringVar(&args.fsname, "fsname", "", "Override the filesystem name")
	flagSet.StringVar(&args.force_owner, "force_owner", "", "uid:gid pair to coerce ownership")
	flagSet.StringVar(&args.trace, "trace", "", "Write execution trace to file")
	flagSet.IntVar(&args.notifypid, "notifypid", 0, "Send USR1 to the specified process after "+
		"successful mount - used internally for daemonization")
	flagSet.IntVar(&args.scryptn, "scryptn", configfile.ScryptDefaultLogN, "scrypt cost parameter logN. Possible values: 10-28. "+
		"A lower value speeds up mounting and reduces its memory needs, but makes the password susceptible to brute-force attacks")
	var dummyString string
	flagSet.StringVar(&dummyString, "o", "", "For compatibility with mount(1), options can be also passed as a comma-separated list to -o on the end.")
	// Actual parsing
	err = flagSet.Parse(os.Args[1:])
	if err == flag.ErrHelp {
		os.Exit(0)
	}
	if err != nil {
		tlog.Warn.Printf("You passed: %s", prettyArgs())
		tlog.Fatal.Printf("%v", err)
		os.Exit(exitcodes.Usage)
	}
	// "-openssl" needs some post-processing
	if opensslAuto == "auto" {
		args.openssl = prefer_openssl.PreferOpenSSL()
	} else {
		args.openssl, err = strconv.ParseBool(opensslAuto)
		if err != nil {
			tlog.Fatal.Printf("Invalid \"-openssl\" setting: %v", err)
			os.Exit(exitcodes.Usage)
		}
	}
	// "-forcedecode" only works with openssl. Check compilation and command line parameters
	if args.forcedecode == true {
		if stupidgcm.BuiltWithoutOpenssl == true {
			tlog.Fatal.Printf("The -forcedecode flag requires openssl support, but gocryptfs was compiled without it!")
			os.Exit(exitcodes.Usage)
		}
		if args.aessiv == true {
			tlog.Fatal.Printf("The -forcedecode and -aessiv flags are incompatible because they use different crypto libs (openssl vs native Go)")
			os.Exit(exitcodes.Usage)
		}
		if args.reverse == true {
			tlog.Fatal.Printf("The reverse mode and the -forcedecode option are not compatible")
			os.Exit(exitcodes.Usage)
		}
		// Has the user explicitly disabled openssl using "-openssl=false/0"?
		if !args.openssl && opensslAuto != "auto" {
			tlog.Fatal.Printf("-forcedecode requires openssl, but is disabled via command-line option")
			os.Exit(exitcodes.Usage)
		}
		args.openssl = true

		// Try to make it harder for the user to shoot himself in the foot.
		args.ro = true
		args.allow_other = false
		args.ko = "noexec"
	}
	// '-passfile FILE' is a shortcut for -extpass='/bin/cat -- FILE'
	if args.passfile != "" {
		args.extpass = "/bin/cat -- " + args.passfile
	}
	if args.extpass != "" && args.masterkey != "" {
		tlog.Fatal.Printf("The options -extpass and -masterkey cannot be used at the same time")
		os.Exit(exitcodes.Usage)
	}
	return args
}

// prettyArgs pretty-prints the command-line arguments.
func prettyArgs() string {
	pa := fmt.Sprintf("%q", os.Args[1:])
	// Get rid of "[" and "]"
	pa = pa[1 : len(pa)-1]
	return pa
}

// countOpFlags counts the number of operation flags we were passed.
func countOpFlags(args *argContainer) int {
	var count int
	if args.info {
		count++
	}
	if args.passwd {
		count++
	}
	if args.init {
		count++
	}
	if args.fsck {
		count++
	}
	return count
}
<commit_msg>main: cut down "flag provided but not defined" error message<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/exitcodes"
	"github.com/rfjakob/gocryptfs/internal/prefer_openssl"
	"github.com/rfjakob/gocryptfs/internal/stupidgcm"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// argContainer stores the parsed CLI options and arguments
type argContainer struct {
	debug, init, zerokey, fusedebug, openssl, passwd, fg, version,
	plaintextnames, quiet, nosyslog, wpanic,
	longnames, allow_other, reverse, aessiv, nonempty, raw64,
	noprealloc, speed, hkdf, serialize_reads, forcedecode, hh, info,
	sharedstorage, devrandom, fsck bool
	// Mount options with opposites
	dev, nodev, suid, nosuid, exec, noexec, rw, ro bool
	masterkey, mountpoint, cipherdir, cpuprofile, extpass,
	memprofile, ko, passfile, ctlsock, fsname, force_owner, trace string
	// Configuration file name override
	config             string
	notifypid, scryptn int
	// Helper variables that are NOT cli options all start with an underscore
	// _configCustom is true when the user sets a custom config file name.
	_configCustom bool
	// _ctlsockFd stores the control socket file descriptor (ctlsock stores the path)
	_ctlsockFd net.Listener
	// _forceOwner is, if non-nil, a parsed, validated Owner (as opposed to the string above)
	_forceOwner *fuse.Owner
}

var flagSet *flag.FlagSet

// prefixOArgs transform options passed via "-o foo,bar" into regular options
// like "-foo -bar" and prefixes them to the command line.
// Testcases in TestPrefixOArgs().
func prefixOArgs(osArgs []string) ([]string, error) {
	// Need at least 3, example: gocryptfs -o    foo,bar
	//                               ^ 0    ^ 1    ^ 2
	if len(osArgs) < 3 {
		return osArgs, nil
	}
	// Passing "--" disables "-o" parsing. Ignore element 0 (program name).
	for _, v := range osArgs[1:] {
		if v == "--" {
			return osArgs, nil
		}
	}
	// Find and extract "-o foo,bar"
	var otherArgs, oOpts []string
	for i := 1; i < len(osArgs); i++ {
		if osArgs[i] == "-o" {
			// Last argument?
			if i+1 >= len(osArgs) {
				return nil, fmt.Errorf("The \"-o\" option requires an argument")
			}
			oOpts = strings.Split(osArgs[i+1], ",")
			// Skip over the arguments to "-o"
			i++
		} else if strings.HasPrefix(osArgs[i], "-o=") {
			oOpts = strings.Split(osArgs[i][3:], ",")
		} else {
			otherArgs = append(otherArgs, osArgs[i])
		}
	}
	// Start with program name
	newArgs := []string{osArgs[0]}
	// Add options from "-o"
	for _, o := range oOpts {
		if o == "" {
			continue
		}
		if o == "o" || o == "-o" {
			tlog.Fatal.Printf("You can't pass \"-o\" to \"-o\"")
			os.Exit(exitcodes.Usage)
		}
		newArgs = append(newArgs, "-"+o)
	}
	// Add other arguments
	newArgs = append(newArgs, otherArgs...)
	return newArgs, nil
}

// parseCliOpts - parse command line options (i.e. arguments that start with "-")
func parseCliOpts() (args argContainer) {
	var err error
	var opensslAuto string

	os.Args, err = prefixOArgs(os.Args)
	if err != nil {
		tlog.Fatal.Println(err)
		os.Exit(exitcodes.Usage)
	}

	flagSet = flag.NewFlagSet(tlog.ProgramName, flag.ContinueOnError)
	flagSet.Usage = func() {}
	flagSet.BoolVar(&args.debug, "d", false, "")
	flagSet.BoolVar(&args.debug, "debug", false, "Enable debug output")
	flagSet.BoolVar(&args.fusedebug, "fusedebug", false, "Enable fuse library debug output")
	flagSet.BoolVar(&args.init, "init", false, "Initialize encrypted directory")
	flagSet.BoolVar(&args.zerokey, "zerokey", false, "Use all-zero dummy master key")
	// Tri-state true/false/auto
	flagSet.StringVar(&opensslAuto, "openssl", "auto", "Use OpenSSL instead of built-in Go crypto")
	flagSet.BoolVar(&args.passwd, "passwd", false, "Change password")
	flagSet.BoolVar(&args.fg, "f", false, "")
	flagSet.BoolVar(&args.fg, "fg", false, "Stay in the foreground")
	flagSet.BoolVar(&args.version, "version", false, "Print version and exit")
	flagSet.BoolVar(&args.plaintextnames, "plaintextnames", false, "Do not encrypt file names")
	flagSet.BoolVar(&args.quiet, "q", false, "")
	flagSet.BoolVar(&args.quiet, "quiet", false, "Quiet - silence informational messages")
	flagSet.BoolVar(&args.nosyslog, "nosyslog", false, "Do not redirect output to syslog when running in the background")
	flagSet.BoolVar(&args.wpanic, "wpanic", false, "When encountering a warning, panic and exit immediately")
	flagSet.BoolVar(&args.longnames, "longnames", true, "Store names longer than 176 bytes in extra files")
	flagSet.BoolVar(&args.allow_other, "allow_other", false, "Allow other users to access the filesystem. "+
		"Only works if user_allow_other is set in /etc/fuse.conf.")
	flagSet.BoolVar(&args.reverse, "reverse", false, "Reverse mode")
	flagSet.BoolVar(&args.aessiv, "aessiv", false, "AES-SIV encryption")
	flagSet.BoolVar(&args.nonempty, "nonempty", false, "Allow mounting over non-empty directories")
	flagSet.BoolVar(&args.raw64, "raw64", true, "Use unpadded base64 for file names")
	flagSet.BoolVar(&args.noprealloc, "noprealloc", false, "Disable preallocation before writing")
	flagSet.BoolVar(&args.speed, "speed", false, "Run crypto speed test")
	flagSet.BoolVar(&args.hkdf, "hkdf", true, "Use HKDF as an additional key derivation step")
	flagSet.BoolVar(&args.serialize_reads, "serialize_reads", false, "Try to serialize read operations")
	flagSet.BoolVar(&args.forcedecode, "forcedecode", false, "Force decode of files even if integrity check fails."+
		" Requires gocryptfs to be compiled with openssl support and implies -openssl true")
	flagSet.BoolVar(&args.hh, "hh", false, "Show this long help text")
	flagSet.BoolVar(&args.info, "info", false, "Display information about CIPHERDIR")
	flagSet.BoolVar(&args.sharedstorage, "sharedstorage", false, "Make concurrent access to a shared CIPHERDIR safer")
	flagSet.BoolVar(&args.devrandom, "devrandom", false, "Use /dev/random for generating master key")
	flagSet.BoolVar(&args.fsck, "fsck", false, "Run a filesystem check on CIPHERDIR")

	// Mount options with opposites
	flagSet.BoolVar(&args.dev, "dev", false, "Allow device files")
	flagSet.BoolVar(&args.nodev, "nodev", false, "Deny device files")
	flagSet.BoolVar(&args.suid, "suid", false, "Allow suid binaries")
	flagSet.BoolVar(&args.nosuid, "nosuid", false, "Deny suid binaries")
	flagSet.BoolVar(&args.exec, "exec", false, "Allow executables")
	flagSet.BoolVar(&args.noexec, "noexec", false, "Deny executables")
	flagSet.BoolVar(&args.rw, "rw", false, "Mount the filesystem read-write")
	flagSet.BoolVar(&args.ro, "ro", false, "Mount the filesystem read-only")

	flagSet.StringVar(&args.masterkey, "masterkey", "", "Mount with explicit master key")
	flagSet.StringVar(&args.cpuprofile, "cpuprofile", "", "Write cpu profile to specified file")
	flagSet.StringVar(&args.memprofile, "memprofile", "", "Write memory profile to specified file")
	flagSet.StringVar(&args.config, "config", "", "Use specified config file instead of CIPHERDIR/gocryptfs.conf")
	flagSet.StringVar(&args.extpass, "extpass", "", "Use external program for the password prompt")
	flagSet.StringVar(&args.passfile, "passfile", "", "Read password from file")
	flagSet.StringVar(&args.ko, "ko", "", "Pass additional options directly to the kernel, comma-separated list")
	flagSet.StringVar(&args.ctlsock, "ctlsock", "", "Create control socket at specified path")
	flagSet.StringVar(&args.fsname, "fsname", "", "Override the filesystem name")
	flagSet.StringVar(&args.force_owner, "force_owner", "", "uid:gid pair to coerce ownership")
	flagSet.StringVar(&args.trace, "trace", "", "Write execution trace to file")
	flagSet.IntVar(&args.notifypid, "notifypid", 0, "Send USR1 to the specified process after "+
		"successful mount - used internally for daemonization")
	flagSet.IntVar(&args.scryptn, "scryptn", configfile.ScryptDefaultLogN, "scrypt cost parameter logN. Possible values: 10-28. "+
		"A lower value speeds up mounting and reduces its memory needs, but makes the password susceptible to brute-force attacks")
	var dummyString string
	flagSet.StringVar(&dummyString, "o", "", "For compatibility with mount(1), options can be also passed as a comma-separated list to -o on the end.")
	// Actual parsing
	err = flagSet.Parse(os.Args[1:])
	if err == flag.ErrHelp {
		helpShort()
		os.Exit(0)
	}
	if err != nil {
		tlog.Fatal.Printf("Invalid command line: %s. Try '%s -help'.", prettyArgs(), tlog.ProgramName)
		os.Exit(exitcodes.Usage)
	}
	// "-openssl" needs some post-processing
	if opensslAuto == "auto" {
		args.openssl = prefer_openssl.PreferOpenSSL()
	} else {
		args.openssl, err = strconv.ParseBool(opensslAuto)
		if err != nil {
			tlog.Fatal.Printf("Invalid \"-openssl\" setting: %v", err)
			os.Exit(exitcodes.Usage)
		}
	}
	// "-forcedecode" only works with openssl. Check compilation and command line parameters
	if args.forcedecode == true {
		if stupidgcm.BuiltWithoutOpenssl == true {
			tlog.Fatal.Printf("The -forcedecode flag requires openssl support, but gocryptfs was compiled without it!")
			os.Exit(exitcodes.Usage)
		}
		if args.aessiv == true {
			tlog.Fatal.Printf("The -forcedecode and -aessiv flags are incompatible because they use different crypto libs (openssl vs native Go)")
			os.Exit(exitcodes.Usage)
		}
		if args.reverse == true {
			tlog.Fatal.Printf("The reverse mode and the -forcedecode option are not compatible")
			os.Exit(exitcodes.Usage)
		}
		// Has the user explicitly disabled openssl using "-openssl=false/0"?
		if !args.openssl && opensslAuto != "auto" {
			tlog.Fatal.Printf("-forcedecode requires openssl, but is disabled via command-line option")
			os.Exit(exitcodes.Usage)
		}
		args.openssl = true

		// Try to make it harder for the user to shoot himself in the foot.
		args.ro = true
		args.allow_other = false
		args.ko = "noexec"
	}
	// '-passfile FILE' is a shortcut for -extpass='/bin/cat -- FILE'
	if args.passfile != "" {
		args.extpass = "/bin/cat -- " + args.passfile
	}
	if args.extpass != "" && args.masterkey != "" {
		tlog.Fatal.Printf("The options -extpass and -masterkey cannot be used at the same time")
		os.Exit(exitcodes.Usage)
	}
	return args
}

// prettyArgs pretty-prints the command-line arguments.
func prettyArgs() string {
	pa := fmt.Sprintf("%v", os.Args)
	// Get rid of "[" and "]"
	pa = pa[1 : len(pa)-1]
	return pa
}

// countOpFlags counts the number of operation flags we were passed.
func countOpFlags(args *argContainer) int {
	var count int
	if args.info {
		count++
	}
	if args.passwd {
		count++
	}
	if args.init {
		count++
	}
	if args.fsck {
		count++
	}
	return count
}
<|endoftext|>


<commit_before>package goad

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/goadapp/goad/infrastructure"
	"github.com/goadapp/goad/queue"
	"github.com/goadapp/goad/version"
)

// TestConfig type
type TestConfig struct {
	URL            string
	Concurrency    uint
	TotalRequests  uint
	RequestTimeout time.Duration
	Regions        []string
	Method         string
	Body           string
	Headers        []string
	AwsProfile     string
}

type invokeArgs struct {
	File string   `json:"file"`
	Args []string `json:"args"`
}

const nano = 1000000000

var supportedRegions = []string{
	"us-east-1",
	"us-west-2",
	"eu-west-1",
	"ap-northeast-1",
	"eu-central-1",
}

// Test type
type Test struct {
	config *TestConfig
}

// NewTest returns a configured Test
func NewTest(config *TestConfig) (*Test, error) {
	err := config.check()
	if err != nil {
		return nil, err
	}
	return &Test{config}, nil
}

// Start a test
func (t *Test) Start() <-chan queue.RegionsAggData {
	awsConfig := aws.NewConfig().WithRegion(t.config.Regions[0])

	if t.config.AwsProfile != "" {
		creds := credentials.NewSharedCredentials("", t.config.AwsProfile)
		if _, err := creds.Get(); err != nil {
			log.Fatal(err)
		}
		awsConfig.WithCredentials(creds)
	}

	infra, err := infrastructure.New(t.config.Regions, awsConfig)
	if err != nil {
		log.Fatal(err)
	}

	t.invokeLambdas(awsConfig, infra.QueueURL())

	results := make(chan queue.RegionsAggData)

	go func() {
		for result := range queue.Aggregate(awsConfig, infra.QueueURL(), t.config.TotalRequests) {
			results <- result
		}
		infra.Clean()
		close(results)
	}()

	return results
}

func (t *Test) invokeLambdas(awsConfig *aws.Config, sqsURL string) {
	lambdas := numberOfLambdas(t.config.Concurrency, len(t.config.Regions))

	for i := 0; i < lambdas; i++ {
		region := t.config.Regions[i%len(t.config.Regions)]
		requests, requestsRemainder := divide(t.config.TotalRequests, lambdas)
		concurrency, _ := divide(t.config.Concurrency, lambdas)

		if requestsRemainder > 0 && i == lambdas-1 {
			requests += requestsRemainder
		}

		c := t.config
		args := []string{
			"-u",
			fmt.Sprintf("%s", c.URL),
			"-c",
			fmt.Sprintf("%s", strconv.Itoa(int(concurrency))),
			"-n",
			fmt.Sprintf("%s", strconv.Itoa(int(requests))),
			"-s",
			fmt.Sprintf("%s", sqsURL),
			"-q",
			fmt.Sprintf("%s", c.Regions[0]),
			"-t",
			fmt.Sprintf("%s", c.RequestTimeout.String()),
			"-f",
			fmt.Sprintf("%s", reportingFrequency(lambdas).String()),
			"-r",
			fmt.Sprintf("%s", region),
			"-m",
			fmt.Sprintf("%s", c.Method),
			"-b",
			fmt.Sprintf("%s", c.Body),
		}

		for _, v := range t.config.Headers {
			args = append(args, "-H", fmt.Sprintf("%s", v))
		}

		invokeargs := invokeArgs{
			File: "./goad-lambda",
			Args: args,
		}

		config := awsConfig.WithRegion(region)
		go t.invokeLambda(config, invokeargs)
	}
}

func (t *Test) invokeLambda(awsConfig *aws.Config, args invokeArgs) {
	svc := lambda.New(session.New(), awsConfig)

	j, _ := json.Marshal(args)

	svc.InvokeAsync(&lambda.InvokeAsyncInput{
		FunctionName: aws.String("goad:" + version.LambdaVersion()),
		InvokeArgs:   bytes.NewReader(j),
	})
}

func numberOfLambdas(concurrency uint, numRegions int) int {
	if numRegions > int(concurrency) {
		return int(concurrency)
	}
	if concurrency/10 > 100 {
		return 100
	}
	if int(concurrency) < 10*numRegions {
		return numRegions
	}
	return int(concurrency-1)/10 + 1
}

func divide(dividend uint, divisor int) (quotient, remainder uint) {
	return dividend / uint(divisor), dividend % uint(divisor)
}

func reportingFrequency(numberOfLambdas int) time.Duration {
	return time.Duration((math.Log2(float64(numberOfLambdas)) + 1)) * time.Second
}

func (c TestConfig) check() error {
	concurrencyLimit := 25000 * uint(len(c.Regions))
	if c.Concurrency < 1 || c.Concurrency > concurrencyLimit {
		return fmt.Errorf("Invalid concurrency (use 1 - %d)", concurrencyLimit)
	}
	if c.TotalRequests < 1 || c.TotalRequests > 1000000 {
		return errors.New("Invalid total requests (use 1 - 1000000)")
	}
	if c.RequestTimeout.Nanoseconds() < nano || c.RequestTimeout.Nanoseconds() > nano*100 {
		return errors.New("Invalid timeout (1s - 100s)")
	}
	for _, region := range c.Regions {
		supportedRegionFound := false
		for _, supported := range supportedRegions {
			if region == supported {
				supportedRegionFound = true
			}
		}
		if !supportedRegionFound {
			return fmt.Errorf("Unsupported region: %s. Supported regions are: %s.", region, strings.Join(supportedRegions, ", "))
		}
	}
	for _, v := range c.Headers {
		header := strings.Split(v, ":")
		if len(header) < 2 {
			return fmt.Errorf("Header %s not valid. Make sure your header is of the form \"Header: value\"", v)
		}
	}
	return nil
}
<commit_msg>Increase limits and number of instances  (#24)<commit_after>package goad

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"math"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/goadapp/goad/infrastructure"
	"github.com/goadapp/goad/queue"
	"github.com/goadapp/goad/version"
)

// TestConfig type
type TestConfig struct {
	URL            string
	Concurrency    uint
	TotalRequests  uint
	RequestTimeout time.Duration
	Regions        []string
	Method         string
	Body           string
	Headers        []string
	AwsProfile     string
}

type invokeArgs struct {
	File string   `json:"file"`
	Args []string `json:"args"`
}

const nano = 1000000000

var supportedRegions = []string{
	"us-east-1",
	"us-west-2",
	"eu-west-1",
	"ap-northeast-1",
	"eu-central-1",
}

// Test type
type Test struct {
	config *TestConfig
}

// NewTest returns a configured Test
func NewTest(config *TestConfig) (*Test, error) {
	err := config.check()
	if err != nil {
		return nil, err
	}
	return &Test{config}, nil
}

// Start a test
func (t *Test) Start() <-chan queue.RegionsAggData {
	awsConfig := aws.NewConfig().WithRegion(t.config.Regions[0])

	if t.config.AwsProfile != "" {
		creds := credentials.NewSharedCredentials("", t.config.AwsProfile)
		if _, err := creds.Get(); err != nil {
			log.Fatal(err)
		}
		awsConfig.WithCredentials(creds)
	}

	infra, err := infrastructure.New(t.config.Regions, awsConfig)
	if err != nil {
		log.Fatal(err)
	}

	t.invokeLambdas(awsConfig, infra.QueueURL())

	results := make(chan queue.RegionsAggData)

	go func() {
		for result := range queue.Aggregate(awsConfig, infra.QueueURL(), t.config.TotalRequests) {
			results <- result
		}
		infra.Clean()
		close(results)
	}()

	return results
}

func (t *Test) invokeLambdas(awsConfig *aws.Config, sqsURL string) {
	lambdas := numberOfLambdas(t.config.Concurrency, len(t.config.Regions))

	for i := 0; i < lambdas; i++ {
		region := t.config.Regions[i%len(t.config.Regions)]
		requests, requestsRemainder := divide(t.config.TotalRequests, lambdas)
		concurrency, _ := divide(t.config.Concurrency, lambdas)

		if requestsRemainder > 0 && i == lambdas-1 {
			requests += requestsRemainder
		}

		c := t.config
		args := []string{
			"-u",
			fmt.Sprintf("%s", c.URL),
			"-c",
			fmt.Sprintf("%s", strconv.Itoa(int(concurrency))),
			"-n",
			fmt.Sprintf("%s", strconv.Itoa(int(requests))),
			"-s",
			fmt.Sprintf("%s", sqsURL),
			"-q",
			fmt.Sprintf("%s", c.Regions[0]),
			"-t",
			fmt.Sprintf("%s", c.RequestTimeout.String()),
			"-f",
			fmt.Sprintf("%s", reportingFrequency(lambdas).String()),
			"-r",
			fmt.Sprintf("%s", region),
			"-m",
			fmt.Sprintf("%s", c.Method),
			"-b",
			fmt.Sprintf("%s", c.Body),
		}

		for _, v := range t.config.Headers {
			args = append(args, "-H", fmt.Sprintf("%s", v))
		}

		invokeargs := invokeArgs{
			File: "./goad-lambda",
			Args: args,
		}

		config := awsConfig.WithRegion(region)
		go t.invokeLambda(config, invokeargs)
	}
}

func (t *Test) invokeLambda(awsConfig *aws.Config, args invokeArgs) {
	svc := lambda.New(session.New(), awsConfig)

	j, _ := json.Marshal(args)

	svc.InvokeAsync(&lambda.InvokeAsyncInput{
		FunctionName: aws.String("goad:" + version.LambdaVersion()),
		InvokeArgs:   bytes.NewReader(j),
	})
}

func numberOfLambdas(concurrency uint, numRegions int) int {
	if numRegions > int(concurrency) {
		return int(concurrency)
	}
	if concurrency/200 > 350 { // > 70.000
		return 500
	} else if concurrency/100 > 100 { // 10.000 <> 70.000
		return 300
	} else if concurrency/10 > 100 { // 1.000 <> 10.000
		return 100
	}
	if int(concurrency) < 10*numRegions {
		return numRegions
	}
	return int(concurrency-1)/10 + 1
}

func divide(dividend uint, divisor int) (quotient, remainder uint) {
	return dividend / uint(divisor), dividend % uint(divisor)
}

func reportingFrequency(numberOfLambdas int) time.Duration {
	return time.Duration((math.Log2(float64(numberOfLambdas)) + 1)) * time.Second
}

func (c TestConfig) check() error {
	concurrencyLimit := 25000 * uint(len(c.Regions))
	if c.Concurrency < 1 || c.Concurrency > concurrencyLimit {
		return fmt.Errorf("Invalid concurrency (use 1 - %d)", concurrencyLimit)
	}
	if c.TotalRequests < 1 || c.TotalRequests > 2000000 {
		return errors.New("Invalid total requests (use 1 - 2000000)")
	}
	if c.RequestTimeout.Nanoseconds() < nano || c.RequestTimeout.Nanoseconds() > nano*100 {
		return errors.New("Invalid timeout (1s - 100s)")
	}
	for _, region := range c.Regions {
		supportedRegionFound := false
		for _, supported := range supportedRegions {
			if region == supported {
				supportedRegionFound = true
			}
		}
		if !supportedRegionFound {
			return fmt.Errorf("Unsupported region: %s. Supported regions are: %s.", region, strings.Join(supportedRegions, ", "))
		}
	}
	for _, v := range c.Headers {
		header := strings.Split(v, ":")
		if len(header) < 2 {
			return fmt.Errorf("Header %s not valid. Make sure your header is of the form \"Header: value\"", v)
		}
	}
	return nil
}
<|endoftext|>