<commit_before>// Copyright © 2016 Asteris, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"os"
	"os/signal"

	"crypto/tls"
	"github.com/Sirupsen/logrus"
	"github.com/asteris-llc/vaultfs/fs"
	"github.com/hashicorp/vault/api"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"net/http"
)

// mountCmd represents the mount command
var mountCmd = &cobra.Command{
	Use:   "mount {mountpoint}",
	Short: "mount a vault FS at the specified mountpoint",
	PreRunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("expected exactly one argument")
		}

		return nil
	},
	Run: func(cmd *cobra.Command, args []string) {
		config := &api.Config{
			Address: viper.GetString("address"),
			HttpClient: &http.Client{
				Transport: &http.Transport{
					Proxy: http.ProxyFromEnvironment,
					TLSClientConfig: &tls.Config{
						InsecureSkipVerify: viper.GetBool("insecure"),
					},
				},
			},
		}

		logrus.WithField("address", viper.GetString("address")).Info("creating FUSE client for Vault")
		fs, err := fs.New(config, viper.GetString("token"), viper.GetString("root"))
		if err != nil {
			logrus.WithError(err).Fatal("error creatinging fs")
		}

		stop, errs := fs.Mount(args[0])

		// handle interrupt
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)

			<-c
			logrus.Info("stopping")
			stop()
		}()

		status := 0
		for err := range errs {
			status = 1
			logrus.WithError(err).Error("error mounting")
		}

		os.Exit(status)
	},
}

func init() {
	RootCmd.AddCommand(mountCmd)

	mountCmd.Flags().StringP("root", "r", "secret", "root path for reads")
	mountCmd.Flags().StringP("address", "a", "https://localhost:8200", "vault address")
	mountCmd.Flags().BoolP("insecure", "i", false, "skip SSL certificate verification")
	mountCmd.Flags().String("token", "", "vault token")
	viper.BindPFlags(mountCmd.Flags())
}
<commit_msg>mount: add short option for token<commit_after>// Copyright © 2016 Asteris, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"errors"
	"os"
	"os/signal"

	"crypto/tls"
	"github.com/Sirupsen/logrus"
	"github.com/asteris-llc/vaultfs/fs"
	"github.com/hashicorp/vault/api"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"net/http"
)

// mountCmd represents the mount command
var mountCmd = &cobra.Command{
	Use:   "mount {mountpoint}",
	Short: "mount a vault FS at the specified mountpoint",
	PreRunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("expected exactly one argument")
		}

		return nil
	},
	Run: func(cmd *cobra.Command, args []string) {
		config := &api.Config{
			Address: viper.GetString("address"),
			HttpClient: &http.Client{
				Transport: &http.Transport{
					Proxy: http.ProxyFromEnvironment,
					TLSClientConfig: &tls.Config{
						InsecureSkipVerify: viper.GetBool("insecure"),
					},
				},
			},
		}

		logrus.WithField("address", viper.GetString("address")).Info("creating FUSE client for Vault")
		fs, err := fs.New(config, viper.GetString("token"), viper.GetString("root"))
		if err != nil {
			logrus.WithError(err).Fatal("error creatinging fs")
		}

		stop, errs := fs.Mount(args[0])

		// handle interrupt
		go func() {
			c := make(chan os.Signal, 1)
			signal.Notify(c, os.Interrupt)

			<-c
			logrus.Info("stopping")
			stop()
		}()

		status := 0
		for err := range errs {
			status = 1
			logrus.WithError(err).Error("error mounting")
		}

		os.Exit(status)
	},
}

func init() {
	RootCmd.AddCommand(mountCmd)

	mountCmd.Flags().StringP("root", "r", "secret", "root path for reads")
	mountCmd.Flags().StringP("address", "a", "https://localhost:8200", "vault address")
	mountCmd.Flags().BoolP("insecure", "i", false, "skip SSL certificate verification")
	mountCmd.Flags().StringP("token", "t", "", "vault token")
	viper.BindPFlags(mountCmd.Flags())
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"regexp"

	"io/ioutil"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docopt/docopt-go"
	"github.com/projectcalico/calico-containers/calicoctl/commands/argutils"
	"github.com/projectcalico/calico-containers/calicoctl/commands/clientmgr"
	"github.com/projectcalico/libcalico-go/lib/api"
)

const (
	ETCD_KEY_NODE_FILE     = "/etc/calico/certs/key.pem"
	ETCD_CERT_NODE_FILE    = "/etc/calico/certs/cert.crt"
	ETCD_CA_CERT_NODE_FILE = "/etc/calico/certs/ca_cert.crt"
)

var (
	checkLogTimeout = 10 * time.Second
)

var VERSION string

// Run function collects diagnostic information and logs
func Run(args []string) {
	var err error
	doc := fmt.Sprintf(`Usage:
  calicoctl node run [--ip=<IP>] [--ip6=<IP6>] [--as=<AS_NUM>]
                     [--name=<NAME>]
                     [--log-dir=<LOG_DIR>]
                     [--node-image=<DOCKER_IMAGE_NAME>]
                     [--backend=(bird|gobgp|none)]
                     [--config=<CONFIG>]
                     [--no-default-ippools]
                     [--dryrun]
                     [--init-system]
                     [--disable-docker-networking]

Options:
  -h --help                Show this screen.
     --as=<AS_NUM>         The default AS number for this node.  If this is not
                           specified, the node will use the global AS number
                           (see 'calicoctl config' for details).
     --name=<NAME>         The name of the Calico node.  If this is not
                           supplied it defaults to the host name.
     --ip=<IP>             The local management address to use.  If this is not
                           specified, the node will attempt to auto-discover
                           the local IP address to use - however, it is
                           recommended to specify the required address to use.
     --ip6=<IP6>           The local IPv6 management address to use.  If this
                           is not specified, the node will not route IPv6.
     --log-dir=<LOG_DIR>   The directory containing Calico logs.
                           [default: /var/log/calico]
     --node-image=<DOCKER_IMAGE_NAME>
                           Docker image to use for Calico's per-node container.
                           [default: calico/node:%s]
     --backend=(bird|gobgp|none)
                           Specify which networking backend to use.  When set
                           to "none", Calico node runs in policy only mode.
                           The option to run with gobgp is currently
                           experimental.
                           [default: bird]
     --dryrun              Output the appropriate command, without starting the
                           container.
     --init-system         Run the appropriate command to use with an init
                           system.
     --no-default-ippools  Do not create default pools upon startup.
                           Default IP pools will be created if this is not set
                           and there are no pre-existing Calico IP pools.
     --disable-docker-networking
                           Disable Docker networking.
  -c --config=<CONFIG>     Path to the file containing connection
                           configuration in YAML or JSON format.
                           [default: /etc/calico/calicoctl.cfg]

Description:
  This command is used to start a calico/node container instance which provides
  Calico networking and network policy on your compute host.
`, VERSION)
	arguments, err := docopt.Parse(doc, args, true, "", false, false)
	if err != nil {
		log.Info(err)
		fmt.Printf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(args, " "))
		os.Exit(1)
	}
	if len(arguments) == 0 {
		return
	}

	// Extract all the parameters.
	ipv4 := argutils.ArgStringOrBlank(arguments, "--ip")
	ipv6 := argutils.ArgStringOrBlank(arguments, "--ip6")
	logDir := argutils.ArgStringOrBlank(arguments, "--log-dir")
	asNumber := argutils.ArgStringOrBlank(arguments, "--as")
	img := argutils.ArgStringOrBlank(arguments, "--node-image")
	backend := argutils.ArgStringOrBlank(arguments, "--backend")
	dryrun := argutils.ArgBoolOrFalse(arguments, "--dryrun")
	name := argutils.ArgStringOrBlank(arguments, "--name")
	nopools := argutils.ArgBoolOrFalse(arguments, "--no-default-ippools")
	config := argutils.ArgStringOrBlank(arguments, "--config")
	disableDockerNw := argutils.ArgBoolOrFalse(arguments, "--disable-docker-networking")
	initSystem := argutils.ArgBoolOrFalse(arguments, "--init-system")

	// Validate parameters.
	if ipv4 != "" {
		ip := argutils.ValidateIP(ipv4)
		if ip.Version() != 4 {
			fmt.Println("Error executing command: --ip is wrong IP version")
			os.Exit(1)
		}
	}
	if ipv6 != "" {
		ip := argutils.ValidateIP(ipv6)
		if ip.Version() != 6 {
			fmt.Println("Error executing command: --ip6 is wrong IP version")
			os.Exit(1)
		}
	}
	if asNumber != "" {
		// The calico/node image does not accept dotted notation for
		// the AS number, so convert.
		asNumber = argutils.ValidateASNumber(asNumber).String()
	}
	backendMatch := regexp.MustCompile("^(none|bird|gobgp)$")
	if !backendMatch.MatchString(backend) {
		fmt.Printf("Error executing command: unknown backend '%s'\n", backend)
		os.Exit(1)
	}

	// Use the hostname if a name is not specified.
	if name == "" {
		name, err = os.Hostname()
		if err != nil || name == "" {
			fmt.Println("Error executing command: unable to determine node name")
			os.Exit(1)
		}
	}

	// Load the etcd configuraiton.
	cfg, err := clientmgr.LoadClientConfig(config)
	if err != nil {
		fmt.Println("Error executing command: invalid config file")
		os.Exit(1)
	}
	if cfg.Spec.DatastoreType != api.EtcdV2 {
		fmt.Println("Error executing command: unsupported backend specified in config")
		os.Exit(1)
	}
	etcdcfg := cfg.Spec.EtcdConfig

	// Convert the nopools boolean to either an empty string or "true".
	noPoolsString := ""
	if nopools {
		noPoolsString = "true"
	}

	// Create a mapping of environment variables to values.
	envs := map[string]string{
		"HOSTNAME": name,
		"IP":       ipv4,
		"IP6":      ipv6,
		"CALICO_NETWORKING_BACKEND": backend,
		"AS":                        asNumber,
		"NO_DEFAULT_POOLS":          noPoolsString,
		"CALICO_LIBNETWORK_ENABLED": fmt.Sprint(!disableDockerNw),
	}

	// Create a map of read only bindings.
	vols := map[string]string{
		logDir:            "/var/log/calico",
		"/var/run/calico": "/var/run/calico",
		"/lib/modules":    "/lib/modules",
	}

	if !disableDockerNw {
		log.Info("Include docker networking volume mounts")
		vols["/run/docker/plugins"] = "/run/docker/plugins"
		vols["/var/run/docker.sock"] = "/var/run/docker.sock"
	}

	if etcdcfg.EtcdEndpoints == "" {
		envs["ETCD_AUTHORITY"] = etcdcfg.EtcdAuthority
		envs["ETCD_SCHEME"] = etcdcfg.EtcdScheme
		envs["ETCD_ENDPOINTS"] = ""
	} else {
		envs["ETCD_ENDPOINTS"] = etcdcfg.EtcdEndpoints
		envs["ETCD_AUTHORITY"] = ""
		envs["ETCD_SCHEME"] = ""
	}
	if etcdcfg.EtcdCACertFile != "" {
		envs["ETCD_CA_CERT_FILE"] = ETCD_CA_CERT_NODE_FILE
		vols[etcdcfg.EtcdCACertFile] = ETCD_CA_CERT_NODE_FILE
	}
	if etcdcfg.EtcdKeyFile != "" && etcdcfg.EtcdCertFile != "" {
		envs["ETCD_KEY_FILE"] = ETCD_KEY_NODE_FILE
		vols[etcdcfg.EtcdKeyFile] = ETCD_KEY_NODE_FILE
		envs["ETCD_CERT_FILE"] = ETCD_CERT_NODE_FILE
		vols[etcdcfg.EtcdCertFile] = ETCD_CERT_NODE_FILE
	}

	// Create the Docker command to execute (or display).  Start with the
	// fixed parts.  If this is a dry-run, then don't include the "detach"
	// flag of the "restart" flag since we write out the command for use
	// in an init system.
	cmd := []string{"docker", "run", "--net=host", "--privileged",
		"--name=calico-node"}
	if initSystem {
		cmd = append(cmd, "--rm")
	} else {
		cmd = append(cmd, "-d", "--restart=always")
	}

	// Add the environment variable pass-through.
	for k, v := range envs {
		cmd = append(cmd, "-e", fmt.Sprintf("%s=%s", k, v))
	}

	// Add the volume mounts.
	for k, v := range vols {
		cmd = append(cmd, "-v", fmt.Sprintf("%s:%s", k, v))
	}

	// Add the container image name
	cmd = append(cmd, img)

	if dryrun {
		fmt.Println("Use the following command to start the calico/node container:")
		fmt.Printf("\n%s\n\n", strings.Join(cmd, " "))

		if !initSystem {
			fmt.Println("If you are running calico/node in an init system, use the --init-system flag")
			fmt.Println("to display the appropriate start and stop commands.")
		} else {
			fmt.Println("Use the following command to stop the calico/node container:")
			fmt.Printf("\ndocker stop calico-node\n\n")
		}
		return
	}

	// This is not a dry run.  Check that we are running as root.
	enforceRoot()

	// Normally, Felix will load the modules it needs, but when running inside a
	// container it might not be able to do so. Ensure the required modules are
	// loaded each time the node starts.
	// We only make a best effort attempt because the command may fail if the
	// modules are built in.
	if !runningInContainer() {
		log.Info("Running in container")
		loadModules()
		setupIPForwarding()
		setNFConntrackMax()
	}

	// Run the docker command.
	fmt.Println("Running the following command:")
	fmt.Printf("\n%s\n\n", strings.Join(cmd, " "))

	err = exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		fmt.Printf("Error executing command: %v\n", err)
		os.Exit(1)
	}

	// Create the command to follow the docker logs for the calico/node
	logCmd := exec.Command("docker", "logs", "calico-node", "--follow")

	// Get the stdout pipe
	outPipe, err := logCmd.StdoutPipe()
	if err != nil {
		fmt.Printf("Error executing command:  unable to check calico/node logs: %v\n", err)
		os.Exit(1)
	}
	outScanner := bufio.NewScanner(outPipe)

	// Start following the logs.
	err = logCmd.Start()
	if err != nil {
		fmt.Printf("Error executing command:  unable to check calico/node logs: %v\n", err)
		fmt.Println(err)
		os.Exit(1)
	}

	// Protect against calico/node taking too long to start, or docker
	// logs hanging without output.
	time.AfterFunc(checkLogTimeout, func() {
		logCmd.Process.Kill()
	})

	// Read stdout until the node fails, or until we see the output
	// indicating success.
	started := false
	for outScanner.Scan() {
		line := outScanner.Text()
		fmt.Println(line)
		if line == "Calico node started successfully" {
			started = true
			break
		}
	}

	// If we didn't successfully start then notify the user.
	if !started {
		fmt.Println("Error executing command: calico/node has terminated, check logs for details")
	}

	// Kill the process if it is still running.
	logCmd.Process.Kill()
	logCmd.Wait()
}

// runningInContainer returns whether we are running calicoctl within a container.
func runningInContainer() bool {
	v := os.Getenv("CALICO_CTL_CONTAINER")
	return v != ""
}

func loadModules() {
	cmd := []string{"modprobe", "-a", "xt_set", "ip6_tables"}
	fmt.Printf("Running command to load modules: %s\n", strings.Join(cmd, " "))
	err := exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		log.Warning(err)
	}
}

func setupIPForwarding() {
	fmt.Println("Enabling IPv4 forwarding")
	err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward",
		[]byte("1"), 0)
	if err != nil {
		fmt.Println("ERROR: Could not enable ipv4 forwarding")
		os.Exit(1)
	}

	if _, err := os.Stat("/proc/sys/net/ipv6"); err == nil {
		fmt.Println("Enabling IPv6 forwarding")
		err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/all/forwarding",
			[]byte("1"), 0)
		if err != nil {
			fmt.Println("ERROR: Could not enable ipv6 forwarding")
			os.Exit(1)
		}
	}
}

func setNFConntrackMax() {
	// A common problem on Linux systems is running out of space in the conntrack
	// table, which can cause poor iptables performance. This can happen if you
	// run a lot of workloads on a given host, or if your workloads create a lot
	// of TCP connections or bidirectional UDP streams.
	//
	// To avoid this becoming a problem, we recommend increasing the conntrack
	// table size. To do so, run the following commands:
	fmt.Println("Increasing conntrack limit")
	err := ioutil.WriteFile("/proc/sys/net/netfilter/nf_conntrack_max",
		[]byte("1000000"), 0)
	if err != nil {
		fmt.Println("WARNING: Could not set nf_contrack_max. This may have an impact at scale.")
	}
}
<commit_msg>code markup<commit_after>// Copyright (c) 2016 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package node

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"regexp"

	"io/ioutil"

	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/docopt/docopt-go"
	"github.com/projectcalico/calico-containers/calicoctl/commands/argutils"
	"github.com/projectcalico/calico-containers/calicoctl/commands/clientmgr"
	"github.com/projectcalico/libcalico-go/lib/api"
)

const (
	ETCD_KEY_NODE_FILE     = "/etc/calico/certs/key.pem"
	ETCD_CERT_NODE_FILE    = "/etc/calico/certs/cert.crt"
	ETCD_CA_CERT_NODE_FILE = "/etc/calico/certs/ca_cert.crt"
)

var (
	checkLogTimeout = 10 * time.Second
)

var VERSION string

// Run function collects diagnostic information and logs
func Run(args []string) {
	var err error
	doc := fmt.Sprintf(`Usage:
  calicoctl node run [--ip=<IP>] [--ip6=<IP6>] [--as=<AS_NUM>]
                     [--name=<NAME>]
                     [--log-dir=<LOG_DIR>]
                     [--node-image=<DOCKER_IMAGE_NAME>]
                     [--backend=(bird|gobgp|none)]
                     [--config=<CONFIG>]
                     [--no-default-ippools]
                     [--dryrun]
                     [--init-system]
                     [--disable-docker-networking]

Options:
  -h --help                Show this screen.
     --as=<AS_NUM>         The default AS number for this node.  If this is not
                           specified, the node will use the global AS number
                           (see 'calicoctl config' for details).
     --name=<NAME>         The name of the Calico node.  If this is not
                           supplied it defaults to the host name.
     --ip=<IP>             The local management address to use.  If this is not
                           specified, the node will attempt to auto-discover
                           the local IP address to use - however, it is
                           recommended to specify the required address to use.
     --ip6=<IP6>           The local IPv6 management address to use.  If this
                           is not specified, the node will not route IPv6.
     --log-dir=<LOG_DIR>   The directory containing Calico logs.
                           [default: /var/log/calico]
     --node-image=<DOCKER_IMAGE_NAME>
                           Docker image to use for Calico's per-node container.
                           [default: calico/node:%s]
     --backend=(bird|gobgp|none)
                           Specify which networking backend to use.  When set
                           to "none", Calico node runs in policy only mode.
                           The option to run with gobgp is currently
                           experimental.
                           [default: bird]
     --dryrun              Output the appropriate command, without starting the
                           container.
     --init-system         Run the appropriate command to use with an init
                           system.
     --no-default-ippools  Do not create default pools upon startup.
                           Default IP pools will be created if this is not set
                           and there are no pre-existing Calico IP pools.
     --disable-docker-networking
                           Disable Docker networking.
  -c --config=<CONFIG>     Path to the file containing connection
                           configuration in YAML or JSON format.
                           [default: /etc/calico/calicoctl.cfg]

Description:
  This command is used to start a calico/node container instance which provides
  Calico networking and network policy on your compute host.
`, VERSION)
	arguments, err := docopt.Parse(doc, args, true, "", false, false)
	if err != nil {
		log.Info(err)
		fmt.Printf("Invalid option: 'calicoctl %s'. Use flag '--help' to read about a specific subcommand.\n", strings.Join(args, " "))
		os.Exit(1)
	}
	if len(arguments) == 0 {
		return
	}

	// Extract all the parameters.
	ipv4 := argutils.ArgStringOrBlank(arguments, "--ip")
	ipv6 := argutils.ArgStringOrBlank(arguments, "--ip6")
	logDir := argutils.ArgStringOrBlank(arguments, "--log-dir")
	asNumber := argutils.ArgStringOrBlank(arguments, "--as")
	img := argutils.ArgStringOrBlank(arguments, "--node-image")
	backend := argutils.ArgStringOrBlank(arguments, "--backend")
	dryrun := argutils.ArgBoolOrFalse(arguments, "--dryrun")
	name := argutils.ArgStringOrBlank(arguments, "--name")
	nopools := argutils.ArgBoolOrFalse(arguments, "--no-default-ippools")
	config := argutils.ArgStringOrBlank(arguments, "--config")
	disableDockerNw := argutils.ArgBoolOrFalse(arguments, "--disable-docker-networking")
	initSystem := argutils.ArgBoolOrFalse(arguments, "--init-system")

	// Validate parameters.
	if ipv4 != "" {
		ip := argutils.ValidateIP(ipv4)
		if ip.Version() != 4 {
			fmt.Println("Error executing command: --ip is wrong IP version")
			os.Exit(1)
		}
	}
	if ipv6 != "" {
		ip := argutils.ValidateIP(ipv6)
		if ip.Version() != 6 {
			fmt.Println("Error executing command: --ip6 is wrong IP version")
			os.Exit(1)
		}
	}
	if asNumber != "" {
		// The calico/node image does not accept dotted notation for
		// the AS number, so convert.
		asNumber = argutils.ValidateASNumber(asNumber).String()
	}
	backendMatch := regexp.MustCompile("^(none|bird|gobgp)$")
	if !backendMatch.MatchString(backend) {
		fmt.Printf("Error executing command: unknown backend '%s'\n", backend)
		os.Exit(1)
	}

	// Use the hostname if a name is not specified.
	if name == "" {
		name, err = os.Hostname()
		if err != nil || name == "" {
			fmt.Println("Error executing command: unable to determine node name")
			os.Exit(1)
		}
	}

	// Load the etcd configuraiton.
	cfg, err := clientmgr.LoadClientConfig(config)
	if err != nil {
		fmt.Println("Error executing command: invalid config file")
		os.Exit(1)
	}
	if cfg.Spec.DatastoreType != api.EtcdV2 {
		fmt.Println("Error executing command: unsupported backend specified in config")
		os.Exit(1)
	}
	etcdcfg := cfg.Spec.EtcdConfig

	// Convert the nopools boolean to either an empty string or "true".
	noPoolsString := ""
	if nopools {
		noPoolsString = "true"
	}

	// Create a mapping of environment variables to values.
	envs := map[string]string{
		"HOSTNAME": name,
		"IP":       ipv4,
		"IP6":      ipv6,
		"CALICO_NETWORKING_BACKEND": backend,
		"AS":                        asNumber,
		"NO_DEFAULT_POOLS":          noPoolsString,
		"CALICO_LIBNETWORK_ENABLED": fmt.Sprint(!disableDockerNw),
	}

	// Create a map of read only bindings.
	vols := map[string]string{
		logDir:            "/var/log/calico",
		"/var/run/calico": "/var/run/calico",
		"/lib/modules":    "/lib/modules",
	}

	if !disableDockerNw {
		log.Info("Include docker networking volume mounts")
		vols["/run/docker/plugins"] = "/run/docker/plugins"
		vols["/var/run/docker.sock"] = "/var/run/docker.sock"
	}

	if etcdcfg.EtcdEndpoints == "" {
		envs["ETCD_AUTHORITY"] = etcdcfg.EtcdAuthority
		envs["ETCD_SCHEME"] = etcdcfg.EtcdScheme
		envs["ETCD_ENDPOINTS"] = ""
	} else {
		envs["ETCD_ENDPOINTS"] = etcdcfg.EtcdEndpoints
		envs["ETCD_AUTHORITY"] = ""
		envs["ETCD_SCHEME"] = ""
	}
	if etcdcfg.EtcdCACertFile != "" {
		envs["ETCD_CA_CERT_FILE"] = ETCD_CA_CERT_NODE_FILE
		vols[etcdcfg.EtcdCACertFile] = ETCD_CA_CERT_NODE_FILE
	}
	if etcdcfg.EtcdKeyFile != "" && etcdcfg.EtcdCertFile != "" {
		envs["ETCD_KEY_FILE"] = ETCD_KEY_NODE_FILE
		vols[etcdcfg.EtcdKeyFile] = ETCD_KEY_NODE_FILE
		envs["ETCD_CERT_FILE"] = ETCD_CERT_NODE_FILE
		vols[etcdcfg.EtcdCertFile] = ETCD_CERT_NODE_FILE
	}

	// Create the Docker command to execute (or display).  Start with the
	// fixed parts.  If this is a dry-run, then don't include the "detach"
	// flag of the "restart" flag since we write out the command for use
	// in an init system.
	cmd := []string{"docker", "run", "--net=host", "--privileged",
		"--name=calico-node"}
	if initSystem {
		cmd = append(cmd, "--rm")
	} else {
		cmd = append(cmd, "-d", "--restart=always")
	}

	// Add the environment variable pass-through.
	for k, v := range envs {
		cmd = append(cmd, "-e", fmt.Sprintf("%s=%s", k, v))
	}

	// Add the volume mounts.
	for k, v := range vols {
		cmd = append(cmd, "-v", fmt.Sprintf("%s:%s", k, v))
	}

	// Add the container image name
	cmd = append(cmd, img)

	if dryrun {
		fmt.Println("Use the following command to start the calico/node container:")
		fmt.Printf("\n%s\n\n", strings.Join(cmd, " "))

		if !initSystem {
			fmt.Println("If you are running calico/node in an init system, use the --init-system flag")
			fmt.Println("to display the appropriate start and stop commands.")
		} else {
			fmt.Println("Use the following command to stop the calico/node container:")
			fmt.Printf("\ndocker stop calico-node\n\n")
		}
		return
	}

	// This is not a dry run.  Check that we are running as root.
	enforceRoot()

	// Normally, Felix will load the modules it needs, but when running inside a
	// container it might not be able to do so. Ensure the required modules are
	// loaded each time the node starts.
	// We only make a best effort attempt because the command may fail if the
	// modules are built in.
	if !runningInContainer() {
		log.Info("Running in container")
		loadModules()
		setupIPForwarding()
		setNFConntrackMax()
	}

	// Run the docker command.
	fmt.Println("Running the following command:")
	fmt.Printf("\n%s\n\n", strings.Join(cmd, " "))

	err = exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		fmt.Printf("Error executing command: %v\n", err)
		os.Exit(1)
	}

	// Create the command to follow the docker logs for the calico/node
	logCmd := exec.Command("docker", "logs", "calico-node", "--follow")

	// Get the stdout pipe
	outPipe, err := logCmd.StdoutPipe()
	if err != nil {
		fmt.Printf("Error executing command:  unable to check calico/node logs: %v\n", err)
		os.Exit(1)
	}
	outScanner := bufio.NewScanner(outPipe)

	// Start following the logs.
	err = logCmd.Start()
	if err != nil {
		fmt.Printf("Error executing command:  unable to check calico/node logs: %v\n", err)
		fmt.Println(err)
		os.Exit(1)
	}

	// Protect against calico processes taking too long to start, or docker
	// logs hanging without output.
	time.AfterFunc(checkLogTimeout, func() {
		logCmd.Process.Kill()
	})

	// Read stdout until the node fails, or until we see the output
	// indicating success.
	started := false
	for outScanner.Scan() {
		line := outScanner.Text()
		fmt.Println(line)
		if line == "Calico node started successfully" {
			started = true
			break
		}
	}

	// If we didn't successfully start then notify the user.
	if outScanner.Err() != nil {
		fmt.Println("Error executing command: error reading calico/node logs, check logs for details")
	} else if !started {
		fmt.Println("Error executing command: calico/node has terminated, check logs for details")
	}

	// Kill the process if it is still running.
	logCmd.Process.Kill()
	logCmd.Wait()
}

// runningInContainer returns whether we are running calicoctl within a container.
func runningInContainer() bool {
	v := os.Getenv("CALICO_CTL_CONTAINER")
	return v != ""
}

func loadModules() {
	cmd := []string{"modprobe", "-a", "xt_set", "ip6_tables"}
	fmt.Printf("Running command to load modules: %s\n", strings.Join(cmd, " "))
	err := exec.Command(cmd[0], cmd[1:]...).Run()
	if err != nil {
		log.Warning(err)
	}
}

func setupIPForwarding() {
	fmt.Println("Enabling IPv4 forwarding")
	err := ioutil.WriteFile("/proc/sys/net/ipv4/ip_forward",
		[]byte("1"), 0)
	if err != nil {
		fmt.Println("ERROR: Could not enable ipv4 forwarding")
		os.Exit(1)
	}

	if _, err := os.Stat("/proc/sys/net/ipv6"); err == nil {
		fmt.Println("Enabling IPv6 forwarding")
		err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/all/forwarding",
			[]byte("1"), 0)
		if err != nil {
			fmt.Println("ERROR: Could not enable ipv6 forwarding")
			os.Exit(1)
		}
	}
}

func setNFConntrackMax() {
	// A common problem on Linux systems is running out of space in the conntrack
	// table, which can cause poor iptables performance. This can happen if you
	// run a lot of workloads on a given host, or if your workloads create a lot
	// of TCP connections or bidirectional UDP streams.
	//
	// To avoid this becoming a problem, we recommend increasing the conntrack
	// table size. To do so, run the following commands:
	fmt.Println("Increasing conntrack limit")
	err := ioutil.WriteFile("/proc/sys/net/netfilter/nf_conntrack_max",
		[]byte("1000000"), 0)
	if err != nil {
		fmt.Println("WARNING: Could not set nf_contrack_max. This may have an impact at scale.")
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows

package net

import (
	"syscall"
	"time"
)

// BUG(mikio): On every POSIX platform, reads from the "ip4" network
// using the ReadFrom or ReadFromIP method might not return a complete
// IPv4 packet, including its header, even if there is space
// available. This can occur even in cases where Read or ReadMsgIP
// could return a complete packet. For this reason, it is recommended
// that you do not uses these methods if it is important to receive a
// full packet.
//
// The Go 1 compatibility guidelines make it impossible for us to
// change the behavior of these methods; use Read or ReadMsgIP
// instead.

func sockaddrToIP(sa syscall.Sockaddr) Addr {
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		return &IPAddr{IP: sa.Addr[0:]}
	case *syscall.SockaddrInet6:
		return &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return nil
}

func (a *IPAddr) family() int {
	if a == nil || len(a.IP) <= IPv4len {
		return syscall.AF_INET
	}
	if a.IP.To4() != nil {
		return syscall.AF_INET
	}
	return syscall.AF_INET6
}

func (a *IPAddr) isWildcard() bool {
	if a == nil || a.IP == nil {
		return true
	}
	return a.IP.IsUnspecified()
}

func (a *IPAddr) sockaddr(family int) (syscall.Sockaddr, error) {
	if a == nil {
		return nil, nil
	}
	return ipToSockaddr(family, a.IP, 0, a.Zone)
}

// IPConn is the implementation of the Conn and PacketConn interfaces
// for IP network connections.
type IPConn struct {
	conn
}

func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} }

// ReadFromIP reads an IP packet from c, copying the payload into b.
// It returns the number of bytes copied into b and the return address
// that was on the packet.
//
// ReadFromIP can be made to time out and return an error with
// Timeout() == true after a fixed time limit; see SetDeadline and
// SetReadDeadline.
func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) {
	if !c.ok() {
		return 0, nil, syscall.EINVAL
	}
	// TODO(cw,rsc): consider using readv if we know the family
	// type to avoid the header trim/copy
	var addr *IPAddr
	n, sa, err := c.fd.readFrom(b)
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		addr = &IPAddr{IP: sa.Addr[0:]}
		if len(b) >= IPv4len { // discard ipv4 header
			hsize := (int(b[0]) & 0xf) * 4
			copy(b, b[hsize:])
			n -= hsize
		}
	case *syscall.SockaddrInet6:
		addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return n, addr, err
}

// ReadFrom implements the PacketConn ReadFrom method.
func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) {
	if !c.ok() {
		return 0, nil, syscall.EINVAL
	}
	n, addr, err := c.ReadFromIP(b)
	return n, addr.toAddr(), err
}

// ReadMsgIP reads a packet from c, copying the payload into b and the
// associated out-of-band data into oob.  It returns the number of
// bytes copied into b, the number of bytes copied into oob, the flags
// that were set on the packet and the source address of the packet.
func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) {
	if !c.ok() {
		return 0, 0, 0, nil, syscall.EINVAL
	}
	var sa syscall.Sockaddr
	n, oobn, flags, sa, err = c.fd.readMsg(b, oob)
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		addr = &IPAddr{IP: sa.Addr[0:]}
	case *syscall.SockaddrInet6:
		addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return
}

// WriteToIP writes an IP packet to addr via c, copying the payload
// from b.
//
// WriteToIP can be made to time out and return an error with
// Timeout() == true after a fixed time limit; see SetDeadline and
// SetWriteDeadline.  On packet-oriented connections, write timeouts
// are rare.
func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	if addr == nil {
		return 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress}
	}
	sa, err := addr.sockaddr(c.fd.family)
	if err != nil {
		return 0, &OpError{"write", c.fd.net, addr, err}
	}
	return c.fd.writeTo(b, sa)
}

// WriteTo implements the PacketConn WriteTo method.
func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	a, ok := addr.(*IPAddr)
	if !ok {
		return 0, &OpError{"write", c.fd.net, addr, syscall.EINVAL}
	}
	return c.WriteToIP(b, a)
}

// WriteMsgIP writes a packet to addr via c, copying the payload from
// b and the associated out-of-band data from oob.  It returns the
// number of payload and out-of-band bytes written.
func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) {
	if !c.ok() {
		return 0, 0, syscall.EINVAL
	}
	if addr == nil {
		return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress}
	}
	sa, err := addr.sockaddr(c.fd.family)
	if err != nil {
		return 0, 0, &OpError{"write", c.fd.net, addr, err}
	}
	return c.fd.writeMsg(b, oob, sa)
}

// DialIP connects to the remote address raddr on the network protocol
// netProto, which must be "ip", "ip4", or "ip6" followed by a colon
// and a protocol number or name.
func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) {
	return dialIP(netProto, laddr, raddr, noDeadline)
}

func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) {
	net, proto, err := parseNetwork(netProto)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err}
	}
	switch net {
	case "ip", "ip4", "ip6":
	default:
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: UnknownNetworkError(netProto)}
	}
	if raddr == nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: nil, Err: errMissingAddress}
	}
	fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_RAW, proto, "dial", sockaddrToIP)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err}
	}
	return newIPConn(fd), nil
}

// ListenIP listens for incoming IP packets addressed to the local
// address laddr.  The returned connection's ReadFrom and WriteTo
// methods can be used to receive and send IP packets with per-packet
// addressing.
func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) {
	net, proto, err := parseNetwork(netProto)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: laddr, Err: err}
	}
	switch net {
	case "ip", "ip4", "ip6":
	default:
		return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: UnknownNetworkError(netProto)}
	}
	fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_RAW, proto, "listen", sockaddrToIP)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: err}
	}
	return newIPConn(fd), nil
}
<commit_msg>net: make WriteTo, WriteToIP and WriteMsgIP fail when IPConn is already connected<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows

package net

import (
	"syscall"
	"time"
)

// BUG(mikio): On every POSIX platform, reads from the "ip4" network
// using the ReadFrom or ReadFromIP method might not return a complete
// IPv4 packet, including its header, even if there is space
// available. This can occur even in cases where Read or ReadMsgIP
// could return a complete packet. For this reason, it is recommended
// that you do not uses these methods if it is important to receive a
// full packet.
//
// The Go 1 compatibility guidelines make it impossible for us to
// change the behavior of these methods; use Read or ReadMsgIP
// instead.

func sockaddrToIP(sa syscall.Sockaddr) Addr {
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		return &IPAddr{IP: sa.Addr[0:]}
	case *syscall.SockaddrInet6:
		return &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return nil
}

func (a *IPAddr) family() int {
	if a == nil || len(a.IP) <= IPv4len {
		return syscall.AF_INET
	}
	if a.IP.To4() != nil {
		return syscall.AF_INET
	}
	return syscall.AF_INET6
}

func (a *IPAddr) isWildcard() bool {
	if a == nil || a.IP == nil {
		return true
	}
	return a.IP.IsUnspecified()
}

func (a *IPAddr) sockaddr(family int) (syscall.Sockaddr, error) {
	if a == nil {
		return nil, nil
	}
	return ipToSockaddr(family, a.IP, 0, a.Zone)
}

// IPConn is the implementation of the Conn and PacketConn interfaces
// for IP network connections.
type IPConn struct {
	conn
}

func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} }

// ReadFromIP reads an IP packet from c, copying the payload into b.
// It returns the number of bytes copied into b and the return address
// that was on the packet.
//
// ReadFromIP can be made to time out and return an error with
// Timeout() == true after a fixed time limit; see SetDeadline and
// SetReadDeadline.
func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) {
	if !c.ok() {
		return 0, nil, syscall.EINVAL
	}
	// TODO(cw,rsc): consider using readv if we know the family
	// type to avoid the header trim/copy
	var addr *IPAddr
	n, sa, err := c.fd.readFrom(b)
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		addr = &IPAddr{IP: sa.Addr[0:]}
		if len(b) >= IPv4len { // discard ipv4 header
			hsize := (int(b[0]) & 0xf) * 4
			copy(b, b[hsize:])
			n -= hsize
		}
	case *syscall.SockaddrInet6:
		addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return n, addr, err
}

// ReadFrom implements the PacketConn ReadFrom method.
func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) {
	if !c.ok() {
		return 0, nil, syscall.EINVAL
	}
	n, addr, err := c.ReadFromIP(b)
	return n, addr.toAddr(), err
}

// ReadMsgIP reads a packet from c, copying the payload into b and the
// associated out-of-band data into oob.  It returns the number of
// bytes copied into b, the number of bytes copied into oob, the flags
// that were set on the packet and the source address of the packet.
func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) {
	if !c.ok() {
		return 0, 0, 0, nil, syscall.EINVAL
	}
	var sa syscall.Sockaddr
	n, oobn, flags, sa, err = c.fd.readMsg(b, oob)
	switch sa := sa.(type) {
	case *syscall.SockaddrInet4:
		addr = &IPAddr{IP: sa.Addr[0:]}
	case *syscall.SockaddrInet6:
		addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))}
	}
	return
}

// WriteToIP writes an IP packet to addr via c, copying the payload
// from b.
//
// WriteToIP can be made to time out and return an error with
// Timeout() == true after a fixed time limit; see SetDeadline and
// SetWriteDeadline.  On packet-oriented connections, write timeouts
// are rare.
func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	if c.fd.isConnected {
		return 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected}
	}
	if addr == nil {
		return 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress}
	}
	sa, err := addr.sockaddr(c.fd.family)
	if err != nil {
		return 0, &OpError{"write", c.fd.net, addr, err}
	}
	return c.fd.writeTo(b, sa)
}

// WriteTo implements the PacketConn WriteTo method.
func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	a, ok := addr.(*IPAddr)
	if !ok {
		return 0, &OpError{"write", c.fd.net, addr, syscall.EINVAL}
	}
	return c.WriteToIP(b, a)
}

// WriteMsgIP writes a packet to addr via c, copying the payload from
// b and the associated out-of-band data from oob.  It returns the
// number of payload and out-of-band bytes written.
func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) {
	if !c.ok() {
		return 0, 0, syscall.EINVAL
	}
	if c.fd.isConnected {
		return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected}
	}
	if addr == nil {
		return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress}
	}
	sa, err := addr.sockaddr(c.fd.family)
	if err != nil {
		return 0, 0, &OpError{"write", c.fd.net, addr, err}
	}
	return c.fd.writeMsg(b, oob, sa)
}

// DialIP connects to the remote address raddr on the network protocol
// netProto, which must be "ip", "ip4", or "ip6" followed by a colon
// and a protocol number or name.
func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) {
	return dialIP(netProto, laddr, raddr, noDeadline)
}

func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) {
	net, proto, err := parseNetwork(netProto)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err}
	}
	switch net {
	case "ip", "ip4", "ip6":
	default:
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: UnknownNetworkError(netProto)}
	}
	if raddr == nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: nil, Err: errMissingAddress}
	}
	fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_RAW, proto, "dial", sockaddrToIP)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err}
	}
	return newIPConn(fd), nil
}

// ListenIP listens for incoming IP packets addressed to the local
// address laddr.  The returned connection's ReadFrom and WriteTo
// methods can be used to receive and send IP packets with per-packet
// addressing.
func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) {
	net, proto, err := parseNetwork(netProto)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: netProto, Addr: laddr, Err: err}
	}
	switch net {
	case "ip", "ip4", "ip6":
	default:
		return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: UnknownNetworkError(netProto)}
	}
	fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_RAW, proto, "listen", sockaddrToIP)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: err}
	}
	return newIPConn(fd), nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func TestAccAWSInstance_normal(t *testing.T) {
	var v ec2.Instance

	testCheck := func(*terraform.State) error {
		if len(v.SecurityGroups) == 0 {
			return fmt.Errorf("no security groups: %#v", v.SecurityGroups)
		}
		if v.SecurityGroups[0].Name != "tf_test_foo" {
			return fmt.Errorf("no security groups: %#v", v.SecurityGroups)
		}

		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck,
					resource.TestCheckResourceAttr(
						"aws_instance.foo",
						"user_data",
						"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"),
				),
			},

			// We repeat the exact same test so that we can be sure
			// that the user data hash stuff is working without generating
			// an incorrect diff.
			resource.TestStep{
				Config: testAccInstanceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck,
					resource.TestCheckResourceAttr(
						"aws_instance.foo",
						"user_data",
						"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"),
				),
			},
		},
	})
}

func TestAccAWSInstance_sourceDestCheck(t *testing.T) {
	var v ec2.Instance

	testCheck := func(enabled bool) resource.TestCheckFunc {
		return func(*terraform.State) error {
			if v.SourceDestCheck != enabled {
				return fmt.Errorf("bad source_dest_check: %#v", v.SourceDestCheck)
			}

			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfigSourceDest,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck(true),
				),
			},

			resource.TestStep{
				Config: testAccInstanceConfigSourceDestDisable,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck(false),
				),
			},
		},
	})
}

func TestAccAWSInstance_vpc(t *testing.T) {
	var v ec2.Instance

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfigVPC,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
				),
			},
		},
	})
}

func testAccCheckInstanceDestroy(s *terraform.State) error {
	conn := testAccProvider.ec2conn

	for _, rs := range s.Resources {
		if rs.Type != "aws_instance" {
			continue
		}

		// Try to find the resource
		resp, err := conn.Instances(
			[]string{rs.ID}, ec2.NewFilter())
		if err == nil {
			if len(resp.Reservations) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		ec2err, ok := err.(*ec2.Error)
		if !ok {
			return err
		}
		if ec2err.Code != "InvalidInstanceID.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckInstanceExists(n string, i *ec2.Instance) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.ec2conn
		resp, err := conn.Instances(
			[]string{rs.ID}, ec2.NewFilter())
		if err != nil {
			return err
		}
		if len(resp.Reservations) == 0 {
			return fmt.Errorf("Instance not found")
		}

		*i = resp.Reservations[0].Instances[0]

		return nil
	}
}

const testAccInstanceConfig = `
resource "aws_security_group" "tf_test_foo" {
	name = "tf_test_foo"
	description = "foo"

	ingress {
		protocol = "icmp"
		from_port = -1
		to_port = -1
		cidr_blocks = ["0.0.0.0/0"]
	}
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	security_groups = ["${aws_security_group.tf_test_foo.name}"]
	user_data = "foo"
}
`

const testAccInstanceConfigSourceDest = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	source_dest_check = true
}
`

const testAccInstanceConfigSourceDestDisable = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	source_dest_check = false
}
`

const testAccInstanceConfigVPC = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	associate_public_ip_address = true
}
`
<commit_msg>providers/aws: add tests for instance availability zone<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/mitchellh/goamz/ec2"
)

func TestAccAWSInstance_normal(t *testing.T) {
	var v ec2.Instance

	testCheck := func(*terraform.State) error {
		if v.AvailZone != "us-west-2a" {
			return fmt.Errorf("bad availability zone: %#v", v.AvailZone)
		}

		if len(v.SecurityGroups) == 0 {
			return fmt.Errorf("no security groups: %#v", v.SecurityGroups)
		}
		if v.SecurityGroups[0].Name != "tf_test_foo" {
			return fmt.Errorf("no security groups: %#v", v.SecurityGroups)
		}

		return nil
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck,
					resource.TestCheckResourceAttr(
						"aws_instance.foo",
						"user_data",
						"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"),
				),
			},

			// We repeat the exact same test so that we can be sure
			// that the user data hash stuff is working without generating
			// an incorrect diff.
			resource.TestStep{
				Config: testAccInstanceConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck,
					resource.TestCheckResourceAttr(
						"aws_instance.foo",
						"user_data",
						"0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"),
				),
			},
		},
	})
}

func TestAccAWSInstance_sourceDestCheck(t *testing.T) {
	var v ec2.Instance

	testCheck := func(enabled bool) resource.TestCheckFunc {
		return func(*terraform.State) error {
			if v.SourceDestCheck != enabled {
				return fmt.Errorf("bad source_dest_check: %#v", v.SourceDestCheck)
			}

			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfigSourceDest,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck(true),
				),
			},

			resource.TestStep{
				Config: testAccInstanceConfigSourceDestDisable,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
					testCheck(false),
				),
			},
		},
	})
}

func TestAccAWSInstance_vpc(t *testing.T) {
	var v ec2.Instance

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckInstanceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccInstanceConfigVPC,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckInstanceExists(
						"aws_instance.foo", &v),
				),
			},
		},
	})
}

func testAccCheckInstanceDestroy(s *terraform.State) error {
	conn := testAccProvider.ec2conn

	for _, rs := range s.Resources {
		if rs.Type != "aws_instance" {
			continue
		}

		// Try to find the resource
		resp, err := conn.Instances(
			[]string{rs.ID}, ec2.NewFilter())
		if err == nil {
			if len(resp.Reservations) > 0 {
				return fmt.Errorf("still exist.")
			}

			return nil
		}

		// Verify the error is what we want
		ec2err, ok := err.(*ec2.Error)
		if !ok {
			return err
		}
		if ec2err.Code != "InvalidInstanceID.NotFound" {
			return err
		}
	}

	return nil
}

func testAccCheckInstanceExists(n string, i *ec2.Instance) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		conn := testAccProvider.ec2conn
		resp, err := conn.Instances(
			[]string{rs.ID}, ec2.NewFilter())
		if err != nil {
			return err
		}
		if len(resp.Reservations) == 0 {
			return fmt.Errorf("Instance not found")
		}

		*i = resp.Reservations[0].Instances[0]

		return nil
	}
}

const testAccInstanceConfig = `
resource "aws_security_group" "tf_test_foo" {
	name = "tf_test_foo"
	description = "foo"

	ingress {
		protocol = "icmp"
		from_port = -1
		to_port = -1
		cidr_blocks = ["0.0.0.0/0"]
	}
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	availability_zone = "us-west-2a"

	instance_type = "m1.small"
	security_groups = ["${aws_security_group.tf_test_foo.name}"]
	user_data = "foo"
}
`

const testAccInstanceConfigSourceDest = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	source_dest_check = true
}
`

const testAccInstanceConfigSourceDestDisable = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	source_dest_check = false
}
`

const testAccInstanceConfigVPC = `
resource "aws_vpc" "foo" {
	cidr_block = "10.1.0.0/16"
}

resource "aws_subnet" "foo" {
	cidr_block = "10.1.1.0/24"
	vpc_id = "${aws_vpc.foo.id}"
}

resource "aws_instance" "foo" {
	# us-west-2
	ami = "ami-4fccb37f"
	instance_type = "m1.small"
	subnet_id = "${aws_subnet.foo.id}"
	associate_public_ip_address = true
}
`
<|endoftext|>


<commit_before>package siegreader

import (
	"io"
	"io/ioutil"
	"os"
	"sync"
)

type stream struct {
	b     *Buffer
	src   io.Reader
	sz    int64
	buf   []byte
	tf    *os.File // temp backing file - used when stream exceeds streamSz
	tfBuf []byte
	eofc  chan struct{}

	mu  sync.Mutex
	i   int // marks how much of buf we have filled
	eof bool
}

func newStream() interface{} {
	return &stream{buf: make([]byte, readSz*2), tfBuf: make([]byte, readSz)}
}

func (s *stream) setSource(src io.Reader, b *Buffer) error {
	s.b = b
	s.src = src
	s.sz = 0
	s.eofc = make(chan struct{})
	s.i = 0
	s.eof = false
	_, err := s.fill()
	return err
}

// close and delete temp file, if exists
func (s *stream) cleanUp() {
	if s.tf == nil {
		return
	}
	s.tf.Close()
	os.Remove(s.tf.Name())
	s.tf = nil
}

// Size returns the buffer's size, which is available immediately for files. Must wait for full read for streams.
func (s *stream) Size() int64 {
	select {
	case <-s.eofc:
		return s.sz
	case <-s.b.Quit:
		return 0
	}
}

// SizeNow is a non-blocking Size(). Will force a full read of a stream.
func (s *stream) SizeNow() int64 {
	var err error
	for _, err = s.fill(); err == nil; _, err = s.fill() {
	}
	return s.sz
}

func (s *stream) grow() error {
	if s.tf != nil { // return if we already have a temp file
		return nil
	}
	c := cap(s.buf) * 2
	if c > streamSz {
		if cap(s.buf) < streamSz {
			c = streamSz
		} else { // if we've exceeded streamSz, use a temp file to copy remainder
			var err error
			s.tf, err = ioutil.TempFile("", "siegfried")
			return err
		}
	}
	buf := make([]byte, c)
	copy(buf, s.buf[:s.i]) // don't care about unlocking as grow() is only called by fill()
	s.buf = buf
	return nil
}

func (s *stream) fill() (int64, error) {
	// have already scanned to the end of the stream
	if s.eof {
		return s.sz, io.EOF
	}
	// if we've run out of room in buf, & there is no backing file, grow the buffer
	if len(s.buf)-readSz < s.i && s.tf == nil {
		s.grow()
	}
	// now let's read
	var err error
	if s.tf != nil {
		// if we have a backing file, fill that
		var wi int64
		wi, err = io.CopyBuffer(s.tf, io.LimitReader(s.src, int64(readSz)), s.tfBuf)
		if wi < int64(readSz) && err == nil {
			err = io.EOF
		}
		// update s.sz
		s.sz += wi
	} else {
		// otherwise, fill the slice
		var i int
		i, err = io.ReadFull(s.src, s.buf[s.i:s.i+readSz])
		s.i += i
		s.sz += int64(i)
		if err == io.ErrUnexpectedEOF {
			err = io.EOF
		}
	}
	if err != nil {
		close(s.eofc)
		s.eof = true
		if err == io.EOF && s.sz == 0 {
			err = ErrEmpty
		}
	}
	return s.sz, err
}

// Slice returns a byte slice from the buffer that begins at offset off and has length l.
func (s *stream) Slice(off int64, l int) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var err error
	var bound int64
	furthest := off + int64(l)
	if furthest > s.sz {
		for bound, err = s.fill(); furthest > bound && err == nil; bound, err = s.fill() {
		}
	}
	if err != nil && err != io.EOF {
		return nil, err
	}
	if err == io.EOF {
		// in the case of an empty file
		if s.sz == 0 {
			return nil, io.EOF
		}
		if furthest > s.sz {
			if off >= s.sz {
				return nil, io.EOF
			}
			l = int(s.sz - off)
		}
	}
	// slice is wholly in buf
	if off+int64(l) <= int64(len(s.buf)) {
		return s.buf[int(off) : int(off)+l], err
	}
	ret := make([]byte, l)
	// if slice crosses border, copy first bit from end of buf
	var ci int
	if off < int64(len(s.buf)) {
		ci = copy(ret, s.buf[int(off):])
		off = 0
	} else {
		off -= int64(len(s.buf))
	}
	_, rerr := s.tf.ReadAt(ret[ci:], off)
	if rerr != nil {
		err = rerr
	}
	return ret, err
}

// EofSlice returns a slice from the end of the buffer that begins at offset s and has length l.
// Blocks until the slice is available (which may be until the full stream is read).
func (s *stream) EofSlice(o int64, l int) ([]byte, error) {
	// block until the EOF is available or we quit
	select {
	case <-s.b.Quit:
		return nil, ErrQuit
	case <-s.eofc:
	}
	if o >= s.sz {
		return nil, io.EOF
	}
	var err error
	if o+int64(l) > s.sz {
		l = int(s.sz - o)
		err = io.EOF
	}
	slc, serr := s.Slice(s.sz-o-int64(l), l)
	if serr != nil {
		err = serr
	}
	return slc, err
}

// fill until a seek to a particular offset is possible, then return true, if it is impossible return false
func (s *stream) CanSeek(o int64, rev bool) (bool, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if rev {
		var err error
		for _, err = s.fill(); err == nil; _, err = s.fill() {
		}
		if err != io.EOF {
			return false, err
		}
		if o >= s.sz {
			return false, nil
		}
		return true, nil
	}
	var err error
	var bound int64
	if o > s.sz {
		for bound, err = s.fill(); o > bound && err == nil; bound, err = s.fill() {
		}
	}
	if err == nil {
		return true, nil
	}
	if err == io.EOF {
		if o > s.sz {
			return false, err
		}
		return true, nil
	}
	return false, err
}
<commit_msg>fix bug in stream reader where EOF returned for reads exactly adjacent to end of file<commit_after>package siegreader

import (
	"io"
	"io/ioutil"
	"os"
	"sync"
)

type stream struct {
	b     *Buffer
	src   io.Reader
	sz    int64
	buf   []byte
	tf    *os.File // temp backing file - used when stream exceeds streamSz
	tfBuf []byte
	eofc  chan struct{}

	mu  sync.Mutex
	i   int // marks how much of buf we have filled
	eof bool
}

func newStream() interface{} {
	return &stream{buf: make([]byte, readSz*2), tfBuf: make([]byte, readSz)}
}

func (s *stream) setSource(src io.Reader, b *Buffer) error {
	s.b = b
	s.src = src
	s.sz = 0
	s.eofc = make(chan struct{})
	s.i = 0
	s.eof = false
	_, err := s.fill()
	return err
}

// close and delete temp file, if exists
func (s *stream) cleanUp() {
	if s.tf == nil {
		return
	}
	s.tf.Close()
	os.Remove(s.tf.Name())
	s.tf = nil
}

// Size returns the buffer's size, which is available immediately for files. Must wait for full read for streams.
func (s *stream) Size() int64 {
	select {
	case <-s.eofc:
		return s.sz
	case <-s.b.Quit:
		return 0
	}
}

// SizeNow is a non-blocking Size(). Will force a full read of a stream.
func (s *stream) SizeNow() int64 {
	var err error
	for _, err = s.fill(); err == nil; _, err = s.fill() {
	}
	return s.sz
}

func (s *stream) grow() error {
	if s.tf != nil { // return if we already have a temp file
		return nil
	}
	c := cap(s.buf) * 2
	if c > streamSz {
		if cap(s.buf) < streamSz {
			c = streamSz
		} else { // if we've exceeded streamSz, use a temp file to copy remainder
			var err error
			s.tf, err = ioutil.TempFile("", "siegfried")
			return err
		}
	}
	buf := make([]byte, c)
	copy(buf, s.buf[:s.i]) // don't care about unlocking as grow() is only called by fill()
	s.buf = buf
	return nil
}

func (s *stream) fill() (int64, error) {
	// have already scanned to the end of the stream
	if s.eof {
		return s.sz, io.EOF
	}
	// if we've run out of room in buf, & there is no backing file, grow the buffer
	if len(s.buf)-readSz < s.i && s.tf == nil {
		s.grow()
	}
	// now let's read
	var err error
	if s.tf != nil {
		// if we have a backing file, fill that
		var wi int64
		wi, err = io.CopyBuffer(s.tf, io.LimitReader(s.src, int64(readSz)), s.tfBuf)
		if wi < int64(readSz) && err == nil {
			err = io.EOF
		}
		// update s.sz
		s.sz += wi
	} else {
		// otherwise, fill the slice
		var i int
		i, err = io.ReadFull(s.src, s.buf[s.i:s.i+readSz])
		s.i += i
		s.sz += int64(i)
		if err == io.ErrUnexpectedEOF {
			err = io.EOF
		}
	}
	if err != nil {
		close(s.eofc)
		s.eof = true
		if err == io.EOF && s.sz == 0 {
			err = ErrEmpty
		}
	}
	return s.sz, err
}

// Slice returns a byte slice from the buffer that begins at offset off and has length l.
func (s *stream) Slice(off int64, l int) ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var err error
	var bound int64
	furthest := off + int64(l)
	if furthest > s.sz {
		for bound, err = s.fill(); furthest > bound && err == nil; bound, err = s.fill() {
		}
	}
	if err != nil && err != io.EOF {
		return nil, err
	}
	if err == io.EOF {
		// in the case of an empty file
		if s.sz == 0 {
			return nil, io.EOF
		}
		if furthest > s.sz {
			if off >= s.sz {
				return nil, io.EOF
			}
			l = int(s.sz - off)
		} else {
			err = nil
		}
	}
	// slice is wholly in buf
	if off+int64(l) <= int64(len(s.buf)) {
		return s.buf[int(off) : int(off)+l], err
	}
	ret := make([]byte, l)
	// if slice crosses border, copy first bit from end of buf
	var ci int
	if off < int64(len(s.buf)) {
		ci = copy(ret, s.buf[int(off):])
		off = 0
	} else {
		off -= int64(len(s.buf))
	}
	_, rerr := s.tf.ReadAt(ret[ci:], off)
	if rerr != nil {
		err = rerr
	}
	return ret, err
}

// EofSlice returns a slice from the end of the buffer that begins at offset s and has length l.
// Blocks until the slice is available (which may be until the full stream is read).
func (s *stream) EofSlice(o int64, l int) ([]byte, error) {
	// block until the EOF is available or we quit
	select {
	case <-s.b.Quit:
		return nil, ErrQuit
	case <-s.eofc:
	}
	if o >= s.sz {
		return nil, io.EOF
	}
	var err error
	if o+int64(l) > s.sz {
		l = int(s.sz - o)
		err = io.EOF
	}
	slc, serr := s.Slice(s.sz-o-int64(l), l)
	if serr != nil {
		err = serr
	}
	return slc, err
}

// fill until a seek to a particular offset is possible, then return true, if it is impossible return false
func (s *stream) CanSeek(o int64, rev bool) (bool, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if rev {
		var err error
		for _, err = s.fill(); err == nil; _, err = s.fill() {
		}
		if err != io.EOF {
			return false, err
		}
		if o >= s.sz {
			return false, nil
		}
		return true, nil
	}
	var err error
	var bound int64
	if o > s.sz {
		for bound, err = s.fill(); o > bound && err == nil; bound, err = s.fill() {
		}
	}
	if err == nil {
		return true, nil
	}
	if err == io.EOF {
		if o > s.sz {
			return false, err
		}
		return true, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>package kite

import (
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/dnode"
	_ "github.com/koding/kite/testutil"
)

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 100

	// number of exp kites that will call mathwork kites
	clientNumber := 100

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	fmt.Printf("Creating %d mathworker kites\n", kiteNumber)
	for i := 0; i < kiteNumber; i++ {
		m := NewKite("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true

		m.HandleFunc("square", Square)

		go http.ListenAndServe("127.0.0.1:"+strconv.Itoa(port+i), m)
	}

	// Wait until it's started
	time.Sleep(time.Second * 2)

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := NewKite("exp"+strconv.Itoa(i), "0.0.1").NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	fmt.Printf("Calling mathworker kites with %d conccurent clients randomly\n", clientNumber)
	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int) {
					defer wg.Done()
					start := time.Now()

					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					result, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Fatal(err)
					}

					elapsedTime := time.Since(start)

					number := result.MustFloat64()

					t.Log("rpc result: %f elapsedTime %f sec\n", number, elapsedTime.Seconds())
				}(i)
			}
		case <-timeout:
			fmt.Println("test stopped")
			t.SkipNow()
		}

	}

	wg.Wait()
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go http.ListenAndServe("127.0.0.1:3637", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// number of exp kites that will call mathworker kite
	clientNumber := 30

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := NewKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go http.ListenAndServe("127.0.0.1:3636", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// Create exp2 kite
	exp2Kite := NewKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func NewKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.ReadEnvironmentVariables()
	return k
}
<commit_msg>kite_test: heavy usage of environment read breaks the test<commit_after>package kite

import (
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/koding/kite/config"
	"github.com/koding/kite/dnode"
	_ "github.com/koding/kite/testutil"
)

func TestMultiple(t *testing.T) {
	testDuration := time.Second * 10

	// number of available mathworker kites to be called
	kiteNumber := 100

	// number of exp kites that will call mathwork kites
	clientNumber := 100

	// ports are starting from 6000 up to 6000 + kiteNumber
	port := 6000

	fmt.Printf("Creating %d mathworker kites\n", kiteNumber)

	var transport config.Transport
	if transportName := os.Getenv("KITE_TRANSPORT"); transportName != "" {
		tr, ok := config.Transports[transportName]
		if !ok {
			t.Fatalf("transport '%s' doesn't exists", transportName)
		}

		transport = tr
	}

	for i := 0; i < kiteNumber; i++ {
		m := New("mathworker"+strconv.Itoa(i), "0.1."+strconv.Itoa(i))
		m.Config.DisableAuthentication = true
		m.Config.Transport = transport

		m.HandleFunc("square", Square)

		go http.ListenAndServe("127.0.0.1:"+strconv.Itoa(port+i), m)
	}

	// Wait until it's started
	time.Sleep(time.Second * 2)

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := NewKite("exp"+strconv.Itoa(i), "0.0.1").NewClient("http://127.0.0.1:" + strconv.Itoa(port+i) + "/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	fmt.Printf("Calling mathworker kites with %d conccurent clients randomly\n", clientNumber)
	timeout := time.After(testDuration)

	// every one second
	for {
		select {
		case <-time.Tick(time.Second):
			for i := 0; i < clientNumber; i++ {
				wg.Add(1)

				go func(i int) {
					defer wg.Done()
					time.Sleep(time.Millisecond * time.Duration(rand.Intn(500)))

					_, err := clients[i].TellWithTimeout("square", 4*time.Second, 2)
					if err != nil {
						t.Fatal(err)
					}
				}(i)
			}
		case <-timeout:
			fmt.Println("test stopped")
			t.SkipNow()
		}

	}

	wg.Wait()
}

// Call a single method with multiple clients. This test is implemented to be
// sure the method is calling back with in the same time and not timing out.
func TestConcurrency(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("ping", func(r *Request) (interface{}, error) {
		time.Sleep(time.Second)
		return "pong", nil
	})
	go http.ListenAndServe("127.0.0.1:3637", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// number of exp kites that will call mathworker kite
	clientNumber := 30

	fmt.Printf("Creating %d exp clients\n", clientNumber)
	clients := make([]*Client, clientNumber)
	for i := 0; i < clientNumber; i++ {
		c := NewKite("exp", "0.0.1").NewClient("http://127.0.0.1:3637/kite")
		if err := c.Dial(); err != nil {
			t.Fatal(err)
		}

		clients[i] = c
	}

	var wg sync.WaitGroup

	for i := range clients {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			result, err := clients[i].TellWithTimeout("ping", 4*time.Second)
			if err != nil {
				t.Fatal(err)
			}

			if result.MustString() != "pong" {
				t.Errorf("Got %s want: pong", result.MustString())
			}
		}(i)
	}

	wg.Wait()
}

// Test 2 way communication between kites.
func TestKite(t *testing.T) {
	// Create a mathworker kite
	mathKite := NewKite("mathworker", "0.0.1")
	mathKite.Config.DisableAuthentication = true
	mathKite.HandleFunc("square", Square)
	mathKite.HandleFunc("squareCB", SquareCB)
	mathKite.HandleFunc("sleep", Sleep)
	go http.ListenAndServe("127.0.0.1:3636", mathKite)

	// Wait until it's started
	time.Sleep(time.Second)

	// Create exp2 kite
	exp2Kite := NewKite("exp2", "0.0.1")
	fooChan := make(chan string)
	exp2Kite.HandleFunc("foo", func(r *Request) (interface{}, error) {
		s := r.Args.One().MustString()
		t.Logf("Message received: %s\n", s)
		fooChan <- s
		return nil, nil
	})

	// exp2 connects to mathworker
	remote := exp2Kite.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		t.Fatal(err)
	}

	result, err := remote.TellWithTimeout("square", 4*time.Second, 2)
	if err != nil {
		t.Fatal(err)
	}

	number := result.MustFloat64()

	t.Logf("rpc result: %f\n", number)

	if number != 4 {
		t.Fatalf("Invalid result: %f", number)
	}

	select {
	case s := <-fooChan:
		if s != "bar" {
			t.Fatalf("Invalid message: %s", s)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	resultChan := make(chan float64, 1)
	resultCallback := func(args *dnode.Partial) {
		n := args.One().MustFloat64()
		resultChan <- n
	}

	result, err = remote.TellWithTimeout("squareCB", 4*time.Second, 3, dnode.Callback(resultCallback))
	if err != nil {
		t.Fatal(err)
	}

	select {
	case n := <-resultChan:
		if n != 9.0 {
			t.Fatalf("Unexpected result: %f", n)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("Did not get the message")
	}

	result, err = remote.TellWithTimeout("sleep", time.Second)
	if err == nil {
		t.Fatal("Did get message in 1 seconds, however the sleep method takes 2 seconds to response")
	}

	result, err = remote.Tell("sleep")
	if err != nil {
		t.Fatal(err)
	}

	if !result.MustBool() {
		t.Fatal("sleep result must be true")
	}

}

// Sleeps for 2 seconds and returns true
func Sleep(r *Request) (interface{}, error) {
	time.Sleep(time.Second * 2)
	return true, nil
}

// Returns the result. Also tests reverse call.
func Square(r *Request) (interface{}, error) {
	a := r.Args.One().MustFloat64()
	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Reverse method call
	r.Client.Go("foo", "bar")

	return result, nil
}

// Calls the callback with the result. For testing requests with Callback.
func SquareCB(r *Request) (interface{}, error) {
	args := r.Args.MustSliceOfLength(2)
	a := args[0].MustFloat64()
	cb := args[1].MustFunction()

	result := a * a

	r.LocalKite.Log.Info("Kite call, sending result '%f' back\n", result)

	// Send the result.
	err := cb.Call(result)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func NewKite(name, version string) *Kite {
	k := New(name, version)
	k.Config.Transport = config.XHRPolling
	return k
}
<|endoftext|>


<commit_before>package soy

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/robfig/soy/data"
	"github.com/robfig/soy/parse"
	"github.com/robfig/soy/parsepasses"
	"github.com/robfig/soy/soyhtml"
	"github.com/robfig/soy/template"
	"github.com/howeyc/fsnotify"
)

// Logger is used to print notifications and compile errors when using the
// "WatchFiles" feature.
var Logger = log.New(os.Stderr, "[soy] ", 0)

type soyFile struct{ name, content string }

// Bundle is a collection of soy content (templates and globals).  It acts as
// input for the soy compiler.
type Bundle struct {
	files   []soyFile
	globals data.Map
	err     error
	watcher *fsnotify.Watcher
}

// NewBundle returns an empty bundle.
func NewBundle() *Bundle {
	return &Bundle{globals: make(data.Map)}
}

// WatchFiles tells soy to watch any template files added to this bundle,
// re-compile as necessary, and propagate the updates to your tofu.  It should
// be called once, before adding any files.
func (b *Bundle) WatchFiles(watch bool) *Bundle {
	if watch && b.err == nil && b.watcher == nil {
		b.watcher, b.err = fsnotify.NewWatcher()
	}
	return b
}

// AddTemplateDir adds all *.soy files found within the given directory
// (including sub-directories) to the bundle.
func (b *Bundle) AddTemplateDir(root string) *Bundle {
	var err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(path, ".soy") {
			return nil
		}
		b.AddTemplateFile(path)
		return nil
	})
	if err != nil {
		b.err = err
	}
	return b
}

// AddTemplateFile adds the given soy template file text to this bundle.
// If WatchFiles is on, it will be subsequently watched for updates.
func (b *Bundle) AddTemplateFile(filename string) *Bundle {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		b.err = err
	}
	if b.err == nil && b.watcher != nil {
		b.err = b.watcher.Watch(filename)
	}
	return b.AddTemplateString(filename, string(content))
}

// AddTemplateString adds the given template to the bundle. The name is only
// used for error messages - it does not need to be provided nor does it need to
// be a real filename.
func (b *Bundle) AddTemplateString(filename, soyfile string) *Bundle {
	b.files = append(b.files, soyFile{filename, soyfile})
	return b
}

// AddGlobalsFile opens and parses the given filename for Soy globals, and adds
// the resulting data map to the bundle.
func (b *Bundle) AddGlobalsFile(filename string) *Bundle {
	var f, err = os.Open(filename)
	if err != nil {
		b.err = err
		return b
	}
	globals, err := ParseGlobals(f)
	if err != nil {
		b.err = err
	}
	f.Close()
	return b.AddGlobalsMap(globals)
}

func (b *Bundle) AddGlobalsMap(globals data.Map) *Bundle {
	for k, v := range globals {
		if existing, ok := b.globals[k]; ok {
			b.err = fmt.Errorf("global %q already defined as %q", k, existing)
			return b
		}
		b.globals[k] = v
	}
	return b
}

// Compile parses all of the soy files in this bundle, verifies a number of
// rules about data references, and returns the completed template registry.
func (b *Bundle) Compile() (*template.Registry, error) {
	if b.err != nil {
		return nil, b.err
	}

	// Compile all the soy (globals are already parsed)
	var registry = template.Registry{}
	for _, soyfile := range b.files {
		var tree, err = parse.SoyFile(soyfile.name, soyfile.content)
		if err != nil {
			return nil, err
		}
		if err = registry.Add(tree); err != nil {
			return nil, err
		}
	}

	// Apply the post-parse processing
	if err := parsepasses.CheckDataRefs(registry); err != nil {
		return nil, err
	}
	if err := parsepasses.SetGlobals(registry, b.globals); err != nil {
		return nil, err
	}
	parsepasses.ProcessMessages(registry)

	if b.watcher != nil {
		go b.recompiler(&registry)
	}
	return &registry, nil
}

// CompileToTofu returns a soyhtml.Tofu object that allows you to render soy
// templates to HTML.
func (b *Bundle) CompileToTofu() (*soyhtml.Tofu, error) {
	var registry, err = b.Compile()
	// TODO: Verify all used funcs exist and have the right # args.
	return soyhtml.NewTofu(registry), err
}

func (b *Bundle) recompiler(reg *template.Registry) {
	for {
		select {
		case ev := <-b.watcher.Event:
			// If it's a rename, then fsnotify has removed the watch.
			// Add it back, after a delay.
			if ev.IsRename() || ev.IsDelete() {
				time.Sleep(10 * time.Millisecond)
				if err := b.watcher.Watch(ev.Name); err != nil {
					Logger.Println(err)
				}
			}

			// Recompile all the soy.
			var bundle = NewBundle().
				AddGlobalsMap(b.globals)
			for _, soyfile := range b.files {
				bundle.AddTemplateFile(soyfile.name)
			}
			var registry, err = bundle.Compile()
			if err != nil {
				Logger.Println(err)
				continue
			}

			// update the existing template registry.
			// (this is not goroutine-safe, but that seems ok for a development aid,
			// as long as it works in practice)
			*reg = *registry
			Logger.Printf("update successful (%v)", ev)

		case err := <-b.watcher.Error:
			// Nothing to do with errors
			Logger.Println(err)
		}
	}
}
<commit_msg>Add recompilationCallback to soy.Bundle<commit_after>package soy

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/robfig/soy/data"
	"github.com/robfig/soy/parse"
	"github.com/robfig/soy/parsepasses"
	"github.com/robfig/soy/soyhtml"
	"github.com/robfig/soy/template"
	"github.com/howeyc/fsnotify"
)

// Logger is used to print notifications and compile errors when using the
// "WatchFiles" feature.
var Logger = log.New(os.Stderr, "[soy] ", 0)

type soyFile struct{ name, content string }

// Bundle is a collection of soy content (templates and globals).  It acts as
// input for the soy compiler.
type Bundle struct {
	files                 []soyFile
	globals               data.Map
	err                   error
	watcher               *fsnotify.Watcher
	recompilationCallback func(*template.Registry)
}

// NewBundle returns an empty bundle.
func NewBundle() *Bundle {
	return &Bundle{globals: make(data.Map)}
}

// WatchFiles tells soy to watch any template files added to this bundle,
// re-compile as necessary, and propagate the updates to your tofu.  It should
// be called once, before adding any files.
func (b *Bundle) WatchFiles(watch bool) *Bundle {
	if watch && b.err == nil && b.watcher == nil {
		b.watcher, b.err = fsnotify.NewWatcher()
	}
	return b
}

// AddTemplateDir adds all *.soy files found within the given directory
// (including sub-directories) to the bundle.
func (b *Bundle) AddTemplateDir(root string) *Bundle {
	var err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}
		if !strings.HasSuffix(path, ".soy") {
			return nil
		}
		b.AddTemplateFile(path)
		return nil
	})
	if err != nil {
		b.err = err
	}
	return b
}

// AddTemplateFile adds the given soy template file text to this bundle.
// If WatchFiles is on, it will be subsequently watched for updates.
func (b *Bundle) AddTemplateFile(filename string) *Bundle {
	content, err := ioutil.ReadFile(filename)
	if err != nil {
		b.err = err
	}
	if b.err == nil && b.watcher != nil {
		b.err = b.watcher.Watch(filename)
	}
	return b.AddTemplateString(filename, string(content))
}

// AddTemplateString adds the given template to the bundle. The name is only
// used for error messages - it does not need to be provided nor does it need to
// be a real filename.
func (b *Bundle) AddTemplateString(filename, soyfile string) *Bundle {
	b.files = append(b.files, soyFile{filename, soyfile})
	return b
}

// AddGlobalsFile opens and parses the given filename for Soy globals, and adds
// the resulting data map to the bundle.
func (b *Bundle) AddGlobalsFile(filename string) *Bundle {
	var f, err = os.Open(filename)
	if err != nil {
		b.err = err
		return b
	}
	globals, err := ParseGlobals(f)
	if err != nil {
		b.err = err
	}
	f.Close()
	return b.AddGlobalsMap(globals)
}

func (b *Bundle) AddGlobalsMap(globals data.Map) *Bundle {
	for k, v := range globals {
		if existing, ok := b.globals[k]; ok {
			b.err = fmt.Errorf("global %q already defined as %q", k, existing)
			return b
		}
		b.globals[k] = v
	}
	return b
}

// SetRecompilationCallback assigns the bundle a function to call after
// recompilation.  This is called before updating the in-use registry.
func (b *Bundle) SetRecompilationCallback(c func(*template.Registry)) *Bundle {
	b.recompilationCallback = c
	return b
}

// Compile parses all of the soy files in this bundle, verifies a number of
// rules about data references, and returns the completed template registry.
func (b *Bundle) Compile() (*template.Registry, error) {
	if b.err != nil {
		return nil, b.err
	}

	// Compile all the soy (globals are already parsed)
	var registry = template.Registry{}
	for _, soyfile := range b.files {
		var tree, err = parse.SoyFile(soyfile.name, soyfile.content)
		if err != nil {
			return nil, err
		}
		if err = registry.Add(tree); err != nil {
			return nil, err
		}
	}

	// Apply the post-parse processing
	if err := parsepasses.CheckDataRefs(registry); err != nil {
		return nil, err
	}
	if err := parsepasses.SetGlobals(registry, b.globals); err != nil {
		return nil, err
	}
	parsepasses.ProcessMessages(registry)

	if b.watcher != nil {
		go b.recompiler(&registry)
	}
	return &registry, nil
}

// CompileToTofu returns a soyhtml.Tofu object that allows you to render soy
// templates to HTML.
func (b *Bundle) CompileToTofu() (*soyhtml.Tofu, error) {
	var registry, err = b.Compile()
	// TODO: Verify all used funcs exist and have the right # args.
	return soyhtml.NewTofu(registry), err
}

func (b *Bundle) recompiler(reg *template.Registry) {
	for {
		select {
		case ev := <-b.watcher.Event:
			// If it's a rename, then fsnotify has removed the watch.
			// Add it back, after a delay.
			if ev.IsRename() || ev.IsDelete() {
				time.Sleep(10 * time.Millisecond)
				if err := b.watcher.Watch(ev.Name); err != nil {
					Logger.Println(err)
				}
			}

			// Recompile all the soy.
			var bundle = NewBundle().
				AddGlobalsMap(b.globals)
			for _, soyfile := range b.files {
				bundle.AddTemplateFile(soyfile.name)
			}
			var registry, err = bundle.Compile()
			if err != nil {
				Logger.Println(err)
				continue
			}

			if b.recompilationCallback != nil {
				b.recompilationCallback(registry)
			}

			// update the existing template registry.
			// (this is not goroutine-safe, but that seems ok for a development aid,
			// as long as it works in practice)
			*reg = *registry
			Logger.Printf("update successful (%v)", ev)

		case err := <-b.watcher.Error:
			// Nothing to do with errors
			Logger.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package delete_vm_test

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	testhelperscpi "github.com/maximilien/bosh-softlayer-cpi/test_helpers"
	slclient "github.com/maximilien/softlayer-go/client"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

const configPath = "test_fixtures/cpi_methods/config.json"

var _ = Describe("BOSH Director Level Integration for delete_vm", func() {
	var (
		err error

		client softlayer.Client

		username, apiKey string

		accountService      softlayer.SoftLayer_Account_Service
		virtualGuestService softlayer.SoftLayer_Virtual_Guest_Service

		virtualGuest  datatypes.SoftLayer_Virtual_Guest
		createdSshKey datatypes.SoftLayer_Security_Ssh_Key

		rootTemplatePath, tmpConfigPath, strVGID string

		replacementMap map[string]string

		output map[string]interface{}
	)

	BeforeEach(func() {
		username = os.Getenv("SL_USERNAME")
		Expect(username).ToNot(Equal(""), "username cannot be empty, set SL_USERNAME")

		apiKey = os.Getenv("SL_API_KEY")
		Expect(apiKey).ToNot(Equal(""), "apiKey cannot be empty, set SL_API_KEY")

		client = slclient.NewSoftLayerClient(username, apiKey)
		Expect(client).ToNot(BeNil())

		accountService, err = testhelpers.CreateAccountService()
		Expect(err).ToNot(HaveOccurred())

		virtualGuestService, err = testhelpers.CreateVirtualGuestService()
		Expect(err).ToNot(HaveOccurred())

		testhelpers.TIMEOUT = 35 * time.Minute
		testhelpers.POLLING_INTERVAL = 10 * time.Second

		pwd, err := os.Getwd()
		Expect(err).ToNot(HaveOccurred())
		rootTemplatePath = filepath.Join(pwd, "..", "..")

		tmpConfigPath, err = testhelperscpi.CreateTmpConfigPath(rootTemplatePath, configPath, username, apiKey)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		err = os.RemoveAll(tmpConfigPath)
		Expect(err).ToNot(HaveOccurred())
	})

	Context("delete_vm with a valid VM ID", func() {
		BeforeEach(func() {
			err = testhelpers.FindAndDeleteTestSshKeys()
			Expect(err).ToNot(HaveOccurred())

			createdSshKey, _ = testhelpers.CreateTestSshKey()
			testhelpers.WaitForCreatedSshKeyToBePresent(createdSshKey.Id)

			virtualGuest = testhelpers.CreateVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{createdSshKey})

			testhelpers.WaitForVirtualGuestToBeRunning(virtualGuest.Id)
			testhelpers.WaitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			strVGID = strconv.Itoa(virtualGuest.Id)

			replacementMap = map[string]string{
				"ID":           strVGID,
				"DirectorUuid": "fake-director-uuid",
			}
		})

		AfterEach(func() {
			testhelpers.DeleteSshKey(createdSshKey.Id)
		})

		It("deletes the VM susseccfully", func() {
			jsonPayload, err := testhelperscpi.GenerateCpiJsonPayload("delete_vm", rootTemplatePath, replacementMap)
			Expect(err).ToNot(HaveOccurred())

			outputBytes, err := testhelperscpi.RunCpi(rootTemplatePath, tmpConfigPath, jsonPayload)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(outputBytes, &output)
			Expect(err).ToNot(HaveOccurred())
			Expect(output["error"]).To(BeNil())
		})

	})

	Context("delete_vm with an invalid VM ID", func() {
		BeforeEach(func() {
			replacementMap = map[string]string{
				"ID":           "123456",
				"DirectorUuid": "fake-director-uuid",
			}
		})

		It("fails to delete the VM", func() {
			jsonPayload, err := testhelperscpi.GenerateCpiJsonPayload("delete_vm", rootTemplatePath, replacementMap)
			Expect(err).ToNot(HaveOccurred())

			outputBytes, err := testhelperscpi.RunCpi(rootTemplatePath, tmpConfigPath, jsonPayload)
			Expect(err).ToNot(HaveOccurred())
			err = json.Unmarshal(outputBytes, &output)
			Expect(err).ToNot(HaveOccurred())
			Expect(output["error"]).ToNot(BeNil())
		})
	})

})
<commit_msg>changed delete_vm_test to wait for no active transactions before deleting ssh key<commit_after>package delete_vm_test

import (
	"encoding/json"
	"os"
	"path/filepath"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	testhelperscpi "github.com/maximilien/bosh-softlayer-cpi/test_helpers"
	slclient "github.com/maximilien/softlayer-go/client"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	softlayer "github.com/maximilien/softlayer-go/softlayer"
	testhelpers "github.com/maximilien/softlayer-go/test_helpers"
)

const configPath = "test_fixtures/cpi_methods/config.json"

var _ = Describe("BOSH Director Level Integration for delete_vm", func() {
	var (
		err error

		client softlayer.Client

		username, apiKey string

		accountService      softlayer.SoftLayer_Account_Service
		virtualGuestService softlayer.SoftLayer_Virtual_Guest_Service

		virtualGuest  datatypes.SoftLayer_Virtual_Guest
		createdSshKey datatypes.SoftLayer_Security_Ssh_Key

		rootTemplatePath, tmpConfigPath, strVGID string

		replacementMap map[string]string

		output map[string]interface{}
	)

	BeforeEach(func() {
		username = os.Getenv("SL_USERNAME")
		Expect(username).ToNot(Equal(""), "username cannot be empty, set SL_USERNAME")

		apiKey = os.Getenv("SL_API_KEY")
		Expect(apiKey).ToNot(Equal(""), "apiKey cannot be empty, set SL_API_KEY")

		client = slclient.NewSoftLayerClient(username, apiKey)
		Expect(client).ToNot(BeNil())

		accountService, err = testhelpers.CreateAccountService()
		Expect(err).ToNot(HaveOccurred())

		virtualGuestService, err = testhelpers.CreateVirtualGuestService()
		Expect(err).ToNot(HaveOccurred())

		testhelpers.TIMEOUT = 35 * time.Minute
		testhelpers.POLLING_INTERVAL = 10 * time.Second

		pwd, err := os.Getwd()
		Expect(err).ToNot(HaveOccurred())
		rootTemplatePath = filepath.Join(pwd, "..", "..")

		tmpConfigPath, err = testhelperscpi.CreateTmpConfigPath(rootTemplatePath, configPath, username, apiKey)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		err = os.RemoveAll(tmpConfigPath)
		Expect(err).ToNot(HaveOccurred())
	})

	Context("delete_vm with a valid VM ID", func() {
		BeforeEach(func() {
			err = testhelpers.FindAndDeleteTestSshKeys()
			Expect(err).ToNot(HaveOccurred())

			createdSshKey, _ = testhelpers.CreateTestSshKey()
			testhelpers.WaitForCreatedSshKeyToBePresent(createdSshKey.Id)

			virtualGuest = testhelpers.CreateVirtualGuestAndMarkItTest([]datatypes.SoftLayer_Security_Ssh_Key{createdSshKey})

			testhelpers.WaitForVirtualGuestToBeRunning(virtualGuest.Id)
			testhelpers.WaitForVirtualGuestToHaveNoActiveTransactions(virtualGuest.Id)

			strVGID = strconv.Itoa(virtualGuest.Id)

			replacementMap = map[string]string{
				"ID":           strVGID,
				"DirectorUuid": "fake-director-uuid",
			}
		})

		AfterEach(func() {
			// assume errors are either because service was already deleted or will be caught later anyway
			testhelpers.WaitForVirtualGuestToHaveNoActiveTransactionsOrToErr(virtualGuest.Id)
			testhelpers.DeleteSshKey(createdSshKey.Id)
		})

		It("deletes the VM susseccfully", func() {
			jsonPayload, err := testhelperscpi.GenerateCpiJsonPayload("delete_vm", rootTemplatePath, replacementMap)
			Expect(err).ToNot(HaveOccurred())

			outputBytes, err := testhelperscpi.RunCpi(rootTemplatePath, tmpConfigPath, jsonPayload)
			Expect(err).ToNot(HaveOccurred())

			err = json.Unmarshal(outputBytes, &output)
			Expect(err).ToNot(HaveOccurred())
			Expect(output["error"]).To(BeNil())
		})

	})

	Context("delete_vm with an invalid VM ID", func() {
		BeforeEach(func() {
			replacementMap = map[string]string{
				"ID":           "123456",
				"DirectorUuid": "fake-director-uuid",
			}
		})

		It("fails to delete the VM", func() {
			jsonPayload, err := testhelperscpi.GenerateCpiJsonPayload("delete_vm", rootTemplatePath, replacementMap)
			Expect(err).ToNot(HaveOccurred())

			outputBytes, err := testhelperscpi.RunCpi(rootTemplatePath, tmpConfigPath, jsonPayload)
			Expect(err).ToNot(HaveOccurred())
			err = json.Unmarshal(outputBytes, &output)
			Expect(err).ToNot(HaveOccurred())
			Expect(output["error"]).ToNot(BeNil())
		})
	})

})
<|endoftext|>