<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// TODO(shyamjvs): Make this exporter work for master too, currently facing
// gcloud auth error when run from within a pod on the master.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/golang/glog"
)

// Initialize the log exporter's configuration related flags.
var (
	nodeName             = flag.String("node-name", "", "Name of the node this log exporter is running on")
	gcsPath              = flag.String("gcs-path", "", "Path to the GCS directory under which to upload logs, for eg: gs://my-logs-bucket/logs")
	journalPath          = flag.String("journal-path", "/var/log/journal", "Path where the systemd journal dir is mounted")
	cloudProvider        = flag.String("cloud-provider", "", "Cloud provider for this node (gce/gke/aws/kubemark/..)")
	gcloudAuthFilePath   = flag.String("gcloud-auth-file-path", "/etc/service-account/service-account.json", "Path to gcloud service account file, for authenticating gsutil to write to GCS bucket")
	enableHollowNodeLogs = flag.Bool("enable-hollow-node-logs", false, "Enable uploading hollow node logs too. Relevant only for kubemark nodes")
	sleepDuration        = flag.Duration("sleep-duration", 60*time.Second, "Duration to sleep before exiting with success. Useful for making pods schedule with hard anti-affinity when run as a job on a k8s cluster")
	dumpSystemdJournal   = flag.Bool("dump-systemd-journal", false, "Whether to dump the full systemd journal")
)

var (
	localLogPath = "/var/log"

	// Node-type specific logfiles.
	// Currently we only handle nodes, and neglect master.
	nodeLogs = []string{"kube-proxy", "node-problem-detector", "fluentd"}

	// Cloud provider specific logfiles.
	awsLogs      = []string{"cloud-init-output"}
	gceLogs      = []string{"startupscript"}
	kubemarkLogs = []string{"*-hollow-node-*"}

	// System services/kernel related logfiles.
	kernelLog            = "kern"
	initdLogs            = []string{"docker"}
	supervisordLogs      = []string{"kubelet", "supervisor/supervisord", "supervisor/kubelet-stdout", "supervisor/kubelet-stderr", "supervisor/docker-stdout", "supervisor/docker-stderr"}
	systemdServices      = []string{"kern", "kubelet", "docker"}
	systemdSetupServices = []string{"kube-node-installation", "kube-node-configuration"}
	nodeSystemdServices  = []string{"node-problem-detector"}
)

// Check if the config provided through the flags take valid values.
func checkConfigValidity() error {
	glog.Info("Verifying if a valid config has been provided through the flags")
	if *nodeName == "" {
		return fmt.Errorf("Flag --node-name has its value unspecified")
	}
	if *gcsPath == "" {
		return fmt.Errorf("Flag --gcs-path has its value unspecified")
	}
	if _, err := os.Stat(*gcloudAuthFilePath); err != nil {
		return fmt.Errorf("Could not find the gcloud service account file: %v", err)
	} else {
		glog.Infof("Running gcloud auth activate-service-account --key-file=%s\n", *gcloudAuthFilePath)
		cmd := exec.Command("gcloud", "auth", "activate-service-account", "--key-file="+*gcloudAuthFilePath)
		var stderr, stdout bytes.Buffer
		cmd.Stderr, cmd.Stdout = &stderr, &stdout
		err = cmd.Run()
		glog.Infof("Stdout:\n%s\n", stdout.String())
		glog.Infof("Stderr:\n%s\n", stderr.String())
		if err != nil {
			return fmt.Errorf("Failed to activate gcloud service account: %v", err)
		}
	}
	return nil
}

// Create logfile for systemd service in outputDir with the given journalctl outputMode.
func createSystemdLogfile(service string, outputMode string, outputDir string) error {
	// Generate the journalctl command.
	journalCmdArgs := []string{fmt.Sprintf("--output=%v", outputMode), "-D", *journalPath}
	if service == "kern" {
		journalCmdArgs = append(journalCmdArgs, "-k")
	} else {
		journalCmdArgs = append(journalCmdArgs, "-u", fmt.Sprintf("%v.service", service))
	}
	cmd := exec.Command("journalctl", journalCmdArgs...)

	// Run the command and record the output to a file.
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("Journalctl command for '%v' service failed: %v", service, err)
	}
	logfile := filepath.Join(outputDir, service+".log")
	if err := ioutil.WriteFile(logfile, output, 0444); err != nil {
		return fmt.Errorf("Writing to file of journalctl logs for '%v' service failed: %v", service, err)
	}
	return nil
}

// createFullSystemdLogfile creates logfile for full systemd journal in the outputDir.
func createFullSystemdLogfile(outputDir string) error {
	cmd := exec.Command("journalctl", "--output=short-precise", "-D", *journalPath)
	// Run the command and record the output to a file.
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("Journalctl command failed: %v", err)
	}
	logfile := filepath.Join(outputDir, "systemd.log")
	if err := ioutil.WriteFile(logfile, output, 0444); err != nil {
		return fmt.Errorf("Writing full journalctl logs to file failed: %v", err)
	}
	return nil
}

// Create logfiles for systemd services in outputDir.
func createSystemdLogfiles(outputDir string) {
	services := append(systemdServices, nodeSystemdServices...)
	for _, service := range services {
		if err := createSystemdLogfile(service, "cat", outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
	// Service logs specific to VM setup.
	for _, service := range systemdSetupServices {
		if err := createSystemdLogfile(service, "short-precise", outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
	if *dumpSystemdJournal {
		if err := createFullSystemdLogfile(outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
}

// Copy logfiles specific to this node based on the cloud-provider, system services, etc
// to a temporary directory. Also create logfiles for systemd services if journalctl is present.
// We do not expect this function to see an error.
func prepareLogfiles(logDir string) {
	glog.Info("Preparing logfiles relevant to this node")
	logfiles := nodeLogs[:]

	switch *cloudProvider {
	case "gce", "gke":
		logfiles = append(logfiles, gceLogs...)
	case "aws":
		logfiles = append(logfiles, awsLogs...)
	default:
		glog.Errorf("Unknown cloud provider '%v' provided, skipping any provider specific logs", *cloudProvider)
	}

	// Grab kubemark logs too, if asked for.
	if *enableHollowNodeLogs {
		logfiles = append(logfiles, kubemarkLogs...)
	}

	// Select system/service specific logs.
	if _, err := os.Stat("/workspace/etc/systemd/journald.conf"); err == nil {
		glog.Info("Journalctl found on host. Collecting systemd logs")
		createSystemdLogfiles(logDir)
	} else {
		glog.Infof("Journalctl not found on host (%v). Collecting supervisord logs instead", err)
		logfiles = append(logfiles, kernelLog)
		logfiles = append(logfiles, initdLogs...)
		logfiles = append(logfiles, supervisordLogs...)
	}

	// Copy all the logfiles that exist, to logDir.
	for _, logfile := range logfiles {
		logfileFullPath := filepath.Join(localLogPath, logfile+".log*") // Append .log* to copy rotated logs too.
		cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("cp %v %v", logfileFullPath, logDir))
		if err := cmd.Run(); err != nil {
			glog.Warningf("Failed to copy any logfiles with pattern '%v': %v", logfileFullPath, err)
		}
	}
}

func uploadLogfilesToGCS(logDir string) error {
	cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("ls %v/*", logDir))
	if output, err := cmd.Output(); err != nil {
		return fmt.Errorf("Could not list any logfiles: %v", err)
	} else {
		glog.Infof("List of logfiles available: %v", string(output))
	}

	gcsLogPath := *gcsPath + "/" + *nodeName
	glog.Infof("Uploading logfiles to GCS at path '%v'", gcsLogPath)
	var err error
	for uploadAttempt := 0; uploadAttempt < 3; uploadAttempt++ {
		// Upload the files with compression (-z) and parallelism (-m) for speeding
		// up, and set their ACL to make them publicly readable.
		cmd := exec.Command("gsutil", "-m", "-q", "cp", "-a", "public-read", "-c",
			"-z", "log,txt,xml", logDir+"/*", gcsLogPath)
		if err = cmd.Run(); err != nil {
			glog.Errorf("Attempt %v to upload to GCS failed: %v", uploadAttempt, err)
			continue
		}
		return writeSuccessMarkerFile()
	}
	return fmt.Errorf("Multiple attempts of gsutil failed, the final one due to: %v", err)
}

// Write a marker file to GCS named after this node to indicate logexporter's success.
// The directory to which we write this file can then be used as a registry to quickly
// fetch the list of nodes on which logexporter succeeded.
func writeSuccessMarkerFile() error {
	markerFilePath := *gcsPath + "/logexported-nodes-registry/" + *nodeName + ".txt"
	cmd := exec.Command("gsutil", "-q", "cp", "-a", "public-read", "-", markerFilePath)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("Failed to get stdin pipe to write marker file: %v", err)
	}
	io.WriteString(stdin, "")
	stdin.Close()
	if err = cmd.Run(); err != nil {
		return fmt.Errorf("Failed to write marker file to GCS: %v", err)
	}
	return nil
}

func main() {
	flag.Parse()
	if err := checkConfigValidity(); err != nil {
		glog.Fatalf("Bad config provided: %v", err)
	}

	localTmpLogPath, err := ioutil.TempDir("/tmp", "k8s-systemd-logs")
	if err != nil {
		glog.Fatalf("Could not create temporary dir locally for copying logs: %v", err)
	}
	defer os.RemoveAll(localTmpLogPath)

	prepareLogfiles(localTmpLogPath)
	if err := uploadLogfilesToGCS(localTmpLogPath); err != nil {
		glog.Fatalf("Could not upload logs to GCS: %v", err)
	}
	glog.Info("Logs successfully uploaded")

	glog.Infof("Entering sleep for a duration of %v seconds", *sleepDuration)
	time.Sleep(*sleepDuration)
}
<commit_msg>logexporter: log stdin, stdout of gsutil command<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// TODO(shyamjvs): Make this exporter work for master too, currently facing
// gcloud auth error when run from within a pod on the master.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/glog"
)

// Initialize the log exporter's configuration related flags.
var (
	nodeName             = flag.String("node-name", "", "Name of the node this log exporter is running on")
	gcsPath              = flag.String("gcs-path", "", "Path to the GCS directory under which to upload logs, for eg: gs://my-logs-bucket/logs")
	journalPath          = flag.String("journal-path", "/var/log/journal", "Path where the systemd journal dir is mounted")
	cloudProvider        = flag.String("cloud-provider", "", "Cloud provider for this node (gce/gke/aws/kubemark/..)")
	gcloudAuthFilePath   = flag.String("gcloud-auth-file-path", "/etc/service-account/service-account.json", "Path to gcloud service account file, for authenticating gsutil to write to GCS bucket")
	enableHollowNodeLogs = flag.Bool("enable-hollow-node-logs", false, "Enable uploading hollow node logs too. Relevant only for kubemark nodes")
	sleepDuration        = flag.Duration("sleep-duration", 60*time.Second, "Duration to sleep before exiting with success. Useful for making pods schedule with hard anti-affinity when run as a job on a k8s cluster")
	dumpSystemdJournal   = flag.Bool("dump-systemd-journal", false, "Whether to dump the full systemd journal")
)

var (
	localLogPath = "/var/log"

	// Node-type specific logfiles.
	// Currently we only handle nodes, and neglect master.
	nodeLogs = []string{"kube-proxy", "node-problem-detector", "fluentd"}

	// Cloud provider specific logfiles.
	awsLogs      = []string{"cloud-init-output"}
	gceLogs      = []string{"startupscript"}
	kubemarkLogs = []string{"*-hollow-node-*"}

	// System services/kernel related logfiles.
	kernelLog            = "kern"
	initdLogs            = []string{"docker"}
	supervisordLogs      = []string{"kubelet", "supervisor/supervisord", "supervisor/kubelet-stdout", "supervisor/kubelet-stderr", "supervisor/docker-stdout", "supervisor/docker-stderr"}
	systemdServices      = []string{"kern", "kubelet", "docker"}
	systemdSetupServices = []string{"kube-node-installation", "kube-node-configuration"}
	nodeSystemdServices  = []string{"node-problem-detector"}
)

// Check if the config provided through the flags take valid values.
func checkConfigValidity() error {
	glog.Info("Verifying if a valid config has been provided through the flags")
	if *nodeName == "" {
		return fmt.Errorf("Flag --node-name has its value unspecified")
	}
	if *gcsPath == "" {
		return fmt.Errorf("Flag --gcs-path has its value unspecified")
	}
	if _, err := os.Stat(*gcloudAuthFilePath); err != nil {
		return fmt.Errorf("Could not find the gcloud service account file: %v", err)
	} else if err := runCommand("gcloud", "auth", "activate-service-account", "--key-file="+*gcloudAuthFilePath); err != nil {
		return fmt.Errorf("Failed to activate gcloud service account: %v", err)
	}
	return nil
}

// Create logfile for systemd service in outputDir with the given journalctl outputMode.
func createSystemdLogfile(service string, outputMode string, outputDir string) error {
	// Generate the journalctl command.
	journalCmdArgs := []string{fmt.Sprintf("--output=%v", outputMode), "-D", *journalPath}
	if service == "kern" {
		journalCmdArgs = append(journalCmdArgs, "-k")
	} else {
		journalCmdArgs = append(journalCmdArgs, "-u", fmt.Sprintf("%v.service", service))
	}
	cmd := exec.Command("journalctl", journalCmdArgs...)

	// Run the command and record the output to a file.
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("Journalctl command for '%v' service failed: %v", service, err)
	}
	logfile := filepath.Join(outputDir, service+".log")
	if err := ioutil.WriteFile(logfile, output, 0444); err != nil {
		return fmt.Errorf("Writing to file of journalctl logs for '%v' service failed: %v", service, err)
	}
	return nil
}

// createFullSystemdLogfile creates logfile for full systemd journal in the outputDir.
func createFullSystemdLogfile(outputDir string) error {
	cmd := exec.Command("journalctl", "--output=short-precise", "-D", *journalPath)
	// Run the command and record the output to a file.
	output, err := cmd.Output()
	if err != nil {
		return fmt.Errorf("Journalctl command failed: %v", err)
	}
	logfile := filepath.Join(outputDir, "systemd.log")
	if err := ioutil.WriteFile(logfile, output, 0444); err != nil {
		return fmt.Errorf("Writing full journalctl logs to file failed: %v", err)
	}
	return nil
}

// Create logfiles for systemd services in outputDir.
func createSystemdLogfiles(outputDir string) {
	services := append(systemdServices, nodeSystemdServices...)
	for _, service := range services {
		if err := createSystemdLogfile(service, "cat", outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
	// Service logs specific to VM setup.
	for _, service := range systemdSetupServices {
		if err := createSystemdLogfile(service, "short-precise", outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
	if *dumpSystemdJournal {
		if err := createFullSystemdLogfile(outputDir); err != nil {
			glog.Warningf("Failed to record journalctl logs: %v", err)
		}
	}
}

// Copy logfiles specific to this node based on the cloud-provider, system services, etc
// to a temporary directory. Also create logfiles for systemd services if journalctl is present.
// We do not expect this function to see an error.
func prepareLogfiles(logDir string) {
	glog.Info("Preparing logfiles relevant to this node")
	logfiles := nodeLogs[:]

	switch *cloudProvider {
	case "gce", "gke":
		logfiles = append(logfiles, gceLogs...)
	case "aws":
		logfiles = append(logfiles, awsLogs...)
	default:
		glog.Errorf("Unknown cloud provider '%v' provided, skipping any provider specific logs", *cloudProvider)
	}

	// Grab kubemark logs too, if asked for.
	if *enableHollowNodeLogs {
		logfiles = append(logfiles, kubemarkLogs...)
	}

	// Select system/service specific logs.
	if _, err := os.Stat("/workspace/etc/systemd/journald.conf"); err == nil {
		glog.Info("Journalctl found on host. Collecting systemd logs")
		createSystemdLogfiles(logDir)
	} else {
		glog.Infof("Journalctl not found on host (%v). Collecting supervisord logs instead", err)
		logfiles = append(logfiles, kernelLog)
		logfiles = append(logfiles, initdLogs...)
		logfiles = append(logfiles, supervisordLogs...)
	}

	// Copy all the logfiles that exist, to logDir.
	for _, logfile := range logfiles {
		logfileFullPath := filepath.Join(localLogPath, logfile+".log*") // Append .log* to copy rotated logs too.
		cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("cp %v %v", logfileFullPath, logDir))
		if err := cmd.Run(); err != nil {
			glog.Warningf("Failed to copy any logfiles with pattern '%v': %v", logfileFullPath, err)
		}
	}
}

func uploadLogfilesToGCS(logDir string) error {
	cmd := exec.Command("/bin/sh", "-c", fmt.Sprintf("ls %v/*", logDir))
	if output, err := cmd.Output(); err != nil {
		return fmt.Errorf("Could not list any logfiles: %v", err)
	} else {
		glog.Infof("List of logfiles available: %v", string(output))
	}

	gcsLogPath := *gcsPath + "/" + *nodeName
	glog.Infof("Uploading logfiles to GCS at path '%v'", gcsLogPath)
	var err error
	for uploadAttempt := 0; uploadAttempt < 3; uploadAttempt++ {
		// Upload the files with compression (-z) and parallelism (-m) for speeding
		// up, and set their ACL to make them publicly readable.
		if err = runCommand("gsutil", "-m", "-q", "cp", "-a", "public-read", "-c",
			"-z", "log,txt,xml", logDir+"/*", gcsLogPath); err != nil {
			glog.Errorf("Attempt %v to upload to GCS failed: %v", uploadAttempt, err)
			continue
		}
		return writeSuccessMarkerFile()
	}
	return fmt.Errorf("Multiple attempts of gsutil failed, the final one due to: %v", err)
}

// Write a marker file to GCS named after this node to indicate logexporter's success.
// The directory to which we write this file can then be used as a registry to quickly
// fetch the list of nodes on which logexporter succeeded.
func writeSuccessMarkerFile() error {
	markerFilePath := *gcsPath + "/logexported-nodes-registry/" + *nodeName + ".txt"
	cmd := exec.Command("gsutil", "-q", "cp", "-a", "public-read", "-", markerFilePath)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return fmt.Errorf("Failed to get stdin pipe to write marker file: %v", err)
	}
	io.WriteString(stdin, "")
	stdin.Close()
	if err = cmd.Run(); err != nil {
		return fmt.Errorf("Failed to write marker file to GCS: %v", err)
	}
	return nil
}

func runCommand(name string, arg ...string) error {
	glog.Infof("Running: %s %s", name, strings.Join(arg, " "))
	cmd := exec.Command(name, arg...)
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	err := cmd.Run()
	glog.Infof("Stdout:\n%s\n", stdout.String())
	glog.Infof("Stderr:\n%s\n", stderr.String())
	return err
}

func main() {
	flag.Parse()
	if err := checkConfigValidity(); err != nil {
		glog.Fatalf("Bad config provided: %v", err)
	}

	localTmpLogPath, err := ioutil.TempDir("/tmp", "k8s-systemd-logs")
	if err != nil {
		glog.Fatalf("Could not create temporary dir locally for copying logs: %v", err)
	}
	defer os.RemoveAll(localTmpLogPath)

	prepareLogfiles(localTmpLogPath)
	if err := uploadLogfilesToGCS(localTmpLogPath); err != nil {
		glog.Fatalf("Could not upload logs to GCS: %v", err)
	}
	glog.Info("Logs successfully uploaded")

	glog.Infof("Entering sleep for a duration of %v seconds", *sleepDuration)
	time.Sleep(*sleepDuration)
}
<|endoftext|>


<commit_before>package envetcd

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/coreos/go-etcd/etcd"
	"github.com/zvelo/zvelo-services/util"
)

// order of precedence:
// global < system < service < host

// Regexp for invalid characters in keys
var invalidRegexp = regexp.MustCompile(`[^a-zA-Z0-9_]`)

// Config contains all of the parameters needed to run GetKeyPairs
type Config struct {
	Etcd              *util.EtcdConfig
	Sanitize          bool
	Upcase            bool
	UseDefaultGateway bool
	Prefix            string
	System            string
	Service           string
	Hostname          string
}

var (
	etcdKeyTemplates = [...]string{
		"{{.Prefix}}/global",
		"{{if .System}}{{.Prefix}}/system/{{.System}}{{end}}",
		"{{if .Service}}{{.Prefix}}/service/{{.Service}}{{end}}",
		"{{if .Hostname}}{{.Prefix}}/host/{{.Hostname}}{{end}}",
	}

	gatewayIP *net.IP
)

// KeyPairs is a slice of KeyPair pointers
type KeyPairs map[string]string

func init() {
	if ip, err := util.DefaultRoute(); err != nil {
		log.Printf("[INFO] envetcd error getting default gateway: %v\n", err)
	} else {
		gatewayIP = &ip
	}
}

// Set modifies the current environment with variables retrieved from etcd. Set
// will not overwrite existing variables.
// On linux systems, the default gateway will be automatically used as the etcd
// endpoint.
// If $ETCD_ENDPOINT is set, it will override the default gateway.
// $ETCD_ENDPOINT should look like "http://127.0.0.1:4001".
// service should be set by the application calling Set and not derived from
// an environment variable.
// Set will also use some other environment variables if they exist.
// $ETCD_PREFIX defaults to "/config"
// $HOSTNAME will be honored if it is set.
// An error is returned only if there was an actual error. Inability to
// determine the etcd endpoint as tolerated and not considered an error. In this
// case Set will simply not do anyting and it is up to the application to ensure
// that it has been configured properly through other means such as environment
// variables or command line flags.
func Set(service string) error {
	util.InitLogger("WARN")

	etcdEndpoint := os.Getenv("ETCD_ENDPOINT")

	useSync := true
	if len(os.Getenv("ETCD_NO_SYNC")) > 0 {
		useSync = false
	}
	useDefaultGateway := true
	if len(os.Getenv("ENVETCD_USE_DEFAULT_GATEWAY")) > 0 {
		if val, err := strconv.ParseBool(os.Getenv("ENVETCD_USE_DEFAULT_GATEWAY")); err != nil {
			log.Printf("[INFO] envetcd.Set could not parse $ENVETCD_USE_DEFAULT_GATEWAY, defaulting to true: %v\n", err)
		} else {
			useDefaultGateway = val
		}
	}

	if gatewayIP != nil && useDefaultGateway && len(etcdEndpoint) == 0 {
		etcdEndpoint = fmt.Sprintf("http://%s:4001", gatewayIP.String())
	}

	if len(etcdEndpoint) == 0 {
		log.Println("[INFO] envetcd.Set returned after it could not determine the etcd endpoint")
		return nil
	}

	config := &Config{
		Etcd: &util.EtcdConfig{
			Peers: []string{etcdEndpoint},
			Sync:  useSync,
		},
		Sanitize:          true,
		Upcase:            true,
		UseDefaultGateway: useDefaultGateway,
		Prefix:            os.Getenv("ETCD_PREFIX"),
		Service:           service,
		Hostname:          os.Getenv("HOSTNAME"),
	}

	if len(config.Etcd.Peers[0]) == 0 {
		config.Etcd.Peers[0] = "http://127.0.0.1:4001"
	}

	if len(config.Prefix) == 0 {
		config.Prefix = "/config"
	}

	keyPairs, err := GetKeyPairs(config)
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] envetcd: %v => %v\n", "ETCD_ENDPOINT", etcdEndpoint)
	keyPairs["ETCD_ENDPOINT"] = etcdEndpoint

	for key, value := range keyPairs {
		if len(os.Getenv(key)) == 0 {
			os.Setenv(key, value)
		}
	}

	return nil
}

// GetKeyPairs takes a given config and client, and returns all key pairs
func GetKeyPairs(config *Config) (KeyPairs, error) {
	const noSort = false
	const recursive = true

	if len(config.Service) > 0 && len(config.System) == 0 {
		if index := strings.Index(config.Service, "-"); index != -1 {
			config.System = config.Service[:index]
		}
	}

	client, err := util.GetEtcdClient(config.Etcd)
	if err != nil {
		return nil, err
	}

	keyPairs := KeyPairs{}

	// For each etcd directory key template
	// fetch from etcd and add to keypairs
	for i, tmpl := range etcdKeyTemplates {
		t := template.Must(template.New("etcdKey" + strconv.Itoa(i)).Parse(tmpl))
		var buf bytes.Buffer
		err := t.Execute(&buf, config)
		if err != nil {
			continue
		}

		if buf.Len() == 0 {
			continue
		}

		dir := buf.String()

		resp, err := client.Get(dir, noSort, recursive)
		if err != nil {
			continue
		}

		addKeyPair(config, keyPairs, dir, resp.Node)
	}

	keyPairs["ETCD_PEERS"] = strings.Join(client.GetCluster(), ", ")

	// Expose --service --system and --hostname command variables to the child process' emvironment,
	// ignoring if these are not set in etcd (or set in etcd. Command line arguments take precedence.)
	if "" != config.Service {
		keyPairs["ENVETCD_SERVICE"] = config.Service
	}

	if "" != config.System {
		keyPairs["ENVETCD_SYSTEM"] = config.System
	}

	if "" != config.Hostname {
		keyPairs["ENVETCD_HOSTNAME"] = config.Hostname
	}

	if config.UseDefaultGateway && gatewayIP != nil {
		keyPairs["ENVETCD_DEFAULT_GATEWAY"] = gatewayIP.String()
	}

	var keys []string
	for key := range keyPairs {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	util.InitLogger(keyPairs["LOG_LEVEL"])

	for _, key := range keys {
		log.Printf("[DEBUG] envetcd: %v => %v\n", key, keyPairs[key])
	}

	return keyPairs, nil
}

func addKeyPair(config *Config, keyPairs KeyPairs, dir string, node *etcd.Node) {
	if node.Dir {
		for _, child := range node.Nodes {
			addKeyPair(config, keyPairs, dir, child)
		}
		return
	}

	key := strings.TrimPrefix(node.Key, dir) // strip the prefix directory from the key
	if key == "" {
		return
	}

	key = strings.TrimLeft(key, "/")         // strip any leading slashes
	key = strings.Replace(key, "/", "_", -1) // convert any remaining slashes to underscores (for any nested keys)

	if config.Sanitize {
		key = invalidRegexp.ReplaceAllString(key, "_")
	}

	if config.Upcase {
		key = strings.ToUpper(key)
	}

	keyPairs[key] = node.Value
}
<commit_msg>Initial log level for Set<commit_after>package envetcd

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"text/template"

	"github.com/coreos/go-etcd/etcd"
	"github.com/zvelo/zvelo-services/util"
)

// order of precedence:
// global < system < service < host

// Regexp for invalid characters in keys
var invalidRegexp = regexp.MustCompile(`[^a-zA-Z0-9_]`)

// Config contains all of the parameters needed to run GetKeyPairs
type Config struct {
	Etcd              *util.EtcdConfig
	Sanitize          bool
	Upcase            bool
	UseDefaultGateway bool
	Prefix            string
	System            string
	Service           string
	Hostname          string
}

var (
	etcdKeyTemplates = [...]string{
		"{{.Prefix}}/global",
		"{{if .System}}{{.Prefix}}/system/{{.System}}{{end}}",
		"{{if .Service}}{{.Prefix}}/service/{{.Service}}{{end}}",
		"{{if .Hostname}}{{.Prefix}}/host/{{.Hostname}}{{end}}",
	}

	gatewayIP *net.IP
)

// KeyPairs is a slice of KeyPair pointers
type KeyPairs map[string]string

func init() {
	if ip, err := util.DefaultRoute(); err != nil {
		log.Printf("[INFO] envetcd error getting default gateway: %v\n", err)
	} else {
		gatewayIP = &ip
	}
}

// Set modifies the current environment with variables retrieved from etcd. Set
// will not overwrite existing variables.
// On linux systems, the default gateway will be automatically used as the etcd
// endpoint.
// If $ETCD_ENDPOINT is set, it will override the default gateway.
// $ETCD_ENDPOINT should look like "http://127.0.0.1:4001".
// service should be set by the application calling Set and not derived from
// an environment variable.
// Set will also use some other environment variables if they exist.
// $ETCD_PREFIX defaults to "/config"
// $HOSTNAME will be honored if it is set.
// An error is returned only if there was an actual error. Inability to
// determine the etcd endpoint as tolerated and not considered an error. In this
// case Set will simply not do anyting and it is up to the application to ensure
// that it has been configured properly through other means such as environment
// variables or command line flags.
func Set(service string) error {
	logLevel := os.Getenv("LOG_LEVEL")
	if logLevel == "" {
		logLevel = "WARN"
	}
	util.InitLogger(logLevel)

	etcdEndpoint := os.Getenv("ETCD_ENDPOINT")

	useSync := true
	if len(os.Getenv("ETCD_NO_SYNC")) > 0 {
		useSync = false
	}
	useDefaultGateway := true
	if len(os.Getenv("ENVETCD_USE_DEFAULT_GATEWAY")) > 0 {
		if val, err := strconv.ParseBool(os.Getenv("ENVETCD_USE_DEFAULT_GATEWAY")); err != nil {
			log.Printf("[INFO] envetcd.Set could not parse $ENVETCD_USE_DEFAULT_GATEWAY, defaulting to true: %v\n", err)
		} else {
			useDefaultGateway = val
		}
	}

	if gatewayIP != nil && useDefaultGateway && len(etcdEndpoint) == 0 {
		etcdEndpoint = fmt.Sprintf("http://%s:4001", gatewayIP.String())
	}

	if len(etcdEndpoint) == 0 {
		log.Println("[INFO] envetcd.Set returned after it could not determine the etcd endpoint")
		return nil
	}

	config := &Config{
		Etcd: &util.EtcdConfig{
			Peers: []string{etcdEndpoint},
			Sync:  useSync,
		},
		Sanitize:          true,
		Upcase:            true,
		UseDefaultGateway: useDefaultGateway,
		Prefix:            os.Getenv("ETCD_PREFIX"),
		Service:           service,
		Hostname:          os.Getenv("HOSTNAME"),
	}

	if len(config.Etcd.Peers[0]) == 0 {
		config.Etcd.Peers[0] = "http://127.0.0.1:4001"
	}

	if len(config.Prefix) == 0 {
		config.Prefix = "/config"
	}

	keyPairs, err := GetKeyPairs(config)
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] envetcd: %v => %v\n", "ETCD_ENDPOINT", etcdEndpoint)
	keyPairs["ETCD_ENDPOINT"] = etcdEndpoint

	for key, value := range keyPairs {
		if len(os.Getenv(key)) == 0 {
			os.Setenv(key, value)
		}
	}

	return nil
}

// GetKeyPairs takes a given config and client, and returns all key pairs
func GetKeyPairs(config *Config) (KeyPairs, error) {
	const noSort = false
	const recursive = true

	if len(config.Service) > 0 && len(config.System) == 0 {
		if index := strings.Index(config.Service, "-"); index != -1 {
			config.System = config.Service[:index]
		}
	}

	client, err := util.GetEtcdClient(config.Etcd)
	if err != nil {
		return nil, err
	}

	keyPairs := KeyPairs{}

	// For each etcd directory key template
	// fetch from etcd and add to keypairs
	for i, tmpl := range etcdKeyTemplates {
		t := template.Must(template.New("etcdKey" + strconv.Itoa(i)).Parse(tmpl))
		var buf bytes.Buffer
		err := t.Execute(&buf, config)
		if err != nil {
			continue
		}

		if buf.Len() == 0 {
			continue
		}

		dir := buf.String()

		resp, err := client.Get(dir, noSort, recursive)
		if err != nil {
			continue
		}

		addKeyPair(config, keyPairs, dir, resp.Node)
	}

	keyPairs["ETCD_PEERS"] = strings.Join(client.GetCluster(), ", ")

	// Expose --service --system and --hostname command variables to the child process' emvironment,
	// ignoring if these are not set in etcd (or set in etcd. Command line arguments take precedence.)
	if "" != config.Service {
		keyPairs["ENVETCD_SERVICE"] = config.Service
	}

	if "" != config.System {
		keyPairs["ENVETCD_SYSTEM"] = config.System
	}

	if "" != config.Hostname {
		keyPairs["ENVETCD_HOSTNAME"] = config.Hostname
	}

	if config.UseDefaultGateway && gatewayIP != nil {
		keyPairs["ENVETCD_DEFAULT_GATEWAY"] = gatewayIP.String()
	}

	var keys []string
	for key := range keyPairs {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	util.InitLogger(keyPairs["LOG_LEVEL"])

	for _, key := range keys {
		log.Printf("[DEBUG] envetcd: %v => %v\n", key, keyPairs[key])
	}

	return keyPairs, nil
}

func addKeyPair(config *Config, keyPairs KeyPairs, dir string, node *etcd.Node) {
	if node.Dir {
		for _, child := range node.Nodes {
			addKeyPair(config, keyPairs, dir, child)
		}
		return
	}

	key := strings.TrimPrefix(node.Key, dir) // strip the prefix directory from the key
	if key == "" {
		return
	}

	key = strings.TrimLeft(key, "/")         // strip any leading slashes
	key = strings.Replace(key, "/", "_", -1) // convert any remaining slashes to underscores (for any nested keys)

	if config.Sanitize {
		key = invalidRegexp.ReplaceAllString(key, "_")
	}

	if config.Upcase {
		key = strings.ToUpper(key)
	}

	keyPairs[key] = node.Value
}
<|endoftext|>


<commit_before>// Copyright 2014 Codehack.com All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package environ is a system to implement the similar functionality as environment
// lists found in all Unix-based OS'. Basically, all the functions found at
// "man 3 setenv" from a Unix prompt. With some additions to support Go's basic types.
package environ

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

// Env is an environment list with name=value values.
type Env []string

// envPool manages Env objects so they can be reused.
var envPool = sync.Pool{
	New: func() interface{} { return new(Env) },
}

// NewEnv returns a new Env list (or from pool).
func NewEnv() *Env { return envPool.Get().(*Env) }

// Free returns this Env e back into the pool for further use.
// It's recommended to use this function when we are done using e.
func (e *Env) Free() { envPool.Put(e) }

// Index returns the index of the first instance of name found in Env e, or
// -1 otherwise.
func (e *Env) Index(name string) int {
	prefix := name + "="
	for k, v := range *e {
		if strings.HasPrefix(v, prefix) {
			return k
		}
	}
	return -1
}

// Contains returns true if name is found in Env e.
func (e *Env) Contains(name string) bool {
	return e.Index(name) >= 0
}

// Get returns the string value matching name in Env e, or empty "" if not found.
func (e *Env) Get(name string) string {
	idx := e.Index(name)
	if idx == -1 {
		return ""
	}
	return (*e)[idx][len(name)+1:]
}

// Get returns the bool value matching name in Env e, or false if not found or the
// value is not a boolean.
func (e *Env) GetBool(name string) bool {
	v, err := strconv.ParseBool(e.Get(name))
	if err != nil {
		return false
	}
	return v
}

// Get returns the float value matching name in Env e, or 0 if not found or the
// value is not a float.
func (e *Env) GetFloat(name string) float64 {
	v, err := strconv.ParseFloat(e.Get(name), 0)
	if err != nil {
		return 0.0
	}
	return v
}

// Get returns the int value matching name in Env e, or 0 if not found or the
// value is not an int.
func (e *Env) GetInt(name string) int {
	v, err := strconv.Atoi(e.Get(name))
	if err != nil {
		return 0
	}
	return v
}

// Put inserts name=value into Env e if a value matching name is not found.
// Otherwise it replaces the current name=value with the new value.
func (e *Env) Put(name, value string) {
	idx := e.Index(name)
	if idx == -1 {
		*e = append(*e, name+"="+value)
		return
	}
	(*e)[idx] = name + "=" + value
}

// Set converts a known value type v to a string value and puts name=value in Env e.
// It returns nil if successful, error otherwise.
// Value types supported: bool, byte, uint (32 & 64), int (32 & 64), float32,
// float64 and string.
func (e *Env) Set(name string, v interface{}) error {
	var value string
	switch v.(type) {
	case bool:
		value = strconv.FormatBool(v.(bool))
	case byte:
		value = string(v.(byte))
	case uint, uint32, uint64:
		value = strconv.FormatUint(uint64(v.(uint)), 10)
	case int, int32, int64:
		value = strconv.FormatInt(int64(v.(int)), 10)
	case float32, float64:
		value = strconv.FormatFloat(v.(float64), 'f', -1, 64)
	case string:
		value = v.(string)
	default:
		t := reflect.TypeOf(v)
		return errors.New("environ.Set: " + name + ": value " + strconv.Quote(t.String()) + " is not supported")
	}
	e.Put(name, value)
	return nil
}

// Unset empties all values with found with name. Note that this function won't
// remove name=value pairs, just set value to empty string "".
func (e *Env) Unset(name string) {
	// we dont really delete, just empty
	e.Put(name, "")
}

// Print will print all name=value pairs to stdout.
func (e *Env) Print() {
	for k, v := range *e {
		fmt.Println(k, "=", v)
	}
}
<commit_msg>added a fix to int/uint conversion panic<commit_after>// Copyright 2014 Codehack.com All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

// Package environ is a system to implement the similar functionality as environment
// lists found in all Unix-based OS'. Basically, all the functions found at
// "man 3 setenv" from a Unix prompt. With some additions to support Go's basic types.
package environ

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"sync"
)

// Env is an environment list with name=value values.
type Env []string

// envPool manages Env objects so they can be reused.
var envPool = sync.Pool{
	New: func() interface{} { return new(Env) },
}

// NewEnv returns a new Env list (or from pool).
func NewEnv() *Env { return envPool.Get().(*Env) }

// Free returns this Env e back into the pool for further use.
// It's recommended to use this function when we are done using e.
func (e *Env) Free() { envPool.Put(e) }

// Index returns the index of the first instance of name found in Env e, or
// -1 otherwise.
func (e *Env) Index(name string) int {
	prefix := name + "="
	for k, v := range *e {
		if strings.HasPrefix(v, prefix) {
			return k
		}
	}
	return -1
}

// Contains returns true if name is found in Env e.
func (e *Env) Contains(name string) bool {
	return e.Index(name) >= 0
}

// Get returns the string value matching name in Env e, or empty "" if not found.
func (e *Env) Get(name string) string {
	idx := e.Index(name)
	if idx == -1 {
		return ""
	}
	return (*e)[idx][len(name)+1:]
}

// Get returns the bool value matching name in Env e, or false if not found or the
// value is not a boolean.
func (e *Env) GetBool(name string) bool {
	v, err := strconv.ParseBool(e.Get(name))
	if err != nil {
		return false
	}
	return v
}

// Get returns the float value matching name in Env e, or 0 if not found or the
// value is not a float.
func (e *Env) GetFloat(name string) float64 {
	v, err := strconv.ParseFloat(e.Get(name), 0)
	if err != nil {
		return 0.0
	}
	return v
}

// Get returns the int value matching name in Env e, or 0 if not found or the
// value is not an int.
func (e *Env) GetInt(name string) int {
	v, err := strconv.Atoi(e.Get(name))
	if err != nil {
		return 0
	}
	return v
}

// Put inserts name=value into Env e if a value matching name is not found.
// Otherwise it replaces the current name=value with the new value.
func (e *Env) Put(name, value string) {
	idx := e.Index(name)
	if idx == -1 {
		*e = append(*e, name+"="+value)
		return
	}
	(*e)[idx] = name + "=" + value
}

// Set converts a known value type v to a string value and puts name=value in Env e.
// It returns nil if successful, error otherwise.
// Value types supported: bool, byte, uint (32 & 64), int (32 & 64), float32,
// float64 and string.
func (e *Env) Set(name string, v interface{}) error {
	var value string
	switch v.(type) {
	case bool:
		value = strconv.FormatBool(v.(bool))
	case byte:
		value = string(v.(byte))
	case uint, uint32:
		value = strconv.FormatUint(uint64(v.(uint)), 10)
	case uint64:
		value = strconv.FormatUint(v.(uint64), 10)
	case int, int32:
		value = strconv.FormatInt(int64(v.(int)), 10)
	case int64:
		value = strconv.FormatInt(v.(int64), 10)
	case float32, float64:
		value = strconv.FormatFloat(v.(float64), 'f', -1, 64)
	case string:
		value = v.(string)
	default:
		t := reflect.TypeOf(v)
		return errors.New("environ.Set: " + name + ": value " + strconv.Quote(t.String()) + " is not supported")
	}
	e.Put(name, value)
	return nil
}

// Unset empties all values with found with name. Note that this function won't
// remove name=value pairs, just set value to empty string "".
func (e *Env) Unset(name string) {
	// we dont really delete, just empty
	e.Put(name, "")
}

// Print will print all name=value pairs to stdout.
func (e *Env) Print() {
	for k, v := range *e {
		fmt.Println(k, "=", v)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

func containerBackupsGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	cname := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, cname)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	recursion := util.IsRecursionRequest(r)

	c, err := containerLoadByProjectAndName(d.State(), project, cname)
	if err != nil {
		return SmartError(err)
	}

	backups, err := c.Backups()
	if err != nil {
		return SmartError(err)
	}

	resultString := []string{}
	resultMap := []*api.ContainerBackup{}

	for _, backup := range backups {
		if !recursion {
			url := fmt.Sprintf("/%s/containers/%s/backups/%s",
				version.APIVersion, cname, strings.Split(backup.name, "/")[1])
			resultString = append(resultString, url)
		} else {
			render := backup.Render()
			resultMap = append(resultMap, render)
		}
	}

	if !recursion {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func containerBackupsPost(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	c, err := containerLoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return SmartError(err)
	}

	req := api.ContainerBackupsPost{
		ExpiryDate: time.Now().Add(30 * time.Minute), // default expiry time of 30 minutes
	}
	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	if req.Name == "" {
		// come up with a name
		backups, err := c.Backups()
		if err != nil {
			return BadRequest(err)
		}

		base := name + shared.SnapshotDelimiter + "backup"
		length := len(base)
		max := 0

		for _, backup := range backups {
			// Ignore backups not containing base
			if !strings.HasPrefix(backup.name, base) {
				continue
			}

			substr := backup.name[length:]
			var num int
			count, err := fmt.Sscanf(substr, "%d", &num)
			if err != nil || count != 1 {
				continue
			}
			if num >= max {
				max = num + 1
			}
		}

		req.Name = fmt.Sprintf("backup%d", max)
	}

	// Validate the name
	if strings.Contains(req.Name, "/") {
		return BadRequest(fmt.Errorf("Backup names may not contain slashes"))
	}

	fullName := name + shared.SnapshotDelimiter + req.Name

	backup := func(op *operation) error {
		args := db.ContainerBackupArgs{
			Name:             fullName,
			ContainerID:      c.Id(),
			CreationDate:     time.Now(),
			ExpiryDate:       req.ExpiryDate,
			ContainerOnly:    req.ContainerOnly,
			OptimizedStorage: req.OptimizedStorage,
		}

		err := backupCreate(d.State(), args, c)
		if err != nil {
			return errors.Wrap(err, "Create backup")
		}

		return nil
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}
	resources["backups"] = []string{req.Name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupCreate, resources, nil, backup, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, backup.Render())
}

func containerBackupPost(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	req := api.ContainerBackupPost{}
	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Validate the name
	if strings.Contains(req.Name, "/") {
		return BadRequest(fmt.Errorf("Backup names may not contain slashes"))
	}

	oldName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, oldName)
	if err != nil {
		SmartError(err)
	}

	newName := name + shared.SnapshotDelimiter + req.Name

	rename := func(op *operation) error {
		err := backup.Rename(newName)
		if err != nil {
			return err
		}

		return nil
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupRename, resources, nil, rename, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupDelete(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	remove := func(op *operation) error {
		err := backup.Delete()
		if err != nil {
			return err
		}

		return nil
	}

	resources := map[string][]string{}
	resources["container"] = []string{name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupRemove, resources, nil, remove, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupExportGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	ent := fileResponseEntry{
		path: shared.VarPath("backups", backup.name),
	}

	return FileResponse(r, []fileResponseEntry{ent}, nil, false)
}
<commit_msg>backup: Allow backups to not expire<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/version"
)

func containerBackupsGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	cname := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, cname)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	recursion := util.IsRecursionRequest(r)

	c, err := containerLoadByProjectAndName(d.State(), project, cname)
	if err != nil {
		return SmartError(err)
	}

	backups, err := c.Backups()
	if err != nil {
		return SmartError(err)
	}

	resultString := []string{}
	resultMap := []*api.ContainerBackup{}

	for _, backup := range backups {
		if !recursion {
			url := fmt.Sprintf("/%s/containers/%s/backups/%s",
				version.APIVersion, cname, strings.Split(backup.name, "/")[1])
			resultString = append(resultString, url)
		} else {
			render := backup.Render()
			resultMap = append(resultMap, render)
		}
	}

	if !recursion {
		return SyncResponse(true, resultString)
	}

	return SyncResponse(true, resultMap)
}

func containerBackupsPost(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	c, err := containerLoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return SmartError(err)
	}

	rj := shared.Jmap{}
	err = json.NewDecoder(r.Body).Decode(&rj)
	if err != nil {
		return InternalError(err)
	}

	expiry, _ := rj.GetString("expiry")
	if expiry == "" {
		// Disable expiration by setting it to zero time
		rj["expiry"] = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)
	}

	// Create body with correct expiry
	body, err := json.Marshal(rj)
	if err != nil {
		return InternalError(err)
	}

	req := api.ContainerBackupsPost{}

	err = json.Unmarshal(body, &req)
	if err != nil {
		return BadRequest(err)
	}

	if req.Name == "" {
		// come up with a name
		backups, err := c.Backups()
		if err != nil {
			return BadRequest(err)
		}

		base := name + shared.SnapshotDelimiter + "backup"
		length := len(base)
		max := 0

		for _, backup := range backups {
			// Ignore backups not containing base
			if !strings.HasPrefix(backup.name, base) {
				continue
			}

			substr := backup.name[length:]
			var num int
			count, err := fmt.Sscanf(substr, "%d", &num)
			if err != nil || count != 1 {
				continue
			}
			if num >= max {
				max = num + 1
			}
		}

		req.Name = fmt.Sprintf("backup%d", max)
	}

	// Validate the name
	if strings.Contains(req.Name, "/") {
		return BadRequest(fmt.Errorf("Backup names may not contain slashes"))
	}

	fullName := name + shared.SnapshotDelimiter + req.Name

	backup := func(op *operation) error {
		args := db.ContainerBackupArgs{
			Name:             fullName,
			ContainerID:      c.Id(),
			CreationDate:     time.Now(),
			ExpiryDate:       req.ExpiryDate,
			ContainerOnly:    req.ContainerOnly,
			OptimizedStorage: req.OptimizedStorage,
		}

		err := backupCreate(d.State(), args, c)
		if err != nil {
			return errors.Wrap(err, "Create backup")
		}

		return nil
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}
	resources["backups"] = []string{req.Name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupCreate, resources, nil, backup, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, backup.Render())
}

func containerBackupPost(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	req := api.ContainerBackupPost{}
	err = json.NewDecoder(r.Body).Decode(&req)
	if err != nil {
		return BadRequest(err)
	}

	// Validate the name
	if strings.Contains(req.Name, "/") {
		return BadRequest(fmt.Errorf("Backup names may not contain slashes"))
	}

	oldName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, oldName)
	if err != nil {
		SmartError(err)
	}

	newName := name + shared.SnapshotDelimiter + req.Name

	rename := func(op *operation) error {
		err := backup.Rename(newName)
		if err != nil {
			return err
		}

		return nil
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupRename, resources, nil, rename, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupDelete(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	remove := func(op *operation) error {
		err := backup.Delete()
		if err != nil {
			return err
		}

		return nil
	}

	resources := map[string][]string{}
	resources["container"] = []string{name}

	op, err := operationCreate(d.cluster, project, operationClassTask,
		db.OperationBackupRemove, resources, nil, remove, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}

func containerBackupExportGet(d *Daemon, r *http.Request) Response {
	project := projectParam(r)
	name := mux.Vars(r)["name"]
	backupName := mux.Vars(r)["backupName"]

	// Handle requests targeted to a container on a different node
	response, err := ForwardedResponseIfContainerIsRemote(d, r, project, name)
	if err != nil {
		return SmartError(err)
	}
	if response != nil {
		return response
	}

	fullName := name + shared.SnapshotDelimiter + backupName
	backup, err := backupLoadByName(d.State(), project, fullName)
	if err != nil {
		return SmartError(err)
	}

	ent := fileResponseEntry{
		path: shared.VarPath("backups", backup.name),
	}

	return FileResponse(r, []fileResponseEntry{ent}, nil, false)
}
<|endoftext|>


<commit_before>// Copyright 2016 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/miekg/dns"

	"github.com/dullgiulio/kuradns/gen"
)

// A record containse the source and destination from the generator,
// a precomputed A and CNAME record and a pointer to the source that
// generated this entry.
type record struct {
	shost, dhost host
	a            dns.RR
	cname        dns.RR
	source       *source
}

// Allocate a new record. For the A record, a resoled IP is needed.
func makeRecord(shost, dhost host, cname bool, ip net.IP, ttl time.Duration, src *source) record {
	r := record{
		shost: shost,
		dhost: dhost,
		a: &dns.A{
			Hdr: dns.RR_Header{
				Name:   shost.dns(),
				Rrtype: dns.TypeA,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl.Seconds()),
			},
			A: ip,
		},
		source: src,
	}
	if cname {
		r.cname = &dns.CNAME{
			Hdr: dns.RR_Header{
				Name:   shost.dns(),
				Rrtype: dns.TypeCNAME,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl.Seconds()),
			},
			Target: host(dhost).dns(),
		}
	}
	return r
}

// target returns the human DNS representation of the destination/target or a record.
func (r record) target() string {
	return r.dhost.browser()
}

// String representation of a record.
func (r record) String() string {
	return fmt.Sprintf("[%s %s]", r.source.String(), r.shost.dns())
}

// Collection of records.
type records struct {
	recs []record
}

// Allocate a new collection of records.
func newRecords() *records {
	return &records{recs: make([]record, 0)}
}

// String representation of a collection of records.
func (r *records) String() string {
	return fmt.Sprintf("%s", r.recs)
}

// deleteSource removes all record that were added by source s.
// Returns the number of records left.
func (r *records) deleteSource(s *source) int {
	res := make([]record, 0)
	for _, rec := range r.recs {
		if rec.source.name != s.name {
			res = append(res, rec)
		}
	}
	// Free up empty arrays
	if len(res) == 0 {
		res = nil
	}
	r.recs = res
	return len(r.recs)
}

// pushFront adds a record to the collection.
func (r *records) pushFront(rec record) {
	r.recs = append([]record{rec}, r.recs...)
}

// clone is the utility function to duplicate a collection.
func (r *records) clone() *records {
	nr := &records{
		recs: make([]record, len(r.recs)),
	}
	for i := 0; i < len(r.recs); i++ {
		nr.recs[i] = r.recs[i]
	}
	return nr
}

// A host is a FQDN or common representation of a DNS address.
type host string

// browser returns the FQDN without the trailing dot.
func (h host) browser() string {
	return strings.TrimSuffix(string(h), ".")
}

// dns returns the FQDN.
func (h host) dns() string {
	return dns.Fqdn(string(h))
}

// hasSuffix returns true if h has suffix h2.
func (h host) hasSuffix(h2 host) bool {
	return strings.HasSuffix(h.browser(), h2.browser())
}

// A repository maps hosts to the records that can resolve them (record collection).
// repository is not thread safe.
type repository map[string]*records

// makeRepository allocates a new repository.
func makeRepository() repository {
	return make(map[string]*records)
}

// add inserts record rec for host as the default record.
func (r repository) add(host host, rec record) {
	key := host.browser()
	recs, ok := r[key]
	if !ok {
		recs = newRecords()
	}
	recs.pushFront(rec)
	r[key] = recs
}

// deleteSource removes all records that were inserted by source s.
// Entries that result in no records are removed completely.
func (r repository) deleteSource(s *source) {
	emptyKeys := make([]string, 0)
	for k, recs := range r {
		if recs.deleteSource(s) == 0 {
			emptyKeys = append(emptyKeys, k)
		}
	}
	// remove entries which have no values
	for _, k := range emptyKeys {
		delete(r, k)
	}
}

// updateSource removes and generate again all records for source src.
func (r repository) updateSource(src *source, zone host, ttl time.Duration) {
	res := newResolver(src, ttl, 6)

	go func() {
		for {
			rentry := src.gen.Generate()
			if rentry.IsEmpty() {
				close(res.rentries)
				// Free up resources used by the generator
				src.gen = nil
				return
			}
			src := host(rentry.Source)
			if !src.hasSuffix(zone) {
				log.Printf("[error] repository: domain %s is not inside zone %s, skipped", src.dns(), zone.dns())
				continue
			}
			res.rentries <- rentry
		}
	}()

	for rec := range res.records {
		r.add(rec.shost, rec)
	}
}

// resolver is a worker that resolves strings into IPs.
type resolver struct {
	cname    bool
	src      *source
	ttl      time.Duration
	rentries chan gen.RawEntry
	records  chan record
	wg       sync.WaitGroup
}

// Allocate a new resolver. Subsequent records will be generated with ttl set as given here.
// workers is number of workers to be run in parallel.
func newResolver(src *source, ttl time.Duration, workers int) *resolver {
	r := &resolver{
		src:      src,
		ttl:      ttl,
		rentries: make(chan gen.RawEntry),
		records:  make(chan record),
	}
	r.wg.Add(workers)
	for i := 0; i < workers; i++ {
		go r.run()
	}
	go func() {
		r.wg.Wait()
		close(r.records)
	}()
	return r
}

// run resolves incoming entries and emits records. It is called automatically.
func (r *resolver) run() {
	for rentry := range r.rentries {
		var cname bool
		ip := net.ParseIP(rentry.Target)
		if ip == nil {
			var err error
			// It's an hostname: resolve it and make both A and CNAME records
			ip, err = lookup(rentry.Target)
			if err != nil {
				log.Printf("[error] repository: failed lookup of %s: %s", rentry.Target, err)
				continue
			}
			cname = true
		}
		r.records <- makeRecord(host(rentry.Source), host(rentry.Target), cname, ip, r.ttl, r.src)
	}
	r.wg.Done()
}

// lookup is a utility to lookup an IP for host (standard format).
func lookup(host string) (net.IP, error) {
	var ip net.IP
	iplist, err := net.LookupIP(host)
	if err == nil {
		ip = iplist[0]
	}
	return ip, err
}

// get returns the default record for host or nil if not found.
func (r repository) get(host host) *record {
	rs, ok := r[host.browser()]
	if !ok {
		return nil
	}
	return &rs.recs[0]
}

// clone duplicates the whole repository.
func (r repository) clone() repository {
	nr := makeRepository()
	for k, recs := range r {
		nr[k] = recs.clone()
	}
	return nr
}

// WriteTo writes the repository contents in hosts format to w.
func (r repository) WriteTo(w io.Writer) error {
	for key, rs := range r {
		if len(rs.recs) == 0 {
			continue
		}
		// TODO: Display IP not aliased target.
		if _, err := fmt.Fprintf(w, "%s\t%s\n", rs.recs[0].target(), key); err != nil {
			return err
		}
		for i := 1; i < len(rs.recs); i++ {
			if _, err := fmt.Fprintf(w, "# %s\t%s\n", rs.recs[i].target(), key); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>dump: Sort list of hosts<commit_after>// Copyright 2016 Giulio Iotti. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/miekg/dns"

	"github.com/dullgiulio/kuradns/gen"
)

// A record containse the source and destination from the generator,
// a precomputed A and CNAME record and a pointer to the source that
// generated this entry.
type record struct {
	shost, dhost host
	a            dns.RR
	cname        dns.RR
	source       *source
}

// Allocate a new record. For the A record, a resoled IP is needed.
func makeRecord(shost, dhost host, cname bool, ip net.IP, ttl time.Duration, src *source) record {
	r := record{
		shost: shost,
		dhost: dhost,
		a: &dns.A{
			Hdr: dns.RR_Header{
				Name:   shost.dns(),
				Rrtype: dns.TypeA,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl.Seconds()),
			},
			A: ip,
		},
		source: src,
	}
	if cname {
		r.cname = &dns.CNAME{
			Hdr: dns.RR_Header{
				Name:   shost.dns(),
				Rrtype: dns.TypeCNAME,
				Class:  dns.ClassINET,
				Ttl:    uint32(ttl.Seconds()),
			},
			Target: host(dhost).dns(),
		}
	}
	return r
}

// target returns the human DNS representation of the destination/target or a record.
func (r record) target() string {
	return r.dhost.browser()
}

// String representation of a record.
func (r record) String() string {
	return fmt.Sprintf("[%s %s]", r.source.String(), r.shost.dns())
}

// Collection of records.
type records struct {
	recs []record
}

// Allocate a new collection of records.
func newRecords() *records {
	return &records{recs: make([]record, 0)}
}

// String representation of a collection of records.
func (r *records) String() string {
	return fmt.Sprintf("%s", r.recs)
}

// deleteSource removes all record that were added by source s.
// Returns the number of records left.
func (r *records) deleteSource(s *source) int {
	res := make([]record, 0)
	for _, rec := range r.recs {
		if rec.source.name != s.name {
			res = append(res, rec)
		}
	}
	// Free up empty arrays
	if len(res) == 0 {
		res = nil
	}
	r.recs = res
	return len(r.recs)
}

// pushFront adds a record to the collection.
func (r *records) pushFront(rec record) {
	r.recs = append([]record{rec}, r.recs...)
}

// clone is the utility function to duplicate a collection.
func (r *records) clone() *records {
	nr := &records{
		recs: make([]record, len(r.recs)),
	}
	for i := 0; i < len(r.recs); i++ {
		nr.recs[i] = r.recs[i]
	}
	return nr
}

// A host is a FQDN or common representation of a DNS address.
type host string

// browser returns the FQDN without the trailing dot.
func (h host) browser() string {
	return strings.TrimSuffix(string(h), ".")
}

// dns returns the FQDN.
func (h host) dns() string {
	return dns.Fqdn(string(h))
}

// hasSuffix returns true if h has suffix h2.
func (h host) hasSuffix(h2 host) bool {
	return strings.HasSuffix(h.browser(), h2.browser())
}

// A repository maps hosts to the records that can resolve them (record collection).
// repository is not thread safe.
type repository map[string]*records

// makeRepository allocates a new repository.
func makeRepository() repository {
	return make(map[string]*records)
}

// add inserts record rec for host as the default record.
func (r repository) add(host host, rec record) {
	key := host.browser()
	recs, ok := r[key]
	if !ok {
		recs = newRecords()
	}
	recs.pushFront(rec)
	r[key] = recs
}

// deleteSource removes all records that were inserted by source s.
// Entries that result in no records are removed completely.
func (r repository) deleteSource(s *source) {
	emptyKeys := make([]string, 0)
	for k, recs := range r {
		if recs.deleteSource(s) == 0 {
			emptyKeys = append(emptyKeys, k)
		}
	}
	// remove entries which have no values
	for _, k := range emptyKeys {
		delete(r, k)
	}
}

// updateSource removes and generate again all records for source src.
func (r repository) updateSource(src *source, zone host, ttl time.Duration) {
	res := newResolver(src, ttl, 6)

	go func() {
		for {
			rentry := src.gen.Generate()
			if rentry.IsEmpty() {
				close(res.rentries)
				// Free up resources used by the generator
				src.gen = nil
				return
			}
			src := host(rentry.Source)
			if !src.hasSuffix(zone) {
				log.Printf("[error] repository: domain %s is not inside zone %s, skipped", src.dns(), zone.dns())
				continue
			}
			res.rentries <- rentry
		}
	}()

	for rec := range res.records {
		r.add(rec.shost, rec)
	}
}

// resolver is a worker that resolves strings into IPs.
type resolver struct {
	cname    bool
	src      *source
	ttl      time.Duration
	rentries chan gen.RawEntry
	records  chan record
	wg       sync.WaitGroup
}

// Allocate a new resolver. Subsequent records will be generated with ttl set as given here.
// workers is number of workers to be run in parallel.
func newResolver(src *source, ttl time.Duration, workers int) *resolver {
	r := &resolver{
		src:      src,
		ttl:      ttl,
		rentries: make(chan gen.RawEntry),
		records:  make(chan record),
	}
	r.wg.Add(workers)
	for i := 0; i < workers; i++ {
		go r.run()
	}
	go func() {
		r.wg.Wait()
		close(r.records)
	}()
	return r
}

// run resolves incoming entries and emits records. It is called automatically.
func (r *resolver) run() {
	for rentry := range r.rentries {
		var cname bool
		ip := net.ParseIP(rentry.Target)
		if ip == nil {
			var err error
			// It's an hostname: resolve it and make both A and CNAME records
			ip, err = lookup(rentry.Target)
			if err != nil {
				log.Printf("[error] repository: failed lookup of %s: %s", rentry.Target, err)
				continue
			}
			cname = true
		}
		r.records <- makeRecord(host(rentry.Source), host(rentry.Target), cname, ip, r.ttl, r.src)
	}
	r.wg.Done()
}

// lookup is a utility to lookup an IP for host (standard format).
func lookup(host string) (net.IP, error) {
	var ip net.IP
	iplist, err := net.LookupIP(host)
	if err == nil {
		ip = iplist[0]
	}
	return ip, err
}

// get returns the default record for host or nil if not found.
func (r repository) get(host host) *record {
	rs, ok := r[host.browser()]
	if !ok {
		return nil
	}
	return &rs.recs[0]
}

// clone duplicates the whole repository.
func (r repository) clone() repository {
	nr := makeRepository()
	for k, recs := range r {
		nr[k] = recs.clone()
	}
	return nr
}

// WriteTo writes the repository contents in hosts format to w.
func (r repository) WriteTo(w io.Writer) error {
	frs := makeFlatRecords()
	for key, rs := range r {
		if len(rs.recs) == 0 {
			continue
		}
		grpKey := rs.recs[0].target()
		for i := range rs.recs {
			frs = append(frs, newFlatRecord(i, grpKey, rs.recs[i].target(), key))
		}
	}
	sort.Sort(frs)
	for i := range frs {
		fmtstr := "%s\t%s\n"
		if frs[i].pos > 0 {
			fmtstr = "# %s\t%s\n"
		}
		if _, err := fmt.Fprintf(w, fmtstr, frs[i].dst, frs[i].src); err != nil {
			return err
		}
	}
	return nil
}

type flatRecord struct {
	pos int
	grp string
	dst string
	src string
}

type flatRecords []*flatRecord

func makeFlatRecords() flatRecords {
	return flatRecords(make([]*flatRecord, 0))
}

func newFlatRecord(i int, grp, dst, src string) *flatRecord {
	return &flatRecord{i, grp, dst, src}
}

func (f flatRecords) Len() int {
	return len(f)
}

func (f flatRecords) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

func (f flatRecords) Less(i, j int) bool {
	if f[i].dst < f[j].dst {
		return true
	}
	if f[i].dst > f[j].dst {
		return false
	}
	if f[i].grp < f[j].grp {
		return true
	}
	if f[i].grp > f[j].grp {
		return false
	}
	if f[i].pos < f[j].pos {
		return true
	}
	if f[i].pos > f[j].pos {
		return false
	}
	return f[i].src < f[j].src

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net"
	"os"
	"time"
)

type DataResponse struct {
	data []byte
	bytesRead int
	err error
}

func ReadToChannel(path string, dataChan, returnChan chan *DataResponse) {
	datares := <- returnChan
	file, err := os.Open(path)
	if err != nil {
		datares.err = err
		dataChan <- datares
		return
	}

	for {
		bytesRead, err := file.Read(datares.data)
		datares.bytesRead = bytesRead
		datares.err = err
		dataChan <- datares
		if err != nil {
			if err := file.Close(); err != nil {
				fmt.Println("Failed to close", path, err)
			}
			break
		}
		datares = <- returnChan
	}

}

func SendFile(path string, blocksize int, addr *net.UDPAddr) {
	started := time.Now()

	rrq, err := NewRRQresponse(addr, blocksize)
	if err != nil {
		fmt.Println("Failed to create rrq", err)
		return
	}
	fmt.Println("GET", path, "blocksize", rrq.blocksize)

	if err := rrq.WriteOACK(); err != nil {
		return
	}

	bufsize := 10
	dataChan := make(chan *DataResponse, bufsize)
	returnChan := make(chan *DataResponse, bufsize)

	datapool := make([]byte, blocksize*bufsize)
	for i := 0; i < bufsize; i++ {
		start := blocksize * i
		end := start + blocksize
		returnChan <- &DataResponse{data: datapool[start:end]}
	}

	go ReadToChannel(path, dataChan, returnChan)

	totalBytes := 0

	for {
		datares := <- dataChan
		totalBytes += datares.bytesRead
		if datares.err == io.EOF {
			rrq.Write(datares.data[:datares.bytesRead])
			rrq.End()
			break
		} else if datares.err != nil {
			fmt.Println("Failed to read data from file:", datares.err)
			break
		} else {
			rrq.Write(datares.data[:datares.bytesRead])
		}

		returnChan <- datares
	}

	took := time.Since(started)

	speed := float64(totalBytes) / took.Seconds() / 1024 / 1024

	fmt.Printf("Sent %v bytes in %v %f MB/s\n", totalBytes, took, speed)
	fmt.Println("END-GET", path)
	fmt.Println()
	fmt.Println()

}

func main() {
	fmt.Println("hello2")
	addr, err := net.ResolveUDPAddr("udp", ":1234")
	if err != nil {
		fmt.Println("Failed to resolve address", err)
		return
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		fmt.Println("Failed to listen UDP", err)
		return
	}

	data := make([]byte, 50)

	for {
		written, client_addr, err := conn.ReadFrom(data)
		if err != nil {
			fmt.Println("Failed to read data from client:", err)
			continue
		}

		raddr, err := net.ResolveUDPAddr("udp", client_addr.String())
		if err != nil {
			fmt.Println("Failed to resolve client address:", err)
			continue
		}

		request, err := ParseRequest(data[:written])
		if err != nil {
			fmt.Println("Failed to parse request:", err)
			continue
		}

		if request.opcode == RRQ {
			go SendFile(request.path, request.blocksize, raddr)
		} else {
			fmt.Println("Unimplemented opcode:", request.opcode)
		}

	}

}
<commit_msg>Revert "Small speed optimization by resusing slices"<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"io"
	"time"
)

func SendFile(path string, blocksize int, addr *net.UDPAddr) {
	started := time.Now()

	rrq, err := NewRRQresponse(addr, blocksize)
	if err != nil {
		fmt.Println("Failed to create rrq", err)
		return
	}
	fmt.Println("GET", path, "blocksize", rrq.blocksize)

	if err := rrq.WriteOACK(); err != nil {
		return
	}

	file, err := os.Open(path)
	if err != nil {
		fmt.Println("Failed to open file", path, err)
		// TODO: write error package
		return
	}

	b := make([]byte, rrq.blocksize)

	totalBytes := 0

	for {
		bytesRead, err := file.Read(b)
		totalBytes += bytesRead

		if err == io.EOF {
			rrq.Write(b[:bytesRead])
			rrq.End()
			break
		} else if err != nil {
			fmt.Println("Error while reading", file, err)
			// TODO: write error package
			return
		}

		rrq.Write(b[:bytesRead])
	}

	file.Close()
	took := time.Since(started)

	speed := float64(totalBytes) / took.Seconds() / 1024 / 1024

	fmt.Printf("Sent %v bytes in %v %f MB/s\n", totalBytes, took, speed)
	fmt.Println("END-GET", path)
	fmt.Println()
	fmt.Println()

}

func main() {
	fmt.Println("hello2")
	addr, err := net.ResolveUDPAddr("udp", ":1234")
	if err != nil {
		fmt.Println("Failed to resolve address", err)
		return
	}

	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		fmt.Println("Failed to listen UDP" , err)
		return
	}

	data := make([]byte, 50)

	for {
		written, client_addr, err := conn.ReadFrom(data)
		if err != nil {
			fmt.Println("Failed to read data from client:", err)
			continue
		}

		raddr, err := net.ResolveUDPAddr("udp", client_addr.String())
		if err != nil {
			fmt.Println("Failed to resolve client address:", err)
			continue
		}

		request, err := ParseRequest(data[:written])
		if err != nil {
			fmt.Println("Failed to parse request:", err)
			continue
		}

		if request.opcode == RRQ {
			go SendFile(request.path, request.blocksize, raddr)
		} else {
			fmt.Println("Unimplemented opcode:", request.opcode)
		}

	}

}
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: Tong Sun (c) 2015-17, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"bytes"
	"fmt"
	ht "html/template"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	tt "text/template"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

// common type for a *(text|html).Template value
type template interface {
	Execute(wr io.Writer, data interface{}) error
	ExecuteTemplate(wr io.Writer, name string, data interface{}) error
	Name() string
}

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

////////////////////////////////////////////////////////////////////////////
// Function definitions

// Generate2 will produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames (for data) respectively.
func Generate2(HTML bool, fileNameTempl string, fileNames ...string) (ret string) {
	Opts.TemplateFile = fileNameTempl

	for _, fileName := range fileNames {
		ret += Generate(HTML, fileName)
	}
	Opts.TemplateFile = ""
	return ret
}

// Generate0 will produce output according from driving data without a template file, using the string from strTempl as the template
func Generate0(HTML bool, strTempl string, fileName string) string {
	Opts.TemplateStr = strTempl
	ret := Generate(HTML, fileName)
	Opts.TemplateStr = ""
	return ret
}

// Generate will produce output from the template according to the corresponding driving data, fileName is for both template and data file name
func Generate(HTML bool, fileName string) string {
	var templates []string

	// Allow to use fileName with and without the @Opts.ExtYaml suffix.
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		fileName += Opts.ExtYaml
	}

	// Allow to use @Opts.TemplateFile without the @Opts.ExtTmpl suffix.
	if len(Opts.TemplateFile) > 0 {
		for _, template := range strings.Split(Opts.TemplateFile, ",") {
			if _, err := os.Stat(template); os.IsNotExist(err) {
				template += Opts.ExtTmpl
			}
			templates = append(templates, template)
		}
	} else if idx := strings.LastIndex(fileName, "."); idx > 0 {
		templates = []string{fileName[:idx] + Opts.ExtTmpl}
	} else {
		templates = []string{fileName + Opts.ExtTmpl}
	}

	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	m["ENV"] = env
	//fmt.Printf("] %+v\n", m)

	buf := new(bytes.Buffer)
	for _, template := range templates {
		t, err := ParseFiles(HTML, template)
		checkError(err)

		err = t.Execute(buf, m)
		checkError(err)
	}
	return buf.String()
}

// ParseFiles wraps parsing text or HTML template files into a single
// function, dictated by the first parameter "HTML".
// By Matt Harden @gmail.com
func ParseFiles(HTML bool, filenames ...string) (template, error) {
	var tname string

	if len(Opts.TemplateStr) > 0 {
		if HTML {
			tname = "HT"
		} else {
			tname = "TT"
		}
	} else if len(filenames) == 0 {
		return nil, fmt.Errorf("ParseFiles called without template filename")
	} else {
		tname = filepath.Base(filenames[0])
	}

	if HTML {
		// use html template
		htmlTemplate := ht.New(tname).Funcs(ht.FuncMap{
			"minus1": minus1,
			"dateI":  dateI,
			"year4":  year4,
			"cls2lc": cls2lc.String,
			"cls2uc": cls2uc.String,
			"cls2ss": cls2ss.String,
			"ck2lc":  ck2lc.String,
			"ck2uc":  ck2uc.String,
			"ck2ls":  ck2ls.String,
			"ck2ss":  ck2ss.String,
			"clc2ss": clc2ss.String,
			"cuc2ss": cuc2ss.String,
		})
		if len(Opts.TemplateStr) > 0 {
			return htmlTemplate.Parse(Opts.TemplateStr)
		}
		return htmlTemplate.ParseFiles(filenames...)
	}

	// use text template
	textTemplate := tt.New(tname).Funcs(tt.FuncMap{
		"eqf":      strings.EqualFold,
		"split":    strings.Fields,
		"minus1":   minus1,
		"dateI":    dateI,
		"year4":    year4,
		"replace":  replace,
		"replacec": replacec,
		"cls2lc":   cls2lc.String,
		"cls2uc":   cls2uc.String,
		"cls2ss":   cls2ss.String,
		"ck2lc":    ck2lc.String,
		"ck2uc":    ck2uc.String,
		"ck2ls":    ck2ls.String,
		"ck2ss":    ck2ss.String,
		"clc2ss":   clc2ss.String,
		"cuc2ss":   cuc2ss.String,
	})

	if len(Opts.TemplateStr) > 0 {
		return textTemplate.Parse(Opts.TemplateStr)
	}
	return textTemplate.ParseFiles(filenames...)
}

// Exit if error occurs
func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "[%s] Fatal error - %s\n", progname, err)
		os.Exit(1)
	}
}
<commit_msg>Resolves #6<commit_after>////////////////////////////////////////////////////////////////////////////
// Package: easygen
// Purpose: Easy to use universal code/text generator
// Authors: Tong Sun (c) 2015-17, All rights reserved
////////////////////////////////////////////////////////////////////////////

/*

Package easygen is an easy to use universal code/text generator library.

It can be used as a text or html generator for arbitrary purposes with arbitrary data and templates.

It can be used as a code generator, or anything that is structurally repetitive. Some command line parameter handling code generator are provided as examples, including the Go's built-in flag package, and the viper & cobra package.

Many examples have been provided to showcase its functionality, and different ways to use it.

*/
package easygen

import (
	"bytes"
	"fmt"
	ht "html/template"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	tt "text/template"

	"gopkg.in/yaml.v2"
)

////////////////////////////////////////////////////////////////////////////
// Constant and data type/structure definitions

// common type for a *(text|html).Template value
type template interface {
	Execute(wr io.Writer, data interface{}) error
	ExecuteTemplate(wr io.Writer, name string, data interface{}) error
	Name() string
}

////////////////////////////////////////////////////////////////////////////
// Global variables definitions

////////////////////////////////////////////////////////////////////////////
// Function definitions

// Generate2 will produce output according to the given template and driving data files,
// specified via fileNameTempl and fileNames (for data) respectively.
func Generate2(HTML bool, fileNameTempl string, fileNames ...string) (ret string) {
	Opts.TemplateFile = fileNameTempl

	for _, fileName := range fileNames {
		ret += Generate(HTML, fileName)
	}
	Opts.TemplateFile = ""
	return ret
}

// Generate0 will produce output according from driving data without a template file, using the string from strTempl as the template
func Generate0(HTML bool, strTempl string, fileName string) string {
	Opts.TemplateStr = strTempl
	ret := Generate(HTML, fileName)
	Opts.TemplateStr = ""
	return ret
}

// Generate will produce output from the template according to the corresponding driving data, fileName is for both template and data file name
func Generate(HTML bool, fileName string) string {
	var templates []string

	// Allow to use fileName with and without the @Opts.ExtYaml suffix.
	// If both <file>.yaml and <file> exist, prefer <file>.yaml.
	if _, err := os.Stat(fileName + Opts.ExtYaml); err == nil {
		fileName += Opts.ExtYaml
	} else if _, err = os.Stat(fileName); os.IsNotExist(err) {
		fileName += Opts.ExtYaml
	}

	// Allow to use @Opts.TemplateFile without the @Opts.ExtTmpl suffix.
	if len(Opts.TemplateFile) > 0 {
		for _, template := range strings.Split(Opts.TemplateFile, ",") {
			if _, err := os.Stat(template); os.IsNotExist(err) {
				template += Opts.ExtTmpl
			}
			templates = append(templates, template)
		}
	} else if idx := strings.LastIndex(fileName, "."); idx > 0 {
		templates = []string{fileName[:idx] + Opts.ExtTmpl}
	} else {
		templates = []string{fileName + Opts.ExtTmpl}
	}

	source, err := ioutil.ReadFile(fileName)
	checkError(err)

	m := make(map[interface{}]interface{})

	err = yaml.Unmarshal(source, &m)
	checkError(err)

	env := make(map[string]string)
	for _, e := range os.Environ() {
		sep := strings.Index(e, "=")
		env[e[0:sep]] = e[sep+1:]
	}
	m["ENV"] = env
	//fmt.Printf("] %+v\n", m)

	buf := new(bytes.Buffer)
	for _, template := range templates {
		t, err := ParseFiles(HTML, template)
		checkError(err)

		err = t.Execute(buf, m)
		checkError(err)
	}
	return buf.String()
}

// ParseFiles wraps parsing text or HTML template files into a single
// function, dictated by the first parameter "HTML".
// By Matt Harden @gmail.com
func ParseFiles(HTML bool, filenames ...string) (template, error) {
	var tname string

	if len(Opts.TemplateStr) > 0 {
		if HTML {
			tname = "HT"
		} else {
			tname = "TT"
		}
	} else if len(filenames) == 0 {
		return nil, fmt.Errorf("ParseFiles called without template filename")
	} else {
		tname = filepath.Base(filenames[0])
	}

	if HTML {
		// use html template
		htmlTemplate := ht.New(tname).Funcs(ht.FuncMap{
			"minus1": minus1,
			"dateI":  dateI,
			"year4":  year4,
			"cls2lc": cls2lc.String,
			"cls2uc": cls2uc.String,
			"cls2ss": cls2ss.String,
			"ck2lc":  ck2lc.String,
			"ck2uc":  ck2uc.String,
			"ck2ls":  ck2ls.String,
			"ck2ss":  ck2ss.String,
			"clc2ss": clc2ss.String,
			"cuc2ss": cuc2ss.String,
		})
		if len(Opts.TemplateStr) > 0 {
			return htmlTemplate.Parse(Opts.TemplateStr)
		}
		return htmlTemplate.ParseFiles(filenames...)
	}

	// use text template
	textTemplate := tt.New(tname).Funcs(tt.FuncMap{
		"eqf":      strings.EqualFold,
		"split":    strings.Fields,
		"minus1":   minus1,
		"dateI":    dateI,
		"year4":    year4,
		"replace":  replace,
		"replacec": replacec,
		"cls2lc":   cls2lc.String,
		"cls2uc":   cls2uc.String,
		"cls2ss":   cls2ss.String,
		"ck2lc":    ck2lc.String,
		"ck2uc":    ck2uc.String,
		"ck2ls":    ck2ls.String,
		"ck2ss":    ck2ss.String,
		"clc2ss":   clc2ss.String,
		"cuc2ss":   cuc2ss.String,
	})

	if len(Opts.TemplateStr) > 0 {
		return textTemplate.Parse(Opts.TemplateStr)
	}
	return textTemplate.ParseFiles(filenames...)
}

// Exit if error occurs
func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "[%s] Fatal error - %s\n", progname, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/flowcommerce/tools/executor"
)

func main() {
	executor := executor.Create("lib-reference-scala")
	executor = executor.Add("git checkout master")
	executor = executor.Add("git pull --rebase")
	executor = executor.Add("sbt test")

	executor = executor.Add("apidoc update")
	executor = executor.Add("go get -u github.com/flowcommerce/json-reference/common")
	executor = executor.Add("go run script/generate.go")
	executor = executor.Add("echo version := \\\"`sem-info tag next`\\\" > VERSION.sbt")
	executor = executor.Add("git commit -a -m 'Auto create version `sem-info tag next`'")
	executor = executor.Add("dev tag")

	executor = executor.Add("sbt publish")
	executor.Run()
}
<commit_msg>Run tests after update<commit_after>package main

import (
	"github.com/flowcommerce/tools/executor"
)

func main() {
	executor := executor.Create("lib-reference-scala")
	executor = executor.Add("git checkout master")
	executor = executor.Add("git pull --rebase")

	executor = executor.Add("apidoc update")
	executor = executor.Add("go get -u github.com/flowcommerce/json-reference/common")
	executor = executor.Add("go run script/generate.go")
	executor = executor.Add("sbt test")

	executor = executor.Add("echo version := \\\"`sem-info tag next`\\\" > VERSION.sbt")
	executor = executor.Add("git commit -a -m 'Auto create version `sem-info tag next`'")
	executor = executor.Add("dev tag")

	executor = executor.Add("sbt publish")
	executor.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"syscall"
	"testing"
	"time"
)

// Test that the handler with no args just returns the data sent
func TestHandlerNoArgs(t *testing.T) {
	handler := createHandler(nil)

	req, err := http.NewRequest("POST", "http://example.com/foo", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		t.Error(err)
	}

	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 200 || w.Body.String() != "{'foo': 3}" {
		t.Error("Did not get an idempotent response")
	}
}

// Test that the handler with no args returns 404 if we try a GET or
// POST with no data
func TestHandlerInvalidRequest(t *testing.T) {
	handler := createHandler(nil)

	req, err := http.NewRequest("GET", "http://foobar.com", nil)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 404 {
		t.Error("Expected an error code of 404 from a GET request.")
	}

	req, err = http.NewRequest("POST", "http://foobar.com", nil)
	if err != nil {
		t.Error(err)
	}
	handler(w, req)

	if w.Code != 404 {
		t.Error("Expected an error code of 404 from an empty POST request.")
	}
}

func makeTestHandler(response string) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		w.Write([]byte(response + buf.String()))
	}
}

// Test that the handler forwards data to servers *in order*
func TestHandlerForwardsData(t *testing.T) {
	ts1 := httptest.NewServer(http.HandlerFunc(makeTestHandler("foo")))
	defer ts1.Close()

	ts2 := httptest.NewServer(http.HandlerFunc(makeTestHandler("first")))
	defer ts2.Close()

	handler := createHandler([]string{ts1.URL, ts2.URL})
	req, err := http.NewRequest("POST", "http://foobar.com", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 200 || w.Body.String() != "firstfoo{'foo': 3}" {
		t.Errorf("Got '%s'; did not get expected response", w.Body.String())
	}
}

func TestMainRunnerExitsGracefully(t *testing.T) {
	// set the interrupt handler to go off after 50 milliseconds
	go func() {
		time.Sleep(1 * time.Second)
		syscall.Kill(syscall.Getpid(), syscall.SIGINT)
	}()
	go func() {
		time.Sleep(50 * time.Millisecond)
		resp, err := http.Post("http://localhost:9800", "application/json", bytes.NewBufferString("{'foo': 3}"))
		if err != nil {
			t.Error(err)
		}
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp.Body)
		result := buf.String()
		if result != "{'foo': 3}" {
			t.Errorf("Got '%s'; did not get expected response", result)
		}
	}()
	main()
}

// Benchmark to see how many requests the "empty" handler can handle
// Obviously, this is not accurate in production since we forward to
// multiple handlers.
func BenchmarkHandler(b *testing.B) {
	handler := createHandler(nil)

	req, err := http.NewRequest("POST", "http://example.com/foo", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		b.Error(err)
	}

	w := httptest.NewRecorder()

	for i := 0; i < b.N; i++ {
		handler(w, req)
	}
}
<commit_msg>updated comment in manager test<commit_after>package main

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"syscall"
	"testing"
	"time"
)

// Test that the handler with no args just returns the data sent
func TestHandlerNoArgs(t *testing.T) {
	handler := createHandler(nil)

	req, err := http.NewRequest("POST", "http://example.com/foo", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		t.Error(err)
	}

	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 200 || w.Body.String() != "{'foo': 3}" {
		t.Error("Did not get an idempotent response")
	}
}

// Test that the handler with no args returns 404 if we try a GET or
// POST with no data
func TestHandlerInvalidRequest(t *testing.T) {
	handler := createHandler(nil)

	req, err := http.NewRequest("GET", "http://foobar.com", nil)
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 404 {
		t.Error("Expected an error code of 404 from a GET request.")
	}

	req, err = http.NewRequest("POST", "http://foobar.com", nil)
	if err != nil {
		t.Error(err)
	}
	handler(w, req)

	if w.Code != 404 {
		t.Error("Expected an error code of 404 from an empty POST request.")
	}
}

func makeTestHandler(response string) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		buf := new(bytes.Buffer)
		buf.ReadFrom(r.Body)
		w.Write([]byte(response + buf.String()))
	}
}

// Test that the handler forwards data to servers *in order*
func TestHandlerForwardsData(t *testing.T) {
	ts1 := httptest.NewServer(http.HandlerFunc(makeTestHandler("foo")))
	defer ts1.Close()

	ts2 := httptest.NewServer(http.HandlerFunc(makeTestHandler("first")))
	defer ts2.Close()

	handler := createHandler([]string{ts1.URL, ts2.URL})
	req, err := http.NewRequest("POST", "http://foobar.com", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		t.Error(err)
	}
	w := httptest.NewRecorder()
	handler(w, req)

	if w.Code != 200 || w.Body.String() != "firstfoo{'foo': 3}" {
		t.Errorf("Got '%s'; did not get expected response", w.Body.String())
	}
}

func TestMainRunnerExitsGracefully(t *testing.T) {
	// set the interrupt handler to go off after 1 second
	go func() {
		time.Sleep(1 * time.Second)
		syscall.Kill(syscall.Getpid(), syscall.SIGINT)
	}()
	go func() {
		time.Sleep(50 * time.Millisecond)
		resp, err := http.Post("http://localhost:9800", "application/json", bytes.NewBufferString("{'foo': 3}"))
		if err != nil {
			t.Error(err)
		}
		buf := new(bytes.Buffer)
		buf.ReadFrom(resp.Body)
		result := buf.String()
		if result != "{'foo': 3}" {
			t.Errorf("Got '%s'; did not get expected response", result)
		}
	}()
	main()
}

// Benchmark to see how many requests the "empty" handler can handle
// Obviously, this is not accurate in production since we forward to
// multiple handlers.
func BenchmarkHandler(b *testing.B) {
	handler := createHandler(nil)

	req, err := http.NewRequest("POST", "http://example.com/foo", bytes.NewBufferString("{'foo': 3}"))
	if err != nil {
		b.Error(err)
	}

	w := httptest.NewRecorder()

	for i := 0; i < b.N; i++ {
		handler(w, req)
	}
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"go/build"
	"log"
	"os"
	"path/filepath"
	"strings"

	gps "github.com/sdboyer/vsolver"
)

func main() {
	// Operate on the current directory
	root, _ := os.Getwd()
	// Assume the current directory is correctly placed on a GOPATH, and derive
	// the ProjectRoot from it
	importroot := strings.TrimPrefix(root, filepath.Join(build.Default.GOPATH, "src")+string(filepath.Separator))

	params := gps.SolveParameters{
		RootDir:     root,
		ImportRoot:  gps.ProjectRoot(importroot),
		Trace:       true,
		TraceLogger: log.New(os.Stdout, "", 0),
	}

	sourcemgr, _ := gps.NewSourceManager(MyAnalyzer{}, ".repocache", false)
	defer sourcemgr.Release()

	solver, _ := gps.Prepare(params, sourcemgr)
	solution, err := solver.Solve()
	if err == nil {
		os.RemoveAll(filepath.Join(root, "vendor"))
		gps.CreateVendorTree(filepath.Join(root, "vendor"), solution, sourcemgr, true)
	}
}

type MyAnalyzer struct{}

func (a MyAnalyzer) GetInfo(path string, n gps.ProjectRoot) (gps.Manifest, gps.Lock, error) {
	return nil, nil, nil
}
<commit_msg>Bit more comments on the example<commit_after>// +build ignore

package main

import (
	"go/build"
	"log"
	"os"
	"path/filepath"
	"strings"

	gps "github.com/sdboyer/vsolver"
)

// This is probably the simplest possible implementation of gps. It does the
// substantive work that `go get` does, except it drops the resulting tree into
// vendor/, and prefers semver tags (if available) over branches.
func main() {
	// Operate on the current directory
	root, _ := os.Getwd()
	// Assume the current directory is correctly placed on a GOPATH, and derive
	// the ProjectRoot from it
	srcprefix := filepath.Join(build.Default.GOPATH, "src") + string(filepath.Separator)
	importroot := filepath.ToSlash(strings.TrimPrefix(root, srcprefix))

	// Set up params, including tracing
	params := gps.SolveParameters{
		RootDir:     root,
		ImportRoot:  gps.ProjectRoot(importroot),
		Trace:       true,
		TraceLogger: log.New(os.Stdout, "", 0),
	}

	// Set up a SourceManager with the NaiveAnalyzer
	sourcemgr, _ := gps.NewSourceManager(NaiveAnalyzer{}, ".repocache", false)
	defer sourcemgr.Release()

	// Prep and run the solver
	solver, _ := gps.Prepare(params, sourcemgr)
	solution, err := solver.Solve()
	if err == nil {
		// If no failure, blow away the vendor dir and write a new one out,
		// stripping nested vendor directories as we go.
		os.RemoveAll(filepath.Join(root, "vendor"))
		gps.CreateVendorTree(filepath.Join(root, "vendor"), solution, sourcemgr, true)
	}
}

type NaiveAnalyzer struct{}

func (a NaiveAnalyzer) GetInfo(path string, n gps.ProjectRoot) (gps.Manifest, gps.Lock, error) {
	return nil, nil, nil
}
<|endoftext|>