<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
)

func appDirExists(application *Application, etcd *Etcd) bool {
	return etcd.HasKey("/paus/users/" + application.Username + "/apps/" + application.AppName)
}

func deploy(dockerHost string, application *Application, composeFilePath string) (string, error) {
	var err error

	compose := NewCompose(dockerHost, composeFilePath, application.ProjectName)

	fmt.Println("=====> Building ...")

	if err = compose.Build(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to build application image. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	fmt.Println("=====> Pulling ...")

	if err = compose.Pull(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to pull application image. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	fmt.Println("=====> Deploying ...")

	if err = compose.Up(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to start application. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	webContainerId, err := compose.GetContainerId("web")

	if err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to web container ID. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	return webContainerId, nil
}

func getSubmodules(repositoryPath string) error {
	dir := filepath.Join(repositoryPath, ".git")

	stat, err := os.Stat(dir)

	if err == nil && stat.IsDir() {
		if e := os.RemoveAll(dir); e != nil {
			return errors.Wrap(e, fmt.Sprintf("Failed to remove %s.", dir))
		}
	}

	cmd := exec.Command("/usr/local/bin/get-submodules")

	if err = RunCommand(cmd); err != nil {
		return errors.Wrap(err, "Failed to get submodules.")
	}

	return nil
}

func injectBuildArgs(application *Application, composeFile *ComposeFile, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		return nil
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		return nil
	}

	buildArgsKey := appDirectoryKey + "/build-args/"

	if !etcd.HasKey(buildArgsKey) {
		return nil
	}

	buildArgKeys, err := etcd.List(buildArgsKey, false)

	if err != nil {
		return errors.Wrap(err, "Failed to get build arg keys.")
	}

	buildArgs := map[string]string{}

	for _, key := range buildArgKeys {
		value, err := etcd.Get(key)

		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to get build arg value. key: %s", key))
		}

		buildArgs[strings.Replace(key, buildArgsKey, "", 1)] = value
	}

	composeFile.InjectBuildArgs(buildArgs)

	return nil
}

func injectEnvironmentVariables(application *Application, composeFile *ComposeFile, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		return nil
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		return nil
	}

	envDirectoryKey := appDirectoryKey + "/envs/"

	if !etcd.HasKey(envDirectoryKey) {
		return nil
	}

	envKeys, err := etcd.List(envDirectoryKey, false)

	if err != nil {
		return errors.Wrap(err, "Failed to get environment variable keys.")
	}

	environmentVariables := map[string]string{}

	for _, key := range envKeys {
		value, err := etcd.Get(key)

		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to get environment variable value. key: %s", key))
		}

		environmentVariables[strings.Replace(key, envDirectoryKey, "", 1)] = value
	}

	composeFile.InjectEnvironmentVariables(environmentVariables)

	return nil
}

func registerApplicationMetadata(application *Application, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		_ = etcd.Mkdir(userDirectoryKey)
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		_ = etcd.Mkdir(appDirectoryKey)
		_ = etcd.Mkdir(appDirectoryKey + "/envs")
		_ = etcd.Mkdir(appDirectoryKey + "/revisions")
	}

	if err := etcd.Set(appDirectoryKey+"/revisions/"+application.Revision, strconv.FormatInt(time.Now().Unix(), 10)); err != nil {
		return errors.Wrap(err, "Failed to set revisdion.")
	}

	return nil
}

func registerVulcandInformation(application *Application, baseDomain string, webContainer *Container, etcd *Etcd) error {
	vulcandDirectoryKeyBase := "/vulcand"

	// {"Type": "http"}
	if err := etcd.Set(vulcandDirectoryKeyBase+"/backends/"+application.ProjectName+"/backend", "{\"Type\": \"http\"}"); err != nil {
		return errors.Wrap(err, "Failed to set vulcand backend.")
	}

	// {"URL": "http://$web_container_host_ip:$web_container_port"}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/backends/"+application.ProjectName+"/servers/"+webContainer.ContainerId,
		"{\"URL\": \"http://"+webContainer.HostIP()+":"+webContainer.HostPort()+"\"}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand backend server.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$PROJECT_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.ProjectName+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.ProjectName+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with project name.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$USER_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.Username+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.Username+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with username.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$APP_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.AppName+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.AppName+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with appName.")
	}

	return nil
}

func removeUnpackedFiles(repositoryPath, newComposeFilePath string) error {
	files, err := ioutil.ReadDir(repositoryPath)

	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("Failed to open %s.", repositoryPath))
	}

	for _, file := range files {
		if filepath.Join(repositoryPath, file.Name()) != newComposeFilePath {
			path := filepath.Join(repositoryPath, file.Name())

			if err = os.RemoveAll(path); err != nil {
				return errors.Wrap(err, fmt.Sprintf("Failed to remove files in %s.", path))
			}
		}
	}

	return nil
}

func unpackReceivedFiles(repositoryDir, username, projectName string, stdin io.Reader) (string, error) {
	repositoryPath := filepath.Join(repositoryDir, username, projectName)

	if err := os.MkdirAll(repositoryPath, 0777); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to create directory %s.", repositoryPath))
	}

	reader := tar.NewReader(stdin)

	for {
		header, err := reader.Next()

		if err == io.EOF {
			break
		}

		if err != nil {
			return "", errors.Wrap(err, "Failed to iterate tarball.")
		}

		buffer := new(bytes.Buffer)
		outPath := filepath.Join(repositoryPath, header.Name)

		switch header.Typeflag {
		case tar.TypeDir:
			if _, err = os.Stat(outPath); err != nil {
				if err = os.MkdirAll(outPath, 0755); err != nil {
					return "", errors.Wrap(err, fmt.Sprintf("Failed to create directory %s from tarball.", outPath))
				}
			}

		case tar.TypeReg, tar.TypeRegA:
			if _, err = io.Copy(buffer, reader); err != nil {
				return "", errors.Wrap(err, fmt.Sprintf("Failed to copy file contents in %s from tarball.", outPath))
			}

			if err = ioutil.WriteFile(outPath, buffer.Bytes(), os.FileMode(header.Mode)); err != nil {
				return "", errors.Wrap(err, fmt.Sprintf("Failed to create file %s from tarball.", outPath))
			}
		}
	}

	return repositoryPath, nil
}

func main() {
	config, err := LoadConfig()

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	etcd, err := NewEtcd(config.EtcdEndpoint)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	application := ApplicationFromArgs(os.Args[1:])

	if !appDirExists(application, etcd) {
		fmt.Fprintln(os.Stderr, "=====> Application not found: "+application.AppName)
		os.Exit(1)
	}

	repositoryPath, err := unpackReceivedFiles(config.RepositoryDir, application.Username, application.ProjectName, os.Stdin)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = os.Chdir(repositoryPath); err != nil {
		errors.Fprint(os.Stderr, errors.Wrap(err, fmt.Sprintf("Failed to chdir to %s.", repositoryPath)))
		os.Exit(1)
	}

	fmt.Println("=====> Getting submodules ...")

	if err = getSubmodules(repositoryPath); err != nil {
		errors.Fprint(os.Stderr, errors.Wrap(err, fmt.Sprintf("Failed to get submodules. path: %s", repositoryPath)))
		os.Exit(1)
	}

	composeFilePath := filepath.Join(repositoryPath, "docker-compose.yml")

	if _, err := os.Stat(composeFilePath); err != nil {
		fmt.Fprintln(os.Stderr, "=====> docker-compose.yml was NOT found!")
		os.Exit(1)
	}

	fmt.Println("=====> docker-compose.yml was found")
	composeFile, err := NewComposeFile(composeFilePath)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = injectBuildArgs(application, composeFile, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = injectEnvironmentVariables(application, composeFile, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	composeFile.RewritePortBindings()
	newComposeFilePath := filepath.Join(repositoryPath, "docker-compose-"+strconv.FormatInt(time.Now().Unix(), 10)+".yml")

	if err = composeFile.SaveAs(newComposeFilePath); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	webContainerId, err := deploy(config.DockerHost, application, newComposeFilePath)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Println("=====> Application container is launched.")

	webContainer, err := ContainerFromID(config.DockerHost, webContainerId)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Println("=====> Registering metadata ...")

	if err = registerApplicationMetadata(application, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = registerVulcandInformation(application, config.BaseDomain, webContainer, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	uriScheme, err := etcd.Get("/paus/uri-scheme")

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	urlList := []string{
		uriScheme + "://" + application.ProjectName + "." + config.BaseDomain, // http://dtan4-docker-service-rails-a7f955b4.pausapp.com
		uriScheme + "://" + application.Username + "." + config.BaseDomain,    // http://dtan4.pausapp.com
		uriScheme + "://" + application.AppName + "." + config.BaseDomain,     // http://docker-service-rails.pausapp.com
	}

	fmt.Println("=====> " + application.Repository + " was successfully deployed at:")

	for _, url := range urlList {
		fmt.Println("         " + url)
	}

	if err = removeUnpackedFiles(repositoryPath, newComposeFilePath); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Rewrite with fmt.Sprintf<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
)

func appDirExists(application *Application, etcd *Etcd) bool {
	return etcd.HasKey("/paus/users/" + application.Username + "/apps/" + application.AppName)
}

func deploy(dockerHost string, application *Application, composeFilePath string) (string, error) {
	var err error

	compose := NewCompose(dockerHost, composeFilePath, application.ProjectName)

	fmt.Println("=====> Building ...")

	if err = compose.Build(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to build application image. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	fmt.Println("=====> Pulling ...")

	if err = compose.Pull(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to pull application image. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	fmt.Println("=====> Deploying ...")

	if err = compose.Up(); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to start application. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	webContainerId, err := compose.GetContainerId("web")

	if err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to web container ID. appName: %s, composeFilePath: %s", application.AppName, composeFilePath))
	}

	return webContainerId, nil
}

func getSubmodules(repositoryPath string) error {
	dir := filepath.Join(repositoryPath, ".git")

	stat, err := os.Stat(dir)

	if err == nil && stat.IsDir() {
		if e := os.RemoveAll(dir); e != nil {
			return errors.Wrap(e, fmt.Sprintf("Failed to remove %s.", dir))
		}
	}

	cmd := exec.Command("/usr/local/bin/get-submodules")

	if err = RunCommand(cmd); err != nil {
		return errors.Wrap(err, "Failed to get submodules.")
	}

	return nil
}

func injectBuildArgs(application *Application, composeFile *ComposeFile, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		return nil
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		return nil
	}

	buildArgsKey := appDirectoryKey + "/build-args/"

	if !etcd.HasKey(buildArgsKey) {
		return nil
	}

	buildArgKeys, err := etcd.List(buildArgsKey, false)

	if err != nil {
		return errors.Wrap(err, "Failed to get build arg keys.")
	}

	buildArgs := map[string]string{}

	for _, key := range buildArgKeys {
		value, err := etcd.Get(key)

		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to get build arg value. key: %s", key))
		}

		buildArgs[strings.Replace(key, buildArgsKey, "", 1)] = value
	}

	composeFile.InjectBuildArgs(buildArgs)

	return nil
}

func injectEnvironmentVariables(application *Application, composeFile *ComposeFile, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		return nil
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		return nil
	}

	envDirectoryKey := appDirectoryKey + "/envs/"

	if !etcd.HasKey(envDirectoryKey) {
		return nil
	}

	envKeys, err := etcd.List(envDirectoryKey, false)

	if err != nil {
		return errors.Wrap(err, "Failed to get environment variable keys.")
	}

	environmentVariables := map[string]string{}

	for _, key := range envKeys {
		value, err := etcd.Get(key)

		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to get environment variable value. key: %s", key))
		}

		environmentVariables[strings.Replace(key, envDirectoryKey, "", 1)] = value
	}

	composeFile.InjectEnvironmentVariables(environmentVariables)

	return nil
}

func registerApplicationMetadata(application *Application, etcd *Etcd) error {
	userDirectoryKey := "/paus/users/" + application.Username

	if !etcd.HasKey(userDirectoryKey) {
		_ = etcd.Mkdir(userDirectoryKey)
	}

	appDirectoryKey := userDirectoryKey + "/apps/" + application.AppName

	if !etcd.HasKey(appDirectoryKey) {
		_ = etcd.Mkdir(appDirectoryKey)
		_ = etcd.Mkdir(appDirectoryKey + "/envs")
		_ = etcd.Mkdir(appDirectoryKey + "/revisions")
	}

	if err := etcd.Set(appDirectoryKey+"/revisions/"+application.Revision, strconv.FormatInt(time.Now().Unix(), 10)); err != nil {
		return errors.Wrap(err, "Failed to set revisdion.")
	}

	return nil
}

func registerVulcandInformation(application *Application, baseDomain string, webContainer *Container, etcd *Etcd) error {
	vulcandDirectoryKeyBase := "/vulcand"

	// {"Type": "http"}
	if err := etcd.Set(vulcandDirectoryKeyBase+"/backends/"+application.ProjectName+"/backend", "{\"Type\": \"http\"}"); err != nil {
		return errors.Wrap(err, "Failed to set vulcand backend.")
	}

	// {"URL": "http://$web_container_host_ip:$web_container_port"}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/backends/"+application.ProjectName+"/servers/"+webContainer.ContainerId,
		"{\"URL\": \"http://"+webContainer.HostIP()+":"+webContainer.HostPort()+"\"}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand backend server.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$PROJECT_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.ProjectName+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.ProjectName+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with project name.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$USER_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.Username+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.Username+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with username.")
	}

	// {"Type": "http", "BackendId": "$PROJECT_NAME", "Route": "Host(`$APP_NAME.$BASE_DOMAIN`) && PathRegexp(`/`)", \"Settings\": {\"TrustForwardHeader\": true}}
	if err := etcd.Set(
		vulcandDirectoryKeyBase+"/frontends/"+application.AppName+"/frontend",
		"{\"Type\": \"http\", \"BackendId\": \""+application.ProjectName+"\", \"Route\": \"Host(`"+application.AppName+"."+baseDomain+"`) && PathRegexp(`/`)\", \"Settings\": {\"TrustForwardHeader\": true}}",
	); err != nil {
		return errors.Wrap(err, "Failed to set vulcand frontend with appName.")
	}

	return nil
}

func removeUnpackedFiles(repositoryPath, newComposeFilePath string) error {
	files, err := ioutil.ReadDir(repositoryPath)

	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("Failed to open %s.", repositoryPath))
	}

	for _, file := range files {
		if filepath.Join(repositoryPath, file.Name()) != newComposeFilePath {
			path := filepath.Join(repositoryPath, file.Name())

			if err = os.RemoveAll(path); err != nil {
				return errors.Wrap(err, fmt.Sprintf("Failed to remove files in %s.", path))
			}
		}
	}

	return nil
}

func unpackReceivedFiles(repositoryDir, username, projectName string, stdin io.Reader) (string, error) {
	repositoryPath := filepath.Join(repositoryDir, username, projectName)

	if err := os.MkdirAll(repositoryPath, 0777); err != nil {
		return "", errors.Wrap(err, fmt.Sprintf("Failed to create directory %s.", repositoryPath))
	}

	reader := tar.NewReader(stdin)

	for {
		header, err := reader.Next()

		if err == io.EOF {
			break
		}

		if err != nil {
			return "", errors.Wrap(err, "Failed to iterate tarball.")
		}

		buffer := new(bytes.Buffer)
		outPath := filepath.Join(repositoryPath, header.Name)

		switch header.Typeflag {
		case tar.TypeDir:
			if _, err = os.Stat(outPath); err != nil {
				if err = os.MkdirAll(outPath, 0755); err != nil {
					return "", errors.Wrap(err, fmt.Sprintf("Failed to create directory %s from tarball.", outPath))
				}
			}

		case tar.TypeReg, tar.TypeRegA:
			if _, err = io.Copy(buffer, reader); err != nil {
				return "", errors.Wrap(err, fmt.Sprintf("Failed to copy file contents in %s from tarball.", outPath))
			}

			if err = ioutil.WriteFile(outPath, buffer.Bytes(), os.FileMode(header.Mode)); err != nil {
				return "", errors.Wrap(err, fmt.Sprintf("Failed to create file %s from tarball.", outPath))
			}
		}
	}

	return repositoryPath, nil
}

func main() {
	config, err := LoadConfig()

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	etcd, err := NewEtcd(config.EtcdEndpoint)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	application := ApplicationFromArgs(os.Args[1:])

	if !appDirExists(application, etcd) {
		fmt.Fprintln(os.Stderr, "=====> Application not found: "+application.AppName)
		os.Exit(1)
	}

	repositoryPath, err := unpackReceivedFiles(config.RepositoryDir, application.Username, application.ProjectName, os.Stdin)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = os.Chdir(repositoryPath); err != nil {
		errors.Fprint(os.Stderr, errors.Wrap(err, fmt.Sprintf("Failed to chdir to %s.", repositoryPath)))
		os.Exit(1)
	}

	fmt.Println("=====> Getting submodules ...")

	if err = getSubmodules(repositoryPath); err != nil {
		errors.Fprint(os.Stderr, errors.Wrap(err, fmt.Sprintf("Failed to get submodules. path: %s", repositoryPath)))
		os.Exit(1)
	}

	composeFilePath := filepath.Join(repositoryPath, "docker-compose.yml")

	if _, err := os.Stat(composeFilePath); err != nil {
		fmt.Fprintln(os.Stderr, "=====> docker-compose.yml was NOT found!")
		os.Exit(1)
	}

	fmt.Println("=====> docker-compose.yml was found")
	composeFile, err := NewComposeFile(composeFilePath)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = injectBuildArgs(application, composeFile, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = injectEnvironmentVariables(application, composeFile, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	composeFile.RewritePortBindings()
	newComposeFilePath := filepath.Join(repositoryPath, "docker-compose-"+strconv.FormatInt(time.Now().Unix(), 10)+".yml")

	if err = composeFile.SaveAs(newComposeFilePath); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	webContainerId, err := deploy(config.DockerHost, application, newComposeFilePath)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Println("=====> Application container is launched.")

	webContainer, err := ContainerFromID(config.DockerHost, webContainerId)

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Println("=====> Registering metadata ...")

	if err = registerApplicationMetadata(application, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	if err = registerVulcandInformation(application, config.BaseDomain, webContainer, etcd); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	uriScheme, err := etcd.Get("/paus/uri-scheme")

	if err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}

	urlList := []string{
		fmt.Sprintf("%s://%s.%s", uriScheme, application.ProjectName, config.BaseDomain), // http://dtan4-docker-service-rails-a7f955b4.pausapp.com
		fmt.Sprintf("%s://%s.%s", uriScheme, application.Username, config.BaseDomain),    // http://dtan4.pausapp.com
		fmt.Sprintf("%s://%s.%s", uriScheme, application.AppName, config.BaseDomain),     // http://docker-service-rails.pausapp.com
	}

	fmt.Println("=====> " + application.Repository + " was successfully deployed at:")

	for _, url := range urlList {
		fmt.Println("         " + url)
	}

	if err = removeUnpackedFiles(repositoryPath, newComposeFilePath); err != nil {
		errors.Fprint(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"sort"

	"log"

	"fmt"

	"strings"

	"github.com/golang-commonmark/markdown"
)

var wikiDir string
var tagDir string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body     template.HTML
	Tags     string
	TagArray []string
	Created  string
	Modified string
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}
type nav struct {
	Wikis []string
	Tags  TagIndex
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func (p *wikiPage) save() error {
	filename := getWikiFilename(wikiDir, p.Title)
	err := ioutil.WriteFile(filename, []byte(p.Body), 0600)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = ioutil.WriteFile(tagsfile, []byte(p.Tags), 0600)
	if err != nil {
		return err
	}
	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	md := markdown.New(markdown.HTML(true))
	page.Body = template.HTML(md.RenderToString([]byte(page.Body)))
	return page, nil

}
func loadPage(p *wikiPage) (*wikiPage, error) {
	filename := getWikiFilename(wikiDir, p.Title)

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	p.Body = template.HTML(body)

	info, err := file.Stat()
	if err != nil {
		fmt.Println(err)
		return p, err
	}

	p.Modified = info.ModTime().String()

	tags, err := ioutil.ReadFile(getWikiTagsFilename(p.Title))
	if err == nil {
		p.Tags = string(tags)
		p.TagArray = strings.Split(p.Tags, ",")
	}

	return p, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, err := convertMarkdown(loadPage(p))
	if err != nil {
		http.Redirect(w, r, "/edit/"+p.Title, http.StatusFound)
		return
	}
	p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, _ = loadPage(p)
	renderTemplate(w, "edit", p)
}

func searchHandler(fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(SearchWikis(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn()}}

		renderTemplate(w, "search", p)
	}
}

type navFunc func() nav

func homeHandler(page string, fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn())
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, wiki string) string {
	body := r.FormValue("body")
	p := wikiPage{basePage: basePage{Title: wiki}, Body: template.HTML(body), Tags: r.FormValue("wikitags")}

	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return ""
	}
	http.Redirect(w, r, "/view/"+p.Title, http.StatusFound)

	return r.FormValue("wikitags")
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ ]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage), navfn navFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				fmt.Println("Dont like " + r.URL.Path)
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn()}}
		fn(w, r, p)
	}
}

func processSave(fn func(http.ResponseWriter, *http.Request, string) string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			fmt.Println("Dont like " + r.URL.Path)
			http.NotFound(w, r)
			return
		}
		tags := fn(w, r, m[2])

		fmt.Println("Tags are :" + tags)
	}
}

type byModTime []os.FileInfo

func (m byModTime) Len() int           { return len(m) }
func (m byModTime) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m byModTime) Less(i, j int) bool { return m[i].ModTime().Before(m[j].ModTime()) }

func getNav() nav {
	return nav{
		Wikis: getWikiList(wikiDir),
		Tags:  IndexTags(tagDir),
	}
}
func getWikiList(path string) []string {
	files, err := ioutil.ReadDir(path)
	checkErr(err)

	sort.Sort(sort.Reverse(byModTime(files)))

	var names []string
	for _, f := range files {
		// Ignore anything that isnt an md file
		if strings.HasSuffix(f.Name(), ".md") {
			names = append(names, strings.TrimSuffix(f.Name(), ".md"))
		}
	}

	return names

}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{([^\}]+)\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func main() {

	config, err := LoadConfig("config.json")
	if err != nil {
		log.Fatal(err)
	}

	wikiDir = config.WikiDir
	tagDir = wikiDir + "/tags/"

	os.Mkdir(config.WikiDir, 0755)
	os.Mkdir(config.WikiDir+"tags", 0755)
	http.HandleFunc("/", homeHandler("home", getNav))
	http.HandleFunc("/search/", searchHandler(getNav))
	http.HandleFunc("/view/", makeHandler(viewHandler, getNav))
	http.HandleFunc("/edit/", makeHandler(editHandler, getNav))
	http.HandleFunc("/save/", processSave(saveHandler))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	http.ListenAndServe(":8080", nil)
}
<commit_msg>Rudimentary SSL support<commit_after>package main

import (
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"sort"

	"log"

	"fmt"

	"strings"

	"github.com/golang-commonmark/markdown"
)

var wikiDir string
var tagDir string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body     template.HTML
	Tags     string
	TagArray []string
	Created  string
	Modified string
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}
type nav struct {
	Wikis []string
	Tags  TagIndex
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func (p *wikiPage) save() error {
	filename := getWikiFilename(wikiDir, p.Title)
	err := ioutil.WriteFile(filename, []byte(p.Body), 0600)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = ioutil.WriteFile(tagsfile, []byte(p.Tags), 0600)
	if err != nil {
		return err
	}
	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	md := markdown.New(markdown.HTML(true))
	page.Body = template.HTML(md.RenderToString([]byte(page.Body)))
	return page, nil

}
func loadPage(p *wikiPage) (*wikiPage, error) {
	filename := getWikiFilename(wikiDir, p.Title)

	file, err := os.Open(filename)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	defer file.Close()

	body, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Println(err)
		return p, err
	}
	p.Body = template.HTML(body)

	info, err := file.Stat()
	if err != nil {
		fmt.Println(err)
		return p, err
	}

	p.Modified = info.ModTime().String()

	tags, err := ioutil.ReadFile(getWikiTagsFilename(p.Title))
	if err == nil {
		p.Tags = string(tags)
		p.TagArray = strings.Split(p.Tags, ",")
	}

	return p, nil
}

func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, err := convertMarkdown(loadPage(p))
	if err != nil {
		http.Redirect(w, r, "/edit/"+p.Title, http.StatusFound)
		return
	}
	p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	renderTemplate(w, "view", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage) {
	p, _ = loadPage(p)
	renderTemplate(w, "edit", p)
}

func searchHandler(fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(SearchWikis(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn()}}

		renderTemplate(w, "search", p)
	}
}

type navFunc func() nav

func homeHandler(page string, fn navFunc) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn())
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, wiki string) string {
	body := r.FormValue("body")
	p := wikiPage{basePage: basePage{Title: wiki}, Body: template.HTML(body), Tags: r.FormValue("wikitags")}

	err := p.save()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return ""
	}
	http.Redirect(w, r, "/view/"+p.Title, http.StatusFound)

	return r.FormValue("wikitags")
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ ]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage), navfn navFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				fmt.Println("Dont like " + r.URL.Path)
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn()}}
		fn(w, r, p)
	}
}

func processSave(fn func(http.ResponseWriter, *http.Request, string) string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			fmt.Println("Dont like " + r.URL.Path)
			http.NotFound(w, r)
			return
		}
		tags := fn(w, r, m[2])

		fmt.Println("Tags are :" + tags)
	}
}

type byModTime []os.FileInfo

func (m byModTime) Len() int           { return len(m) }
func (m byModTime) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m byModTime) Less(i, j int) bool { return m[i].ModTime().Before(m[j].ModTime()) }

func getNav() nav {
	return nav{
		Wikis: getWikiList(wikiDir),
		Tags:  IndexTags(tagDir),
	}
}
func getWikiList(path string) []string {
	files, err := ioutil.ReadDir(path)
	checkErr(err)

	sort.Sort(sort.Reverse(byModTime(files)))

	var names []string
	for _, f := range files {
		// Ignore anything that isnt an md file
		if strings.HasSuffix(f.Name(), ".md") {
			names = append(names, strings.TrimSuffix(f.Name(), ".md"))
		}
	}

	return names

}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{([^\}]+)\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func main() {

	config, err := LoadConfig("config.json")
	if err != nil {
		log.Fatal(err)
	}

	wikiDir = config.WikiDir
	tagDir = wikiDir + "/tags/"

	os.Mkdir(config.WikiDir, 0755)
	os.Mkdir(config.WikiDir+"tags", 0755)
	http.HandleFunc("/", homeHandler("home", getNav))
	http.HandleFunc("/search/", searchHandler(getNav))
	http.HandleFunc("/view/", makeHandler(viewHandler, getNav))
	http.HandleFunc("/edit/", makeHandler(editHandler, getNav))
	http.HandleFunc("/save/", processSave(saveHandler))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	err = http.ListenAndServeTLS(":443", "/Users/gawth/ssl/server.crt", "/Users/gawth/ssl/server.key", nil)
	checkErr(err)
}
<|endoftext|>


<commit_before>package orbitus

import (
	"github.com/bmizerany/assert"
	"testing"
	"time"
)

func TestNewReceiverOrbiter(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var i uint64 = 4

	// Ensure all the indexes are initialized to zero
	assert.Equal(t, i, orbiter.GetReceiverIndex())
	assert.Equal(t, i-1, orbiter.GetJournalerIndex())
	assert.Equal(t, i-2, orbiter.GetReplicatorIndex())
	assert.Equal(t, i-3, orbiter.GetUnmarshallerIndex())
	assert.Equal(t, i-4, orbiter.GetExecutorIndex())

	// Ensure buffer has been fully allocated
	for i = 0; i < buffer_size; i++ {
		msg := orbiter.GetMessage(i)
		msg.marshalled = []byte(test + string(i))
	}

	for i = 0; i < buffer_size; i++ {
		msg := orbiter.GetMessage(i)
		assert.Equal(t, msg.marshalled, []byte(test+string(i)))
	}
}

func TestReceiverOrbiterReset(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var i uint64
	testvals := []uint64{1, buffer_size - 1, buffer_size, buffer_size + 1}

	for _, i = range testvals {
		err := orbiter.Reset(i)
		assert.Equal(t, nil, err)

		// Ensure all indexes have been set to the given value
		assert.Equal(t, i, orbiter.GetReceiverIndex())
		assert.Equal(t, i-1, orbiter.GetJournalerIndex())
		assert.Equal(t, i-2, orbiter.GetReplicatorIndex())
		assert.Equal(t, i-3, orbiter.GetUnmarshallerIndex())
		assert.Equal(t, i-4, orbiter.GetExecutorIndex())
	}

	// Ensure orbiter does not reset if running
	orbiter.running = true
	err := orbiter.Reset(buffer_size)
	assert.Equal(t, "Cannot reset a running Orbiter", err.Error())
}

func TestReceiverOrbiterStart(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, journaler, replicator,
		unmarshaller, executor)
	journalerRan, replicatorRan, unmarshallerRan, executorRan =
		false, false, false, false
	orbiter.Start()

	// Manually insert a new Message
	orbiter.buffer[4] = &Message{
		id:         4,
		marshalled: []byte("This is a test message"),
	}
	orbiter.receiverIndex += 1

	time.Sleep(1 * time.Millisecond)

	assert.Equal(t, true, journalerRan)
	assert.Equal(t, true, replicatorRan)
	assert.Equal(t, true, unmarshallerRan)
	assert.Equal(t, true, executorRan)
}

func TestReceiverOrbiterSetReceiverIndex(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var err error
	var old uint64

	// Setting to value lower than current index should fail
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(orbiter.GetReceiverIndex() - 1)
	assert.Equal(t, "New receiver index cannot be less than current index",
		err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Setting to same value as current should work (although do nothing really)
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(orbiter.GetReceiverIndex())
	assert.Equal(t, nil, err)
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Basic incrementing by one (where there is room in front) should work.
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(old + 1)
	assert.Equal(t, nil, err)
	assert.Equal(t, old+1, orbiter.GetReceiverIndex())

	// Wrapping around should work, even though new modulus of index is lower
	// than the index of the consumer in front of it.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	err = orbiter.SetReceiverIndex(buffer_size)
	assert.Equal(t, nil, err)
	assert.Equal(t, buffer_size, orbiter.GetReceiverIndex())

	// Wrapping around and setting index to the same location as the Business
	// Logic Consumer should not work.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(buffer_size + 1)
	assert.Equal(t, "The Receiver Consumer cannot pass the Business Logic "+
		"Consumer", err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Wrapping around and passing the Business Logic Consumer should not work.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(buffer_size + (buffer_size / 2))
	assert.Equal(t, "The Receiver Consumer cannot pass the Business Logic "+
		"Consumer", err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())
}
<commit_msg>Added ReceiverOrbiter.SetJournalerIndex tests<commit_after>package orbitus

import (
	"github.com/bmizerany/assert"
	"testing"
	"time"
)

func TestNewReceiverOrbiter(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var i uint64 = 4

	// Ensure all the indexes are initialized to zero
	assert.Equal(t, i, orbiter.GetReceiverIndex())
	assert.Equal(t, i-1, orbiter.GetJournalerIndex())
	assert.Equal(t, i-2, orbiter.GetReplicatorIndex())
	assert.Equal(t, i-3, orbiter.GetUnmarshallerIndex())
	assert.Equal(t, i-4, orbiter.GetExecutorIndex())

	// Ensure buffer has been fully allocated
	for i = 0; i < buffer_size; i++ {
		msg := orbiter.GetMessage(i)
		msg.marshalled = []byte(test + string(i))
	}

	for i = 0; i < buffer_size; i++ {
		msg := orbiter.GetMessage(i)
		assert.Equal(t, msg.marshalled, []byte(test+string(i)))
	}
}

func TestReceiverOrbiterReset(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var i uint64
	testvals := []uint64{1, buffer_size - 1, buffer_size, buffer_size + 1}

	for _, i = range testvals {
		err := orbiter.Reset(i)
		assert.Equal(t, nil, err)

		// Ensure all indexes have been set to the given value
		assert.Equal(t, i, orbiter.GetReceiverIndex())
		assert.Equal(t, i-1, orbiter.GetJournalerIndex())
		assert.Equal(t, i-2, orbiter.GetReplicatorIndex())
		assert.Equal(t, i-3, orbiter.GetUnmarshallerIndex())
		assert.Equal(t, i-4, orbiter.GetExecutorIndex())
	}

	// Ensure orbiter does not reset if running
	orbiter.running = true
	err := orbiter.Reset(buffer_size)
	assert.Equal(t, "Cannot reset a running Orbiter", err.Error())
}

func TestReceiverOrbiterStart(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, journaler, replicator,
		unmarshaller, executor)
	journalerRan, replicatorRan, unmarshallerRan, executorRan =
		false, false, false, false
	orbiter.Start()

	// Manually insert a new Message
	orbiter.buffer[4] = &Message{
		id:         4,
		marshalled: []byte("This is a test message"),
	}
	orbiter.receiverIndex += 1

	time.Sleep(1 * time.Millisecond)

	assert.Equal(t, true, journalerRan)
	assert.Equal(t, true, replicatorRan)
	assert.Equal(t, true, unmarshallerRan)
	assert.Equal(t, true, executorRan)
}

func TestReceiverOrbiterSetReceiverIndex(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var err error
	var old uint64

	// Setting to value lower than current index should fail
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(orbiter.GetReceiverIndex() - 1)
	assert.Equal(t, "New receiver index cannot be less than current index",
		err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Setting to same value as current should work (although do nothing really)
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(orbiter.GetReceiverIndex())
	assert.Equal(t, nil, err)
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Basic incrementing by one (where there is room in front) should work.
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(old + 1)
	assert.Equal(t, nil, err)
	assert.Equal(t, old+1, orbiter.GetReceiverIndex())

	// Wrapping around should work, even though new modulus of index is lower
	// than the index of the consumer in front of it.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	err = orbiter.SetReceiverIndex(buffer_size)
	assert.Equal(t, nil, err)
	assert.Equal(t, buffer_size, orbiter.GetReceiverIndex())

	// Wrapping around and setting index to the same location as the Business
	// Logic Consumer should not work.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(buffer_size + 1)
	assert.Equal(t, "The Receiver Consumer cannot pass the Business Logic "+
		"Consumer", err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())

	// Wrapping around and passing the Business Logic Consumer should not work.
	orbiter.Reset(5)
	assert.Equal(t, uint64(1), orbiter.GetExecutorIndex())
	old = orbiter.GetReceiverIndex()
	err = orbiter.SetReceiverIndex(buffer_size + (buffer_size / 2))
	assert.Equal(t, "The Receiver Consumer cannot pass the Business Logic "+
		"Consumer", err.Error())
	assert.Equal(t, old, orbiter.GetReceiverIndex())
}

func TestReceiverOrbiterSetJournalerIndex(t *testing.T) {
	orbiter := NewReceiverOrbiter(buffer_size, nil, nil, nil, nil, nil)
	var err error
	var old uint64

	// Setting to value lower than current index should fail
	old = orbiter.GetJournalerIndex()
	err = orbiter.SetJournalerIndex(orbiter.GetJournalerIndex() - 1)
	assert.Equal(t, "New journaler index cannot be less than current index",
		err.Error())
	assert.Equal(t, old, orbiter.GetJournalerIndex())

	// Setting to same value as current should work (although do nothing really)
	old = orbiter.GetJournalerIndex()
	err = orbiter.SetJournalerIndex(orbiter.GetJournalerIndex())
	assert.Equal(t, nil, err)
	assert.Equal(t, old, orbiter.GetJournalerIndex())

	// Basic incrementing by one (where there is room in front) should work.
	orbiter.SetReceiverIndex(orbiter.GetReceiverIndex() + 1)
	old = orbiter.GetJournalerIndex()
	err = orbiter.SetJournalerIndex(old + 1)
	assert.Equal(t, nil, err)
	assert.Equal(t, old+1, orbiter.GetJournalerIndex())

	// Setting index to the same as the Receiver Consumer should not work.
	old = orbiter.GetJournalerIndex()
	err = orbiter.SetJournalerIndex(orbiter.GetReceiverIndex())
	assert.Equal(t, "New journaler index cannot be greater than the current "+
		"receiver index", err.Error())
	assert.Equal(t, old, orbiter.GetJournalerIndex())

	// Padding the Receiver Consumer should not work
	old = orbiter.GetJournalerIndex()
	err = orbiter.SetJournalerIndex(orbiter.GetReceiverIndex() +
		orbiter.GetBufferSize() - 1)
	assert.Equal(t, "New journaler index cannot be greater than the current "+
		"receiver index", err.Error())
	assert.Equal(t, old, orbiter.GetJournalerIndex())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

const moved = `

The present tool has moved to the go.tools repository.

Please install it from its new location:

	go get code.google.com/p/go.tools/cmd/present

`

func main() {
	fmt.Print(moved)
	os.Exit(1)
}
<commit_msg>talks/present: update "moved" message to golang.org/x name<commit_after>package main

import (
	"fmt"
	"os"
)

const moved = `

The present tool has moved to the Go tools repository.

Please install it from its new location:

	go get golang.org/x/tools/cmd/present

`

func main() {
	fmt.Print(moved)
	os.Exit(1)
}
<|endoftext|>


<commit_before>package dsc

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/viant/toolbox"
)

type metaRecordMapper struct {
	converter        toolbox.Converter
	structType       interface{}
	columnToFieldMap map[string](map[string]string)
	usePointer       bool
}

//NewMetaRecordMapped creates a new MetaRecordMapped to map a data record to a struct, it takes target struct and flag if it is a pointer as parameters.
func NewMetaRecordMapped(targetType interface{}, usePointer bool) RecordMapper {
	structType := targetType
	if usePointer {
		var originalType = targetType.(reflect.Type).Elem()
		structType = originalType
	}
	var result = &metaRecordMapper{
		converter:        *toolbox.NewColumnConverter(""),
		structType:       structType,
		usePointer:       usePointer,
		columnToFieldMap: toolbox.NewFieldSettingByKey(targetType, "column")}
	return result
}

func normalizeColumnKey(column string) string {
	var result = strings.ToLower(column)
	result = strings.Replace(result, "_", "", len(result))
	return result
}

func (rm *metaRecordMapper) getValueMappingCount(columns []string) int {
	result := 0
	for _, key := range columns {
		mapping, ok := rm.columnToFieldMap[key]
		if !ok {
			mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			if _, found := mapping["valueMap"]; found {
				result++
			}
		}
	}
	return result
}

func (rm *metaRecordMapper) allocateValueMapByKey(columns []string) map[string]interface{} {
	var valuesPointers = make([]interface{}, rm.getValueMappingCount(columns))
	index := 0
	var result = make(map[string]interface{})
	for _, key := range columns {
		mapping, ok := rm.columnToFieldMap[key]
		if !ok {
			mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			if _, found := mapping["valueMap"]; found {
				result[key] = &valuesPointers[index]
				index++
			}
		}
	}
	return result
}

func (rm *metaRecordMapper) applyFieldMapValuesIfNeeded(fieldsValueMap map[string]interface{}, structPointer reflect.Value) error {
	for key, rawValue := range fieldsValueMap {
		valueMapping, ok := rm.columnToFieldMap[key]
		if !ok {
			valueMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		fieldName := valueMapping["fieldName"]
		field := structPointer.Elem().FieldByName(fieldName)
		unwrappedValue := reflect.ValueOf(rawValue).Elem()
		if unwrappedValue.IsNil() {
			if field.Kind() != reflect.Ptr {
				return fmt.Errorf("failed to apply value map on %v, unable to set nil", fieldName)
			}
			continue
		}

		rawValue = unwrappedValue.Interface()
		var value string
		if valueAsBytes, ok := rawValue.([]byte); ok {
			value = string(valueAsBytes)
		} else {
			value = toolbox.AsString(rawValue)
		}

		valueMap := toolbox.MakeStringMap(valueMapping["valueMap"], ":", ",")
		stringValue := toolbox.AsString(value)
		if mappedValue, found := valueMap[stringValue]; found {
			fieldValuePointer := field.Addr().Interface()
			err := rm.converter.AssignConverted(fieldValuePointer, mappedValue)
			if err != nil {
				return fmt.Errorf("failed to map record, unable convert,dur to %v", err)

			}
		} else {
			return fmt.Errorf("failed to map record, unable to find valid mapping, want one of %s, but had %v", valueMap, stringValue)
		}

	}
	return nil
}

func (rm *metaRecordMapper) scanData(scanner Scanner) (result interface{}, err error) {
	structType := toolbox.DiscoverTypeByKind(rm.structType, reflect.Struct)
	structPointer := reflect.New(structType)
	resultStruct := structPointer.Elem()
	columns, _ := scanner.Columns()
	var fieldValuePointers = make([]interface{}, len(columns))
	var fieldsValueMap map[string]interface{}

	hasFieldValueMap := rm.getValueMappingCount(columns) > 0
	if hasFieldValueMap {
		fieldsValueMap = rm.allocateValueMapByKey(columns)
	}
	for i, key := range columns {

		fieldMapping, ok := rm.columnToFieldMap[key]
		if !ok {
			fieldMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			fieldName := fieldMapping["fieldName"]
			field := resultStruct.FieldByName(fieldName)

			if _, found := fieldMapping["valueMap"]; found {
				fieldValuePointers[i] = fieldsValueMap[key]
				continue
			}
			fieldValuePointers[i] = field.Addr().Interface()

		} else {
			return nil, fmt.Errorf("unable to map column %v to %v, avaialble: %v", key, rm.columnToFieldMap[key], rm.columnToFieldMap)
		}
	}
	err = scanner.Scan(fieldValuePointers...)
	if err != nil {
		return nil, fmt.Errorf("failed to scan data: %v\n", err)
	}

	if hasFieldValueMap {
		err := rm.applyFieldMapValuesIfNeeded(fieldsValueMap, structPointer)
		if err != nil {
			return nil, err
		}
	}

	if !rm.usePointer {
		result = structPointer.Elem().Interface()
		return result, err
	}
	return structPointer.Interface(), err
}

func (rm *metaRecordMapper) mapFromValues(vaues []interface{}) (result interface{}, err error) {
	return nil, nil
}

func (rm *metaRecordMapper) Map(scanner Scanner) (result interface{}, err error) {
	return rm.scanData(scanner)
}

type columnarRecordMapper struct {
	usePointer  bool
	targetSlice reflect.Type
}

//NewColumnarRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice.
func NewColumnarRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper {
	return &columnarRecordMapper{usePointer, targetSlice}
}

func (rm *columnarRecordMapper) Map(scanner Scanner) (interface{}, error) {
	result, _, err := ScanRow(scanner)
	if err != nil {
		return nil, err
	}
	if rm.usePointer {
		return result, nil
	}
	return result, nil
}

type mapRecordMapper struct {
	usePointer  bool
	targetSlice reflect.Type
}

func (rm *mapRecordMapper) Map(scanner Scanner) (interface{}, error) {
	result, _, err := ScanRow(scanner)
	if err != nil {
		return nil, err
	}
	columns, _ := scanner.Columns()

	aMap := make(map[string]interface{})
	for i, column := range columns {
		aMap[column] = result[i]
	}
	if rm.usePointer {
		return &aMap, nil
	}
	return aMap, nil
}

//NewMapRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice.
func NewMapRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper {
	return &mapRecordMapper{usePointer, targetSlice}
}

//NewRecordMapper create a new record mapper, if struct is passed it would be MetaRecordMapper, or for slice ColumnRecordMapper
func NewRecordMapper(targetType reflect.Type) RecordMapper {
	switch targetType.Kind() {
	case reflect.Struct:
		var mapper = NewMetaRecordMapped(targetType, false)
		return mapper

	case reflect.Map:
		var mapper = NewMapRecordMapper(false, targetType)
		return mapper
	case reflect.Slice:
		var mapper = NewColumnarRecordMapper(false, targetType)
		return mapper
	case reflect.Ptr:
		if targetType.Elem().Kind() == reflect.Slice {
			var mapper = NewColumnarRecordMapper(true, targetType.Elem())
			return mapper
		} else if targetType.Elem().Kind() == reflect.Struct {
			var mapper = NewMetaRecordMapped(targetType, true)
			return mapper
		}
	default:
		panic(fmt.Sprintf("unsupported type: %v ", targetType.Name()))
	}
	return nil
}

//NewRecordMapperIfNeeded create a new mapper if passed in mapper is nil. It takes target type for the record mapper.
func NewRecordMapperIfNeeded(mapper RecordMapper, targetType reflect.Type) RecordMapper {
	if mapper != nil {
		return mapper
	}
	return NewRecordMapper(targetType)
}

//ScanRow takes scanner to scans row.
func ScanRow(scanner Scanner) ([]interface{}, []string, error) {
	columns, _ := scanner.Columns()
	count := len(columns)

	var err error
	var rowValues = make([]interface{}, count)

	provider, ok := scanner.(ColumnValueProvider)
	if ok {
		if values, err := provider.ColumnValues(); err == nil {
			if err = scanner.Scan(values...); err != nil {
				return nil, nil, fmt.Errorf("failed to scan row due to %v", err)
			}
			for i, v := range values {
				if v == nil {
					continue
				}
				rowValues[i] = toolbox.DereferenceValue(v)
			}
			return rowValues, columns, nil
		}
	}

	var valuePointers = make([]interface{}, count)
	for i := range rowValues {
		valuePointers[i] = &rowValues[i]
	}

	err = scanner.Scan(valuePointers...)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to scan row due to %v", err)
	}

	for i := range rowValues {
		var value interface{}
		rawValue := rowValues[i]
		b, ok := rawValue.([]byte)
		if ok {
			value = string(b)
		} else {
			value = rawValue
		}
		rowValues[i] = value
	}

	return rowValues, columns, nil
}
<commit_msg>patched record mapping<commit_after>package dsc

import (
	"fmt"
	"reflect"
	"strings"

	"github.com/viant/toolbox"
)

type metaRecordMapper struct {
	converter        toolbox.Converter
	structType       interface{}
	columnToFieldMap map[string](map[string]string)
	usePointer       bool
}

//NewMetaRecordMapped creates a new MetaRecordMapped to map a data record to a struct, it takes target struct and flag if it is a pointer as parameters.
func NewMetaRecordMapped(targetType interface{}, usePointer bool) RecordMapper {
	structType := targetType
	if usePointer {
		var originalType = targetType.(reflect.Type).Elem()
		structType = originalType
	}
	var result = &metaRecordMapper{
		converter:        *toolbox.NewColumnConverter(""),
		structType:       structType,
		usePointer:       usePointer,
		columnToFieldMap: toolbox.NewFieldSettingByKey(targetType, "column")}
	return result
}

func normalizeColumnKey(column string) string {
	var result = strings.ToLower(column)
	result = strings.Replace(result, "_", "", len(result))
	return result
}

func (rm *metaRecordMapper) getValueMappingCount(columns []string) int {
	result := 0
	for _, key := range columns {
		mapping, ok := rm.columnToFieldMap[key]
		if !ok {
			mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			if _, found := mapping["valueMap"]; found {
				result++
			}
		}
	}
	return result
}

func (rm *metaRecordMapper) allocateValueMapByKey(columns []string) map[string]interface{} {
	var valuesPointers = make([]interface{}, rm.getValueMappingCount(columns))
	index := 0
	var result = make(map[string]interface{})
	for _, key := range columns {
		mapping, ok := rm.columnToFieldMap[key]
		if !ok {
			mapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			if _, found := mapping["valueMap"]; found {
				result[key] = &valuesPointers[index]
				index++
			}
		}
	}
	return result
}

func (rm *metaRecordMapper) applyFieldMapValuesIfNeeded(fieldsValueMap map[string]interface{}, structPointer reflect.Value) error {
	for key, rawValue := range fieldsValueMap {
		valueMapping, ok := rm.columnToFieldMap[key]
		if !ok {
			valueMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		fieldName := valueMapping["fieldName"]
		field := structPointer.Elem().FieldByName(fieldName)
		unwrappedValue := reflect.ValueOf(rawValue).Elem()
		if unwrappedValue.IsNil() {
			if field.Kind() != reflect.Ptr {
				return fmt.Errorf("failed to apply value map on %v, unable to set nil", fieldName)
			}
			continue
		}

		rawValue = unwrappedValue.Interface()
		var value string
		if valueAsBytes, ok := rawValue.([]byte); ok {
			value = string(valueAsBytes)
		} else {
			value = toolbox.AsString(rawValue)
		}

		valueMap := toolbox.MakeStringMap(valueMapping["valueMap"], ":", ",")
		stringValue := toolbox.AsString(value)
		if mappedValue, found := valueMap[stringValue]; found {
			fieldValuePointer := field.Addr().Interface()
			err := rm.converter.AssignConverted(fieldValuePointer, mappedValue)
			if err != nil {
				return fmt.Errorf("failed to map record, unable convert,dur to %v", err)

			}
		} else {
			return fmt.Errorf("failed to map record, unable to find valid mapping, want one of %s, but had %v", valueMap, stringValue)
		}

	}
	return nil
}

func (rm *metaRecordMapper) scanData(scanner Scanner) (result interface{}, err error) {
	structType := toolbox.DiscoverTypeByKind(rm.structType, reflect.Struct)
	structPointer := reflect.New(structType)
	resultStruct := structPointer.Elem()
	columns, _ := scanner.Columns()
	var fieldValuePointers = make([]interface{}, len(columns))
	var fieldsValueMap map[string]interface{}

	hasFieldValueMap := rm.getValueMappingCount(columns) > 0
	if hasFieldValueMap {
		fieldsValueMap = rm.allocateValueMapByKey(columns)
	}
	for i, key := range columns {

		fieldMapping, ok := rm.columnToFieldMap[key]
		if !ok {
			fieldMapping, ok = rm.columnToFieldMap[normalizeColumnKey(key)]
		}
		if ok {
			fieldName := fieldMapping["fieldName"]
			field := resultStruct.FieldByName(fieldName)

			if _, found := fieldMapping["valueMap"]; found {
				fieldValuePointers[i] = fieldsValueMap[key]
				continue
			}
			fieldValuePointers[i] = field.Addr().Interface()

		} else {
			return nil, fmt.Errorf("unable to map column %v to %v, avaialble: %v", key, rm.columnToFieldMap[key], rm.columnToFieldMap)
		}
	}
	err = scanner.Scan(fieldValuePointers...)
	if err != nil {
		return nil, fmt.Errorf("failed to scan data: %v\n", err)
	}

	if hasFieldValueMap {
		err := rm.applyFieldMapValuesIfNeeded(fieldsValueMap, structPointer)
		if err != nil {
			return nil, err
		}
	}

	if !rm.usePointer {
		result = structPointer.Elem().Interface()
		return result, err
	}
	return structPointer.Interface(), err
}

func (rm *metaRecordMapper) mapFromValues(vaues []interface{}) (result interface{}, err error) {
	return nil, nil
}

func (rm *metaRecordMapper) Map(scanner Scanner) (result interface{}, err error) {
	return rm.scanData(scanner)
}

type columnarRecordMapper struct {
	usePointer  bool
	targetSlice reflect.Type
}

//NewColumnarRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice.
func NewColumnarRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper {
	return &columnarRecordMapper{usePointer, targetSlice}
}

func (rm *columnarRecordMapper) Map(scanner Scanner) (interface{}, error) {
	result, _, err := ScanRow(scanner)
	if err != nil {
		return nil, err
	}
	if rm.usePointer {
		return result, nil
	}
	return result, nil
}

type mapRecordMapper struct {
	usePointer  bool
	targetSlice reflect.Type
}

func (rm *mapRecordMapper) Map(scanner Scanner) (interface{}, error) {
	result, _, err := ScanRow(scanner)
	if err != nil {
		return nil, err
	}
	columns, _ := scanner.Columns()

	aMap := make(map[string]interface{})
	for i, column := range columns {
		aMap[column] = result[i]
	}
	if rm.usePointer {
		return &aMap, nil
	}
	return aMap, nil
}

//NewMapRecordMapper creates a new ColumnarRecordMapper, to map a data record to slice.
func NewMapRecordMapper(usePointer bool, targetSlice reflect.Type) RecordMapper {
	return &mapRecordMapper{usePointer, targetSlice}
}

//NewRecordMapper create a new record mapper, if struct is passed it would be MetaRecordMapper, or for slice ColumnRecordMapper
func NewRecordMapper(targetType reflect.Type) RecordMapper {
	switch targetType.Kind() {
	case reflect.Struct:
		var mapper = NewMetaRecordMapped(targetType, false)
		return mapper

	case reflect.Map:
		var mapper = NewMapRecordMapper(false, targetType)
		return mapper
	case reflect.Slice:
		var mapper = NewColumnarRecordMapper(false, targetType)
		return mapper
	case reflect.Ptr:
		if targetType.Elem().Kind() == reflect.Slice {
			var mapper = NewColumnarRecordMapper(true, targetType.Elem())
			return mapper
		} else if targetType.Elem().Kind() == reflect.Struct {
			var mapper = NewMetaRecordMapped(targetType, true)
			return mapper
		}
	default:
		panic(fmt.Sprintf("unsupported type: %v ", targetType.Name()))
	}
	return nil
}

//NewRecordMapperIfNeeded create a new mapper if passed in mapper is nil. It takes target type for the record mapper.
func NewRecordMapperIfNeeded(mapper RecordMapper, targetType reflect.Type) RecordMapper {
	if mapper != nil {
		return mapper
	}
	return NewRecordMapper(targetType)
}

//ScanRow takes scanner to scans row.
func ScanRow(scanner Scanner) ([]interface{}, []string, error) {
	columns, _ := scanner.Columns()
	count := len(columns)

	var err error
	var rowValues = make([]interface{}, count)

	provider, ok := scanner.(ColumnValueProvider)
	if ok {
		if values, err := provider.ColumnValues(); err == nil {
			if err = scanner.Scan(values...); err != nil {
				return nil, nil, fmt.Errorf("failed to scan row due to %v", err)
			}
			for i, v := range values {
				if v == nil {
					continue
				}
				valuePtr := reflect.ValueOf(v).Elem()
				pointer := v
				if valuePtr.Kind() ==  reflect.Ptr {
					pointer = valuePtr.Elem().Interface()
				}
				if pointer == nil {
					continue
				}
				value := pointer
				valuePtr =  reflect.ValueOf(pointer)
				if valuePtr.Kind() ==  reflect.Ptr {
					value = valuePtr.Elem().Interface()
				}
				rowValues[i] = value
			}
			return rowValues, columns, nil
		}
	}

	var valuePointers = make([]interface{}, count)
	for i := range rowValues {
		valuePointers[i] = &rowValues[i]
	}

	err = scanner.Scan(valuePointers...)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to scan row due to %v", err)
	}

	for i := range rowValues {
		var value interface{}
		rawValue := rowValues[i]
		b, ok := rawValue.([]byte)
		if ok {
			value = string(b)
		} else {
			value = rawValue
		}
		rowValues[i] = value
	}

	return rowValues, columns, nil
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"
	"testing"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/client/restclient"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/client/unversioned/fake"
)

var (
	TRUE  = true
	FALSE = false
)

func TestLimits(t *testing.T) {
	tests := []struct {
		CPU, Memory string
		Expected    api.ResourceList
		Error       bool
	}{
		{
			CPU:    "100m",
			Memory: "100Mi",
			Expected: api.ResourceList{
				api.ResourceLimitsCPU:    resource.MustParse("100m"),
				api.ResourceLimitsMemory: resource.MustParse("100Mi"),
			},
		},
		{
			CPU: "100m",
			Expected: api.ResourceList{
				api.ResourceLimitsCPU: resource.MustParse("100m"),
			},
		},
		{
			Memory: "100Mi",
			Expected: api.ResourceList{
				api.ResourceLimitsMemory: resource.MustParse("100Mi"),
			},
		},
		{
			CPU:      "100j",
			Expected: api.ResourceList{},
			Error:    true,
		},
		{
			Memory:   "100j",
			Expected: api.ResourceList{},
			Error:    true,
		},
		{
			Expected: api.ResourceList{},
		},
	}

	for _, test := range tests {
		res, err := limits(test.CPU, test.Memory)

		if err != nil && !test.Error {
			t.Errorf("got error but expected '%v': %s", test.Expected, err)
			continue
		}

		if !reflect.DeepEqual(res, test.Expected) {
			t.Errorf("got: '%v' but expected: '%v'", res, test.Expected)
			continue
		}
	}
}

func TestBuildContainer(t *testing.T) {
	tests := []struct {
		Name, Image, BuildDir string
		Privileged            bool
		Command               []string
		Environment           []string
		Limits                api.ResourceList

		Expected api.Container
	}{
		{
			Name:        "test",
			Image:       "image",
			BuildDir:    "/test/build",
			Privileged:  true,
			Command:     []string{"test", "command"},
			Environment: nil,
			Limits:      nil,

			Expected: api.Container{
				Name:    "test",
				Image:   "image",
				Command: []string{"test", "command"},
				Env: []api.EnvVar{
					{Name: "CI", Value: "true"}, {Name: "CI_BUILD_REF"}, {Name: "CI_BUILD_BEFORE_SHA"},
					{Name: "CI_BUILD_REF_NAME"}, {Name: "CI_BUILD_ID", Value: "0"}, {Name: "CI_BUILD_REPO"},
					{Name: "CI_BUILD_TOKEN"}, {Name: "CI_PROJECT_ID", Value: "0"}, {Name: "CI_PROJECT_DIR", Value: "/test/build"},
					{Name: "CI_SERVER", Value: "yes"}, {Name: "CI_SERVER_NAME", Value: "GitLab CI"}, {Name: "CI_SERVER_VERSION"},
					{Name: "CI_SERVER_REVISION"}, {Name: "GITLAB_CI", Value: "true"},
				},
				Resources: api.ResourceRequirements{
					Limits: nil,
				},
				VolumeMounts: []api.VolumeMount{
					api.VolumeMount{
						Name:      "repo",
						MountPath: "/test",
					},
				},
				SecurityContext: &api.SecurityContext{
					Privileged: &TRUE,
				},
				Stdin: true,
			},
		},
	}

	for _, test := range tests {
		e := executor{
			extraOptions: FakeOptions{
				privileged: test.Privileged,
			},
			AbstractExecutor: executors.AbstractExecutor{
				Build: &common.Build{
					BuildDir: test.BuildDir,
					Runner: &common.RunnerConfig{
						RunnerSettings: common.RunnerSettings{
							Environment: test.Environment,
						},
					},
				},
				ExecutorOptions: executors.ExecutorOptions{
					Shell: common.ShellScriptInfo{
						Build: &common.Build{
							BuildDir: test.BuildDir,
						},
					},
				},
			},
		}
		if bc := e.buildContainer(test.Name, test.Image, test.Limits, test.Command...); !reflect.DeepEqual(bc, test.Expected) {
			t.Errorf("error testing buildContainer. expected '%v', got '%v'", test.Expected, bc)
			continue
		}
	}
}

func TestCleanup(t *testing.T) {
	version := testapi.Default.GroupVersion().Version
	codec := testapi.Default.Codec()

	tests := []struct {
		Pod        *api.Pod
		ClientFunc func(*http.Request) (*http.Response, error)
		Error      bool
	}{
		{
			Pod: &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:      "test-pod",
					Namespace: "test-ns",
				},
			},
			ClientFunc: func(req *http.Request) (*http.Response, error) {
				switch p, m := req.URL.Path, req.Method; {
				case m == "DELETE" && p == "/api/"+version+"/namespaces/test-ns/pods/test-pod":
					return &http.Response{StatusCode: 200, Body: FakeReadCloser{
						Reader: strings.NewReader(""),
					}}, nil
				default:
					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
				}
			},
		},
		{
			Pod: &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:      "test-pod",
					Namespace: "test-ns",
				},
			},
			ClientFunc: func(req *http.Request) (*http.Response, error) {
				return nil, fmt.Errorf("delete failed")
			},
			Error: true,
		},
	}

	for _, test := range tests {
		c := client.NewOrDie(&restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &unversioned.GroupVersion{Version: version}}})
		fakeClient := fake.RESTClient{
			Codec:  codec,
			Client: fake.CreateHTTPClient(test.ClientFunc),
		}
		c.Client = fakeClient.Client

		ex := executor{
			kubeClient: c,
			pod:        test.Pod,
		}
		errored := false
		ex.AbstractExecutor.BuildLog = FakeBuildTrace{
			testWriter{
				call: func(b []byte) (int, error) {
					if test.Error && !errored {
						if strings.Contains(string(b), "Error cleaning up") {
							errored = true
						} else {
							t.Errorf("expected failure. got: '%s'", string(b))
						}
					}
					return len(b), nil
				},
			},
		}
		ex.Cleanup()
		if test.Error && !errored {
			t.Errorf("expected cleanup to error but it didn't")
		}
	}
}

func TestPrepare(t *testing.T) {
	tests := []struct {
		GlobalConfig *common.Config
		RunnerConfig *common.RunnerConfig
		Build        *common.Build

		Expected *executor
		Error    bool
	}{
		{
			GlobalConfig: &common.Config{},
			RunnerConfig: &common.RunnerConfig{
				RunnerSettings: common.RunnerSettings{
					Kubernetes: &common.KubernetesConfig{
						Host:            "test-server",
						ServiceCPUs:     "0.5",
						ServiceMemory:   "200Mi",
						CPUs:            "1.5",
						Memory:          "4Gi",
						AllowPrivileged: true,
					},
				},
			},
			Build: &common.Build{
				GetBuildResponse: common.GetBuildResponse{
					Sha: "1234567890",
					Options: common.BuildOptions{
						"image": "test-image",
					},
					Variables: common.BuildVariables{
						{Key: "privileged", Value: "true"},
					},
				},
				Runner: &common.RunnerConfig{},
			},
			Expected: &executor{
				options: &kubernetesOptions{
					Image: "test-image",
				},
				extraOptions: DefaultOptions{
					BuildVariables: common.BuildVariables{
						{Key: "privileged", Value: "true"},
					},
				},
				serviceLimits: api.ResourceList{
					api.ResourceLimitsCPU:    resource.MustParse("0.5"),
					api.ResourceLimitsMemory: resource.MustParse("200Mi"),
				},
				buildLimits: api.ResourceList{
					api.ResourceLimitsCPU:    resource.MustParse("1.5"),
					api.ResourceLimitsMemory: resource.MustParse("4Gi"),
				},
			},
		},
		{
			GlobalConfig: &common.Config{},
			RunnerConfig: &common.RunnerConfig{
				RunnerSettings: common.RunnerSettings{
					Kubernetes: &common.KubernetesConfig{
						Host:            "test-server",
						ServiceCPUs:     "0.5",
						ServiceMemory:   "200Mi",
						CPUs:            "1.5",
						Memory:          "4Gi",
						AllowPrivileged: false,
					},
				},
			},
			Build: &common.Build{
				GetBuildResponse: common.GetBuildResponse{
					Sha: "1234567890",
					Options: common.BuildOptions{
						"image": "test-image",
					},
					Variables: common.BuildVariables{
						{Key: "privileged", Value: "true"},
					},
				},
				Runner: &common.RunnerConfig{},
			},
			Error: true,
		},
	}

	for _, test := range tests {
		e := &executor{
			AbstractExecutor: executors.AbstractExecutor{
				ExecutorOptions: executorOptions,
			},
		}

		err := e.Prepare(test.GlobalConfig, test.RunnerConfig, test.Build)

		if err != nil {
			if !test.Error {
				t.Errorf("Got error. Expected: %v", test.Expected)
			}
			continue
		}

		// Set this to nil so we aren't testing the functionality of the
		// base AbstractExecutor's Prepare method
		e.AbstractExecutor = executors.AbstractExecutor{}

		// TODO: Improve this so we don't have to nil-ify the kubeClient.
		// It currently contains some moving parts that are failing, meaning
		// we'll need to mock _something_
		e.kubeClient = nil
		if !reflect.DeepEqual(e, test.Expected) {
			t.Errorf("Got executor '%+v' but expected '%+v'", e, test.Expected)
			continue
		}
	}
}

type FakeReadCloser struct {
	io.Reader
}

func (f FakeReadCloser) Close() error { return nil }

type FakeBuildTrace struct {
	testWriter
}

func (f FakeBuildTrace) Success()       {}
func (f FakeBuildTrace) Fail(error)     {}
func (f FakeBuildTrace) Notify(func())  {}
func (f FakeBuildTrace) IsStdout() bool { return false }
<commit_msg>Fix go vet<commit_after>package kubernetes

import (
	"fmt"
	"io"
	"net/http"
	"reflect"
	"strings"
	"testing"

	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/common"
	"gitlab.com/gitlab-org/gitlab-ci-multi-runner/executors"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/client/restclient"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/client/unversioned/fake"
)

var (
	TRUE  = true
	FALSE = false
)

func TestLimits(t *testing.T) {
	tests := []struct {
		CPU, Memory string
		Expected    api.ResourceList
		Error       bool
	}{
		{
			CPU:    "100m",
			Memory: "100Mi",
			Expected: api.ResourceList{
				api.ResourceLimitsCPU:    resource.MustParse("100m"),
				api.ResourceLimitsMemory: resource.MustParse("100Mi"),
			},
		},
		{
			CPU: "100m",
			Expected: api.ResourceList{
				api.ResourceLimitsCPU: resource.MustParse("100m"),
			},
		},
		{
			Memory: "100Mi",
			Expected: api.ResourceList{
				api.ResourceLimitsMemory: resource.MustParse("100Mi"),
			},
		},
		{
			CPU:      "100j",
			Expected: api.ResourceList{},
			Error:    true,
		},
		{
			Memory:   "100j",
			Expected: api.ResourceList{},
			Error:    true,
		},
		{
			Expected: api.ResourceList{},
		},
	}

	for _, test := range tests {
		res, err := limits(test.CPU, test.Memory)

		if err != nil && !test.Error {
			t.Errorf("got error but expected '%v': %s", test.Expected, err)
			continue
		}

		if !reflect.DeepEqual(res, test.Expected) {
			t.Errorf("got: '%v' but expected: '%v'", res, test.Expected)
			continue
		}
	}
}

func TestBuildContainer(t *testing.T) {
	tests := []struct {
		Name, Image, BuildDir string
		Privileged            bool
		Command               []string
		Environment           []string
		Limits                api.ResourceList

		Expected api.Container
	}{
		{
			Name:        "test",
			Image:       "image",
			BuildDir:    "/test/build",
			Privileged:  true,
			Command:     []string{"test", "command"},
			Environment: nil,
			Limits:      nil,

			Expected: api.Container{
				Name:    "test",
				Image:   "image",
				Command: []string{"test", "command"},
				Env: []api.EnvVar{
					{Name: "CI", Value: "true"}, {Name: "CI_BUILD_REF"}, {Name: "CI_BUILD_BEFORE_SHA"},
					{Name: "CI_BUILD_REF_NAME"}, {Name: "CI_BUILD_ID", Value: "0"}, {Name: "CI_BUILD_REPO"},
					{Name: "CI_BUILD_TOKEN"}, {Name: "CI_PROJECT_ID", Value: "0"}, {Name: "CI_PROJECT_DIR", Value: "/test/build"},
					{Name: "CI_SERVER", Value: "yes"}, {Name: "CI_SERVER_NAME", Value: "GitLab CI"}, {Name: "CI_SERVER_VERSION"},
					{Name: "CI_SERVER_REVISION"}, {Name: "GITLAB_CI", Value: "true"},
				},
				Resources: api.ResourceRequirements{
					Limits: nil,
				},
				VolumeMounts: []api.VolumeMount{
					api.VolumeMount{
						Name:      "repo",
						MountPath: "/test",
					},
				},
				SecurityContext: &api.SecurityContext{
					Privileged: &TRUE,
				},
				Stdin: true,
			},
		},
	}

	for _, test := range tests {
		e := executor{
			extraOptions: FakeOptions{
				privileged: test.Privileged,
			},
			AbstractExecutor: executors.AbstractExecutor{
				Build: &common.Build{
					BuildDir: test.BuildDir,
					Runner: &common.RunnerConfig{
						RunnerSettings: common.RunnerSettings{
							Environment: test.Environment,
						},
					},
				},
				ExecutorOptions: executors.ExecutorOptions{
					Shell: common.ShellScriptInfo{
						Build: &common.Build{
							BuildDir: test.BuildDir,
						},
					},
				},
			},
		}
		if bc := e.buildContainer(test.Name, test.Image, test.Limits, test.Command...); !reflect.DeepEqual(bc, test.Expected) {
			t.Errorf("error testing buildContainer. expected '%v', got '%v'", test.Expected, bc)
			continue
		}
	}
}

func TestCleanup(t *testing.T) {
	version := testapi.Default.GroupVersion().Version
	codec := testapi.Default.Codec()

	tests := []struct {
		Pod        *api.Pod
		ClientFunc func(*http.Request) (*http.Response, error)
		Error      bool
	}{
		{
			Pod: &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:      "test-pod",
					Namespace: "test-ns",
				},
			},
			ClientFunc: func(req *http.Request) (*http.Response, error) {
				switch p, m := req.URL.Path, req.Method; {
				case m == "DELETE" && p == "/api/"+version+"/namespaces/test-ns/pods/test-pod":
					return &http.Response{StatusCode: 200, Body: FakeReadCloser{
						Reader: strings.NewReader(""),
					}}, nil
				default:
					return nil, fmt.Errorf("unexpected request. method: %s, path: %s", m, p)
				}
			},
		},
		{
			Pod: &api.Pod{
				ObjectMeta: api.ObjectMeta{
					Name:      "test-pod",
					Namespace: "test-ns",
				},
			},
			ClientFunc: func(req *http.Request) (*http.Response, error) {
				return nil, fmt.Errorf("delete failed")
			},
			Error: true,
		},
	}

	for _, test := range tests {
		c := client.NewOrDie(&restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &unversioned.GroupVersion{Version: version}}})
		fakeClient := fake.RESTClient{
			Codec:  codec,
			Client: fake.CreateHTTPClient(test.ClientFunc),
		}
		c.Client = fakeClient.Client

		ex := executor{
			kubeClient: c,
			pod:        test.Pod,
		}
		errored := false
		ex.AbstractExecutor.BuildLog = FakeBuildTrace{
			testWriter{
				call: func(b []byte) (int, error) {
					if test.Error && !errored {
						if strings.Contains(string(b), "Error cleaning up") {
							errored = true
						} else {
							t.Errorf("expected failure. got: '%s'", string(b))
						}
					}
					return len(b), nil
				},
			},
		}
		ex.Cleanup()
		if test.Error && !errored {
			t.Errorf("expected cleanup to error but it didn't")
		}
	}
}

func TestPrepare(t *testing.T) {
	tests := []struct {
		GlobalConfig *common.Config
		RunnerConfig *common.RunnerConfig
		Build        *common.Build

		Expected *executor
		Error    bool
	}{
		{
			GlobalConfig: &common.Config{},
			RunnerConfig: &common.RunnerConfig{
				RunnerSettings: common.RunnerSettings{
					Kubernetes: &common.KubernetesConfig{
						Host:            "test-server",
						ServiceCPUs:     "0.5",
						ServiceMemory:   "200Mi",
						CPUs:            "1.5",
						Memory:          "4Gi",
						AllowPrivileged: true,
					},
				},
			},
			Build: &common.Build{
				GetBuildResponse: common.GetBuildResponse{
					Sha: "1234567890",
					Options: common.BuildOptions{
						"image": "test-image",
					},
					Variables: []common.BuildVariable{
						common.BuildVariable{Key: "privileged", Value: "true"},
					},
				},
				Runner: &common.RunnerConfig{},
			},
			Expected: &executor{
				options: &kubernetesOptions{
					Image: "test-image",
				},
				extraOptions: DefaultOptions{
					BuildVariables: []common.BuildVariable{
						common.BuildVariable{Key: "privileged", Value: "true"},
					},
				},
				serviceLimits: api.ResourceList{
					api.ResourceLimitsCPU:    resource.MustParse("0.5"),
					api.ResourceLimitsMemory: resource.MustParse("200Mi"),
				},
				buildLimits: api.ResourceList{
					api.ResourceLimitsCPU:    resource.MustParse("1.5"),
					api.ResourceLimitsMemory: resource.MustParse("4Gi"),
				},
			},
		},
		{
			GlobalConfig: &common.Config{},
			RunnerConfig: &common.RunnerConfig{
				RunnerSettings: common.RunnerSettings{
					Kubernetes: &common.KubernetesConfig{
						Host:            "test-server",
						ServiceCPUs:     "0.5",
						ServiceMemory:   "200Mi",
						CPUs:            "1.5",
						Memory:          "4Gi",
						AllowPrivileged: false,
					},
				},
			},
			Build: &common.Build{
				GetBuildResponse: common.GetBuildResponse{
					Sha: "1234567890",
					Options: common.BuildOptions{
						"image": "test-image",
					},
					Variables: []common.BuildVariable{
						common.BuildVariable{Key: "privileged", Value: "true"},
					},
				},
				Runner: &common.RunnerConfig{},
			},
			Error: true,
		},
	}

	for _, test := range tests {
		e := &executor{
			AbstractExecutor: executors.AbstractExecutor{
				ExecutorOptions: executorOptions,
			},
		}

		err := e.Prepare(test.GlobalConfig, test.RunnerConfig, test.Build)

		if err != nil {
			if !test.Error {
				t.Errorf("Got error. Expected: %v", test.Expected)
			}
			continue
		}

		// Set this to nil so we aren't testing the functionality of the
		// base AbstractExecutor's Prepare method
		e.AbstractExecutor = executors.AbstractExecutor{}

		// TODO: Improve this so we don't have to nil-ify the kubeClient.
		// It currently contains some moving parts that are failing, meaning
		// we'll need to mock _something_
		e.kubeClient = nil
		if !reflect.DeepEqual(e, test.Expected) {
			t.Errorf("Got executor '%+v' but expected '%+v'", e, test.Expected)
			continue
		}
	}
}

type FakeReadCloser struct {
	io.Reader
}

func (f FakeReadCloser) Close() error { return nil }

type FakeBuildTrace struct {
	testWriter
}

func (f FakeBuildTrace) Success()       {}
func (f FakeBuildTrace) Fail(error)     {}
func (f FakeBuildTrace) Notify(func())  {}
func (f FakeBuildTrace) IsStdout() bool { return false }
<|endoftext|>