<commit_before>/*
Copyright (c) 2015, UPMC Enterprises
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api/v1"
	"k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/1.4/rest"
)

var (
	argListenPort        = flag.Int("listen-port", 9080, "port to have API listen")
	argDockerRegistry    = flag.String("docker-registry", "", "docker registry to use")
	argKubecfgFile       = flag.String("kubecfg-file", "", "Location of kubecfg file for access to kubernetes master service; --kube_master_url overrides the URL part of this; if neither this nor --kube_master_url are provided, defaults to service account tokens")
	argKubeMasterURL     = flag.String("kube-master-url", "", "URL to reach kubernetes master. Env variables in this flag will be expanded.")
	argTemplateNamespace = flag.String("template-namespace", "template", "Namespace to 'clone from when creating new deployments'")
	argPathToTokens      = flag.String("path-to-tokens", "", "Full path including file name to tokens file for authorization, setting to empty string will disable.")
	argSubDomain         = flag.String("subdomain", "k8s.local.com", "Subdomain used to configure external routing to branch (e.g. namespace.ci.k8s.local)")
	client               *kubernetes.Clientset
	defaultReplicaCount  *int32
)

const (
	appVersion = "0.0.3"
)

// Default (GET "/")
func indexRoute(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, %s", "welcome to Emmie!")
}

// Version (GET "/version")
func versionRoute(w http.ResponseWriter, r *http.Request) {
	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	fmt.Fprintf(w, "%q", appVersion)
}

// Deploy (POST "/deploy/namespace/branchName")
func deployRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	imageNamespace := vars["namespace"]

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)
	log.Println("[Emmie] is deploying branch:", branchName)

	// create namespace
	err := createNamespace(branchName)

	if err != nil {
		// TODO: Don't use error for logic
		// Existing namespace, do an update
		log.Println("Existing namespace found: ", branchName, " deleting pods.")

		deletePodsByNamespace(branchName)
	} else {
		log.Println("Namespace created, deploying new app...")

		// copy controllers / services based on label query
		rcs, _ := listReplicationControllersByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(rcs.Items), " template replication controllers to copy.")

		deployments, _ := listDeploymentsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(deployments.Items), " template deployments to copy.")

		svcs, _ := listServicesByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(svcs.Items), " template services to copy.")

		secrets, _ := listSecretsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(secrets.Items), " template secrets to copy.")

		configmaps, _ := listConfigMapsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(configmaps.Items), " template configmaps to copy.")

		ingresses, _ := listIngresssByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(ingresses.Items), " template ingresses to copy.")

		// create configmaps
		for _, configmap := range configmaps.Items {

			requestConfigMap := &v1.ConfigMap{
				ObjectMeta: v1.ObjectMeta{
					Name:      configmap.Name,
					Namespace: branchName,
				},
				Data: configmap.Data,
			}

			createConfigMap(branchName, requestConfigMap)
		}

		// create secrets
		for _, secret := range secrets.Items {

			// skip service accounts
			if secret.Type != "kubernetes.io/service-account-token" {

				requestSecret := &v1.Secret{
					ObjectMeta: v1.ObjectMeta{
						Name:      secret.Name,
						Namespace: branchName,
					},
					Type: secret.Type,
					Data: secret.Data,
				}

				createSecret(branchName, requestSecret)
			}
		}

		// create services
		for _, svc := range svcs.Items {

			requestService := &v1.Service{
				ObjectMeta: v1.ObjectMeta{
					Name:      svc.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			ports := []v1.ServicePort{}
			for _, port := range svc.Spec.Ports {
				newPort := v1.ServicePort{
					Name:       port.Name,
					Protocol:   port.Protocol,
					Port:       port.Port,
					TargetPort: port.TargetPort,
				}

				ports = append(ports, newPort)
			}

			requestService.Spec.Ports = ports
			requestService.Spec.Selector = svc.Spec.Selector
			requestService.Spec.Type = svc.Spec.Type
			requestService.Labels = svc.Labels

			createService(branchName, requestService)
		}

		// now that we have all replicationControllers, update them to have new image name
		for _, rc := range rcs.Items {

			containerNameToUpdate := ""

			// Looks for annotations to know which container to replace
			for key, value := range rc.Annotations {
				if key == "emmie-update" {
					containerNameToUpdate = value
				}
			}

			// Find the container which matches the annotation
			for i, container := range rc.Spec.Template.Spec.Containers {

				imageName := ""

				if containerNameToUpdate == "" {
					//default to current image tag if no annotations found
					imageName = container.Image
				} else {
					imageName = fmt.Sprintf("%s%s/%s:%s", *argDockerRegistry, imageNamespace, rc.ObjectMeta.Labels["name"], branchName)
				}

				rc.Spec.Template.Spec.Containers[i].Image = imageName

				// Set the image pull policy to "Always"
				rc.Spec.Template.Spec.Containers[i].ImagePullPolicy = "Always"
			}

			requestController := &v1.ReplicationController{
				ObjectMeta: v1.ObjectMeta{
					Name:      rc.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			requestController.Spec = rc.Spec
			requestController.Spec.Replicas = defaultReplicaCount

			// create new replication controller
			createReplicationController(branchName, requestController)
		}

		// now that we have all deployments, update them to have new image name
		for _, dply := range deployments.Items {

			containerNameToUpdate := ""

			// Looks for annotations to know which container to replace
			for key, value := range dply.Annotations {
				if key == "emmie-update" {
					containerNameToUpdate = value
				}
			}

			// Find the container which matches the annotation
			for i, container := range dply.Spec.Template.Spec.Containers {

				imageName := ""

				if containerNameToUpdate == "" {
					//default to current image tag if no annotations found
					imageName = container.Image
				} else {
					imageName = fmt.Sprintf("%s%s/%s:%s", *argDockerRegistry, imageNamespace, dply.ObjectMeta.Labels["name"], branchName)
				}

				dply.Spec.Template.Spec.Containers[i].Image = imageName

				// Set the image pull policy to "Always"
				dply.Spec.Template.Spec.Containers[i].ImagePullPolicy = "Always"
			}

			deployment := &v1beta1.Deployment{
				ObjectMeta: v1.ObjectMeta{
					Name:      dply.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			deployment.Spec = dply.Spec
			deployment.Spec.Replicas = defaultReplicaCount

			// create new replication controller
			createDeployment(branchName, deployment)
		}

		// create ingress
		for _, ingress := range ingresses.Items {

			rules := ingress.Spec.Rules
			rules[0].Host = fmt.Sprintf("%s.%s", branchName, *argSubDomain)

			requestIngress := &v1beta1.Ingress{
				ObjectMeta: v1.ObjectMeta{
					Name:      ingress.Name,
					Namespace: branchName,
				},
				Spec: v1beta1.IngressSpec{
					Rules: rules,
				},
			}

			createIngress(branchName, requestIngress)
		}
	}
	log.Println("[Emmie] is finished deploying branch!")
}

// Put (PUT "/deploy")
func updateRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	log.Println(w, "[Emmie] is updating branch:", branchName)

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)

	deletePodsByNamespace(branchName)

	log.Println("Finished updating branch!")
}

// Delete (DELETE "/deploy")
func deleteRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	log.Println("[Emmie] is deleting branch:", branchName)

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)

	// get controllers / services / secrets in namespace
	rcs, _ := listReplicationControllersByNamespace(*argTemplateNamespace)
	for _, rc := range rcs.Items {
		deleteReplicationController(branchName, rc.ObjectMeta.Name)
		log.Println("Deleted replicationController:", rc.ObjectMeta.Name)
	}

	deployments, _ := listDeploymentsByNamespace(*argTemplateNamespace)
	for _, dply := range deployments.Items {
		deleteDeployment(branchName, dply.ObjectMeta.Name)
		log.Println("Deleted deployment:", dply.ObjectMeta.Name)
	}

	svcs, _ := listServicesByNamespace(*argTemplateNamespace)
	for _, svc := range svcs.Items {
		deleteService(branchName, svc.ObjectMeta.Name)
		log.Println("Deleted service:", svc.ObjectMeta.Name)
	}

	secrets, _ := listSecretsByNamespace(*argTemplateNamespace)
	for _, secret := range secrets.Items {
		deleteSecret(branchName, secret.ObjectMeta.Name)
		log.Println("Deleted secret:", secret.ObjectMeta.Name)
	}

	configmaps, _ := listConfigMapsByNamespace(*argTemplateNamespace)
	for _, configmap := range configmaps.Items {
		deleteSecret(branchName, configmap.ObjectMeta.Name)
		log.Println("Deleted configmap:", configmap.ObjectMeta.Name)
	}

	ingresses, _ := listIngresssByNamespace(*argTemplateNamespace)
	for _, ingress := range ingresses.Items {
		deleteIngress(branchName, ingress.ObjectMeta.Name)
		log.Println("Deleted ingress:", ingress.ObjectMeta.Name)
	}

	deleteNamespace(branchName)
	log.Println("[Emmie] is done deleting branch.")
}

func tokenIsValid(token string) bool {
	// If no path is passed, then auth is disabled
	if *argPathToTokens == "" {
		return true
	}

	file, err := os.Open(*argPathToTokens)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if token == scanner.Text() {
			fmt.Println("Token IS valid!")
			return true
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Token is NOT valid! =(")
	return false
}

func main() {
	flag.Parse()
	log.Println("[Emmie] is up and running!", time.Now())

	// Sanitize docker registry
	if *argDockerRegistry != "" {
		*argDockerRegistry = fmt.Sprintf("%s/", *argDockerRegistry)
	}

	// Configure router
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/", indexRoute)
	router.HandleFunc("/deploy/{namespace}/{branchName}", deployRoute).Methods("POST")
	router.HandleFunc("/deploy/{branchName}", deleteRoute).Methods("DELETE")
	router.HandleFunc("/deploy/{branchName}", updateRoute).Methods("PUT")
	router.HandleFunc("/deploy", getDeploymentsRoute).Methods("GET")

	// Services
	// router.HandleFunc("/services/{namespace}/{serviceName}", getServiceRoute).Methods("GET")
	// router.HandleFunc("/services/{namespace}/{key}/{value}", getServicesRoute).Methods("GET")

	// ReplicationControllers
	// router.HandleFunc("/replicationControllers/{namespace}/{rcName}", getReplicationControllerRoute).Methods("GET")
	// router.HandleFunc("/replicationControllers/{namespace}/{key}/{value}", getReplicationControllersRoute).Methods("GET")

	// Deployments
	// router.HandleFunc("/deployments/{namespace}/{deploymentName}", getDeploymentRoute).Methods("GET")
	// router.HandleFunc("/deployments/{namespace}/{key}/{value}", getDeploymentsRoute).Methods("GET")

	// Version
	router.HandleFunc("/version", versionRoute)

	// Create k8s client
	config, err := rest.InClusterConfig()
	//config, err := clientcmd.BuildConfigFromFlags("", *argKubecfgFile)
	if err != nil {
		panic(err.Error())
	}

	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	client = clientset

	// Start server
	log.Fatal(http.ListenAndServeTLS(fmt.Sprintf(":%d", *argListenPort), "certs/cert.pem", "certs/key.pem", router))
	//log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *argListenPort), router))
}
<commit_msg>Fixed image replace logic<commit_after>/*
Copyright (c) 2015, UPMC Enterprises
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name UPMC Enterprises nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL UPMC ENTERPRISES BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
*/

package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api/v1"
	"k8s.io/client-go/1.4/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/1.4/rest"
)

var (
	argListenPort        = flag.Int("listen-port", 9080, "port to have API listen")
	argDockerRegistry    = flag.String("docker-registry", "", "docker registry to use")
	argKubecfgFile       = flag.String("kubecfg-file", "", "Location of kubecfg file for access to kubernetes master service; --kube_master_url overrides the URL part of this; if neither this nor --kube_master_url are provided, defaults to service account tokens")
	argKubeMasterURL     = flag.String("kube-master-url", "", "URL to reach kubernetes master. Env variables in this flag will be expanded.")
	argTemplateNamespace = flag.String("template-namespace", "template", "Namespace to 'clone from when creating new deployments'")
	argPathToTokens      = flag.String("path-to-tokens", "", "Full path including file name to tokens file for authorization, setting to empty string will disable.")
	argSubDomain         = flag.String("subdomain", "k8s.local.com", "Subdomain used to configure external routing to branch (e.g. namespace.ci.k8s.local)")
	client               *kubernetes.Clientset
	defaultReplicaCount  *int32
)

const (
	appVersion = "0.0.3"
)

// Default (GET "/")
func indexRoute(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello, %s", "welcome to Emmie!")
}

// Version (GET "/version")
func versionRoute(w http.ResponseWriter, r *http.Request) {
	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	fmt.Fprintf(w, "%q", appVersion)
}

// Deploy (POST "/deploy/namespace/branchName")
func deployRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	imageNamespace := vars["namespace"]

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)
	log.Println("[Emmie] is deploying branch:", branchName)

	// create namespace
	err := createNamespace(branchName)

	if err != nil {
		// TODO: Don't use error for logic
		// Existing namespace, do an update
		log.Println("Existing namespace found: ", branchName, " deleting pods.")

		deletePodsByNamespace(branchName)
	} else {
		log.Println("Namespace created, deploying new app...")

		// copy controllers / services based on label query
		rcs, _ := listReplicationControllersByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(rcs.Items), " template replication controllers to copy.")

		deployments, _ := listDeploymentsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(deployments.Items), " template deployments to copy.")

		svcs, _ := listServicesByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(svcs.Items), " template services to copy.")

		secrets, _ := listSecretsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(secrets.Items), " template secrets to copy.")

		configmaps, _ := listConfigMapsByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(configmaps.Items), " template configmaps to copy.")

		ingresses, _ := listIngresssByNamespace(*argTemplateNamespace)
		log.Println("Found ", len(ingresses.Items), " template ingresses to copy.")

		// create configmaps
		for _, configmap := range configmaps.Items {

			requestConfigMap := &v1.ConfigMap{
				ObjectMeta: v1.ObjectMeta{
					Name:      configmap.Name,
					Namespace: branchName,
				},
				Data: configmap.Data,
			}

			createConfigMap(branchName, requestConfigMap)
		}

		// create secrets
		for _, secret := range secrets.Items {

			// skip service accounts
			if secret.Type != "kubernetes.io/service-account-token" {

				requestSecret := &v1.Secret{
					ObjectMeta: v1.ObjectMeta{
						Name:      secret.Name,
						Namespace: branchName,
					},
					Type: secret.Type,
					Data: secret.Data,
				}

				createSecret(branchName, requestSecret)
			}
		}

		// create services
		for _, svc := range svcs.Items {

			requestService := &v1.Service{
				ObjectMeta: v1.ObjectMeta{
					Name:      svc.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			ports := []v1.ServicePort{}
			for _, port := range svc.Spec.Ports {
				newPort := v1.ServicePort{
					Name:       port.Name,
					Protocol:   port.Protocol,
					Port:       port.Port,
					TargetPort: port.TargetPort,
				}

				ports = append(ports, newPort)
			}

			requestService.Spec.Ports = ports
			requestService.Spec.Selector = svc.Spec.Selector
			requestService.Spec.Type = svc.Spec.Type
			requestService.Labels = svc.Labels

			createService(branchName, requestService)
		}

		// now that we have all replicationControllers, update them to have new image name
		for _, rc := range rcs.Items {

			containerNameToUpdate := ""

			// Looks for annotations to know which container to replace
			for key, value := range rc.Annotations {
				if key == "emmie-update" {
					containerNameToUpdate = value
				}
			}

			// Find the container which matches the annotation
			for i, container := range rc.Spec.Template.Spec.Containers {

				imageName := ""

				if containerNameToUpdate == rc.ObjectMeta.Name {
					imageName = fmt.Sprintf("%s%s/%s:%s", *argDockerRegistry, imageNamespace, rc.ObjectMeta.Name, branchName)
				} else {
					//default to current image tag if no annotations found
					imageName = container.Image
				}

				rc.Spec.Template.Spec.Containers[i].Image = imageName

				// Set the image pull policy to "Always"
				rc.Spec.Template.Spec.Containers[i].ImagePullPolicy = "Always"
			}

			requestController := &v1.ReplicationController{
				ObjectMeta: v1.ObjectMeta{
					Name:      rc.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			requestController.Spec = rc.Spec
			requestController.Spec.Replicas = defaultReplicaCount

			// create new replication controller
			createReplicationController(branchName, requestController)
		}

		// now that we have all deployments, update them to have new image name
		for _, dply := range deployments.Items {

			containerNameToUpdate := ""

			// Looks for annotations to know which container to replace
			for key, value := range dply.Annotations {
				if key == "emmie-update" {
					containerNameToUpdate = value
				}
			}

			// Find the container which matches the annotation
			for i, container := range dply.Spec.Template.Spec.Containers {

				imageName := ""

				if containerNameToUpdate == container.Name {
					imageName = fmt.Sprintf("%s%s/%s:%s", *argDockerRegistry, imageNamespace, container.Name, branchName)
				} else {
					//default to current image tag if no annotations found
					imageName = container.Image
				}

				dply.Spec.Template.Spec.Containers[i].Image = imageName

				// Set the image pull policy to "Always"
				dply.Spec.Template.Spec.Containers[i].ImagePullPolicy = "Always"
			}

			deployment := &v1beta1.Deployment{
				ObjectMeta: v1.ObjectMeta{
					Name:      dply.ObjectMeta.Name,
					Namespace: branchName,
				},
			}

			deployment.Spec = dply.Spec
			deployment.Spec.Replicas = defaultReplicaCount

			// create new replication controller
			createDeployment(branchName, deployment)
		}

		// create ingress
		for _, ingress := range ingresses.Items {

			rules := ingress.Spec.Rules
			rules[0].Host = fmt.Sprintf("%s.%s", branchName, *argSubDomain)

			requestIngress := &v1beta1.Ingress{
				ObjectMeta: v1.ObjectMeta{
					Name:      ingress.Name,
					Namespace: branchName,
				},
				Spec: v1beta1.IngressSpec{
					Rules: rules,
				},
			}

			createIngress(branchName, requestIngress)
		}
	}
	log.Println("[Emmie] is finished deploying branch!")
}

// Put (PUT "/deploy")
func updateRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	log.Println(w, "[Emmie] is updating branch:", branchName)

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)

	deletePodsByNamespace(branchName)

	log.Println("Finished updating branch!")
}

// Delete (DELETE "/deploy")
func deleteRoute(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	branchName := vars["branchName"]
	log.Println("[Emmie] is deleting branch:", branchName)

	if !tokenIsValid(r.FormValue("token")) {
		w.WriteHeader(http.StatusUnauthorized)
		return
	}

	// sanitize BranchName
	branchName = strings.Replace(branchName, "_", "-", -1)

	// get controllers / services / secrets in namespace
	rcs, _ := listReplicationControllersByNamespace(*argTemplateNamespace)
	for _, rc := range rcs.Items {
		deleteReplicationController(branchName, rc.ObjectMeta.Name)
		log.Println("Deleted replicationController:", rc.ObjectMeta.Name)
	}

	deployments, _ := listDeploymentsByNamespace(*argTemplateNamespace)
	for _, dply := range deployments.Items {
		deleteDeployment(branchName, dply.ObjectMeta.Name)
		log.Println("Deleted deployment:", dply.ObjectMeta.Name)
	}

	svcs, _ := listServicesByNamespace(*argTemplateNamespace)
	for _, svc := range svcs.Items {
		deleteService(branchName, svc.ObjectMeta.Name)
		log.Println("Deleted service:", svc.ObjectMeta.Name)
	}

	secrets, _ := listSecretsByNamespace(*argTemplateNamespace)
	for _, secret := range secrets.Items {
		deleteSecret(branchName, secret.ObjectMeta.Name)
		log.Println("Deleted secret:", secret.ObjectMeta.Name)
	}

	configmaps, _ := listConfigMapsByNamespace(*argTemplateNamespace)
	for _, configmap := range configmaps.Items {
		deleteSecret(branchName, configmap.ObjectMeta.Name)
		log.Println("Deleted configmap:", configmap.ObjectMeta.Name)
	}

	ingresses, _ := listIngresssByNamespace(*argTemplateNamespace)
	for _, ingress := range ingresses.Items {
		deleteIngress(branchName, ingress.ObjectMeta.Name)
		log.Println("Deleted ingress:", ingress.ObjectMeta.Name)
	}

	deleteNamespace(branchName)
	log.Println("[Emmie] is done deleting branch.")
}

func tokenIsValid(token string) bool {
	// If no path is passed, then auth is disabled
	if *argPathToTokens == "" {
		return true
	}

	file, err := os.Open(*argPathToTokens)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if token == scanner.Text() {
			fmt.Println("Token IS valid!")
			return true
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	fmt.Println("Token is NOT valid! =(")
	return false
}

func main() {
	flag.Parse()
	log.Println("[Emmie] is up and running!", time.Now())

	// Sanitize docker registry
	if *argDockerRegistry != "" {
		*argDockerRegistry = fmt.Sprintf("%s/", *argDockerRegistry)
	}

	// Configure router
	router := mux.NewRouter().StrictSlash(true)
	router.HandleFunc("/", indexRoute)
	router.HandleFunc("/deploy/{namespace}/{branchName}", deployRoute).Methods("POST")
	router.HandleFunc("/deploy/{branchName}", deleteRoute).Methods("DELETE")
	router.HandleFunc("/deploy/{branchName}", updateRoute).Methods("PUT")
	router.HandleFunc("/deploy", getDeploymentsRoute).Methods("GET")

	// Services
	// router.HandleFunc("/services/{namespace}/{serviceName}", getServiceRoute).Methods("GET")
	// router.HandleFunc("/services/{namespace}/{key}/{value}", getServicesRoute).Methods("GET")

	// ReplicationControllers
	// router.HandleFunc("/replicationControllers/{namespace}/{rcName}", getReplicationControllerRoute).Methods("GET")
	// router.HandleFunc("/replicationControllers/{namespace}/{key}/{value}", getReplicationControllersRoute).Methods("GET")

	// Deployments
	// router.HandleFunc("/deployments/{namespace}/{deploymentName}", getDeploymentRoute).Methods("GET")
	// router.HandleFunc("/deployments/{namespace}/{key}/{value}", getDeploymentsRoute).Methods("GET")

	// Version
	router.HandleFunc("/version", versionRoute)

	// Create k8s client
	config, err := rest.InClusterConfig()
	//config, err := clientcmd.BuildConfigFromFlags("", *argKubecfgFile)
	if err != nil {
		panic(err.Error())
	}

	// creates the clientset
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	client = clientset

	// Start server
	log.Fatal(http.ListenAndServeTLS(fmt.Sprintf(":%d", *argListenPort), "certs/cert.pem", "certs/key.pem", router))
	//log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *argListenPort), router))
}
<|endoftext|>


<commit_before>package httputilmore

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
)

// GetWriteFile performs a HTTP GET request and saves the response body
// to the file path specified
func GetWriteFile(url string, filename string, perm os.FileMode) ([]byte, error) {
	_, bytes, err := GetResponseAndBytes(url)
	if err != nil {
		return bytes, err
	}
	err = ioutil.WriteFile(filename, bytes, perm)
	return bytes, err
}

func GetJsonSimple(requrl string, header http.Header, data interface{}) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodGet, requrl, nil)
	if err != nil {
		return nil, err
	}
	req.Header = header
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	_, err = jsonutil.UnmarshalIoReader(resp.Body, data)
	return resp, err
}

func PostJsonBytes(client *http.Client, requrl string, headers map[string]string, bodyBytes []byte) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodPost, requrl, bytes.NewBuffer(bodyBytes))
	if err != nil {
		return nil, err
	}
	for k, v := range headers {
		k = strings.TrimSpace(k)
		kMatch := strings.ToLower(k)
		if kMatch == strings.ToLower(HeaderContentType) {
			continue
		}
		req.Header.Set(k, v)
	}
	req.Header.Set(HeaderContentType, ContentTypeAppJsonUtf8)
	if client == nil {
		client = &http.Client{}
	}
	return client.Do(req)
}

func PostJsonMarshal(client *http.Client, requrl string, headers map[string]string, body interface{}) (*http.Response, error) {
	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	return PostJsonMarshal(client, requrl, headers, bodyBytes)
}

// PostJsonSimple performs a HTTP POST request converting a body interface{} to
// JSON and adding the appropriate JSON Content-Type header.
func PostJsonSimple(requrl string, body interface{}) (*http.Response, error) {
	return PostJsonMarshal(nil, requrl, map[string]string{}, body)
}

// GetResponseAndBytes retreives a URL and returns the response body
// as a byte array in addition to the *http.Response.
func GetResponseAndBytes(url string) (*http.Response, []byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return resp, []byte{}, err
	}
	bytes, err := ioutil.ReadAll(resp.Body)
	return resp, bytes, err
}

func SendWwwFormUrlEncodedSimple(method, urlStr string, data url.Values) (*http.Response, error) {
	req, err := http.NewRequest(
		method,
		urlStr,
		strings.NewReader(data.Encode())) // URL-encoded payload
	if err != nil {
		return nil, err
	}
	req.Header.Add(HeaderContentType, ContentTypeAppFormUrlEncoded)
	req.Header.Add(HeaderContentLength, strconv.Itoa(len(data.Encode())))
	client := &http.Client{}
	return client.Do(req)
}
<commit_msg>update GetWriteFile functionality<commit_after>package httputilmore

import (
	"bytes"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/grokify/gotilla/encoding/jsonutil"
	"github.com/pkg/errors"
)

// GetWriteFile gets the conents of a URL and stores the body in
// the desired filename location.
func GetWriteFile(client *http.Client, url, filename string) error {
	resp, err := client.Get(url)
	if err != nil {
		return errors.Wrap(err, "httputilmore.GetStoreURL.client.Get()")
	}
	defer resp.Body.Close()
	dir, file := filepath.Split(filename)
	if len(strings.TrimSpace(dir)) > 0 {
		os.Chdir(dir)
	}
	f, err := os.Create(file)
	if err != nil {
		return errors.Wrap(err, "httputilmore.GetStoreURL.os.Create()")
	}
	defer f.Close()
	_, err = io.Copy(f, resp.Body)
	if err != nil {
		err = errors.Wrap(err, "httputilmore.GetStoreURL.io.Copy()")
	}
	return err
}

// GetWriteFile performs a HTTP GET request and saves the response body
// to the file path specified. It reeads the entire file into memory
// which is not ideal for large files.
func GetWriteFileSimple(url string, filename string, perm os.FileMode) ([]byte, error) {
	_, bytes, err := GetResponseAndBytes(url)
	if err != nil {
		return bytes, err
	}
	err = ioutil.WriteFile(filename, bytes, perm)
	return bytes, err
}

func GetJsonSimple(requrl string, header http.Header, data interface{}) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodGet, requrl, nil)
	if err != nil {
		return nil, err
	}
	req.Header = header
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	_, err = jsonutil.UnmarshalIoReader(resp.Body, data)
	return resp, err
}

func PostJsonBytes(client *http.Client, requrl string, headers map[string]string, bodyBytes []byte) (*http.Response, error) {
	req, err := http.NewRequest(http.MethodPost, requrl, bytes.NewBuffer(bodyBytes))
	if err != nil {
		return nil, err
	}
	for k, v := range headers {
		k = strings.TrimSpace(k)
		kMatch := strings.ToLower(k)
		if kMatch == strings.ToLower(HeaderContentType) {
			continue
		}
		req.Header.Set(k, v)
	}
	req.Header.Set(HeaderContentType, ContentTypeAppJsonUtf8)
	if client == nil {
		client = &http.Client{}
	}
	return client.Do(req)
}

func PostJsonMarshal(client *http.Client, requrl string, headers map[string]string, body interface{}) (*http.Response, error) {
	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return nil, err
	}
	return PostJsonMarshal(client, requrl, headers, bodyBytes)
}

// PostJsonSimple performs a HTTP POST request converting a body interface{} to
// JSON and adding the appropriate JSON Content-Type header.
func PostJsonSimple(requrl string, body interface{}) (*http.Response, error) {
	return PostJsonMarshal(nil, requrl, map[string]string{}, body)
}

// GetResponseAndBytes retreives a URL and returns the response body
// as a byte array in addition to the *http.Response.
func GetResponseAndBytes(url string) (*http.Response, []byte, error) {
	resp, err := http.Get(url)
	if err != nil {
		return resp, []byte{}, err
	}
	bytes, err := ioutil.ReadAll(resp.Body)
	return resp, bytes, err
}

func SendWwwFormUrlEncodedSimple(method, urlStr string, data url.Values) (*http.Response, error) {
	req, err := http.NewRequest(
		method,
		urlStr,
		strings.NewReader(data.Encode())) // URL-encoded payload
	if err != nil {
		return nil, err
	}
	req.Header.Add(HeaderContentType, ContentTypeAppFormUrlEncoded)
	req.Header.Add(HeaderContentLength, strconv.Itoa(len(data.Encode())))
	client := &http.Client{}
	return client.Do(req)
}
<|endoftext|>


<commit_before>package gorocksdb

// #cgo LDFLAGS: -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy
// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"

import (
    "errors"
    "unsafe"
)

/* BackupEngineInfo represents the information about the backups
    in a backup engine instance. Use this get the state of the
    backup like number of backups and their ids and timestamps etc */
type BackupEngineInfo struct {
    c *C.rocksdb_backup_engine_info_t
}

// GetCount gets the number backsup available
func (self *BackupEngineInfo) GetCount() int {
    return int(C.rocksdb_backup_engine_info_count(self.c))
}

// GetTimestamp gets the timestamp at which the backup @index was taken
func (self *BackupEngineInfo) GetTimestamp(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_timestamp(self.c, C.int(index)))
}

// GetBackupId gets an id that uniquely identifies a backup
// regardless of its position
func (self *BackupEngineInfo) GetBackupId(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_backup_id(self.c, C.int(index)))
}

// GetSize get the size of the backup in bytes
func (self *BackupEngineInfo) GetSize(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_size(self.c, C.int(index)))
}

// GetNumFiles gets the number of files in the backup @index
func (self *BackupEngineInfo) GetNumFiles(index int) int32 {
    return int32(C.rocksdb_backup_engine_info_number_files(self.c, C.int(index)))
}

// Destroy destroys the backup engine info instance
func (self *BackupEngineInfo) Destroy() {
    C.rocksdb_backup_engine_info_destroy(self.c)
    self.c = nil
}

// RestoreOptions captures the options to be used during
// restoration of a backup
type RestoreOptions struct {
    c *C.rocksdb_restore_options_t
}

// NewRestoreOptions creates a RestoreOptions instance
func NewRestoreOptions() *RestoreOptions {
    return &RestoreOptions {
        c: C.rocksdb_restore_options_create(),
    }
}

func (self *RestoreOptions) SetKeepLogFiles(v int) {
    C.rocksdb_restore_options_set_keep_log_files(self.c, C.int(v))
}

// Destroy destroys this RestoreOptions instance
func (self *RestoreOptions) Destroy() {
    C.rocksdb_restore_options_destroy(self.c)
}

// BackupEngine is a reusable handle to a RocksDB Backup, created by
// OpenBackupEngine
type BackupEngine struct {
    c *C.rocksdb_backup_engine_t
    path string
    opts *Options
}

// OpenBackupEngine opens a backup engine with specified options
func OpenBackupEngine(opts *Options, path string) (*BackupEngine, error) {
    var cErr *C.char
    cpath := C.CString(path)
    defer C.free(unsafe.Pointer(cpath))

    be := C.rocksdb_backup_engine_open(opts.c, cpath, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return nil, errors.New(C.GoString(cErr))
    }

    return &BackupEngine{
        c: be,
        path: path,
        opts: opts,
    }, nil
}

// CreateNewBackup takes a new backup from @db
func (self *BackupEngine) CreateNewBackup(db *DB) error {
    var cErr *C.char

    C.rocksdb_backup_engine_create_new_backup(self.c, db.c, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return errors.New(C.GoString(cErr))
    }

    return nil
}

// GetInfo gets an object that gives information about
// the backups that have already been taken
func (self *BackupEngine) GetInfo() *BackupEngineInfo {
    return &BackupEngineInfo{
        c: C.rocksdb_backup_engine_get_backup_info(self.c),
    }
}

// RestoreDBFromLatestBackup restores the latest backup to @db_dir. @wal_dir
// is where the write ahead logs are restored to and usually the same as @db_dir.
func (self *BackupEngine) RestoreDBFromLatestBackup(db_dir string, wal_dir string,
    opts *RestoreOptions) error {
    var cErr *C.char
    c_db_dir := C.CString(db_dir)
    c_wal_dir := C.CString(wal_dir)
    defer func() {
        C.free(unsafe.Pointer(c_db_dir))
        C.free(unsafe.Pointer(c_wal_dir))
    }()

    C.rocksdb_backup_engine_restore_db_from_latest_backup(self.c,
        c_db_dir, c_wal_dir, opts.c, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return errors.New(C.GoString(cErr))
    }

    return nil
}

// Close close the backup engine and cleans up state
// The backups already taken remain on storage.
func (self *BackupEngine) Close() {
    C.rocksdb_backup_engine_close(self.c)
    self.c = nil
}

<commit_msg>references #25, fixed multiline comments format and added comment for SetKeepLogFiles function<commit_after>package gorocksdb

// #cgo LDFLAGS: -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy
// #include <stdlib.h>
// #include "rocksdb/c.h"
import "C"

import (
    "errors"
    "unsafe"
)

// BackupEngineInfo represents the information about the backups
// in a backup engine instance. Use this get the state of the
// backup like number of backups and their ids and timestamps etc
type BackupEngineInfo struct {
    c *C.rocksdb_backup_engine_info_t
}

// GetCount gets the number backsup available
func (self *BackupEngineInfo) GetCount() int {
    return int(C.rocksdb_backup_engine_info_count(self.c))
}

// GetTimestamp gets the timestamp at which the backup @index was taken
func (self *BackupEngineInfo) GetTimestamp(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_timestamp(self.c, C.int(index)))
}

// GetBackupId gets an id that uniquely identifies a backup
// regardless of its position
func (self *BackupEngineInfo) GetBackupId(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_backup_id(self.c, C.int(index)))
}

// GetSize get the size of the backup in bytes
func (self *BackupEngineInfo) GetSize(index int) int64 {
    return int64(C.rocksdb_backup_engine_info_size(self.c, C.int(index)))
}

// GetNumFiles gets the number of files in the backup @index
func (self *BackupEngineInfo) GetNumFiles(index int) int32 {
    return int32(C.rocksdb_backup_engine_info_number_files(self.c, C.int(index)))
}

// Destroy destroys the backup engine info instance
func (self *BackupEngineInfo) Destroy() {
    C.rocksdb_backup_engine_info_destroy(self.c)
    self.c = nil
}

// RestoreOptions captures the options to be used during
// restoration of a backup
type RestoreOptions struct {
    c *C.rocksdb_restore_options_t
}

// NewRestoreOptions creates a RestoreOptions instance
func NewRestoreOptions() *RestoreOptions {
    return &RestoreOptions {
        c: C.rocksdb_restore_options_create(),
    }
}

// SetKeepLogFiles is used to set or unset the keep_log_files option
// If true, restore won't overwrite the existing log files in wal_dir. It will
// also move all log files from archive directory to wal_dir. By default, this
// is false
func (self *RestoreOptions) SetKeepLogFiles(v int) {
    C.rocksdb_restore_options_set_keep_log_files(self.c, C.int(v))
}

// Destroy destroys this RestoreOptions instance
func (self *RestoreOptions) Destroy() {
    C.rocksdb_restore_options_destroy(self.c)
}

// BackupEngine is a reusable handle to a RocksDB Backup, created by
// OpenBackupEngine
type BackupEngine struct {
    c *C.rocksdb_backup_engine_t
    path string
    opts *Options
}

// OpenBackupEngine opens a backup engine with specified options
func OpenBackupEngine(opts *Options, path string) (*BackupEngine, error) {
    var cErr *C.char
    cpath := C.CString(path)
    defer C.free(unsafe.Pointer(cpath))

    be := C.rocksdb_backup_engine_open(opts.c, cpath, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return nil, errors.New(C.GoString(cErr))
    }

    return &BackupEngine{
        c: be,
        path: path,
        opts: opts,
    }, nil
}

// CreateNewBackup takes a new backup from @db
func (self *BackupEngine) CreateNewBackup(db *DB) error {
    var cErr *C.char

    C.rocksdb_backup_engine_create_new_backup(self.c, db.c, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return errors.New(C.GoString(cErr))
    }

    return nil
}

// GetInfo gets an object that gives information about
// the backups that have already been taken
func (self *BackupEngine) GetInfo() *BackupEngineInfo {
    return &BackupEngineInfo{
        c: C.rocksdb_backup_engine_get_backup_info(self.c),
    }
}

// RestoreDBFromLatestBackup restores the latest backup to @db_dir. @wal_dir
// is where the write ahead logs are restored to and usually the same as @db_dir.
func (self *BackupEngine) RestoreDBFromLatestBackup(db_dir string, wal_dir string,
    opts *RestoreOptions) error {
    var cErr *C.char
    c_db_dir := C.CString(db_dir)
    c_wal_dir := C.CString(wal_dir)
    defer func() {
        C.free(unsafe.Pointer(c_db_dir))
        C.free(unsafe.Pointer(c_wal_dir))
    }()

    C.rocksdb_backup_engine_restore_db_from_latest_backup(self.c,
        c_db_dir, c_wal_dir, opts.c, &cErr)
    if cErr != nil {
        defer C.free(unsafe.Pointer(cErr))
        return errors.New(C.GoString(cErr))
    }

    return nil
}

// Close close the backup engine and cleans up state
// The backups already taken remain on storage.
func (self *BackupEngine) Close() {
    C.rocksdb_backup_engine_close(self.c)
    self.c = nil
}

<|endoftext|>


<commit_before>// This file contains tests that only compile/work in Go 1.13 and forward
// +build go1.13

package e2etest

import (
	"context"
	"errors"
	"os"
	"testing"

	"github.com/hashicorp/go-version"

	"github.com/hashicorp/terraform-exec/tfexec"
)

func TestUnparsedError(t *testing.T) {
	// This simulates an unparsed error from the Cmd.Run method (in this case file not found). This
	// is to ensure we don't miss raising unexpected errors in addition to parsed / well known ones.
	runTest(t, "", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {

		// force delete the working dir to cause an os.PathError
		err := os.RemoveAll(tf.WorkingDir())
		if err != nil {
			t.Fatal(err)
		}

		err = tf.Init(context.Background())
		if err == nil {
			t.Fatalf("expected error running Init, none returned")
		}
		var e *os.PathError
		if !errors.As(err, &e) {
			t.Fatalf("expected os.PathError, got %T, %s", err, err)
		}
	})
}

func TestMissingVar(t *testing.T) {
	runTest(t, "var", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
		err := tf.Init(context.Background())
		if err != nil {
			t.Fatalf("err during init: %s", err)
		}

		err = tf.Plan(context.Background())
		if err == nil {
			t.Fatalf("expected error running Plan, none returned")
		}
		var e *tfexec.ErrMissingVar
		if !errors.As(err, &e) {
			t.Fatalf("expected ErrMissingVar, got %T, %s", err, err)
		}

		if e.VariableName != "no_default" {
			t.Fatalf("expected missing no_default, got %q", e.VariableName)
		}

		err = tf.Plan(context.Background(), tfexec.Var("no_default=foo"))
		if err != nil {
			t.Fatalf("expected no error, got %s", err)
		}
	})
}
<commit_msg>Updating function call<commit_after>// This file contains tests that only compile/work in Go 1.13 and forward
// +build go1.13

package e2etest

import (
	"context"
	"errors"
	"os"
	"testing"

	"github.com/hashicorp/go-version"

	"github.com/hashicorp/terraform-exec/tfexec"
)

func TestUnparsedError(t *testing.T) {
	// This simulates an unparsed error from the Cmd.Run method (in this case file not found). This
	// is to ensure we don't miss raising unexpected errors in addition to parsed / well known ones.
	runTest(t, "", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {

		// force delete the working dir to cause an os.PathError
		err := os.RemoveAll(tf.WorkingDir())
		if err != nil {
			t.Fatal(err)
		}

		err = tf.Init(context.Background())
		if err == nil {
			t.Fatalf("expected error running Init, none returned")
		}
		var e *os.PathError
		if !errors.As(err, &e) {
			t.Fatalf("expected os.PathError, got %T, %s", err, err)
		}
	})
}

func TestMissingVar(t *testing.T) {
	runTest(t, "var", func(t *testing.T, tfv *version.Version, tf *tfexec.Terraform) {
		err := tf.Init(context.Background())
		if err != nil {
			t.Fatalf("err during init: %s", err)
		}

		_, err = tf.Plan(context.Background())
		if err == nil {
			t.Fatalf("expected error running Plan, none returned")
		}
		var e *tfexec.ErrMissingVar
		if !errors.As(err, &e) {
			t.Fatalf("expected ErrMissingVar, got %T, %s", err, err)
		}

		if e.VariableName != "no_default" {
			t.Fatalf("expected missing no_default, got %q", e.VariableName)
		}

		_, err = tf.Plan(context.Background(), tfexec.Var("no_default=foo"))
		if err != nil {
			t.Fatalf("expected no error, got %s", err)
		}
	})
}
<|endoftext|>


<commit_before>package configuration

import "flag"

func FromCommandLineArgs() *ApplicationConfiguration {
	hostPort := flag.String("hostPort", ":9000", "Host:port of the greenwall HTTP server")
	staticDir := flag.String("staticDir", "frontend", "Path to frontend static resources")
	flag.Parse()

	return &ApplicationConfiguration{
		HostPort:  *hostPort,
		StaticDir: *staticDir,
	}
}
<commit_msg>Change listening port to 9001<commit_after>package configuration

import "flag"

func FromCommandLineArgs() *ApplicationConfiguration {
	hostPort := flag.String("hostPort", ":9001", "Host:port of the greenwall HTTP server")
	staticDir := flag.String("staticDir", "frontend", "Path to frontend static resources")
	flag.Parse()

	return &ApplicationConfiguration{
		HostPort:  *hostPort,
		StaticDir: *staticDir,
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"context"
	"net/http"
	"sort"
	"strconv"

	"google.golang.org/api/googleapi"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/buildbucket"
	bbv1 "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/server/auth"

	"go.chromium.org/luci/milo/api/buildbot"
)

// Ternary has 3 defined values: either (zero), yes and no.
type Ternary int

const (
	Either Ternary = iota
	Yes
	No
)

func (t Ternary) filter(q *datastore.Query, fieldName string) *datastore.Query {
	switch t {
	case Yes:
		return q.Eq(fieldName, true)
	case No:
		return q.Eq(fieldName, false)
	default:
		return q
	}
}

// Query is a build query.
type Query struct {
	Master   string
	Builder  string
	Limit    int
	Finished Ternary
	Cursor   string

	// The following fields are tuning parameters specific to a buildstore
	// implementation. Their usage implies understanding of how emulation
	// works.

	// KeyOnly, if true, makes the datastore query keys-only.
	// Loaded Buildbot builds will have only master, builder and number.
	KeyOnly bool // make the data

	// NoAnnotationFetch, if true, will not fetch annotation proto from LogDog.
	// Loaded LUCI builds will not have properties, steps, logs or text.
	NoAnnotationFetch bool

	// NoChangeFetch, if true, will not load change history from Gitiles.
	// Loaded LUCI builds will not have Blame or SourceStamp.Changes.
	NoChangeFetch bool
}

func (q *Query) dsQuery() *datastore.Query {
	dsq := datastore.NewQuery(buildKind)
	if q.Master != "" {
		dsq = dsq.Eq("master", q.Master)
	}
	if q.Builder != "" {
		dsq = dsq.Eq("builder", q.Builder)
	}
	dsq = q.Finished.filter(dsq, "finished")
	if q.Limit > 0 {
		dsq = dsq.Limit(int32(q.Limit))
	}
	if q.KeyOnly {
		dsq = dsq.KeysOnly(true)
	}
	return dsq
}

// QueryResult is a result of running a Query.
type QueryResult struct {
	Builds     []*buildbot.Build // ordered from greater-number to lower-number
	NextCursor string
	PrevCursor string
}

// GetBuilds executes a build query and returns results.
// Does not check access.
func GetBuilds(c context.Context, q Query) (*QueryResult, error) {
	switch {
	case q.Master == "":
		return nil, errors.New("master is required")
	case q.Builder == "":
		return nil, errors.New("builder is required")
	}

	if !EmulationEnabled(c) {
		return getDatastoreBuilds(c, q, true)
	}

	var emulatedBuilds, buildbotBuilds []*buildbot.Build
	err := parallel.FanOutIn(func(work chan<- func() error) {
		work <- func() (err error) {
			res, err := getDatastoreBuilds(c, q, false)
			if res != nil {
				buildbotBuilds = res.Builds
			}
			return
		}
		work <- func() (err error) {
			emulatedBuilds, err = getEmulatedBuilds(c, q)
			return
		}
	})
	if err != nil {
		return nil, errors.Annotate(err, "could not load builds").Err()
	}

	mergedBuilds := mergeBuilds(emulatedBuilds, buildbotBuilds)
	if q.Limit > 0 && len(mergedBuilds) > q.Limit {
		mergedBuilds = mergedBuilds[:q.Limit]
	}
	return &QueryResult{Builds: mergedBuilds}, nil
}

// mergeBuilds merges builds from a and b to one slice.
// The returned builds are ordered by build numbers, descending.
//
// If a build number is present in both a and b, b's build is ignored.
func mergeBuilds(a, b []*buildbot.Build) []*buildbot.Build {
	ret := make([]*buildbot.Build, len(a), len(a)+len(b))
	copy(ret, a)

	// add builds from b that have unique build numbers.
	aNumbers := make(map[int]struct{}, len(a))
	for _, build := range a {
		aNumbers[build.Number] = struct{}{}
	}
	for _, build := range b {
		if _, ok := aNumbers[build.Number]; !ok {
			ret = append(ret, build)
		}
	}
	sort.Slice(ret, func(i, j int) bool {
		return ret[i].Number > ret[j].Number
	})
	return ret
}

func getEmulatedBuilds(c context.Context, q Query) ([]*buildbot.Build, error) {
	if q.Cursor != "" {
		// build query emulation does not support cursors
		logging.Warningf(c, "ignoring cursor %q", q.Cursor)
		q.Cursor = ""
	}

	bb, err := buildbucketClient(c)
	if err != nil {
		return nil, err
	}

	bucket, err := BucketOf(c, q.Master)
	switch {
	case err != nil:
		return nil, errors.Annotate(err, "could not get bucket of %q", q.Master).Err()
	case bucket == "":
		return nil, nil
	}

	search := bb.Search().
		Bucket(bucket).
		Tag(strpair.Format(bbv1.TagBuilder, q.Builder)).
		Context(c)
	switch q.Finished {
	case Yes:
		search.Status(bbv1.StatusCompleted)
	case No:
		search.Status(bbv1.StatusFilterIncomplete)
	}

	start := clock.Now(c)
	msgs, _, err := search.Fetch(q.Limit, nil)
	switch apiErr, _ := err.(*googleapi.Error); {
	case apiErr != nil && apiErr.Code == http.StatusForbidden:
		logging.Warningf(c, "%q does not have access to bucket %q. Returning 0 builds.",
			auth.CurrentIdentity(c),
			bucket)
		return nil, nil
	case err != nil:
		return nil, errors.Annotate(err, "searching on buildbucket").Err()
	}

	logging.Infof(c, "buildbucket search took %s", clock.Since(c, start))

	buildsTemp := make([]*buildbot.Build, len(msgs))
	start = clock.Now(c)
	err = parallel.WorkPool(10, func(work chan<- func() error) {
		for i, msg := range msgs {
			i := i
			msg := msg
			work <- func() error {
				var buildbucketBuild buildbucket.Build
				if err := buildbucketBuild.ParseMessage(msg); err != nil {
					return err
				}
				// may load annotations from logdog, that's why parallelized.
				b, err := buildFromBuildbucket(c, q.Master, &buildbucketBuild, !q.NoAnnotationFetch)
				switch {
				case ErrNoBuildNumber.In(err):
					return nil
				case err != nil:
					return err
				}
				buildsTemp[i] = b
				return nil
			}
		}
	})
	if err != nil {
		return nil, err
	}
	logging.Infof(c, "conversion from buildbucket builds took %s", clock.Since(c, start))

	// Remove nil builds. I.E. The ones without build numbers.
	builds := make([]*buildbot.Build, 0, len(buildsTemp))
	for _, b := range buildsTemp {
		if b != nil {
			builds = append(builds, b)
		}
	}

	if !q.NoChangeFetch && len(builds) > 0 {
		start = clock.Now(c)
		// We need to compute blamelist for multiple builds.
		// 1) We don't have a guarantee that the numbers are contiguous
		// 2) For some builds, we may have cached changes
		// => compute blamelist for each build individually

		// cache build revisions before fetching changes
		// in case build numbers are contiguous.
		caches := make([]memcache.Item, len(builds))
		for i, b := range builds {
			caches[i] = buildRevCache(c, b)
		}
		memcache.Set(c, caches...)

		// compute blamelist serially so that git cache is reused.
		for _, b := range builds {
			if err := blame(c, b); err != nil {
				return nil, errors.Annotate(err, "blamelist computation for build #%d failed", b.Number).Err()
			}
		}

		logging.Infof(c, "blamelist computation took %s", clock.Since(c, start))
	}
	return builds, nil
}

func getDatastoreBuilds(c context.Context, q Query, includeExperimental bool) (*QueryResult, error) {
	var builds []*buildEntity
	if q.Limit > 0 {
		builds = make([]*buildEntity, 0, q.Limit)
	}

	dsq := q.dsQuery()

	if !includeExperimental {
		dsq = dsq.Eq("is_experimental", false)
	}

	// CUSTOM CURSOR.
	// This function uses a custom cursor based on build numbers.
	// A cursor is a build number that defines a page boundary.
	// If >=0, it is the inclusive lower boundary.
	//   Example: cursor="10", means return builds ...12, 11, 10.
	// If <0, it is the exclusive upper boundary, negated.
	//   Example: -10, means return builds 9, 8, 7...
	cursorNumber := 0
	order := "-number"
	reverse := false
	hasCursor := false
	if q.Cursor != "" {
		var err error
		if cursorNumber, err = strconv.Atoi(q.Cursor); err == nil {
			hasCursor = true
			if cursorNumber >= 0 {
				dsq = dsq.Gte("number", cursorNumber)
				order = "number"
				reverse = true
			} else {
				dsq = dsq.Lt("number", -cursorNumber)
			}
		} else {
			return nil, errors.New("bad cursor")
		}
	}
	dsq = dsq.Order(order)

	logging.Debugf(c, "running datastore query: %s", dsq)
	err := datastore.GetAll(c, dsq, &builds)
	if err != nil {
		return nil, err
	}
	if reverse {
		for i, j := 0, len(builds)-1; i < j; i, j = i+1, j-1 {
			builds[i], builds[j] = builds[j], builds[i]
		}
	}
	res := &QueryResult{
		Builds: make([]*buildbot.Build, len(builds)),
	}
	for i, b := range builds {
		b.addViewPath()
		res.Builds[i] = (*buildbot.Build)(b)
	}

	// Compute prev and next cursors.
	switch {
	case len(res.Builds) > 0:
		// res.Builds are ordered by numbers descending.

		// previous page must display builds with higher numbers.
		if !hasCursor {
			// do not generate a prev cursor for a non-cursor query
		} else {
			// positive cursors are inclusive
			res.PrevCursor = strconv.Itoa(res.Builds[0].Number + 1)
		}

		// next page must display builds with lower numbers.

		if lastNum := res.Builds[len(res.Builds)-1].Number; lastNum == 0 {
			// this is the first ever build, 0, do not generate a cursor
		} else {
			// negative cursors are exclusive.
			res.NextCursor = strconv.Itoa(-lastNum)
		}

	case cursorNumber > 0:
		// no builds and cursor is the inclusive lower boundary
		// e.g. cursor asks for builds after 10,
		// but there are only 0..5 builds.
		// Make the next cursor for builds <10.
		res.NextCursor = strconv.Itoa(-cursorNumber)

	default:
		// there can't be any builds.
	}

	return res, nil
}
<commit_msg>[milo] Extract 'project' from master->bucket mapping<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"context"
	"net/http"
	"sort"
	"strconv"

	"google.golang.org/api/googleapi"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/buildbucket"
	bbv1 "go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/server/auth"

	"go.chromium.org/luci/milo/api/buildbot"
	"go.chromium.org/luci/milo/git"
)

// Ternary has 3 defined values: either (zero), yes and no.
type Ternary int

const (
	Either Ternary = iota
	Yes
	No
)

func (t Ternary) filter(q *datastore.Query, fieldName string) *datastore.Query {
	switch t {
	case Yes:
		return q.Eq(fieldName, true)
	case No:
		return q.Eq(fieldName, false)
	default:
		return q
	}
}

// Query is a build query.
type Query struct {
	Master   string
	Builder  string
	Limit    int
	Finished Ternary
	Cursor   string

	// The following fields are tuning parameters specific to a buildstore
	// implementation. Their usage implies understanding of how emulation
	// works.

	// KeyOnly, if true, makes the datastore query keys-only.
	// Loaded Buildbot builds will have only master, builder and number.
	KeyOnly bool // make the data

	// NoAnnotationFetch, if true, will not fetch annotation proto from LogDog.
	// Loaded LUCI builds will not have properties, steps, logs or text.
	NoAnnotationFetch bool

	// NoChangeFetch, if true, will not load change history from Gitiles.
	// Loaded LUCI builds will not have Blame or SourceStamp.Changes.
	NoChangeFetch bool
}

func (q *Query) dsQuery() *datastore.Query {
	dsq := datastore.NewQuery(buildKind)
	if q.Master != "" {
		dsq = dsq.Eq("master", q.Master)
	}
	if q.Builder != "" {
		dsq = dsq.Eq("builder", q.Builder)
	}
	dsq = q.Finished.filter(dsq, "finished")
	if q.Limit > 0 {
		dsq = dsq.Limit(int32(q.Limit))
	}
	if q.KeyOnly {
		dsq = dsq.KeysOnly(true)
	}
	return dsq
}

// QueryResult is a result of running a Query.
type QueryResult struct {
	Builds     []*buildbot.Build // ordered from greater-number to lower-number
	NextCursor string
	PrevCursor string
}

// GetBuilds executes a build query and returns results.
// Does not check access.
func GetBuilds(c context.Context, q Query) (*QueryResult, error) {
	switch {
	case q.Master == "":
		return nil, errors.New("master is required")
	case q.Builder == "":
		return nil, errors.New("builder is required")
	}

	if !EmulationEnabled(c) {
		return getDatastoreBuilds(c, q, true)
	}

	var emulatedBuilds, buildbotBuilds []*buildbot.Build
	err := parallel.FanOutIn(func(work chan<- func() error) {
		work <- func() (err error) {
			res, err := getDatastoreBuilds(c, q, false)
			if res != nil {
				buildbotBuilds = res.Builds
			}
			return
		}
		work <- func() (err error) {
			emulatedBuilds, err = getEmulatedBuilds(c, q)
			return
		}
	})
	if err != nil {
		return nil, errors.Annotate(err, "could not load builds").Err()
	}

	mergedBuilds := mergeBuilds(emulatedBuilds, buildbotBuilds)
	if q.Limit > 0 && len(mergedBuilds) > q.Limit {
		mergedBuilds = mergedBuilds[:q.Limit]
	}
	return &QueryResult{Builds: mergedBuilds}, nil
}

// mergeBuilds merges builds from a and b to one slice.
// The returned builds are ordered by build numbers, descending.
//
// If a build number is present in both a and b, b's build is ignored.
func mergeBuilds(a, b []*buildbot.Build) []*buildbot.Build {
	ret := make([]*buildbot.Build, len(a), len(a)+len(b))
	copy(ret, a)

	// add builds from b that have unique build numbers.
	aNumbers := make(map[int]struct{}, len(a))
	for _, build := range a {
		aNumbers[build.Number] = struct{}{}
	}
	for _, build := range b {
		if _, ok := aNumbers[build.Number]; !ok {
			ret = append(ret, build)
		}
	}
	sort.Slice(ret, func(i, j int) bool {
		return ret[i].Number > ret[j].Number
	})
	return ret
}

func getEmulatedBuilds(c context.Context, q Query) ([]*buildbot.Build, error) {
	if q.Cursor != "" {
		// build query emulation does not support cursors
		logging.Warningf(c, "ignoring cursor %q", q.Cursor)
		q.Cursor = ""
	}

	bb, err := buildbucketClient(c)
	if err != nil {
		return nil, err
	}

	bucket, err := BucketOf(c, q.Master)
	switch {
	case err != nil:
		return nil, errors.Annotate(err, "could not get bucket of %q", q.Master).Err()
	case bucket == "":
		return nil, nil
	}

	// Extract project and annotate context to use project scoped account.
	project, _ := buildbucket.BucketNameToV2(bucket)
	if project == "" {
		return nil, errors.Annotate(err, "unable to extract project from bucket name").Err()
	}
	c = git.WithProject(c, project)

	search := bb.Search().
		Bucket(bucket).
		Tag(strpair.Format(bbv1.TagBuilder, q.Builder)).
		Context(c)
	switch q.Finished {
	case Yes:
		search.Status(bbv1.StatusCompleted)
	case No:
		search.Status(bbv1.StatusFilterIncomplete)
	}

	start := clock.Now(c)
	msgs, _, err := search.Fetch(q.Limit, nil)
	switch apiErr, _ := err.(*googleapi.Error); {
	case apiErr != nil && apiErr.Code == http.StatusForbidden:
		logging.Warningf(c, "%q does not have access to bucket %q. Returning 0 builds.",
			auth.CurrentIdentity(c),
			bucket)
		return nil, nil
	case err != nil:
		return nil, errors.Annotate(err, "searching on buildbucket").Err()
	}

	logging.Infof(c, "buildbucket search took %s", clock.Since(c, start))

	buildsTemp := make([]*buildbot.Build, len(msgs))
	start = clock.Now(c)
	err = parallel.WorkPool(10, func(work chan<- func() error) {
		for i, msg := range msgs {
			i := i
			msg := msg
			work <- func() error {
				var buildbucketBuild buildbucket.Build
				if err := buildbucketBuild.ParseMessage(msg); err != nil {
					return err
				}
				// may load annotations from logdog, that's why parallelized.
				b, err := buildFromBuildbucket(c, q.Master, &buildbucketBuild, !q.NoAnnotationFetch)
				switch {
				case ErrNoBuildNumber.In(err):
					return nil
				case err != nil:
					return err
				}
				buildsTemp[i] = b
				return nil
			}
		}
	})
	if err != nil {
		return nil, err
	}
	logging.Infof(c, "conversion from buildbucket builds took %s", clock.Since(c, start))

	// Remove nil builds. I.E. The ones without build numbers.
	builds := make([]*buildbot.Build, 0, len(buildsTemp))
	for _, b := range buildsTemp {
		if b != nil {
			builds = append(builds, b)
		}
	}

	if !q.NoChangeFetch && len(builds) > 0 {
		start = clock.Now(c)
		// We need to compute blamelist for multiple builds.
		// 1) We don't have a guarantee that the numbers are contiguous
		// 2) For some builds, we may have cached changes
		// => compute blamelist for each build individually

		// cache build revisions before fetching changes
		// in case build numbers are contiguous.
		caches := make([]memcache.Item, len(builds))
		for i, b := range builds {
			caches[i] = buildRevCache(c, b)
		}
		memcache.Set(c, caches...)

		// compute blamelist serially so that git cache is reused.
		for _, b := range builds {
			if err := blame(c, b); err != nil {
				return nil, errors.Annotate(err, "blamelist computation for build #%d failed", b.Number).Err()
			}
		}

		logging.Infof(c, "blamelist computation took %s", clock.Since(c, start))
	}
	return builds, nil
}

func getDatastoreBuilds(c context.Context, q Query, includeExperimental bool) (*QueryResult, error) {
	var builds []*buildEntity
	if q.Limit > 0 {
		builds = make([]*buildEntity, 0, q.Limit)
	}

	dsq := q.dsQuery()

	if !includeExperimental {
		dsq = dsq.Eq("is_experimental", false)
	}

	// CUSTOM CURSOR.
	// This function uses a custom cursor based on build numbers.
	// A cursor is a build number that defines a page boundary.
	// If >=0, it is the inclusive lower boundary.
	//   Example: cursor="10", means return builds ...12, 11, 10.
	// If <0, it is the exclusive upper boundary, negated.
	//   Example: -10, means return builds 9, 8, 7...
	cursorNumber := 0
	order := "-number"
	reverse := false
	hasCursor := false
	if q.Cursor != "" {
		var err error
		if cursorNumber, err = strconv.Atoi(q.Cursor); err == nil {
			hasCursor = true
			if cursorNumber >= 0 {
				dsq = dsq.Gte("number", cursorNumber)
				order = "number"
				reverse = true
			} else {
				dsq = dsq.Lt("number", -cursorNumber)
			}
		} else {
			return nil, errors.New("bad cursor")
		}
	}
	dsq = dsq.Order(order)

	logging.Debugf(c, "running datastore query: %s", dsq)
	err := datastore.GetAll(c, dsq, &builds)
	if err != nil {
		return nil, err
	}
	if reverse {
		for i, j := 0, len(builds)-1; i < j; i, j = i+1, j-1 {
			builds[i], builds[j] = builds[j], builds[i]
		}
	}
	res := &QueryResult{
		Builds: make([]*buildbot.Build, len(builds)),
	}
	for i, b := range builds {
		b.addViewPath()
		res.Builds[i] = (*buildbot.Build)(b)
	}

	// Compute prev and next cursors.
	switch {
	case len(res.Builds) > 0:
		// res.Builds are ordered by numbers descending.

		// previous page must display builds with higher numbers.
		if !hasCursor {
			// do not generate a prev cursor for a non-cursor query
		} else {
			// positive cursors are inclusive
			res.PrevCursor = strconv.Itoa(res.Builds[0].Number + 1)
		}

		// next page must display builds with lower numbers.

		if lastNum := res.Builds[len(res.Builds)-1].Number; lastNum == 0 {
			// this is the first ever build, 0, do not generate a cursor
		} else {
			// negative cursors are exclusive.
			res.NextCursor = strconv.Itoa(-lastNum)
		}

	case cursorNumber > 0:
		// no builds and cursor is the inclusive lower boundary
		// e.g. cursor asks for builds after 10,
		// but there are only 0..5 builds.
		// Make the next cursor for builds <10.
		res.NextCursor = strconv.Itoa(-cursorNumber)

	default:
		// there can't be any builds.
	}

	return res, nil
}
<|endoftext|>


<commit_before>package classifier

import (
	"fmt"
	"os"
	"runtime"
	"sort"
	"sync"

	"github.com/syou6162/go-active-learning/lib/evaluation"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/util"
)

type MIRAClassifier struct {
	weight map[string]float64
	c      float64
}

func newMIRAClassifier(c float64) *MIRAClassifier {
	return &MIRAClassifier{make(map[string]float64), c}
}

func NewMIRAClassifier(examples example.Examples, c float64) *MIRAClassifier {
	train := util.FilterLabeledExamples(examples)
	model := newMIRAClassifier(c)
	for iter := 0; iter < 30; iter++ {
		util.Shuffle(train)
		for _, example := range train {
			model.learn(*example)
		}
	}
	return model
}

func OverSamplingPositiveExamples(examples example.Examples) example.Examples {
	overSampled := example.Examples{}
	posExamples := example.Examples{}
	negExamples := example.Examples{}

	numNeg := 0

	for _, e := range examples {
		if e.Label == example.NEGATIVE {
			numNeg += 1
			negExamples = append(negExamples, e)
		} else if e.Label == example.POSITIVE {
			posExamples = append(posExamples, e)
		}
	}

	for len(overSampled) <= numNeg {
		util.Shuffle(posExamples)
		overSampled = append(overSampled, posExamples[0])
	}
	overSampled = append(overSampled, negExamples...)
	util.Shuffle(overSampled)

	return overSampled
}

type MIRAResult struct {
	mira   MIRAClassifier
	FValue float64
}

type MIRAResultList []MIRAResult

func (l MIRAResultList) Len() int           { return len(l) }
func (l MIRAResultList) Less(i, j int) bool { return l[i].FValue < l[j].FValue }
func (l MIRAResultList) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }

func NewMIRAClassifierByCrossValidation(examples example.Examples) *MIRAClassifier {
	train, dev := util.SplitTrainAndDev(util.FilterLabeledExamples(examples))
	train = OverSamplingPositiveExamples(train)

	params := []float64{100, 50, 10.0, 5.0, 1.0, 0.5, 0.1, 0.05, 0.01}
	miraResults := MIRAResultList{}

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)

	models := make([]*MIRAClassifier, len(params))
	for idx, c := range params {
		wg.Add(1)
		go func(idx int, c float64) {
			defer wg.Done()
			model := NewMIRAClassifier(train, c)
			models[idx] = model
		}(idx, c)
	}
	wg.Wait()

	for _, model := range models {
		c := model.c
		devPredicts := make([]example.LabelType, len(dev))
		for i, example := range dev {
			devPredicts[i] = model.Predict(example.Fv)
		}
		accuracy := evaluation.GetAccuracy(ExtractGoldLabels(dev), devPredicts)
		precision := evaluation.GetPrecision(ExtractGoldLabels(dev), devPredicts)
		recall := evaluation.GetRecall(ExtractGoldLabels(dev), devPredicts)
		f := (2 * recall * precision) / (recall + precision)
		fmt.Fprintln(os.Stderr, fmt.Sprintf("C:%0.03f\tAccuracy:%0.03f\tPrecision:%0.03f\tRecall:%0.03f\tF-value:%0.03f", c, accuracy, precision, recall, f))
		miraResults = append(miraResults, MIRAResult{*model, f})
	}

	sort.Sort(sort.Reverse(miraResults))
	bestModel := &miraResults[0].mira
	return NewMIRAClassifier(util.FilterLabeledExamples(examples), bestModel.c)
}

func (model *MIRAClassifier) learn(example example.Example) {
	tmp := float64(example.Label) * model.PredictScore(example.Fv) // y w^T x
	loss := 0.0
	if tmp < 1.0 {
		loss = 1 - tmp
	}

	norm := float64(len(example.Fv) * len(example.Fv))
	// tau := math.Min(model.c, loss/norm) // update by PA-I
	tau := loss / (norm + 1.0/model.c) // update by PA-II

	if tau != 0.0 {
		for _, f := range example.Fv {
			w, _ := model.weight[f]
			model.weight[f] = w + tau*float64(example.Label)
		}
	}
}

func (model MIRAClassifier) PredictScore(features feature.FeatureVector) float64 {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}
	}
	return result
}

func (model MIRAClassifier) Predict(features feature.FeatureVector) example.LabelType {
	if model.PredictScore(features) > 0 {
		return example.POSITIVE
	}
	return example.NEGATIVE
}

func (model MIRAClassifier) SortByScore(examples example.Examples) example.Examples {
	return SortByScore(model, examples)
}

func (model MIRAClassifier) GetWeight(f string) float64 {
	w, ok := model.weight[f]
	if ok {
		return w
	}
	return 0.0
}

func (model MIRAClassifier) GetActiveFeatures() []string {
	result := make([]string, 0)
	for f := range model.weight {
		result = append(result, f)
	}
	return result
}
<commit_msg>環境変数が設定されていればmackerelのサービスメトリックに評価指標をpostする<commit_after>package classifier

import (
	"fmt"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"

	mkr "github.com/mackerelio/mackerel-client-go"
	"github.com/syou6162/go-active-learning/lib/evaluation"
	"github.com/syou6162/go-active-learning/lib/example"
	"github.com/syou6162/go-active-learning/lib/feature"
	"github.com/syou6162/go-active-learning/lib/util"
)

type MIRAClassifier struct {
	weight map[string]float64
	c      float64
}

func newMIRAClassifier(c float64) *MIRAClassifier {
	return &MIRAClassifier{make(map[string]float64), c}
}

func NewMIRAClassifier(examples example.Examples, c float64) *MIRAClassifier {
	train := util.FilterLabeledExamples(examples)
	model := newMIRAClassifier(c)
	for iter := 0; iter < 30; iter++ {
		util.Shuffle(train)
		for _, example := range train {
			model.learn(*example)
		}
	}
	return model
}

func OverSamplingPositiveExamples(examples example.Examples) example.Examples {
	overSampled := example.Examples{}
	posExamples := example.Examples{}
	negExamples := example.Examples{}

	numNeg := 0

	for _, e := range examples {
		if e.Label == example.NEGATIVE {
			numNeg += 1
			negExamples = append(negExamples, e)
		} else if e.Label == example.POSITIVE {
			posExamples = append(posExamples, e)
		}
	}

	for len(overSampled) <= numNeg {
		util.Shuffle(posExamples)
		overSampled = append(overSampled, posExamples[0])
	}
	overSampled = append(overSampled, negExamples...)
	util.Shuffle(overSampled)

	return overSampled
}

type MIRAResult struct {
	mira   MIRAClassifier
	FValue float64
}

type MIRAResultList []MIRAResult

func (l MIRAResultList) Len() int           { return len(l) }
func (l MIRAResultList) Less(i, j int) bool { return l[i].FValue < l[j].FValue }
func (l MIRAResultList) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }

func NewMIRAClassifierByCrossValidation(examples example.Examples) *MIRAClassifier {
	train, dev := util.SplitTrainAndDev(util.FilterLabeledExamples(examples))
	train = OverSamplingPositiveExamples(train)

	params := []float64{100, 50, 10.0, 5.0, 1.0, 0.5, 0.1, 0.05, 0.01}
	miraResults := MIRAResultList{}

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)

	models := make([]*MIRAClassifier, len(params))
	for idx, c := range params {
		wg.Add(1)
		go func(idx int, c float64) {
			defer wg.Done()
			model := NewMIRAClassifier(train, c)
			models[idx] = model
		}(idx, c)
	}
	wg.Wait()

	for _, model := range models {
		c := model.c
		devPredicts := make([]example.LabelType, len(dev))
		for i, example := range dev {
			devPredicts[i] = model.Predict(example.Fv)
		}
		accuracy := evaluation.GetAccuracy(ExtractGoldLabels(dev), devPredicts)
		precision := evaluation.GetPrecision(ExtractGoldLabels(dev), devPredicts)
		recall := evaluation.GetRecall(ExtractGoldLabels(dev), devPredicts)
		f := (2 * recall * precision) / (recall + precision)
		fmt.Fprintln(os.Stderr, fmt.Sprintf("C:%0.03f\tAccuracy:%0.03f\tPrecision:%0.03f\tRecall:%0.03f\tF-value:%0.03f", c, accuracy, precision, recall, f))
		err := postEvaluatedMetricsToMackerel(accuracy, precision, recall, f)
		if err != nil {
			fmt.Println(err.Error())
		}
		miraResults = append(miraResults, MIRAResult{*model, f})
	}

	sort.Sort(sort.Reverse(miraResults))
	bestModel := &miraResults[0].mira
	return NewMIRAClassifier(util.FilterLabeledExamples(examples), bestModel.c)
}

func postEvaluatedMetricsToMackerel(accuracy float64, precision float64, recall float64, fvalue float64) error {
	apiKey := os.Getenv("MACKEREL_API_KEY")
	serviceName := os.Getenv("MACKEREL_SERVICE_NAME")
	if apiKey == "" || serviceName == "" {
		return nil
	}

	client := mkr.NewClient(apiKey)
	now := time.Now().Unix()
	err := client.PostServiceMetricValues(serviceName, []*mkr.MetricValue{
		&mkr.MetricValue{
			Name:  "evaluation.accuracy",
			Time:  now,
			Value: accuracy,
		},
		&mkr.MetricValue{
			Name:  "evaluation.precision",
			Time:  now,
			Value: precision,
		},
		&mkr.MetricValue{
			Name:  "evaluation.recall",
			Time:  now,
			Value: recall,
		},
		&mkr.MetricValue{
			Name:  "evaluation.fvalue",
			Time:  now,
			Value: fvalue,
		},
	})
	return err
}

func (model *MIRAClassifier) learn(example example.Example) {
	tmp := float64(example.Label) * model.PredictScore(example.Fv) // y w^T x
	loss := 0.0
	if tmp < 1.0 {
		loss = 1 - tmp
	}

	norm := float64(len(example.Fv) * len(example.Fv))
	// tau := math.Min(model.c, loss/norm) // update by PA-I
	tau := loss / (norm + 1.0/model.c) // update by PA-II

	if tau != 0.0 {
		for _, f := range example.Fv {
			w, _ := model.weight[f]
			model.weight[f] = w + tau*float64(example.Label)
		}
	}
}

func (model MIRAClassifier) PredictScore(features feature.FeatureVector) float64 {
	result := 0.0
	for _, f := range features {
		w, ok := model.weight[f]
		if ok {
			result = result + w*1.0
		}
	}
	return result
}

func (model MIRAClassifier) Predict(features feature.FeatureVector) example.LabelType {
	if model.PredictScore(features) > 0 {
		return example.POSITIVE
	}
	return example.NEGATIVE
}

func (model MIRAClassifier) SortByScore(examples example.Examples) example.Examples {
	return SortByScore(model, examples)
}

func (model MIRAClassifier) GetWeight(f string) float64 {
	w, ok := model.weight[f]
	if ok {
		return w
	}
	return 0.0
}

func (model MIRAClassifier) GetActiveFeatures() []string {
	result := make([]string, 0)
	for f := range model.weight {
		result = append(result, f)
	}
	return result
}
<|endoftext|>


<commit_before>package echo

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	// Binder is the interface that wraps the Bind method.
	Binder interface {
		Bind(*http.Request, interface{}) error
	}

	binder struct {
		maxMemory int64
	}
)

const (
	defaultMaxMemory = 32 << 20 // 32 MB
)

// SetMaxBodySize sets multipart forms max body size
func (b *binder) SetMaxMemory(size int64) {
	b.maxMemory = size
}

// MaxBodySize return multipart forms max body size
func (b *binder) MaxMemory() int64 {
	return b.maxMemory
}

func (b *binder) Bind(r *http.Request, i interface{}) (err error) {
	ct := r.Header.Get(ContentType)
	err = ErrUnsupportedMediaType
	switch {
	case strings.HasPrefix(ct, ApplicationJSON):
		if err = json.NewDecoder(r.Body).Decode(i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, ApplicationXML):
		if err = xml.NewDecoder(r.Body).Decode(i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, ApplicationForm):
		if err = b.bindForm(r, i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, MultipartForm):
		if err = b.bindMultiPartForm(r, i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	}
	return
}

func (binder) bindForm(r *http.Request, i interface{}) error {
	if err := r.ParseForm(); err != nil {
		return err
	}
	return mapForm(i, r.Form)
}

func (b binder) bindMultiPartForm(r *http.Request, i interface{}) error {
	if b.maxMemory == 0 {
		b.maxMemory = defaultMaxMemory
	}
	if err := r.ParseMultipartForm(b.maxMemory); err != nil {
		return err
	}
	return mapForm(i, r.Form)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	val := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		structField := val.Field(i)
		if !structField.CanSet() {
			continue
		}

		structFieldKind := structField.Kind()
		inputFieldName := typeField.Tag.Get("form")
		if inputFieldName == "" {
			inputFieldName = typeField.Name

			// if "form" tag is nil, we inspect if the field is a struct.
			// this would not make sense for JSON parsing but it does for a form
			// since data is flatten
			if structFieldKind == reflect.Struct {
				err := mapForm(structField.Addr().Interface(), form)
				if err != nil {
					return err
				}
				continue
			}
		}
		inputValue, exists := form[inputFieldName]
		if !exists {
			continue
		}

		numElems := len(inputValue)
		if structFieldKind == reflect.Slice && numElems > 0 {
			sliceOf := structField.Type().Elem().Kind()
			slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
			for i := 0; i < numElems; i++ {
				if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
					return err
				}
			}
			val.Field(i).Set(slice)
		} else {
			if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
				return err
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return err
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}
<commit_msg>fix memory leaks #356<commit_after>package echo

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

type (
	// Binder is the interface that wraps the Bind method.
	Binder interface {
		Bind(*http.Request, interface{}) error
	}

	binder struct {
		maxMemory int64
	}
)

const (
	defaultMaxMemory = 32 << 20 // 32 MB
)

// SetMaxBodySize sets multipart forms max body size
func (b *binder) SetMaxMemory(size int64) {
	b.maxMemory = size
}

// MaxBodySize return multipart forms max body size
func (b *binder) MaxMemory() int64 {
	return b.maxMemory
}

func (b *binder) Bind(r *http.Request, i interface{}) (err error) {
	if r.Body == nil {
		err = NewHTTPError(http.StatusBadRequest, "Requesr body can't be nil")
		return
	}
	defer r.Body.Close()
	ct := r.Header.Get(ContentType)
	err = ErrUnsupportedMediaType
	switch {
	case strings.HasPrefix(ct, ApplicationJSON):
		if err = json.NewDecoder(r.Body).Decode(i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, ApplicationXML):
		if err = xml.NewDecoder(r.Body).Decode(i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, ApplicationForm):
		if err = b.bindForm(r, i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	case strings.HasPrefix(ct, MultipartForm):
		if err = b.bindMultiPartForm(r, i); err != nil {
			err = NewHTTPError(http.StatusBadRequest, err.Error())
		}
	}
	return
}

func (binder) bindForm(r *http.Request, i interface{}) error {
	if err := r.ParseForm(); err != nil {
		return err
	}
	return mapForm(i, r.Form)
}

func (b binder) bindMultiPartForm(r *http.Request, i interface{}) error {
	if b.maxMemory == 0 {
		b.maxMemory = defaultMaxMemory
	}
	if err := r.ParseMultipartForm(b.maxMemory); err != nil {
		return err
	}
	return mapForm(i, r.Form)
}

func mapForm(ptr interface{}, form map[string][]string) error {
	typ := reflect.TypeOf(ptr).Elem()
	val := reflect.ValueOf(ptr).Elem()
	for i := 0; i < typ.NumField(); i++ {
		typeField := typ.Field(i)
		structField := val.Field(i)
		if !structField.CanSet() {
			continue
		}

		structFieldKind := structField.Kind()
		inputFieldName := typeField.Tag.Get("form")
		if inputFieldName == "" {
			inputFieldName = typeField.Name

			// if "form" tag is nil, we inspect if the field is a struct.
			// this would not make sense for JSON parsing but it does for a form
			// since data is flatten
			if structFieldKind == reflect.Struct {
				err := mapForm(structField.Addr().Interface(), form)
				if err != nil {
					return err
				}
				continue
			}
		}
		inputValue, exists := form[inputFieldName]
		if !exists {
			continue
		}

		numElems := len(inputValue)
		if structFieldKind == reflect.Slice && numElems > 0 {
			sliceOf := structField.Type().Elem().Kind()
			slice := reflect.MakeSlice(structField.Type(), numElems, numElems)
			for i := 0; i < numElems; i++ {
				if err := setWithProperType(sliceOf, inputValue[i], slice.Index(i)); err != nil {
					return err
				}
			}
			val.Field(i).Set(slice)
		} else {
			if err := setWithProperType(typeField.Type.Kind(), inputValue[0], structField); err != nil {
				return err
			}
		}
	}
	return nil
}

func setWithProperType(valueKind reflect.Kind, val string, structField reflect.Value) error {
	switch valueKind {
	case reflect.Int:
		return setIntField(val, 0, structField)
	case reflect.Int8:
		return setIntField(val, 8, structField)
	case reflect.Int16:
		return setIntField(val, 16, structField)
	case reflect.Int32:
		return setIntField(val, 32, structField)
	case reflect.Int64:
		return setIntField(val, 64, structField)
	case reflect.Uint:
		return setUintField(val, 0, structField)
	case reflect.Uint8:
		return setUintField(val, 8, structField)
	case reflect.Uint16:
		return setUintField(val, 16, structField)
	case reflect.Uint32:
		return setUintField(val, 32, structField)
	case reflect.Uint64:
		return setUintField(val, 64, structField)
	case reflect.Bool:
		return setBoolField(val, structField)
	case reflect.Float32:
		return setFloatField(val, 32, structField)
	case reflect.Float64:
		return setFloatField(val, 64, structField)
	case reflect.String:
		structField.SetString(val)
	default:
		return errors.New("Unknown type")
	}
	return nil
}

func setIntField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	intVal, err := strconv.ParseInt(val, 10, bitSize)
	if err == nil {
		field.SetInt(intVal)
	}
	return err
}

func setUintField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0"
	}
	uintVal, err := strconv.ParseUint(val, 10, bitSize)
	if err == nil {
		field.SetUint(uintVal)
	}
	return err
}

func setBoolField(val string, field reflect.Value) error {
	if val == "" {
		val = "false"
	}
	boolVal, err := strconv.ParseBool(val)
	if err == nil {
		field.SetBool(boolVal)
	}
	return err
}

func setFloatField(val string, bitSize int, field reflect.Value) error {
	if val == "" {
		val = "0.0"
	}
	floatVal, err := strconv.ParseFloat(val, bitSize)
	if err == nil {
		field.SetFloat(floatVal)
	}
	return err
}
<|endoftext|>