<commit_before>package handlers

import (
	"net/http"
	"encoding/json"
	"fmt"
	"strings"
)

type stateChillHandler struct {
	imageUrls map[string]string
}

// NewStateChillHandler creates an http handler that will return chill images of states. The state image returned is
// determined by "state" parameter provided in the request.
//
// Example request:
// http://localhost:8008/states_chill?state=MN
//
// Example response:
//   {"chill image":"http://blahblahblh.cloudfront.net/CA.jpg"}
func NewStateChillHandler(stateImagesLocation string) http.Handler {
	imageUrls := make(map[string]string)
	for state := range validStates {
		imageUrls[state] = fmt.Sprintf("%s/%s.jpg", stateImagesLocation, state)
	}
	return stateChillHandler{
		imageUrls: imageUrls,
	}
}

func (sc stateChillHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/json; charset=utf-8")
	sc.processRequest(rw, req)
}

func (sc stateChillHandler) processRequest(rw http.ResponseWriter, req *http.Request) {
	state := strings.ToUpper(req.FormValue("state"))
	if state == "" {
		sc.writeBadRequestResponse(rw)
		return
	}

	if !validStates[state] {
		sc.writeInvalidStateResponse(rw, state)
	}

	sc.writeChillResponse(rw, state)
}

func (sc stateChillHandler) writeBadRequestResponse(rw http.ResponseWriter) {
	rw.WriteHeader(http.StatusBadRequest)
	response := map[string]string{"error": "missing \"state\" parameter"}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

func (sc stateChillHandler) writeInvalidStateResponse(rw http.ResponseWriter, missingState string) {
	rw.WriteHeader(http.StatusNotFound)
	response := map[string]string{"error": fmt.Sprintf("Could not find state %q", missingState)}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

func (sc stateChillHandler) writeChillResponse(rw http.ResponseWriter, state string) {
	rw.WriteHeader(http.StatusOK)
	response := map[string]string{"chill image": sc.imageUrls[state]}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

<commit_msg>Modify chill state response<commit_after>package handlers

import (
	"net/http"
	"encoding/json"
	"fmt"
	"strings"
)

type stateChillHandler struct {
	imageUrls map[string]string
}

// NewStateChillHandler creates an http handler that will return chill images of states. The state image returned is
// determined by "state" parameter provided in the request.
//
// Example request:
// http://localhost:8008/states_chill?state=MN
//
// Example response:
//   {"chill_image":"http://blahblahblh.cloudfront.net/CA.jpg"}
func NewStateChillHandler(stateImagesLocation string) http.Handler {
	imageUrls := make(map[string]string)
	for state := range validStates {
		imageUrls[state] = fmt.Sprintf("%s/%s.jpg", stateImagesLocation, state)
	}
	return stateChillHandler{
		imageUrls: imageUrls,
	}
}

func (sc stateChillHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	rw.Header().Set("Content-Type", "text/json; charset=utf-8")
	sc.processRequest(rw, req)
}

func (sc stateChillHandler) processRequest(rw http.ResponseWriter, req *http.Request) {
	state := strings.ToUpper(req.FormValue("state"))
	if state == "" {
		sc.writeBadRequestResponse(rw)
		return
	}

	if !validStates[state] {
		sc.writeInvalidStateResponse(rw, state)
	}

	sc.writeChillResponse(rw, state)
}

func (sc stateChillHandler) writeBadRequestResponse(rw http.ResponseWriter) {
	rw.WriteHeader(http.StatusBadRequest)
	response := map[string]string{"error": "missing \"state\" parameter"}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

func (sc stateChillHandler) writeInvalidStateResponse(rw http.ResponseWriter, missingState string) {
	rw.WriteHeader(http.StatusNotFound)
	response := map[string]string{"error": fmt.Sprintf("Could not find state %q", missingState)}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

func (sc stateChillHandler) writeChillResponse(rw http.ResponseWriter, state string) {
	rw.WriteHeader(http.StatusOK)
	response := map[string]string{"chill_image": sc.imageUrls[state]}
	encoder := json.NewEncoder(rw)
	encoder.Encode(response)
}

<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"
	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"
)

type Step struct {
	Name string `yaml:"name"`
	Args []string
}

// struct for images/<image>/cloudbuild.yaml
// Example: images/alpine/cloudbuild.yaml
type CloudBuildYAMLFile struct {
	Steps         []Step `yaml:"steps"`
	Substitutions map[string]string
	Images        []string
}

func getProjectID() (string, error) {
	cmd := exec.Command("gcloud", "config", "get-value", "project")
	projectID, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("failed to get project_id: %v", err)
	}
	return string(projectID), nil
}

func getImageName(o options, tag string, config string) (string, error) {
	var cloudbuildyamlFile CloudBuildYAMLFile
	buf, _ := ioutil.ReadFile(o.cloudbuildFile)
	if err := yaml.Unmarshal(buf, &cloudbuildyamlFile); err != nil {
		return "", fmt.Errorf("failed to get image name: %v", err)
	}
	var projectID, _ = getProjectID()
	var imageNames = cloudbuildyamlFile.Images
	r := strings.NewReplacer("$PROJECT_ID", strings.TrimSpace(projectID), "$_GIT_TAG", tag, "$_CONFIG", config)
	var result string
	for _, name := range imageNames {
		result = result + r.Replace(name) + " "
	}
	return result, nil
}

func runCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func getVersion(flags []string) (string, error) {
	describeCmd := append([]string{"describe"}, flags...)
	cmd := exec.Command("git", describeCmd...)
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	validTagRegexp, err := regexp.Compile("[^-_.a-zA-Z0-9]+")
	if err != nil {
		return "", err
	}
	sanitizedOutput := validTagRegexp.ReplaceAllString(string(output), "")
	t := time.Now().Format("20060102")
	return fmt.Sprintf("v%s-%s", t, sanitizedOutput), nil
}

func (o *options) validateConfigDir() error {
	configDir := o.configDir
	dirInfo, err := os.Stat(o.configDir)
	if os.IsNotExist(err) {
		log.Fatalf("Config directory (%s) does not exist", configDir)
	}

	if !dirInfo.IsDir() {
		log.Fatalf("Config directory (%s) is not actually a directory", configDir)
	}

	_, err = os.Stat(o.cloudbuildFile)
	if os.IsNotExist(err) {
		log.Fatalf("%s does not exist", o.cloudbuildFile)
	}

	return nil
}

func (o *options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	log.Printf("Creating source tarball at %s...\n", name)
	var args []string
	if !o.withGitDirectory {
		args = append(args, "--exclude", ".git")
	}
	args = append(args, "-czf", name, ".")
	if err := runCmd("tar", args...); err != nil {
		return "", fmt.Errorf("failed to tar files: %s", err)
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	log.Printf("Uploading %s to %s...\n", name, uploaded)
	if err := runCmd("gsutil", "cp", name, uploaded); err != nil {
		return "", fmt.Errorf("failed to upload files: %s", err)
	}

	return uploaded, nil
}

func getExtraSubs(o options) map[string]string {
	envs := strings.Split(o.envPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func runSingleJob(o options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.cloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.project != "" {
		args = append(args, "--project", o.project)
	}

	if o.scratchBucket != "" {
		args = append(args, "--gcs-log-dir", o.scratchBucket+gcsLogsDir)
		args = append(args, "--gcs-source-staging-dir", o.scratchBucket+gcsSourceDir)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.noSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	cmd := exec.Command("gcloud", args...)

	var logFilePath string
	if o.logDir != "" {
		logFilePath = path.Join(o.logDir, strings.Replace(jobName, "/", "-", -1)+".log")
		f, err := os.Create(logFilePath)

		if err != nil {
			return fmt.Errorf("couldn't create %s: %v", logFilePath, err)
		}

		defer f.Sync()
		defer f.Close()

		cmd.Stdout = f
		cmd.Stderr = f
	} else {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err := cmd.Run(); err != nil {
		if o.logDir != "" {
			buildLog, _ := ioutil.ReadFile(logFilePath)
			fmt.Println(string(buildLog))
		}
		return fmt.Errorf("error running %s: %v", cmd.Args, err)
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.configDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("failed to load variants.yaml: %v", err)
		}
		if o.variant != "" {
			return nil, fmt.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, fmt.Errorf("failed to read variants.yaml: %v", err)
	}
	if o.variant != "" {
		va, ok := v.Variants[o.variant]
		if !ok {
			return nil, fmt.Errorf("requested variant %q, which is not present in variants.yaml", o.variant)
		}
		return variants{o.variant: va}, nil
	}
	return v.Variants, nil
}

func runBuildJobs(o options) []error {
	var uploaded string
	if o.scratchBucket != "" {
		if !o.noSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.scratchBucket + gcsSourceDir)
			if err != nil {
				return []error{fmt.Errorf("failed to upload source: %v", err)}
			}
		}
	} else {
		log.Println("Skipping advance upload and relying on gcloud...")
	}

	log.Println("Running build jobs...")
	tagFlags := []string{"--tags", "--always", "--dirty"}
	if len(o.tagMatch) > 0 {
		tagFlags = append(tagFlags, "--match "+o.tagMatch)
	}
	tag, err := getVersion(tagFlags)
	if err != nil {
		return []error{fmt.Errorf("failed to get current tag: %v", err)}
	}

	if !o.allowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{fmt.Errorf("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}

	if len(vs) == 0 {
		log.Println("No variants.yaml, starting single build job...")
		if err := runSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		var imageName, _ = getImageName(o, tag, "")
		log.Printf("Successfully built image: %v \n", imageName)
		return nil
	}

	log.Printf("Found variants.yaml, starting %d build jobs...\n", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var errors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			log.Printf("Starting job %q...\n", job)
			if err := runSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				errors = append(errors, fmt.Errorf("job %q failed: %v", job, err))
				log.Printf("Job %q failed: %v\n", job, err)
			} else {
				var imageName, _ = getImageName(o, tag, job)
				log.Printf("Successfully built image: %v \n", imageName)
				log.Printf("Job %q completed.\n", job)
			}
		}(k, v)
	}
	w.Wait()
	return errors
}

type options struct {
	buildDir       string
	configDir      string
	cloudbuildFile string
	logDir         string
	scratchBucket  string
	project        string
	allowDirty     bool
	noSource       bool
	variant        string
	envPassthrough string
	tagMatch       string

	// withGitDirectory will include the .git directory when uploading the source to GCB
	withGitDirectory bool
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}

func parseFlags() options {
	o := options{}
	flag.StringVar(&o.buildDir, "build-dir", "", "If provided, this directory will be uploaded as the source for the Google Cloud Build run.")
	flag.StringVar(&o.cloudbuildFile, "gcb-config", "cloudbuild.yaml", "If provided, this will be used as the name of the Google Cloud Build config file.")
	flag.StringVar(&o.logDir, "log-dir", "", "If provided, build logs will be sent to files in this directory instead of to stdout/stderr.")
	flag.StringVar(&o.scratchBucket, "scratch-bucket", "", "The complete GCS path for Cloud Build to store scratch files (sources, logs).")
	flag.StringVar(&o.project, "project", "", "If specified, use a non-default GCP project.")
	flag.BoolVar(&o.allowDirty, "allow-dirty", false, "If true, allow pushing dirty builds.")
	flag.BoolVar(&o.noSource, "no-source", false, "If true, no source will be uploaded with this build.")
	flag.StringVar(&o.variant, "variant", "", "If specified, build only the given variant. An error if no variants are defined.")
	flag.StringVar(&o.envPassthrough, "env-passthrough", "", "Comma-separated list of specified environment variables to be passed to GCB as substitutions with an _ prefix. If the variable doesn't exist, the substitution will exist but be empty.")
	flag.BoolVar(&o.withGitDirectory, "with-git-dir", o.withGitDirectory, "If true, upload the .git directory to GCB, so we can e.g. get the git log and tag.")
	flag.StringVar(&o.tagMatch, "tag-match", "", "If specified, use the latest tag matching this pattern (see `git describe --tags --match`)")

	flag.Parse()

	if flag.NArg() < 1 {
		_, _ = fmt.Fprintln(os.Stderr, "expected a config directory to be provided")
		os.Exit(1)
	}

	o.configDir = strings.TrimSuffix(flag.Arg(0), "/")

	return o
}

func main() {
	o := parseFlags()

	if bazelWorkspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); bazelWorkspace != "" {
		if err := os.Chdir(bazelWorkspace); err != nil {
			log.Fatalf("Failed to chdir to bazel workspace (%s): %v", bazelWorkspace, err)
		}
	}

	if o.buildDir == "" {
		o.buildDir = o.configDir
	}

	log.Printf("Build directory: %s\n", o.buildDir)

	// Canonicalize the config directory to be an absolute path.
	// As we're about to cd into the build directory, we need a consistent way to reference the config files
	// when the config directory is not the same as the build directory.
	absConfigDir, absErr := filepath.Abs(o.configDir)
	if absErr != nil {
		log.Fatalf("Could not resolve absolute path for config directory: %v", absErr)
	}

	o.configDir = absConfigDir
	o.cloudbuildFile = path.Join(o.configDir, o.cloudbuildFile)

	configDirErr := o.validateConfigDir()
	if configDirErr != nil {
		log.Fatalf("Could not validate config directory: %v", configDirErr)
	}

	log.Printf("Config directory: %s\n", o.configDir)

	log.Printf("cd-ing to build directory: %s\n", o.buildDir)
	if err := os.Chdir(o.buildDir); err != nil {
		log.Fatalf("Failed to chdir to build directory (%s): %v", o.buildDir, err)
	}

	errors := runBuildJobs(o)
	if len(errors) != 0 {
		log.Fatalf("Failed to run some build jobs: %v", errors)
	}
	log.Println("Finished.")
}
<commit_msg>Fix parsing of --tag-match image builder flag<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"
	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"
)

type Step struct {
	Name string `yaml:"name"`
	Args []string
}

// struct for images/<image>/cloudbuild.yaml
// Example: images/alpine/cloudbuild.yaml
type CloudBuildYAMLFile struct {
	Steps         []Step `yaml:"steps"`
	Substitutions map[string]string
	Images        []string
}

func getProjectID() (string, error) {
	cmd := exec.Command("gcloud", "config", "get-value", "project")
	projectID, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("failed to get project_id: %v", err)
	}
	return string(projectID), nil
}

func getImageName(o options, tag string, config string) (string, error) {
	var cloudbuildyamlFile CloudBuildYAMLFile
	buf, _ := ioutil.ReadFile(o.cloudbuildFile)
	if err := yaml.Unmarshal(buf, &cloudbuildyamlFile); err != nil {
		return "", fmt.Errorf("failed to get image name: %v", err)
	}
	var projectID, _ = getProjectID()
	var imageNames = cloudbuildyamlFile.Images
	r := strings.NewReplacer("$PROJECT_ID", strings.TrimSpace(projectID), "$_GIT_TAG", tag, "$_CONFIG", config)
	var result string
	for _, name := range imageNames {
		result = result + r.Replace(name) + " "
	}
	return result, nil
}

func runCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func getVersion(flags []string) (string, error) {
	describeCmd := append([]string{"describe"}, flags...)
	cmd := exec.Command("git", describeCmd...)
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	validTagRegexp, err := regexp.Compile("[^-_.a-zA-Z0-9]+")
	if err != nil {
		return "", err
	}
	sanitizedOutput := validTagRegexp.ReplaceAllString(string(output), "")
	t := time.Now().Format("20060102")
	return fmt.Sprintf("v%s-%s", t, sanitizedOutput), nil
}

func (o *options) validateConfigDir() error {
	configDir := o.configDir
	dirInfo, err := os.Stat(o.configDir)
	if os.IsNotExist(err) {
		log.Fatalf("Config directory (%s) does not exist", configDir)
	}

	if !dirInfo.IsDir() {
		log.Fatalf("Config directory (%s) is not actually a directory", configDir)
	}

	_, err = os.Stat(o.cloudbuildFile)
	if os.IsNotExist(err) {
		log.Fatalf("%s does not exist", o.cloudbuildFile)
	}

	return nil
}

func (o *options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	log.Printf("Creating source tarball at %s...\n", name)
	var args []string
	if !o.withGitDirectory {
		args = append(args, "--exclude", ".git")
	}
	args = append(args, "-czf", name, ".")
	if err := runCmd("tar", args...); err != nil {
		return "", fmt.Errorf("failed to tar files: %s", err)
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	log.Printf("Uploading %s to %s...\n", name, uploaded)
	if err := runCmd("gsutil", "cp", name, uploaded); err != nil {
		return "", fmt.Errorf("failed to upload files: %s", err)
	}

	return uploaded, nil
}

func getExtraSubs(o options) map[string]string {
	envs := strings.Split(o.envPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func runSingleJob(o options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.cloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.project != "" {
		args = append(args, "--project", o.project)
	}

	if o.scratchBucket != "" {
		args = append(args, "--gcs-log-dir", o.scratchBucket+gcsLogsDir)
		args = append(args, "--gcs-source-staging-dir", o.scratchBucket+gcsSourceDir)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.noSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	cmd := exec.Command("gcloud", args...)

	var logFilePath string
	if o.logDir != "" {
		logFilePath = path.Join(o.logDir, strings.Replace(jobName, "/", "-", -1)+".log")
		f, err := os.Create(logFilePath)

		if err != nil {
			return fmt.Errorf("couldn't create %s: %v", logFilePath, err)
		}

		defer f.Sync()
		defer f.Close()

		cmd.Stdout = f
		cmd.Stderr = f
	} else {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err := cmd.Run(); err != nil {
		if o.logDir != "" {
			buildLog, _ := ioutil.ReadFile(logFilePath)
			fmt.Println(string(buildLog))
		}
		return fmt.Errorf("error running %s: %v", cmd.Args, err)
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.configDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("failed to load variants.yaml: %v", err)
		}
		if o.variant != "" {
			return nil, fmt.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, fmt.Errorf("failed to read variants.yaml: %v", err)
	}
	if o.variant != "" {
		va, ok := v.Variants[o.variant]
		if !ok {
			return nil, fmt.Errorf("requested variant %q, which is not present in variants.yaml", o.variant)
		}
		return variants{o.variant: va}, nil
	}
	return v.Variants, nil
}

func runBuildJobs(o options) []error {
	var uploaded string
	if o.scratchBucket != "" {
		if !o.noSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.scratchBucket + gcsSourceDir)
			if err != nil {
				return []error{fmt.Errorf("failed to upload source: %v", err)}
			}
		}
	} else {
		log.Println("Skipping advance upload and relying on gcloud...")
	}

	log.Println("Running build jobs...")
	tagFlags := []string{"--tags", "--always", "--dirty"}
	if len(o.tagMatch) > 0 {
		tagFlags = append(tagFlags, fmt.Sprintf(`--match "%s"`, o.tagMatch))
	}
	tag, err := getVersion(tagFlags)
	if err != nil {
		return []error{fmt.Errorf("failed to get current tag: %v", err)}
	}

	if !o.allowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{fmt.Errorf("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}

	if len(vs) == 0 {
		log.Println("No variants.yaml, starting single build job...")
		if err := runSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		var imageName, _ = getImageName(o, tag, "")
		log.Printf("Successfully built image: %v \n", imageName)
		return nil
	}

	log.Printf("Found variants.yaml, starting %d build jobs...\n", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var errors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			log.Printf("Starting job %q...\n", job)
			if err := runSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				errors = append(errors, fmt.Errorf("job %q failed: %v", job, err))
				log.Printf("Job %q failed: %v\n", job, err)
			} else {
				var imageName, _ = getImageName(o, tag, job)
				log.Printf("Successfully built image: %v \n", imageName)
				log.Printf("Job %q completed.\n", job)
			}
		}(k, v)
	}
	w.Wait()
	return errors
}

type options struct {
	buildDir       string
	configDir      string
	cloudbuildFile string
	logDir         string
	scratchBucket  string
	project        string
	allowDirty     bool
	noSource       bool
	variant        string
	envPassthrough string
	tagMatch       string

	// withGitDirectory will include the .git directory when uploading the source to GCB
	withGitDirectory bool
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}

func parseFlags() options {
	o := options{}
	flag.StringVar(&o.buildDir, "build-dir", "", "If provided, this directory will be uploaded as the source for the Google Cloud Build run.")
	flag.StringVar(&o.cloudbuildFile, "gcb-config", "cloudbuild.yaml", "If provided, this will be used as the name of the Google Cloud Build config file.")
	flag.StringVar(&o.logDir, "log-dir", "", "If provided, build logs will be sent to files in this directory instead of to stdout/stderr.")
	flag.StringVar(&o.scratchBucket, "scratch-bucket", "", "The complete GCS path for Cloud Build to store scratch files (sources, logs).")
	flag.StringVar(&o.project, "project", "", "If specified, use a non-default GCP project.")
	flag.BoolVar(&o.allowDirty, "allow-dirty", false, "If true, allow pushing dirty builds.")
	flag.BoolVar(&o.noSource, "no-source", false, "If true, no source will be uploaded with this build.")
	flag.StringVar(&o.variant, "variant", "", "If specified, build only the given variant. An error if no variants are defined.")
	flag.StringVar(&o.envPassthrough, "env-passthrough", "", "Comma-separated list of specified environment variables to be passed to GCB as substitutions with an _ prefix. If the variable doesn't exist, the substitution will exist but be empty.")
	flag.BoolVar(&o.withGitDirectory, "with-git-dir", o.withGitDirectory, "If true, upload the .git directory to GCB, so we can e.g. get the git log and tag.")
	flag.StringVar(&o.tagMatch, "tag-match", "", "If specified, use the latest tag matching this pattern (see `git describe --tags --match`)")

	flag.Parse()

	if flag.NArg() < 1 {
		_, _ = fmt.Fprintln(os.Stderr, "expected a config directory to be provided")
		os.Exit(1)
	}

	o.configDir = strings.TrimSuffix(flag.Arg(0), "/")

	return o
}

func main() {
	o := parseFlags()

	if bazelWorkspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY"); bazelWorkspace != "" {
		if err := os.Chdir(bazelWorkspace); err != nil {
			log.Fatalf("Failed to chdir to bazel workspace (%s): %v", bazelWorkspace, err)
		}
	}

	if o.buildDir == "" {
		o.buildDir = o.configDir
	}

	log.Printf("Build directory: %s\n", o.buildDir)

	// Canonicalize the config directory to be an absolute path.
	// As we're about to cd into the build directory, we need a consistent way to reference the config files
	// when the config directory is not the same as the build directory.
	absConfigDir, absErr := filepath.Abs(o.configDir)
	if absErr != nil {
		log.Fatalf("Could not resolve absolute path for config directory: %v", absErr)
	}

	o.configDir = absConfigDir
	o.cloudbuildFile = path.Join(o.configDir, o.cloudbuildFile)

	configDirErr := o.validateConfigDir()
	if configDirErr != nil {
		log.Fatalf("Could not validate config directory: %v", configDirErr)
	}

	log.Printf("Config directory: %s\n", o.configDir)

	log.Printf("cd-ing to build directory: %s\n", o.buildDir)
	if err := os.Chdir(o.buildDir); err != nil {
		log.Fatalf("Failed to chdir to build directory (%s): %v", o.buildDir, err)
	}

	errors := runBuildJobs(o)
	if len(errors) != 0 {
		log.Fatalf("Failed to run some build jobs: %v", errors)
	}
	log.Println("Finished.")
}
<|endoftext|>


<commit_before>package haproxystats

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io"
	"net"
	"strings"

	"github.com/gocarina/gocsv"
)

const (
	socketSchema = "unix:///"
	tcpSchema    = "tcp://"
)

type HAProxyClient struct {
	conn net.Conn
}

func (h *HAProxyClient) RunCommand(cmd string) *bytes.Buffer {
	done := make(chan bool)
	result := bytes.NewBuffer(nil)

	go func() {
		io.Copy(result, h.conn)
		defer func() { done <- true }()
	}()

	go func() {
		h.conn.Write([]byte(cmd + "\n"))
		defer func() { done <- true }()
	}()

	// Wait for both io streams to close
	for i := 0; i < 2; i++ {
		select {
		case <-done:
		}
	}

	return result
}

func (h *HAProxyClient) Stats() (services Services, err error) {
	res := h.RunCommand("show stat")

	allStats := []*Stat{}
	reader := csv.NewReader(res)
	reader.TrailingComma = true
	err = gocsv.UnmarshalCSV(reader, &allStats)
	if err != nil {
		return services, fmt.Errorf("error reading csv: %s", err)
	}

	for _, s := range allStats {
		switch s.SvName {
		case "FRONTEND":
			services.Frontends = append(services.Frontends, s)
		case "BACKEND":
			services.Backends = append(services.Backends, s)
		default:
			services.Listeners = append(services.Listeners, s)
		}
	}

	return services, nil
}

func New(addr string) (*HAProxyClient, error) {
	var err error
	client := &HAProxyClient{}

	if strings.HasPrefix(addr, socketSchema) {
		client.conn, err = net.Dial("unix", strings.Replace(addr, socketSchema, "", 1))
		if err != nil {
			panic(err)
		}
	}

	if strings.HasPrefix(addr, tcpSchema) {
		client.conn, err = net.Dial("tcp", strings.Replace(addr, tcpSchema, "", 1))
		if err != nil {
			panic(err)
		}
	}

	if client.conn == nil {
		return nil, fmt.Errorf("unknown schema")
	}

	return client, nil
}
<commit_msg>return error on unknown command<commit_after>package haproxystats

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"io"
	"net"
	"strings"

	"github.com/gocarina/gocsv"
)

const (
	socketSchema = "unix:///"
	tcpSchema    = "tcp://"
)

type HAProxyClient struct {
	conn net.Conn
}

func (h *HAProxyClient) RunCommand(cmd string) (*bytes.Buffer, error) {
	done := make(chan bool)
	result := bytes.NewBuffer(nil)

	go func() {
		io.Copy(result, h.conn)
		defer func() { done <- true }()
	}()

	go func() {
		h.conn.Write([]byte(cmd + "\n"))
		defer func() { done <- true }()
	}()

	// Wait for both io streams to close
	for i := 0; i < 2; i++ {
		select {
		case <-done:
		}
	}

	fmt.Println(result.String())
	if strings.HasPrefix(result.String(), "Unknown command") {
		return nil, fmt.Errorf("Uknown command: %s", cmd)
	}

	return result, nil
}

func (h *HAProxyClient) Stats() (services Services, err error) {
	res, err := h.RunCommand("show stat")
	if err != nil {
		return services, err
	}

	allStats := []*Stat{}
	reader := csv.NewReader(res)
	reader.TrailingComma = true
	err = gocsv.UnmarshalCSV(reader, &allStats)
	if err != nil {
		return services, fmt.Errorf("error reading csv: %s", err)
	}

	for _, s := range allStats {
		switch s.SvName {
		case "FRONTEND":
			services.Frontends = append(services.Frontends, s)
		case "BACKEND":
			services.Backends = append(services.Backends, s)
		default:
			services.Listeners = append(services.Listeners, s)
		}
	}

	return services, nil
}

func New(addr string) (*HAProxyClient, error) {
	var err error
	client := &HAProxyClient{}

	if strings.HasPrefix(addr, socketSchema) {
		client.conn, err = net.Dial("unix", strings.Replace(addr, socketSchema, "", 1))
		if err != nil {
			panic(err)
		}
	}

	if strings.HasPrefix(addr, tcpSchema) {
		client.conn, err = net.Dial("tcp", strings.Replace(addr, tcpSchema, "", 1))
		if err != nil {
			panic(err)
		}
	}

	if client.conn == nil {
		return nil, fmt.Errorf("unknown schema")
	}

	return client, nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql/driver"
	"log"
	"time"

	"github.com/kshvakov/clickhouse"
)

func main() {
	connect, err := clickhouse.Open("tcp://127.0.0.1:9000?username=&debug=true")
	if err != nil {
		log.Fatal(err)
	}
	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare(`
			CREATE TABLE IF NOT EXISTS example (
				country_code FixedString(2),
				os_id        UInt8,
				browser_id   UInt8,
				action_day   Date,
				action_time  DateTime
			) engine=Memory
		`)

		if _, err := stmt.Exec([]driver.Value{}); err != nil {
			log.Fatal(err)
		}
		tx.Commit()
	}
	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare("INSERT INTO example (country_code, os_id, browser_id, action_day, action_time) VALUES (?, ?, ?, ?, ?)")
		for i := 0; i < 100; i++ {
			if _, err := stmt.Exec([]driver.Value{
				"CZ",
				uint8(10 + i),
				uint8(100 + i),
				time.Now(),
				time.Now(),
			}); err != nil {
				log.Fatal(err)
			}
		}

		if err := tx.Commit(); err != nil {
			log.Fatal(err)
		}
	}

	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare(`DROP TABLE example`)

		if _, err := stmt.Exec([]driver.Value{}); err != nil {
			log.Fatal(err)
		}
		tx.Commit()
	}
}
<commit_msg>example: direct array was added<commit_after>package main

import (
	"database/sql/driver"
	"log"
	"time"

	"github.com/kshvakov/clickhouse"
)

func main() {
	connect, err := clickhouse.Open("tcp://127.0.0.1:9000?username=&debug=true")
	if err != nil {
		log.Fatal(err)
	}
	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare(`
			CREATE TABLE IF NOT EXISTS example (
				country_code FixedString(2),
				os_id        UInt8,
				browser_id   UInt8,
				categories   Array(Int16),
				action_day   Date,
				action_time  DateTime
			) engine=Memory
		`)

		if _, err := stmt.Exec([]driver.Value{}); err != nil {
			log.Fatal(err)
		}
		tx.Commit()
	}
	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare("INSERT INTO example (country_code, os_id, browser_id, categories, action_day, action_time) VALUES (?, ?, ?, ?, ?, ?)")
		for i := 0; i < 100; i++ {
			if _, err := stmt.Exec([]driver.Value{
				"CZ",
				uint8(10 + i),
				uint8(100 + i),
				clickhouse.Array([]int16{1, 2, 3}),
				time.Now(),
				time.Now(),
			}); err != nil {
				log.Fatal(err)
			}
		}

		if err := tx.Commit(); err != nil {
			log.Fatal(err)
		}
	}

	{
		tx, _ := connect.Begin()
		stmt, _ := connect.Prepare(`DROP TABLE example`)

		if _, err := stmt.Exec([]driver.Value{}); err != nil {
			log.Fatal(err)
		}
		tx.Commit()
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/CrowdSurge/banner"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/spf13/cobra"
)

// job var used as a central point for command data
var job = struct {
	cfgFile      string
	tplFile      string
	profile      string
	tplFiles     []string
	stacks       map[string]string
	terminateAll bool
	version      bool
	request      string
	debug        bool
	funcEvent    string
	changeName   string
	stackName    string
	rollback     bool
}{}

// Wait Group for handling goroutines
var wg sync.WaitGroup

// RootCmd command (calls all other commands)
var RootCmd = &cobra.Command{
	Use:   "qaz",
	Short: fmt.Sprintf("%s\n--> Shut up & deploy my templates...!", colorString(banner.PrintS("qaz"), "magenta")),
	Run: func(cmd *cobra.Command, args []string) {

		if job.version {
			fmt.Printf("qaz - Version %s"+"\n", version)
			return
		}

		cmd.Help()
	},
}

var initCmd = &cobra.Command{
	Use:   "init [target directory]",
	Short: "Creates a basic qaz project",
	Run: func(cmd *cobra.Command, args []string) {

		// Print Banner
		banner.Print("qaz")
		fmt.Printf("\n--\n")

		var target string
		switch len(args) {
		case 0:
			target, _ = os.Getwd()
		default:
			target = args[0]
		}

		// Get Project & AWS Region
		project = getInput("-> Enter your Project name", "MyqazProject")
		region = getInput("-> Enter AWS Region", "eu-west-1")

		// set target paths
		c := filepath.Join(target, "config.yml")
		t := filepath.Join(target, "templates")
		f := filepath.Join(target, "files")

		// Check if config file exists
		var overwrite string
		if _, err := os.Stat(c); err == nil {
			overwrite = getInput(
				fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", colorString("->", "yellow"), c, colorString("Overwrite", "red")),
				"N",
			)

			if overwrite == "Y" {
				fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", colorString("->", "yellow"), c))
			}
		}

		// Create template file
		if overwrite != "N" {
			if err := ioutil.WriteFile(c, configTemplate(project, region), 0644); err != nil {
				fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, colorString("->", "red"))
				return
			}
		}

		// Create template folder
		for _, dir := range []string{t, f} {
			if err := os.Mkdir(dir, os.ModePerm); err != nil {
				fmt.Printf("%s [%s] folder not created: %s"+"\n--\n", colorString("->", "yellow"), dir, err)
				return
			}
		}

		fmt.Println("--")
	},
}

var generateCmd = &cobra.Command{
	Use:   "generate",
	Short: "Generates template from configuration values",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "generate"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source
		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		name := fmt.Sprintf("%s-%s", project, s)
		Log(fmt.Sprintln("Generating a template for ", name), "debug")

		tpl, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}
		fmt.Println(tpl)
	},
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploys stack(s) to AWS",
	Example: strings.Join([]string{
		"qaz deploy -c path/to/config -t path/to/template",
		"qaz deploy -c path/to/config -t stack::s3//bucket/key",
		"qaz deploy -c path/to/config -t stack::http://someurl",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "deploy"

		job.stacks = make(map[string]string)

		sourceCopy := job.tplFiles

		// creating empty template list for re-population later
		job.tplFiles = []string{}

		for _, src := range sourceCopy {
			if strings.Contains(src, `*`) {
				glob, _ := filepath.Glob(src)

				for _, f := range glob {
					job.tplFiles = append(job.tplFiles, f)
				}
				continue
			}

			job.tplFiles = append(job.tplFiles, src)
		}

		for _, f := range job.tplFiles {
			s, source, err := getSource(f)
			if err != nil {
				handleError(err)
				return
			}
			job.stacks[s] = source
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		for s, f := range job.stacks {
			if v, err := genTimeParser(f); err != nil {
				handleError(err)
			} else {

				// Handle missing stacks
				if stacks[s] == nil {
					handleError(fmt.Errorf("Missing Stack in %s: [%s]", job.cfgFile, s))
					return
				}

				stacks[s].template = v
			}
		}

		// Deploy Stacks
		DeployHandler()

	},
}

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates a given stack",
	Example: strings.Join([]string{
		"qaz update -c path/to/config -t stack::path/to/template",
		"qaz update -c path/to/config -t stack::s3//bucket/key",
		"qaz update -c path/to/config -t stack::http://someurl",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "update"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source

		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		v, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		// Handle missing stacks
		if stacks[s] == nil {
			handleError(fmt.Errorf("Missing Stack in %s: [%s]", job.cfgFile, s))
			return
		}

		stacks[s].template = v

		// resolve deploy time function
		if err = stacks[s].deployTimeParser(); err != nil {
			handleError(err)
		}

		// Update stack
		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}
		stacks[s].update(sess)

	},
}

var terminateCmd = &cobra.Command{
	Use:   "terminate [stacks]",
	Short: "Terminates stacks",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "terminate"

		if !job.terminateAll {
			job.stacks = make(map[string]string)
			for _, stk := range args {
				job.stacks[stk] = ""
			}

			if len(job.stacks) == 0 {
				Log("No stack specified for termination", level.warn)
				return
			}
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		// Terminate Stacks
		TerminateHandler()
	},
}

var statusCmd = &cobra.Command{
	Use:   "status",
	Short: "Prints status of deployed/un-deployed stacks",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "status"

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		for _, v := range stacks {
			wg.Add(1)
			go func(s *stack) {
				if err := s.status(sess); err != nil {
					handleError(err)
				}
				wg.Done()
			}(v)

		}
		wg.Wait()
	},
}

var outputsCmd = &cobra.Command{
	Use:     "outputs [stack]",
	Short:   "Prints stack outputs",
	Example: "qaz outputs vpc subnets --config path/to/config",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "outputs"

		if len(args) < 1 {
			fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help")
			return
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		for _, s := range args {
			// check if stack exists
			if _, ok := stacks[s]; !ok {
				handleError(fmt.Errorf("%s: does not Exist in Config", s))
				continue
			}

			wg.Add(1)
			go func(s string) {
				if err := stacks[s].outputs(sess); err != nil {
					handleError(err)
				}

				for _, i := range stacks[s].output.Stacks {
					fmt.Printf("\n"+"[%s]"+"\n", *i.StackName)
					for _, o := range i.Outputs {
						fmt.Printf("  Description: %s\n  %s: %s\n\n", *o.Description, colorString(*o.OutputKey, "magenta"), *o.OutputValue)
					}
				}

				wg.Done()
			}(s)
		}
		wg.Wait()

	},
}

var exportsCmd = &cobra.Command{
	Use:     "exports",
	Short:   "Prints stack exports",
	Example: "qaz exports",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "exports"

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		Exports(sess)

	},
}

var checkCmd = &cobra.Command{
	Use:   "check",
	Short: "Validates Cloudformation Templates",
	Example: strings.Join([]string{
		"qaz check -c path/to/config.yml -t path/to/template -c path/to/config",
		"qaz check -c path/to/config.yml -t stack::http://someurl.example",
		"qaz check -c path/to/config.yml -t stack::s3://bucket/key",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "validate"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source

		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		name := fmt.Sprintf("%s-%s", project, s)
		fmt.Println("Validating template for", name)

		tpl, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		stk := stack{name: s}
		stk.setStackName()
		stk.template = tpl

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		if err := stk.check(sess); err != nil {
			handleError(err)
			return
		}
	},
}

var invokeCmd = &cobra.Command{
	Use:   "invoke",
	Short: "Invoke AWS Lambda Functions",
	Run: func(cmd *cobra.Command, args []string) {
		job.request = "lambda_invoke"
		// fmt.Println(colorString("Coming Soon!", "magenta"))

		if len(args) < 1 {
			fmt.Println("No Lambda Function specified")
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		f := function{name: args[0]}

		if job.funcEvent != "" {
			f.payload = []byte(job.funcEvent)
		}

		if err := f.Invoke(sess); err != nil {
			if strings.Contains(err.Error(), "Unhandled") {
				handleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name))
			}
			handleError(err)
			return
		}

		fmt.Println(f.response)

	},
}

var tailCmd = &cobra.Command{
	Use:     "tail",
	Short:   "Tail Real-Time AWS Cloudformation events",
	Example: "qaz tail -r eu-west-1",
	Run: func(cmd *cobra.Command, args []string) {
		job.request = "tail"

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
		}

		// Tail each stack on it's own goroutine.
		for _, s := range stacks {
			wg.Add(1)
			go func(s *stack, sess *session.Session) {
				verbose(s.stackname, "", sess)
				wg.Done()
			}(s, sess)
		}

		wg.Wait() // Will probably wait forevery
	},
}
<commit_msg>added set policy command<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/CrowdSurge/banner"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/spf13/cobra"
)

// job var used as a central point for command data
var job = struct {
	cfgFile      string
	tplFile      string
	profile      string
	tplFiles     []string
	stacks       map[string]string
	terminateAll bool
	version      bool
	request      string
	debug        bool
	funcEvent    string
	changeName   string
	stackName    string
	rollback     bool
}{}

// Wait Group for handling goroutines
var wg sync.WaitGroup

// RootCmd command (calls all other commands)
var RootCmd = &cobra.Command{
	Use:   "qaz",
	Short: fmt.Sprintf("%s\n--> Shut up & deploy my templates...!", colorString(banner.PrintS("qaz"), "magenta")),
	Run: func(cmd *cobra.Command, args []string) {

		if job.version {
			fmt.Printf("qaz - Version %s"+"\n", version)
			return
		}

		cmd.Help()
	},
}

var initCmd = &cobra.Command{
	Use:   "init [target directory]",
	Short: "Creates a basic qaz project",
	Run: func(cmd *cobra.Command, args []string) {

		// Print Banner
		banner.Print("qaz")
		fmt.Printf("\n--\n")

		var target string
		switch len(args) {
		case 0:
			target, _ = os.Getwd()
		default:
			target = args[0]
		}

		// Get Project & AWS Region
		project = getInput("-> Enter your Project name", "MyqazProject")
		region = getInput("-> Enter AWS Region", "eu-west-1")

		// set target paths
		c := filepath.Join(target, "config.yml")
		t := filepath.Join(target, "templates")
		f := filepath.Join(target, "files")

		// Check if config file exists
		var overwrite string
		if _, err := os.Stat(c); err == nil {
			overwrite = getInput(
				fmt.Sprintf("%s [%s] already exist, Do you want to %s?(Y/N) ", colorString("->", "yellow"), c, colorString("Overwrite", "red")),
				"N",
			)

			if overwrite == "Y" {
				fmt.Println(fmt.Sprintf("%s Overwriting: [%s]..", colorString("->", "yellow"), c))
			}
		}

		// Create template file
		if overwrite != "N" {
			if err := ioutil.WriteFile(c, configTemplate(project, region), 0644); err != nil {
				fmt.Printf("%s Error, unable to create config.yml file: %s"+"\n", err, colorString("->", "red"))
				return
			}
		}

		// Create template folder
		for _, dir := range []string{t, f} {
			if err := os.Mkdir(dir, os.ModePerm); err != nil {
				fmt.Printf("%s [%s] folder not created: %s"+"\n--\n", colorString("->", "yellow"), dir, err)
				return
			}
		}

		fmt.Println("--")
	},
}

var generateCmd = &cobra.Command{
	Use:   "generate",
	Short: "Generates template from configuration values",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "generate"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source
		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		name := fmt.Sprintf("%s-%s", project, s)
		Log(fmt.Sprintln("Generating a template for ", name), "debug")

		tpl, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}
		fmt.Println(tpl)
	},
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploys stack(s) to AWS",
	Example: strings.Join([]string{
		"qaz deploy -c path/to/config -t path/to/template",
		"qaz deploy -c path/to/config -t stack::s3//bucket/key",
		"qaz deploy -c path/to/config -t stack::http://someurl",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "deploy"

		job.stacks = make(map[string]string)

		sourceCopy := job.tplFiles

		// creating empty template list for re-population later
		job.tplFiles = []string{}

		for _, src := range sourceCopy {
			if strings.Contains(src, `*`) {
				glob, _ := filepath.Glob(src)

				for _, f := range glob {
					job.tplFiles = append(job.tplFiles, f)
				}
				continue
			}

			job.tplFiles = append(job.tplFiles, src)
		}

		for _, f := range job.tplFiles {
			s, source, err := getSource(f)
			if err != nil {
				handleError(err)
				return
			}
			job.stacks[s] = source
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		for s, f := range job.stacks {
			if v, err := genTimeParser(f); err != nil {
				handleError(err)
			} else {

				// Handle missing stacks
				if stacks[s] == nil {
					handleError(fmt.Errorf("Missing Stack in %s: [%s]", job.cfgFile, s))
					return
				}

				stacks[s].template = v
			}
		}

		// Deploy Stacks
		DeployHandler()

	},
}

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates a given stack",
	Example: strings.Join([]string{
		"qaz update -c path/to/config -t stack::path/to/template",
		"qaz update -c path/to/config -t stack::s3//bucket/key",
		"qaz update -c path/to/config -t stack::http://someurl",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "update"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source

		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		v, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		// Handle missing stacks
		if stacks[s] == nil {
			handleError(fmt.Errorf("Missing Stack in %s: [%s]", job.cfgFile, s))
			return
		}

		stacks[s].template = v

		// resolve deploy time function
		if err = stacks[s].deployTimeParser(); err != nil {
			handleError(err)
		}

		// Update stack
		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}
		stacks[s].update(sess)

	},
}

var terminateCmd = &cobra.Command{
	Use:   "terminate [stacks]",
	Short: "Terminates stacks",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "terminate"

		if !job.terminateAll {
			job.stacks = make(map[string]string)
			for _, stk := range args {
				job.stacks[stk] = ""
			}

			if len(job.stacks) == 0 {
				Log("No stack specified for termination", level.warn)
				return
			}
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		// Terminate Stacks
		TerminateHandler()
	},
}

var statusCmd = &cobra.Command{
	Use:   "status",
	Short: "Prints status of deployed/un-deployed stacks",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "status"

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		for _, v := range stacks {
			wg.Add(1)
			go func(s *stack) {
				if err := s.status(sess); err != nil {
					handleError(err)
				}
				wg.Done()
			}(v)

		}
		wg.Wait()
	},
}

var outputsCmd = &cobra.Command{
	Use:     "outputs [stack]",
	Short:   "Prints stack outputs",
	Example: "qaz outputs vpc subnets --config path/to/config",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "outputs"

		if len(args) < 1 {
			fmt.Println("Please specify stack(s) to check, For details try --> qaz outputs --help")
			return
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		for _, s := range args {
			// check if stack exists
			if _, ok := stacks[s]; !ok {
				handleError(fmt.Errorf("%s: does not Exist in Config", s))
				continue
			}

			wg.Add(1)
			go func(s string) {
				if err := stacks[s].outputs(sess); err != nil {
					handleError(err)
				}

				for _, i := range stacks[s].output.Stacks {
					fmt.Printf("\n"+"[%s]"+"\n", *i.StackName)
					for _, o := range i.Outputs {
						fmt.Printf("  Description: %s\n  %s: %s\n\n", *o.Description, colorString(*o.OutputKey, "magenta"), *o.OutputValue)
					}
				}

				wg.Done()
			}(s)
		}
		wg.Wait()

	},
}

var exportsCmd = &cobra.Command{
	Use:     "exports",
	Short:   "Prints stack exports",
	Example: "qaz exports",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "exports"

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		Exports(sess)

	},
}

var checkCmd = &cobra.Command{
	Use:   "check",
	Short: "Validates Cloudformation Templates",
	Example: strings.Join([]string{
		"qaz check -c path/to/config.yml -t path/to/template -c path/to/config",
		"qaz check -c path/to/config.yml -t stack::http://someurl.example",
		"qaz check -c path/to/config.yml -t stack::s3://bucket/key",
	}, "\n"),
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "validate"

		s, source, err := getSource(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		job.tplFile = source

		err = configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		name := fmt.Sprintf("%s-%s", project, s)
		fmt.Println("Validating template for", name)

		tpl, err := genTimeParser(job.tplFile)
		if err != nil {
			handleError(err)
			return
		}

		stk := stack{name: s}
		stk.setStackName()
		stk.template = tpl

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		if err := stk.check(sess); err != nil {
			handleError(err)
			return
		}
	},
}

var invokeCmd = &cobra.Command{
	Use:   "invoke",
	Short: "Invoke AWS Lambda Functions",
	Run: func(cmd *cobra.Command, args []string) {
		job.request = "lambda_invoke"
		// fmt.Println(colorString("Coming Soon!", "magenta"))

		if len(args) < 1 {
			fmt.Println("No Lambda Function specified")
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
			return
		}

		f := function{name: args[0]}

		if job.funcEvent != "" {
			f.payload = []byte(job.funcEvent)
		}

		if err := f.Invoke(sess); err != nil {
			if strings.Contains(err.Error(), "Unhandled") {
				handleError(fmt.Errorf("Unhandled Exception: Potential Issue with Lambda Function Logic for %s...\n", f.name))
			}
			handleError(err)
			return
		}

		fmt.Println(f.response)

	},
}

var tailCmd = &cobra.Command{
	Use:     "tail",
	Short:   "Tail Real-Time AWS Cloudformation events",
	Example: "qaz tail -r eu-west-1",
	Run: func(cmd *cobra.Command, args []string) {
		job.request = "tail"

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
		}

		// Tail each stack on it's own goroutine.
		for _, s := range stacks {
			wg.Add(1)
			go func(s *stack, sess *session.Session) {
				verbose(s.stackname, "", sess)
				wg.Done()
			}(s, sess)
		}

		wg.Wait() // Will probably wait forevery
	},
}

var policyCmd = &cobra.Command{
	Use:     "set-policy",
	Short:   "Set Stack Policies based on configured value",
	Example: "qaz set-policy <stack name>",
	Run: func(cmd *cobra.Command, args []string) {

		job.request = "set-policy"

		if len(args) == 0 {
			handleError(fmt.Errorf("Please specify stack name..."))
			return
		}

		err := configReader(job.cfgFile)
		if err != nil {
			handleError(err)
			return
		}

		sess, err := awsSession()
		if err != nil {
			handleError(err)
		}

		for _, s := range args {
			wg.Add(1)
			go func(s string, sess *session.Session) {

				if _, ok := stacks[s]; !ok {
					handleError(fmt.Errorf("Stack [%s] not found in config", s))

				} else {
					if err := stacks[s].stackPolicy(sess); err != nil {
						handleError(err)
					}
				}

				wg.Done()
				return

			}(s, sess)
		}

		wg.Wait()

	},
}
<|endoftext|>


<commit_before>package execute

import (
	"fmt"
	"github.com/ConSol/sakuli-go-wrapper/helper"
	"runtime"
)

//RunHandler runs external program with no parameters
func RunHandler(executable string) {
	fmt.Printf("Calling Handler executable: %s\n", executable)
	var returnCode int
	var err error
	if helper.IsRunningOnWindows() {
		returnCode, err = Execute("cmd", "/c", executable)
	} else if helper.IsRunningOnLinux() {
		returnCode, err = Execute("sh", "-c", executable)
	} else {
		panic("Unkown OS: " + runtime.GOOS)
	}
	if err != nil {
		fmt.Printf("Error while calling: %s\n", executable)
	}
	fmt.Printf("Handler [%s] finished with returncode: %d\n", executable, returnCode)
}
<commit_msg>sakuli stopps if handler returned with an errorcode != 0<commit_after>package execute

import (
	"fmt"
	"github.com/ConSol/sakuli-go-wrapper/helper"
	"runtime"
	"os"
)

//RunHandler runs external program with no parameters
func RunHandler(executable string) {
	fmt.Printf("Calling Handler executable: %s\n", executable)
	var returnCode int
	var err error
	if helper.IsRunningOnWindows() {
		returnCode, err = Execute("cmd", "/c", executable)
	} else if helper.IsRunningOnLinux() {
		returnCode, err = Execute("sh", "-c", executable)
	} else {
		panic("Unkown OS: " + runtime.GOOS)
	}
	if err != nil {
		fmt.Printf("Error while calling: %s\n", executable)
	}
	fmt.Printf("Handler [%s] finished with returncode: %d\n", executable, returnCode)
	if returnCode != 0{
		os.Exit(returnCode)
	}
}
<|endoftext|>


<commit_before>package homecloud

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"time"

	"git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/go-ninja/rpc/json2"
	"github.com/ninjasphere/redigo/redis"
)

var log = logger.GetLogger("HomeCloud")

var conn *ninja.Connection

var thingModel *ThingModel
var deviceModel *DeviceModel
var channelModel *ChannelModel
var roomModel *RoomModel
var driverModel *DriverModel

var locationRegexp = regexp.MustCompile("\\$device\\/([A-F0-9]*)\\/[^\\/]*\\/location")

type incomingLocationUpdate struct {
	Zone *string `json:"zone,omitempty"`
}

type outgoingLocationUpdate struct {
	ID         *string `json:"id"`
	HasChanged bool    `json:"hasChanged"`
}

var RedisPool = &redis.Pool{
	MaxIdle:     3,
	IdleTimeout: 240 * time.Second,
	Dial: func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", ":6379")
		if err != nil {
			return nil, err
		}
		/*if _, err := c.Do("AUTH", password); err != nil {
			c.Close()
			return nil, err
		}*/
		return c, err
	},
	TestOnBorrow: func(c redis.Conn, t time.Time) error {
		_, err := c.Do("PING")
		return err
	},
}

func Start(c *ninja.Connection) {

	//FIXME
	conn = c

	thingModel = NewThingModel(RedisPool, conn)
	conn.MustExportService(thingModel, "$home/services/ThingModel", &model.ServiceAnnouncement{
		Schema: "/service/thing-model",
	})

	deviceModel = NewDeviceModel(RedisPool, conn)
	conn.MustExportService(deviceModel, "$home/services/DeviceModel", &model.ServiceAnnouncement{
		Schema: "/service/device-model",
	})

	roomModel = NewRoomModel(RedisPool, conn)
	conn.MustExportService(roomModel, "$home/services/RoomModel", &model.ServiceAnnouncement{
		Schema: "/service/room-model",
	})

	driverModel = NewDriverModel(RedisPool, conn)
	channelModel = NewChannelModel(RedisPool, conn)

	if config.Bool(false, "clearcloud") {
		log.Infof("Clearing all cloud data in 5 seconds")

		time.Sleep(time.Second * 5)

		thingModel.ClearCloud()
		channelModel.ClearCloud()
		deviceModel.ClearCloud()
		roomModel.ClearCloud()

		log.Infof("All cloud data cleared? Probably.")

		os.Exit(0)

		return
	}

	go func() {
		for {
			log.Infof("\n\n\n------ Timed model syncing started (every 30 min) ------ ")

			roomResult := roomModel.sync()
			deviceResult := deviceModel.sync()
			channelResult := channelModel.sync()
			thingResult := thingModel.sync()

			log.Infof("Room sync error: %s", roomResult)
			log.Infof("Device sync error: %s", deviceResult)
			log.Infof("Channel sync error: %s", channelResult)
			log.Infof("Thing sync error: %s", thingResult)

			log.Infof("------ Timed model syncing complete ------\n\n\n")

			time.Sleep(time.Minute * 30)
		}
	}()

	startManagingDrivers()
	startManagingDevices()
	startMonitoringLocations()

	ensureNodeDeviceExists()

	go func() {
		// Give it a chance to sync first...
		time.Sleep(time.Second * 10)
		startDrivers()
	}()

}

func startDrivers() {

	do := func(name string, task string) error {
		return conn.SendNotification("$node/"+config.Serial()+"/module/"+task, name)
	}

	for _, name := range []string{"driver-go-zigbee", "driver-go-sonos", "driver-go-lifx", "driver-go-blecombined", "driver-go-hue", "driver-go-wemo"} {
		log.Infof("-- (Re)starting '%s'", name)

		err := do(name, "stop")
		if err != nil {
			log.Fatalf("Failed to send %s stop message! %s", name, err)
		}

		time.Sleep(time.Second * 2)

		err = do(name, "start")
		if err != nil {
			log.Fatalf("Failed to send %s start message! %s", name, err)
		}
	}

}

func startDriver(node string, driverID string, config *string) error {

	var rawConfig json.RawMessage
	if config != nil {
		rawConfig = []byte(*config)
	} else {
		rawConfig = []byte("{}")
	}

	client := conn.GetServiceClient(fmt.Sprintf("$node/%s/driver/%s", node, driverID))
	err := client.Call("start", &rawConfig, nil, 10*time.Second)

	if err != nil {
		jsonError, ok := err.(*json2.Error)
		if ok {
			if jsonError.Code == json2.E_INVALID_REQ {

				err := driverModel.DeleteConfig(driverID)
				if err != nil {
					log.Warningf("Driver %s could not parse its config. Also, we couldn't clear it! errors:%s and %s", driverID, jsonError.Message, err)
				} else {
					log.Warningf("Driver %s could not parse its config, so we cleared it from redis. error:%s", driverID, jsonError.Message)
				}

				return startDriver(node, driverID, nil)
			}
		}
	}

	return err
}

func startManagingDrivers() {

	conn.Subscribe("$node/:node/driver/:driver/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		node, driver := values["node"], values["driver"]

		log.Infof("Got driver announcement node:%s driver:%s", node, driver)

		if announcement == nil {
			log.Warningf("Nil driver announcement from node:%s driver:%s", node, driver)
			return true
		}

		module := &model.Module{}
		err := json.Unmarshal(*announcement, module)

		if announcement == nil {
			log.Warningf("Could not parse announcement from node:%s driver:%s error:%s", node, driver, err)
			return true
		}

		err = driverModel.Create(module)
		if err != nil {
			log.Warningf("Failed to save driver announcement for %s error:%s", driver, err)
		}

		config, err := driverModel.GetConfig(values["driver"])

		if err != nil {
			log.Warningf("Failed to retrieve config for driver %s error:%s", driver, err)
		} else {
			err = startDriver(node, driver, config)
			if err != nil {
				log.Warningf("Failed to start driver: %s error:%s", driver, err)
			}
		}

		return true
	})

	conn.Subscribe("$node/:node/driver/:driver/event/config", func(config *json.RawMessage, values map[string]string) bool {
		log.Infof("Got driver config node:%s driver:%s config:%s", values["node"], values["driver"], *config)

		if config != nil {
			err := driverModel.SetConfig(values["driver"], string(*config))

			if err != nil {
				log.Warningf("Failed to save config for driver: %s error: %s", values["driver"], err)
			}
		} else {
			log.Infof("Nil config recevied from node:%s driver:%s", values["node"], values["driver"])
		}

		return true
	})

}

func startManagingDevices() {

	conn.Subscribe("$device/:id/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		id := values["id"]

		log.Infof("Got device announcement device:%s", id)

		if announcement == nil {
			log.Warningf("Nil driver announcement from device:%s", id)
			return true
		}

		device := &model.Device{}
		err := json.Unmarshal(*announcement, device)

		if announcement == nil {
			log.Warningf("Could not parse announcement from device:%s error:%s", id, err)
			return true
		}

		err = deviceModel.Create(device)
		if err != nil {
			log.Warningf("Failed to save device announcement for device:%s error:%s", id, err)
		}

		return true
	})

	conn.Subscribe("$device/:device/channel/:channel/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		deviceID, channelID := values["device"], values["channel"]

		log.Infof("Got channel announcement device:%s channel:%s", deviceID, channelID)

		if announcement == nil {
			log.Warningf("Nil channel announcement from device:%s channel:%s", deviceID, channelID)
			return true
		}

		channel := &model.Channel{}
		err := json.Unmarshal(*announcement, channel)

		if err != nil {
			log.Warningf("Could not parse channel announcement from device:%s channel:%s error:%s", deviceID, channelID, err)
			return true
		}

		err = channelModel.Create(deviceID, channel)
		if err != nil {
			log.Warningf("Failed to save channel announcement for device:%s channel:%s error:%s", deviceID, channelID, err)
		}

		return true
	})

}

func startMonitoringLocations() {

	filter, err := mqtt.NewTopicFilter("$device/+/+/location", 0)
	if err != nil {
		log.FatalError(err, "Failed to subscribe to device locations")
	}

	receipt, err := conn.GetMqttClient().StartSubscription(func(_ *mqtt.MqttClient, message mqtt.Message) {

		deviceID := locationRegexp.FindAllStringSubmatch(message.Topic(), -1)[0][1]

		update := &incomingLocationUpdate{}
		err := json.Unmarshal(message.Payload(), update)
		if err != nil {
			log.Errorf("Failed to parse location update %s to %s : %s", message.Payload(), message.Topic(), err)
			return
		}

		thing, err := thingModel.FetchByDeviceId(deviceID)
		if err != nil && err != RecordNotFound {
			log.Warningf("Failed to fetch thing by device id %s", deviceID)
		}

		if update.Zone == nil {
			log.Debugf("< Incoming location update: device %s not in a zone", deviceID)
		} else {
			log.Debugf("< Incoming location update: device %s is in zone %s", deviceID, *update.Zone)
		}

		hasChangedZone := true

		if err == RecordNotFound {
			log.Debugf("Device %s is not attached to a thing. Ignoring.", deviceID)
		} else {

			if (thing.Location != nil && update.Zone != nil && *thing.Location == *update.Zone) || (thing.Location == nil && update.Zone == nil) {
				// It's already there
				log.Debugf("Thing %s (%s) (Device %s) was already in that zone.", thing.ID, thing.Name, deviceID)
				hasChangedZone = true
			} else {

				log.Debugf("Thing %s (%s) (Device %s) moved from %s to %s", thing.ID, thing.Name, deviceID, thing.Location, update.Zone)

				err = thingModel.SetLocation(thing.ID, update.Zone)
				if err != nil {
					log.FatalError(err, fmt.Sprintf("Failed to update location property of thing %s", thing.ID))
				}

				if update.Zone != nil {
					_, err := roomModel.Fetch(*update.Zone)
					if err != nil && err != RecordNotFound {
						log.FatalError(err, fmt.Sprintf("Failed to fetch room %s", *update.Zone))
					}

					if err != RecordNotFound {
						// XXX: TODO: Remove me once the cloud room model is sync'd and locatino service uses it
						log.Infof("Unknown room %s. Advising remote location service to forget it.", *update.Zone)

						pubReceipt := conn.GetMqttClient().Publish(mqtt.QoS(0), "$location/delete", message.Payload())
						<-pubReceipt
					}
				}
			}

			topic := fmt.Sprintf("$device/%s/channel/%s/%s/event/state", deviceID, "location", "location")

			payload, _ := json.Marshal(&outgoingLocationUpdate{
				ID:         update.Zone,
				HasChanged: hasChangedZone,
			})

			pubReceipt := conn.GetMqttClient().Publish(mqtt.QoS(0), topic, payload)
			<-pubReceipt

		}

	}, filter)

	if err != nil {
		log.Fatalf("Failed to subscribe to device locations: %s", err)
	}

	<-receipt
}
<commit_msg>Don't autostart driver-go-zigbee<commit_after>package homecloud

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"time"

	"git.eclipse.org/gitroot/paho/org.eclipse.paho.mqtt.golang.git"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/go-ninja/rpc/json2"
	"github.com/ninjasphere/redigo/redis"
)

var log = logger.GetLogger("HomeCloud")

var conn *ninja.Connection

var thingModel *ThingModel
var deviceModel *DeviceModel
var channelModel *ChannelModel
var roomModel *RoomModel
var driverModel *DriverModel

var locationRegexp = regexp.MustCompile("\\$device\\/([A-F0-9]*)\\/[^\\/]*\\/location")

type incomingLocationUpdate struct {
	Zone *string `json:"zone,omitempty"`
}

type outgoingLocationUpdate struct {
	ID         *string `json:"id"`
	HasChanged bool    `json:"hasChanged"`
}

var RedisPool = &redis.Pool{
	MaxIdle:     3,
	IdleTimeout: 240 * time.Second,
	Dial: func() (redis.Conn, error) {
		c, err := redis.Dial("tcp", ":6379")
		if err != nil {
			return nil, err
		}
		/*if _, err := c.Do("AUTH", password); err != nil {
			c.Close()
			return nil, err
		}*/
		return c, err
	},
	TestOnBorrow: func(c redis.Conn, t time.Time) error {
		_, err := c.Do("PING")
		return err
	},
}

func Start(c *ninja.Connection) {

	//FIXME
	conn = c

	thingModel = NewThingModel(RedisPool, conn)
	conn.MustExportService(thingModel, "$home/services/ThingModel", &model.ServiceAnnouncement{
		Schema: "/service/thing-model",
	})

	deviceModel = NewDeviceModel(RedisPool, conn)
	conn.MustExportService(deviceModel, "$home/services/DeviceModel", &model.ServiceAnnouncement{
		Schema: "/service/device-model",
	})

	roomModel = NewRoomModel(RedisPool, conn)
	conn.MustExportService(roomModel, "$home/services/RoomModel", &model.ServiceAnnouncement{
		Schema: "/service/room-model",
	})

	driverModel = NewDriverModel(RedisPool, conn)
	channelModel = NewChannelModel(RedisPool, conn)

	if config.Bool(false, "clearcloud") {
		log.Infof("Clearing all cloud data in 5 seconds")

		time.Sleep(time.Second * 5)

		thingModel.ClearCloud()
		channelModel.ClearCloud()
		deviceModel.ClearCloud()
		roomModel.ClearCloud()

		log.Infof("All cloud data cleared? Probably.")

		os.Exit(0)

		return
	}

	go func() {
		for {
			log.Infof("\n\n\n------ Timed model syncing started (every 30 min) ------ ")

			roomResult := roomModel.sync()
			deviceResult := deviceModel.sync()
			channelResult := channelModel.sync()
			thingResult := thingModel.sync()

			log.Infof("Room sync error: %s", roomResult)
			log.Infof("Device sync error: %s", deviceResult)
			log.Infof("Channel sync error: %s", channelResult)
			log.Infof("Thing sync error: %s", thingResult)

			log.Infof("------ Timed model syncing complete ------\n\n\n")

			time.Sleep(time.Minute * 30)
		}
	}()

	startManagingDrivers()
	startManagingDevices()
	startMonitoringLocations()

	ensureNodeDeviceExists()

	go func() {
		// Give it a chance to sync first...
		time.Sleep(time.Second * 10)
		startDrivers()
	}()

}

func startDrivers() {

	do := func(name string, task string) error {
		return conn.SendNotification("$node/"+config.Serial()+"/module/"+task, name)
	}

	for _, name := range []string{"driver-go-sonos", "driver-go-lifx", "driver-go-blecombined", "driver-go-hue", "driver-go-wemo"} {
		log.Infof("-- (Re)starting '%s'", name)

		err := do(name, "stop")
		if err != nil {
			log.Fatalf("Failed to send %s stop message! %s", name, err)
		}

		time.Sleep(time.Second * 2)

		err = do(name, "start")
		if err != nil {
			log.Fatalf("Failed to send %s start message! %s", name, err)
		}
	}

}

func startDriver(node string, driverID string, config *string) error {

	var rawConfig json.RawMessage
	if config != nil {
		rawConfig = []byte(*config)
	} else {
		rawConfig = []byte("{}")
	}

	client := conn.GetServiceClient(fmt.Sprintf("$node/%s/driver/%s", node, driverID))
	err := client.Call("start", &rawConfig, nil, 10*time.Second)

	if err != nil {
		jsonError, ok := err.(*json2.Error)
		if ok {
			if jsonError.Code == json2.E_INVALID_REQ {

				err := driverModel.DeleteConfig(driverID)
				if err != nil {
					log.Warningf("Driver %s could not parse its config. Also, we couldn't clear it! errors:%s and %s", driverID, jsonError.Message, err)
				} else {
					log.Warningf("Driver %s could not parse its config, so we cleared it from redis. error:%s", driverID, jsonError.Message)
				}

				return startDriver(node, driverID, nil)
			}
		}
	}

	return err
}

func startManagingDrivers() {

	conn.Subscribe("$node/:node/driver/:driver/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		node, driver := values["node"], values["driver"]

		log.Infof("Got driver announcement node:%s driver:%s", node, driver)

		if announcement == nil {
			log.Warningf("Nil driver announcement from node:%s driver:%s", node, driver)
			return true
		}

		module := &model.Module{}
		err := json.Unmarshal(*announcement, module)

		if announcement == nil {
			log.Warningf("Could not parse announcement from node:%s driver:%s error:%s", node, driver, err)
			return true
		}

		err = driverModel.Create(module)
		if err != nil {
			log.Warningf("Failed to save driver announcement for %s error:%s", driver, err)
		}

		config, err := driverModel.GetConfig(values["driver"])

		if err != nil {
			log.Warningf("Failed to retrieve config for driver %s error:%s", driver, err)
		} else {
			err = startDriver(node, driver, config)
			if err != nil {
				log.Warningf("Failed to start driver: %s error:%s", driver, err)
			}
		}

		return true
	})

	conn.Subscribe("$node/:node/driver/:driver/event/config", func(config *json.RawMessage, values map[string]string) bool {
		log.Infof("Got driver config node:%s driver:%s config:%s", values["node"], values["driver"], *config)

		if config != nil {
			err := driverModel.SetConfig(values["driver"], string(*config))

			if err != nil {
				log.Warningf("Failed to save config for driver: %s error: %s", values["driver"], err)
			}
		} else {
			log.Infof("Nil config recevied from node:%s driver:%s", values["node"], values["driver"])
		}

		return true
	})

}

func startManagingDevices() {

	conn.Subscribe("$device/:id/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		id := values["id"]

		log.Infof("Got device announcement device:%s", id)

		if announcement == nil {
			log.Warningf("Nil driver announcement from device:%s", id)
			return true
		}

		device := &model.Device{}
		err := json.Unmarshal(*announcement, device)

		if announcement == nil {
			log.Warningf("Could not parse announcement from device:%s error:%s", id, err)
			return true
		}

		err = deviceModel.Create(device)
		if err != nil {
			log.Warningf("Failed to save device announcement for device:%s error:%s", id, err)
		}

		return true
	})

	conn.Subscribe("$device/:device/channel/:channel/event/announce", func(announcement *json.RawMessage, values map[string]string) bool {

		deviceID, channelID := values["device"], values["channel"]

		log.Infof("Got channel announcement device:%s channel:%s", deviceID, channelID)

		if announcement == nil {
			log.Warningf("Nil channel announcement from device:%s channel:%s", deviceID, channelID)
			return true
		}

		channel := &model.Channel{}
		err := json.Unmarshal(*announcement, channel)

		if err != nil {
			log.Warningf("Could not parse channel announcement from device:%s channel:%s error:%s", deviceID, channelID, err)
			return true
		}

		err = channelModel.Create(deviceID, channel)
		if err != nil {
			log.Warningf("Failed to save channel announcement for device:%s channel:%s error:%s", deviceID, channelID, err)
		}

		return true
	})

}

func startMonitoringLocations() {

	filter, err := mqtt.NewTopicFilter("$device/+/+/location", 0)
	if err != nil {
		log.FatalError(err, "Failed to subscribe to device locations")
	}

	receipt, err := conn.GetMqttClient().StartSubscription(func(_ *mqtt.MqttClient, message mqtt.Message) {

		deviceID := locationRegexp.FindAllStringSubmatch(message.Topic(), -1)[0][1]

		update := &incomingLocationUpdate{}
		err := json.Unmarshal(message.Payload(), update)
		if err != nil {
			log.Errorf("Failed to parse location update %s to %s : %s", message.Payload(), message.Topic(), err)
			return
		}

		thing, err := thingModel.FetchByDeviceId(deviceID)
		if err != nil && err != RecordNotFound {
			log.Warningf("Failed to fetch thing by device id %s", deviceID)
		}

		if update.Zone == nil {
			log.Debugf("< Incoming location update: device %s not in a zone", deviceID)
		} else {
			log.Debugf("< Incoming location update: device %s is in zone %s", deviceID, *update.Zone)
		}

		hasChangedZone := true

		if err == RecordNotFound {
			log.Debugf("Device %s is not attached to a thing. Ignoring.", deviceID)
		} else {

			if (thing.Location != nil && update.Zone != nil && *thing.Location == *update.Zone) || (thing.Location == nil && update.Zone == nil) {
				// It's already there
				log.Debugf("Thing %s (%s) (Device %s) was already in that zone.", thing.ID, thing.Name, deviceID)
				hasChangedZone = true
			} else {

				log.Debugf("Thing %s (%s) (Device %s) moved from %s to %s", thing.ID, thing.Name, deviceID, thing.Location, update.Zone)

				err = thingModel.SetLocation(thing.ID, update.Zone)
				if err != nil {
					log.FatalError(err, fmt.Sprintf("Failed to update location property of thing %s", thing.ID))
				}

				if update.Zone != nil {
					_, err := roomModel.Fetch(*update.Zone)
					if err != nil && err != RecordNotFound {
						log.FatalError(err, fmt.Sprintf("Failed to fetch room %s", *update.Zone))
					}

					if err != RecordNotFound {
						// XXX: TODO: Remove me once the cloud room model is sync'd and locatino service uses it
						log.Infof("Unknown room %s. Advising remote location service to forget it.", *update.Zone)

						pubReceipt := conn.GetMqttClient().Publish(mqtt.QoS(0), "$location/delete", message.Payload())
						<-pubReceipt
					}
				}
			}

			topic := fmt.Sprintf("$device/%s/channel/%s/%s/event/state", deviceID, "location", "location")

			payload, _ := json.Marshal(&outgoingLocationUpdate{
				ID:         update.Zone,
				HasChanged: hasChangedZone,
			})

			pubReceipt := conn.GetMqttClient().Publish(mqtt.QoS(0), topic, payload)
			<-pubReceipt

		}

	}, filter)

	if err != nil {
		log.Fatalf("Failed to subscribe to device locations: %s", err)
	}

	<-receipt
}
<|endoftext|>


<commit_before>package githubtest

import (
	"github.com/bmatsuo/go-jsontree"
	"io/ioutil"
	"net/http"
	"strconv"
)

type Commit struct {
	Owner  string
	Repo   string
	Branch string
	Commit string
}

func (c *Commit) String() (output string) {
	output += "owner:  " + c.Owner + "\n"
	output += "repo:   " + c.Repo + "\n"
	output += "branch: " + c.Branch + "\n"
	output += "commit: " + c.Commit + "\n"
	return
}

type Server struct {
	Port   int
	Path   string
	Events chan Commit
}

func NewServer() *Server {
	return &Server{
		Port:   80,
		Path:   "githubtest",
		Events: make(chan Commit, 10), // buffered to 10 items
	}
}

func (s *Server) ListenAndServe() error {
	return http.ListenAndServe(":"+strconv.Itoa(s.Port), s)
}

func (s *Server) GoListenAndServe() {
	go func() {
		s.ListenAndServe()
	}()
}

func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()

	if req.Method != "POST" {
		http.Error(w, "405 Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	if req.URL.Path != s.Path {
		http.Error(w, "404 Not found", http.StatusNotFound)
		return
	}

	eventType := req.Header.Get("X-GitHub-Event")
	if eventType == "" {
		http.Error(w, "400 Bad Request - Missing X-GitHub-Event Header", http.StatusBadRequest)
		return
	}
	if eventType != "push" && eventType != "pull_request" {
		http.Error(w, "400 Bad Request - Unknown Event Type "+eventType, http.StatusBadRequest)
		return
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	request := jsontree.New()
	err = request.UnmarshalJSON(body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Parse the request and build the Commit
	commit := Commit{}

	if eventType == "push" {
		rawRef, err := request.Get("ref").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// If the ref is not a branch, we don't care about it
		if rawRef[:11] != "refs/heads/" || request.Get("head_commit").IsNull() {
			return
		}

		// Fill in values
		commit.Branch = rawRef[11:]
		commit.Repo, err = request.Get("repository").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Commit, err = request.Get("head_commit").Get("id").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Owner, err = request.Get("repository").Get("owner").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	if eventType == "pull_request" {
		action, err := request.Get("action").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// If the action is not to open or to synchronize we don't care about it
		if action != "synchronize" && action != "opened" {
			return
		}

		// Fill in values
		commit.Repo, err = request.Get("pull_request").Get("head").Get("repo").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Commit, err = request.Get("pull_request").Get("head").Get("sha").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Branch, err = request.Get("pull_request").Get("head").Get("ref").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Owner, err = request.Get("pull_request").Get("head").Get("repo").Get("owner").Get("login").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	// We've built our Commit - put it into the channel and we're done
	go func() {
		s.Events <- commit
	}()

	w.Write([]byte(commit.String()))
	w.Write([]byte("\n\nOK"))
}
<commit_msg>Adding IPs<commit_after>package githubtest

import (
	"github.com/bmatsuo/go-jsontree"
	"io/ioutil"
	"net/http"
	"strconv"
)

// A list of valid github webhook IP addresses
// A request from an IP address not in this list will fail
var ValidIP []string = []string{
	"207.97.227.253",
	"50.57.128.197",
	"108.171.174.178",
	"50.57.231.61",
}

type Commit struct {
	Owner  string
	Repo   string
	Branch string
	Commit string
}

func (c *Commit) String() (output string) {
	output += "owner:  " + c.Owner + "\n"
	output += "repo:   " + c.Repo + "\n"
	output += "branch: " + c.Branch + "\n"
	output += "commit: " + c.Commit + "\n"
	return
}

type Server struct {
	Port   int
	Path   string
	Events chan Commit
}

func NewServer() *Server {
	return &Server{
		Port:   80,
		Path:   "postreceive",
		Events: make(chan Commit, 10), // buffered to 10 items
	}
}

func (s *Server) ListenAndServe() error {
	return http.ListenAndServe(":"+strconv.Itoa(s.Port), s)
}

func (s *Server) GoListenAndServe() {
	go func() {
		s.ListenAndServe()
	}()
}

func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	defer req.Body.Close()

	if req.Method != "POST" {
		http.Error(w, "405 Method not allowed", http.StatusMethodNotAllowed)
		return
	}
	if req.URL.Path != s.Path {
		http.Error(w, "404 Not found", http.StatusNotFound)
		return
	}

	eventType := req.Header.Get("X-GitHub-Event")
	if eventType == "" {
		http.Error(w, "400 Bad Request - Missing X-GitHub-Event Header", http.StatusBadRequest)
		return
	}
	if eventType != "push" && eventType != "pull_request" {
		http.Error(w, "400 Bad Request - Unknown Event Type "+eventType, http.StatusBadRequest)
		return
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	request := jsontree.New()
	err = request.UnmarshalJSON(body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Parse the request and build the Commit
	commit := Commit{}

	if eventType == "push" {
		rawRef, err := request.Get("ref").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// If the ref is not a branch, we don't care about it
		if rawRef[:11] != "refs/heads/" || request.Get("head_commit").IsNull() {
			return
		}

		// Fill in values
		commit.Branch = rawRef[11:]
		commit.Repo, err = request.Get("repository").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Commit, err = request.Get("head_commit").Get("id").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Owner, err = request.Get("repository").Get("owner").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}
	if eventType == "pull_request" {
		action, err := request.Get("action").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		// If the action is not to open or to synchronize we don't care about it
		if action != "synchronize" && action != "opened" {
			return
		}

		// Fill in values
		commit.Repo, err = request.Get("pull_request").Get("head").Get("repo").Get("name").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Commit, err = request.Get("pull_request").Get("head").Get("sha").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Branch, err = request.Get("pull_request").Get("head").Get("ref").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		commit.Owner, err = request.Get("pull_request").Get("head").Get("repo").Get("owner").Get("login").String()
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	}

	// We've built our Commit - put it into the channel and we're done
	go func() {
		s.Events <- commit
	}()

	w.Write([]byte(commit.String()))
	w.Write([]byte("\n\nOK"))
}
<|endoftext|>