<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"text/template"

	"code.google.com/p/go.crypto/ssh"
	"github.com/flynn/flynn-test/cluster"
	"gopkg.in/check.v1"
)

var listen = flag.Bool("listen", false, "listen for repository events")
var username = flag.String("user", "ubuntu", "user to run QEMU as")
var rootfs = flag.String("rootfs", "rootfs/rootfs.img", "fs image to use with QEMU")
var kernel = flag.String("kernel", "rootfs/vmlinuz", "path to the Linux binary")
var flagCLI = flag.String("cli", "flynn", "path to flynn-cli binary")
var debug = flag.Bool("debug", false, "enable debug output")
var natIface = flag.String("nat", "eth0", "the interface to provide NAT to vms")
var killCluster = flag.Bool("kill", true, "kill the cluster after running the tests")

var sshWrapper = template.Must(template.New("ssh").Parse(`
#!/bin/bash

ssh -o LogLevel=FATAL -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{.SSHKey}} "$@"
`[1:]))

var gitEnv []string
var dockerfs string
var flynnrc string
var bootConfig cluster.BootConfig
var events chan Event

var repos = map[string]string{
	"flynn-host":       "master",
	"docker-etcd":      "master",
	"discoverd":        "master",
	"flynn-bootstrap":  "master",
	"flynn-controller": "master",
	"flynn-postgres":   "master",
	"flynn-receive":    "master",
	"shelf":            "master",
	"strowger":         "master",
	"slugbuilder":      "master",
	"slugrunner":       "master",
}

func init() {
	flag.StringVar(&dockerfs, "dockerfs", "", "docker fs")
	flag.StringVar(&flynnrc, "flynnrc", "", "path to flynnrc file")
	flag.Parse()
	log.SetFlags(log.Lshortfile)
}

func main() {
	bootConfig = cluster.BootConfig{
		User:     *username,
		RootFS:   *rootfs,
		Kernel:   *kernel,
		NatIface: *natIface,
	}

	if *listen == true {
		if dockerfs == "" {
			var err error
			if dockerfs, err = cluster.BuildFlynn(bootConfig, "", repos); err != nil {
				log.Fatal("could not build flynn:", err)
			}
		}
		events = make(chan Event, 10)
		go handleEvents(dockerfs)

		http.HandleFunc("/", webhookHandler)
		fmt.Println("Listening on :80...")
		if err := http.ListenAndServe(":80", nil); err != nil {
			log.Fatal("ListenAndServer: ", err)
		}
	}

	if flynnrc == "" {
		c := cluster.New(bootConfig)
		if dockerfs == "" {
			var err error
			if dockerfs, err = c.BuildFlynn("", repos); err != nil {
				log.Fatal("could not build flynn:", err)
			}
		}
		if err := c.Boot(dockerfs, 1); err != nil {
			log.Fatal("could not boot cluster: ", err)
		}
		if *killCluster {
			defer c.Shutdown()
		}

		if err := createFlynnrc(c); err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(flynnrc)
	}

	ssh, err := genSSHKey()
	if err != nil {
		log.Fatal(err)
	}
	defer ssh.Cleanup()
	gitEnv = ssh.Env

	keyAdd := flynn("", "key-add", ssh.Pub)
	if keyAdd.Err != nil {
		log.Fatalf("Error during `%s`:\n%s%s", strings.Join(keyAdd.Cmd, " "), keyAdd.Output, keyAdd.Err)
	}

	res := check.RunAll(&check.RunConf{
		Stream:      true,
		Verbose:     true,
		KeepWorkDir: *debug,
	})
	fmt.Println(res)
}

func webhookHandler(w http.ResponseWriter, r *http.Request) {
	header, ok := r.Header["X-Github-Event"]
	if !ok {
		log.Println("webhook: request missing X-Github-Event header")
		http.Error(w, "missing X-Github-Event header\n", 400)
		return
	}

	name := strings.Join(header, " ")
	var event Event
	switch name {
	case "push":
		event = &PushEvent{}
	case "pull_request":
		event = &PullRequestEvent{}
	default:
		log.Println("webhook: unknown X-Github-Event:", name)
		http.Error(w, fmt.Sprintf("Unknown X-Github-Event: %s\n", name), 400)
		return
	}

	dec := json.NewDecoder(r.Body)
	if err := dec.Decode(&event); err != nil && err != io.EOF {
		log.Println("webhook: error decoding JSON", err)
		http.Error(w, fmt.Sprintf("invalid JSON payload for %s event", name), 400)
		return
	}
	repo := event.Repo()
	if _, ok := repos[repo]; !ok {
		log.Println("webhook: unknown repo", repo)
		http.Error(w, fmt.Sprintf("unknown repo %s", repo), 400)
		return
	}
	logEvent(event)
	events <- event
	io.WriteString(w, "ok\n")
}

func handleEvents(dockerfs string) {
	for event := range events {
		if !needsBuild(event) {
			continue
		}
		log.Printf("building %s[%s]\n", event.Repo(), event.Commit())
		repos := map[string]string{event.Repo(): event.Commit()}
		newDockerfs, err := cluster.BuildFlynn(bootConfig, dockerfs, repos)
		if err != nil {
			fmt.Printf("could not build flynn: %s\n", err)
			continue
		}

		cmd := exec.Command(
			os.Args[0],
			"--user", *username,
			"--rootfs", *rootfs,
			"--dockerfs", newDockerfs,
			"--kernel", *kernel,
			"--cli", *flagCLI,
			"--nat", *natIface,
			"--kill", "false",
		)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			fmt.Printf("build failed: %s\n", err)
			continue
		}
		fmt.Printf("build passed!\n")
	}
}

func logEvent(event Event) {
	switch event.(type) {
	case *PushEvent:
		e := event.(*PushEvent)
		log.Printf(
			"received push of %s[%s] by %s: %s => %s\n",
			e.Repo(),
			e.Ref,
			e.Pusher.Name,
			e.Before,
			e.After,
		)
	case *PullRequestEvent:
		e := event.(*PullRequestEvent)
		log.Printf(
			"pull request %s/%d %s by %s\n",
			e.Repo(),
			e.Number,
			e.Action,
			e.Sender.Login,
		)
	}
}

func needsBuild(event Event) bool {
	if e, ok := event.(*PullRequestEvent); ok && e.Action == "closed" {
		return false
	}
	return true
}

type sshData struct {
	Key     string
	Pub     string
	Env     []string
	Cleanup func()
}

func genSSHKey() (*sshData, error) {
	keyFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err

	}
	defer keyFile.Close()
	rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	pem.Encode(keyFile, &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
	})

	pubFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	defer pubFile.Close()
	rsaPubKey, err := ssh.NewPublicKey(&rsaKey.PublicKey)
	if err != nil {
		return nil, err
	}
	if _, err := pubFile.Write(ssh.MarshalAuthorizedKey(rsaPubKey)); err != nil {
		return nil, err
	}

	wrapperFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	defer wrapperFile.Close()
	if err := sshWrapper.Execute(wrapperFile, map[string]string{"SSHKey": keyFile.Name()}); err != nil {
		return nil, err
	}
	if err := wrapperFile.Chmod(0700); err != nil {
		return nil, err
	}

	return &sshData{
		Key: keyFile.Name(),
		Pub: pubFile.Name(),
		Env: []string{"GIT_SSH=" + wrapperFile.Name()},
		Cleanup: func() {
			os.RemoveAll(keyFile.Name())
			os.RemoveAll(pubFile.Name())
			os.RemoveAll(wrapperFile.Name())
		},
	}, nil
}

func createFlynnrc(c *cluster.Cluster) error {
	tmpfile, err := ioutil.TempFile("", "flynnrc-")
	if err != nil {
		return err
	}
	flynnrc = tmpfile.Name()

	githost := fmt.Sprintf("%s:2222", c.ControllerDomain)
	url := fmt.Sprintf("https://%s:443", c.ControllerDomain)
	return flynn("", "server-add", "-g", githost, "-p", c.ControllerPin, "default", url, c.ControllerKey).Err
}

type CmdResult struct {
	Cmd    []string
	Output string
	Err    error
}

func flynn(dir string, args ...string) *CmdResult {
	cmd := exec.Command(*flagCLI, args...)
	cmd.Env = append(os.Environ(), "FLYNNRC="+flynnrc)
	cmd.Dir = dir
	return run(cmd)
}

func git(dir string, args ...string) *CmdResult {
	cmd := exec.Command("git", args...)
	cmd.Env = append(os.Environ(), gitEnv...)
	cmd.Dir = dir
	return run(cmd)
}

func run(cmd *exec.Cmd) *CmdResult {
	var out bytes.Buffer
	if *debug {
		fmt.Println("++", cmd.Path, strings.Join(cmd.Args[1:], " "))
		cmd.Stdout = io.MultiWriter(os.Stdout, &out)
		cmd.Stderr = io.MultiWriter(os.Stderr, &out)
	} else {
		cmd.Stdout = &out
		cmd.Stderr = &out
	}
	err := cmd.Run()
	res := &CmdResult{
		Cmd:    cmd.Args,
		Err:    err,
		Output: out.String(),
	}
	return res
}

var Outputs check.Checker = outputChecker{
	&check.CheckerInfo{
		Name:   "Outputs",
		Params: []string{"result", "output"},
	},
}

type outputChecker struct {
	*check.CheckerInfo
}

func (outputChecker) Check(params []interface{}, names []string) (bool, string) {
	ok, msg, s, res := checkCmdResult(params, names)
	if !ok {
		return ok, msg
	}
	return s == res.Output, ""
}

func checkCmdResult(params []interface{}, names []string) (ok bool, msg, s string, res *CmdResult) {
	res, ok = params[0].(*CmdResult)
	if !ok {
		msg = "result must be a *CmdResult"
		return
	}
	switch v := params[1].(type) {
	case []byte:
		s = string(v)
	case string:
		s = v
	default:
		msg = "output must be a []byte or string"
		return
	}
	if res.Err != nil {
		return false, "", "", nil
	}
	ok = true
	return
}

var OutputContains check.Checker = outputContainsChecker{
	&check.CheckerInfo{
		Name:   "OutputContains",
		Params: []string{"result", "contains"},
	},
}

type outputContainsChecker struct {
	*check.CheckerInfo
}

func (outputContainsChecker) Check(params []interface{}, names []string) (bool, string) {
	ok, msg, s, res := checkCmdResult(params, names)
	if !ok {
		return ok, msg
	}
	return strings.Contains(res.Output, s), ""
}

var Succeeds check.Checker = succeedsChecker{
	&check.CheckerInfo{
		Name:   "Succeeds",
		Params: []string{"result"},
	},
}

type succeedsChecker struct {
	*check.CheckerInfo
}

func (succeedsChecker) Check(params []interface{}, names []string) (bool, string) {
	res, ok := params[0].(*CmdResult)
	if !ok {
		return false, "result must be a *CmdResult"
	}
	return res.Err == nil, ""
}
<commit_msg>test: Update Github commit statuses<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"text/template"

	"code.google.com/p/go.crypto/ssh"
	"github.com/flynn/flynn-test/cluster"
	"gopkg.in/check.v1"
)

var listen = flag.Bool("listen", false, "listen for repository events")
var username = flag.String("user", "ubuntu", "user to run QEMU as")
var rootfs = flag.String("rootfs", "rootfs/rootfs.img", "fs image to use with QEMU")
var kernel = flag.String("kernel", "rootfs/vmlinuz", "path to the Linux binary")
var flagCLI = flag.String("cli", "flynn", "path to flynn-cli binary")
var debug = flag.Bool("debug", false, "enable debug output")
var natIface = flag.String("nat", "eth0", "the interface to provide NAT to vms")
var killCluster = flag.Bool("kill", true, "kill the cluster after running the tests")

var sshWrapper = template.Must(template.New("ssh").Parse(`
#!/bin/bash

ssh -o LogLevel=FATAL -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{.SSHKey}} "$@"
`[1:]))

var gitEnv []string
var dockerfs string
var flynnrc string
var bootConfig cluster.BootConfig
var events chan Event
var githubToken string

var repos = map[string]string{
	"flynn-host":       "master",
	"docker-etcd":      "master",
	"discoverd":        "master",
	"flynn-bootstrap":  "master",
	"flynn-controller": "master",
	"flynn-postgres":   "master",
	"flynn-receive":    "master",
	"shelf":            "master",
	"strowger":         "master",
	"slugbuilder":      "master",
	"slugrunner":       "master",
}

func init() {
	flag.StringVar(&dockerfs, "dockerfs", "", "docker fs")
	flag.StringVar(&flynnrc, "flynnrc", "", "path to flynnrc file")
	flag.Parse()
	log.SetFlags(log.Lshortfile)
}

func main() {
	bootConfig = cluster.BootConfig{
		User:     *username,
		RootFS:   *rootfs,
		Kernel:   *kernel,
		NatIface: *natIface,
	}

	if *listen == true {
		githubToken = os.Getenv("GITHUB_TOKEN")
		if githubToken == "" {
			log.Fatal("GITHUB_TOKEN not set")
		}
		if dockerfs == "" {
			var err error
			if dockerfs, err = cluster.BuildFlynn(bootConfig, "", repos); err != nil {
				log.Fatal("could not build flynn:", err)
			}
		}
		events = make(chan Event, 10)
		go handleEvents(dockerfs)

		http.HandleFunc("/", webhookHandler)
		fmt.Println("Listening on :80...")
		if err := http.ListenAndServe(":80", nil); err != nil {
			log.Fatal("ListenAndServer: ", err)
		}
	}

	if flynnrc == "" {
		c := cluster.New(bootConfig)
		if dockerfs == "" {
			var err error
			if dockerfs, err = c.BuildFlynn("", repos); err != nil {
				log.Fatal("could not build flynn:", err)
			}
		}
		if err := c.Boot(dockerfs, 1); err != nil {
			log.Fatal("could not boot cluster: ", err)
		}
		if *killCluster {
			defer c.Shutdown()
		}

		if err := createFlynnrc(c); err != nil {
			log.Fatal(err)
		}
		defer os.RemoveAll(flynnrc)
	}

	ssh, err := genSSHKey()
	if err != nil {
		log.Fatal(err)
	}
	defer ssh.Cleanup()
	gitEnv = ssh.Env

	keyAdd := flynn("", "key-add", ssh.Pub)
	if keyAdd.Err != nil {
		log.Fatalf("Error during `%s`:\n%s%s", strings.Join(keyAdd.Cmd, " "), keyAdd.Output, keyAdd.Err)
	}

	res := check.RunAll(&check.RunConf{
		Stream:      true,
		Verbose:     true,
		KeepWorkDir: *debug,
	})
	fmt.Println(res)
}

func webhookHandler(w http.ResponseWriter, r *http.Request) {
	header, ok := r.Header["X-Github-Event"]
	if !ok {
		log.Println("webhook: request missing X-Github-Event header")
		http.Error(w, "missing X-Github-Event header\n", 400)
		return
	}

	name := strings.Join(header, " ")
	var event Event
	switch name {
	case "push":
		event = &PushEvent{}
	case "pull_request":
		event = &PullRequestEvent{}
	default:
		log.Println("webhook: unknown X-Github-Event:", name)
		http.Error(w, fmt.Sprintf("Unknown X-Github-Event: %s\n", name), 400)
		return
	}

	dec := json.NewDecoder(r.Body)
	if err := dec.Decode(&event); err != nil && err != io.EOF {
		log.Println("webhook: error decoding JSON", err)
		http.Error(w, fmt.Sprintf("invalid JSON payload for %s event", name), 400)
		return
	}
	repo := event.Repo()
	if _, ok := repos[repo]; !ok {
		log.Println("webhook: unknown repo", repo)
		http.Error(w, fmt.Sprintf("unknown repo %s", repo), 400)
		return
	}
	logEvent(event)
	events <- event
	io.WriteString(w, "ok\n")
}

func handleEvents(dockerfs string) {
	for event := range events {
		if !needsBuild(event) {
			continue
		}
		updateStatus(event, "pending")
		log.Printf("building %s[%s]\n", event.Repo(), event.Commit())
		repos := map[string]string{event.Repo(): event.Commit()}
		newDockerfs, err := cluster.BuildFlynn(bootConfig, dockerfs, repos)
		if err != nil {
			updateStatus(event, "failure")
			fmt.Printf("could not build flynn: %s\n", err)
			continue
		}

		cmd := exec.Command(
			os.Args[0],
			"--user", *username,
			"--rootfs", *rootfs,
			"--dockerfs", newDockerfs,
			"--kernel", *kernel,
			"--cli", *flagCLI,
			"--nat", *natIface,
			"--kill", "false",
		)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			updateStatus(event, "failure")
			fmt.Printf("build failed: %s\n", err)
			continue
		}
		updateStatus(event, "success")
		fmt.Printf("build passed!\n")
	}
}

func logEvent(event Event) {
	switch event.(type) {
	case *PushEvent:
		e := event.(*PushEvent)
		log.Printf(
			"received push of %s[%s] by %s: %s => %s\n",
			e.Repo(),
			e.Ref,
			e.Pusher.Name,
			e.Before,
			e.After,
		)
	case *PullRequestEvent:
		e := event.(*PullRequestEvent)
		log.Printf(
			"pull request %s/%d %s by %s\n",
			e.Repo(),
			e.Number,
			e.Action,
			e.Sender.Login,
		)
	}
}

func needsBuild(event Event) bool {
	if e, ok := event.(*PullRequestEvent); ok && e.Action == "closed" {
		return false
	}
	return true
}

type Status struct {
	State       string `json:"state"`
	TargetUrl   string `json:"target_url,omitempty"`
	Description string `json:"description,omitempty"`
	Context     string `json:"context,omitempty"`
}

var descriptions = map[string]string{
	"pending": "The Flynn CI build is in progress",
	"success": "The Flynn CI build passed",
	"failure": "The Flynn CI build failed",
}

func updateStatus(event Event, state string) {
	go func() {
		log.Printf("updateStatus: %s %s[%s]\n", state, event.Repo(), event.Commit())

		url := fmt.Sprintf("https://api.github.com/repos/flynn/%s/statuses/%s", event.Repo(), event.Commit())
		status := Status{State: state, Description: descriptions[state], Context: "flynn"}
		body := bytes.NewBufferString("")
		if err := json.NewEncoder(body).Encode(status); err != nil {
			log.Printf("updateStatus: could not encode status: %+v\n", status)
			return
		}

		req, err := http.NewRequest("POST", url, body)
		if err != nil {
			log.Printf("updateStatus: could not create request: %s\n", err)
			return
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("Authorization", fmt.Sprintf("token %s", githubToken))

		res, err := http.DefaultClient.Do(req)
		defer res.Body.Close()
		if err != nil {
			log.Printf("updateStatus: could not send request: %s\n", err)
			return
		}
		if res.StatusCode != 201 {
			log.Printf("updateStatus: request failed: %d\n", res.StatusCode)
		}
	}()
}

type sshData struct {
	Key     string
	Pub     string
	Env     []string
	Cleanup func()
}

func genSSHKey() (*sshData, error) {
	keyFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err

	}
	defer keyFile.Close()
	rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	pem.Encode(keyFile, &pem.Block{
		Type:  "RSA PRIVATE KEY",
		Bytes: x509.MarshalPKCS1PrivateKey(rsaKey),
	})

	pubFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	defer pubFile.Close()
	rsaPubKey, err := ssh.NewPublicKey(&rsaKey.PublicKey)
	if err != nil {
		return nil, err
	}
	if _, err := pubFile.Write(ssh.MarshalAuthorizedKey(rsaPubKey)); err != nil {
		return nil, err
	}

	wrapperFile, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, err
	}
	defer wrapperFile.Close()
	if err := sshWrapper.Execute(wrapperFile, map[string]string{"SSHKey": keyFile.Name()}); err != nil {
		return nil, err
	}
	if err := wrapperFile.Chmod(0700); err != nil {
		return nil, err
	}

	return &sshData{
		Key: keyFile.Name(),
		Pub: pubFile.Name(),
		Env: []string{"GIT_SSH=" + wrapperFile.Name()},
		Cleanup: func() {
			os.RemoveAll(keyFile.Name())
			os.RemoveAll(pubFile.Name())
			os.RemoveAll(wrapperFile.Name())
		},
	}, nil
}

func createFlynnrc(c *cluster.Cluster) error {
	tmpfile, err := ioutil.TempFile("", "flynnrc-")
	if err != nil {
		return err
	}
	flynnrc = tmpfile.Name()

	githost := fmt.Sprintf("%s:2222", c.ControllerDomain)
	url := fmt.Sprintf("https://%s:443", c.ControllerDomain)
	return flynn("", "server-add", "-g", githost, "-p", c.ControllerPin, "default", url, c.ControllerKey).Err
}

type CmdResult struct {
	Cmd    []string
	Output string
	Err    error
}

func flynn(dir string, args ...string) *CmdResult {
	cmd := exec.Command(*flagCLI, args...)
	cmd.Env = append(os.Environ(), "FLYNNRC="+flynnrc)
	cmd.Dir = dir
	return run(cmd)
}

func git(dir string, args ...string) *CmdResult {
	cmd := exec.Command("git", args...)
	cmd.Env = append(os.Environ(), gitEnv...)
	cmd.Dir = dir
	return run(cmd)
}

func run(cmd *exec.Cmd) *CmdResult {
	var out bytes.Buffer
	if *debug {
		fmt.Println("++", cmd.Path, strings.Join(cmd.Args[1:], " "))
		cmd.Stdout = io.MultiWriter(os.Stdout, &out)
		cmd.Stderr = io.MultiWriter(os.Stderr, &out)
	} else {
		cmd.Stdout = &out
		cmd.Stderr = &out
	}
	err := cmd.Run()
	res := &CmdResult{
		Cmd:    cmd.Args,
		Err:    err,
		Output: out.String(),
	}
	return res
}

var Outputs check.Checker = outputChecker{
	&check.CheckerInfo{
		Name:   "Outputs",
		Params: []string{"result", "output"},
	},
}

type outputChecker struct {
	*check.CheckerInfo
}

func (outputChecker) Check(params []interface{}, names []string) (bool, string) {
	ok, msg, s, res := checkCmdResult(params, names)
	if !ok {
		return ok, msg
	}
	return s == res.Output, ""
}

func checkCmdResult(params []interface{}, names []string) (ok bool, msg, s string, res *CmdResult) {
	res, ok = params[0].(*CmdResult)
	if !ok {
		msg = "result must be a *CmdResult"
		return
	}
	switch v := params[1].(type) {
	case []byte:
		s = string(v)
	case string:
		s = v
	default:
		msg = "output must be a []byte or string"
		return
	}
	if res.Err != nil {
		return false, "", "", nil
	}
	ok = true
	return
}

var OutputContains check.Checker = outputContainsChecker{
	&check.CheckerInfo{
		Name:   "OutputContains",
		Params: []string{"result", "contains"},
	},
}

type outputContainsChecker struct {
	*check.CheckerInfo
}

func (outputContainsChecker) Check(params []interface{}, names []string) (bool, string) {
	ok, msg, s, res := checkCmdResult(params, names)
	if !ok {
		return ok, msg
	}
	return strings.Contains(res.Output, s), ""
}

var Succeeds check.Checker = succeedsChecker{
	&check.CheckerInfo{
		Name:   "Succeeds",
		Params: []string{"result"},
	},
}

type succeedsChecker struct {
	*check.CheckerInfo
}

func (succeedsChecker) Check(params []interface{}, names []string) (bool, string) {
	res, ok := params[0].(*CmdResult)
	if !ok {
		return false, "result must be a *CmdResult"
	}
	return res.Err == nil, ""
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/garyburd/redigo/redis"
)

func authHandler(w http.ResponseWriter, r *http.Request) {

	//w.Write([]byte(r.URL.Path + "FOO"))

	//enforce POST
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte(http.StatusText(http.StatusMethodNotAllowed)))
	}

	bodyBytes, bodyError := ioutil.ReadAll(r.Body)

	if bodyError != nil {
		log.Fatalln(bodyError)
	}

	w.Write(bodyBytes)

}

func setupRedis(RedisSchemeURL *string) (connection redis.Conn) {
	connection, error := redis.DialURL(*RedisSchemeURL)
	if error != nil {
		log.Fatalln("Could not connect to REDIS:", error)
	} else {
		fmt.Println("Connected to REDIS:", *RedisSchemeURL)
	}

	return connection
}

func setupFlags() (RedisSchemeURL *string, HTTPAddress *string) {
	RedisSchemeURL = flag.String("redis-url", "redis://@localhost:6379/0", "Url to connect to redis")
	HTTPAddress = flag.String("http-address", ":8080", "Http address to bind to")

	flag.Parse()

	return RedisSchemeURL, HTTPAddress
}

func main() {
	RedisSchemeURL, HTTPAddress := setupFlags()
	redisConnection := setupRedis(RedisSchemeURL)

	http.HandleFunc("/auth", authHandler)
	log.Fatalln(http.ListenAndServe(*HTTPAddress, nil))

	defer redisConnection.Close()
}
<commit_msg>Going<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/garyburd/redigo/redis"
)

func authHandler(w http.ResponseWriter, r *http.Request) {

	//w.Write([]byte(r.URL.Path + "FOO"))

	//enforce POST
	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		w.Write([]byte(http.StatusText(http.StatusMethodNotAllowed)))
	}

	bodyBytes, bodyError := ioutil.ReadAll(r.Body)

	if bodyError != nil {
		log.Fatalln(bodyError)
	} else {
		bodyJson := json.NewDecoder(bodyBytes).Decode()
		w.Write(bodyBytes)
	}
}

func setupRedis(RedisSchemeURL *string) (connection redis.Conn) {
	connection, error := redis.DialURL(*RedisSchemeURL)
	if error != nil {
		log.Fatalln("Could not connect to REDIS:", error)
	} else {
		fmt.Println("Connected to REDIS:", *RedisSchemeURL)
	}

	return connection
}

func setupFlags() (RedisSchemeURL *string, HTTPAddress *string) {
	RedisSchemeURL = flag.String("redis-url", "redis://@localhost:6379/0", "Url to connect to redis")
	HTTPAddress = flag.String("http-address", ":8080", "Http address to bind to")

	flag.Parse()

	return RedisSchemeURL, HTTPAddress
}

func main() {
	RedisSchemeURL, HTTPAddress := setupFlags()
	redisConnection := setupRedis(RedisSchemeURL)

	http.HandleFunc("/auth", authHandler)
	log.Fatalln(http.ListenAndServe(*HTTPAddress, nil))

	defer redisConnection.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/99designs/aws-vault/keyring"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	Version string
)

type Ui struct {
	*log.Logger
	Error, Debug *log.Logger
	Exit         func(code int)
}

type logWriter struct{ *log.Logger }

func (w logWriter) Write(b []byte) (int, error) {
	w.Printf("%s", b)
	return len(b), nil
}

func main() {
	var (
		debug            = kingpin.Flag("debug", "Show debugging output").Bool()
		add              = kingpin.Command("add", "Adds credentials, prompts if none provided")
		addProfile       = add.Arg("profile", "Name of the profile").Required().String()
		addFromEnv       = add.Flag("env", "Read the credentials from the environment").Bool()
		ls               = kingpin.Command("ls", "List profiles")
		exec             = kingpin.Command("exec", "Executes a command with AWS credentials in the environment")
		execProfile      = exec.Arg("profile", "Name of the profile").Required().String()
		execSessDuration = exec.Flag("session-ttl", "Expiration time for aws session").Default("8h").OverrideDefaultFromEnvar("AWS_SESSION_TTL").Short('t').Duration()
		execCmd          = exec.Arg("cmd", "Command to execute").Required().String()
		execCmdArgs      = exec.Arg("args", "Command arguments").Strings()
		rm               = kingpin.Command("rm", "Removes credentials")
		rmProfile        = rm.Arg("profile", "Name of the profile").Required().String()
	)

	kingpin.Version(Version)
	kingpin.CommandLine.Help =
		`A vault for securely storing and accessing AWS credentials in development environments.`

	ui := Ui{
		Logger: log.New(os.Stdout, "", 0),
		Error:  log.New(os.Stderr, "", 0),
		Debug:  log.New(ioutil.Discard, "", 0),
		Exit:   os.Exit,
	}

	keyring, err := keyring.ForPlatform()
	if err != nil {
		ui.Error.Fatal(err)
	}

	cmd := kingpin.Parse()

	if *debug {
		ui.Debug = log.New(os.Stderr, "DEBUG ", log.LstdFlags)
		log.SetFlags(0)
		log.SetOutput(&logWriter{ui.Debug})
	} else {
		log.SetOutput(ioutil.Discard)
	}

	switch cmd {
	case ls.FullCommand():
		LsCommand(ui, LsCommandInput{
			Keyring: keyring,
		})

	case rm.FullCommand():
		RemoveCommand(ui, RemoveCommandInput{
			Profile: *rmProfile,
			Keyring: keyring,
		})

	case add.FullCommand():
		AddCommand(ui, AddCommandInput{
			Profile: *addProfile,
			Keyring: keyring,
			FromEnv: *addFromEnv,
		})

	case exec.FullCommand():
		ExecCommand(ui, ExecCommandInput{
			Profile:  *execProfile,
			Command:  *execCmd,
			Args:     *execCmdArgs,
			Keyring:  keyring,
			Duration: *execSessDuration,
		})
	}
}
<commit_msg>Default exec's command to $SHELL<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"

	"github.com/99designs/aws-vault/keyring"

	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	Version string
)

type Ui struct {
	*log.Logger
	Error, Debug *log.Logger
	Exit         func(code int)
}

type logWriter struct{ *log.Logger }

func (w logWriter) Write(b []byte) (int, error) {
	w.Printf("%s", b)
	return len(b), nil
}

func main() {
	var (
		debug            = kingpin.Flag("debug", "Show debugging output").Bool()
		add              = kingpin.Command("add", "Adds credentials, prompts if none provided")
		addProfile       = add.Arg("profile", "Name of the profile").Required().String()
		addFromEnv       = add.Flag("env", "Read the credentials from the environment").Bool()
		ls               = kingpin.Command("ls", "List profiles")
		exec             = kingpin.Command("exec", "Executes a command with AWS credentials in the environment")
		execProfile      = exec.Arg("profile", "Name of the profile").Required().String()
		execSessDuration = exec.Flag("session-ttl", "Expiration time for aws session").Default("8h").OverrideDefaultFromEnvar("AWS_SESSION_TTL").Short('t').Duration()
		execCmd          = exec.Arg("cmd", "Command to execute").Default(os.Getenv("SHELL")).String()
		execCmdArgs      = exec.Arg("args", "Command arguments").Strings()
		rm               = kingpin.Command("rm", "Removes credentials")
		rmProfile        = rm.Arg("profile", "Name of the profile").Required().String()
	)

	kingpin.Version(Version)
	kingpin.CommandLine.Help =
		`A vault for securely storing and accessing AWS credentials in development environments.`

	ui := Ui{
		Logger: log.New(os.Stdout, "", 0),
		Error:  log.New(os.Stderr, "", 0),
		Debug:  log.New(ioutil.Discard, "", 0),
		Exit:   os.Exit,
	}

	keyring, err := keyring.ForPlatform()
	if err != nil {
		ui.Error.Fatal(err)
	}

	cmd := kingpin.Parse()

	if *debug {
		ui.Debug = log.New(os.Stderr, "DEBUG ", log.LstdFlags)
		log.SetFlags(0)
		log.SetOutput(&logWriter{ui.Debug})
	} else {
		log.SetOutput(ioutil.Discard)
	}

	switch cmd {
	case ls.FullCommand():
		LsCommand(ui, LsCommandInput{
			Keyring: keyring,
		})

	case rm.FullCommand():
		RemoveCommand(ui, RemoveCommandInput{
			Profile: *rmProfile,
			Keyring: keyring,
		})

	case add.FullCommand():
		AddCommand(ui, AddCommandInput{
			Profile: *addProfile,
			Keyring: keyring,
			FromEnv: *addFromEnv,
		})

	case exec.FullCommand():
		ExecCommand(ui, ExecCommandInput{
			Profile:  *execProfile,
			Command:  *execCmd,
			Args:     *execCmdArgs,
			Keyring:  keyring,
			Duration: *execSessDuration,
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"

	"github.com/digitalocean/godo"
	"github.com/miekg/dns"
	"golang.org/x/oauth2"

	"github.com/abrander/dozone/proxy"
)

const (
	patEnvName = "DIGITALOCEAN_ACCESS_TOKEN"
)

var (
	pat tokenSource

	client    *godo.Client
	debugHTTP = true
	yes       = true
	download  string
)

func init() {
	pat = tokenSource(os.Getenv(patEnvName))

	if pat == "" {
		bail("Please set the environment variable: %s\n", patEnvName)
	}

	flag.BoolVar(&debugHTTP, "debugHTTP", false, "Output Digital Ocean API requests for debugging")
	flag.BoolVar(&yes, "yes", false, "Don't ask before committing")
	flag.StringVar(&download, "download", "", "Download zone")

	flag.Parse()
}

func getClient() *godo.Client {
	if client != nil {
		return client
	}

	oauthClient := oauth2.NewClient(oauth2.NoContext, pat)
	client = godo.NewClient(oauthClient)

	if debugHTTP {
		client.OnRequestCompleted(func(req *http.Request, resp *http.Response) {
			data, err := httputil.DumpRequestOut(req, true)
			if err == nil {
				fmt.Printf("\033[33m%s\033[0m\n", data)
			} else {
				log.Fatalf("\033[33mERROR: %s\033[0m\n", err.Error())
			}

			data, err = httputil.DumpResponse(resp, true)
			if err == nil {
				fmt.Printf("\033[32m%s\033[0m\n", data)
			} else {
				log.Fatalf("\033[32mERROR: %s\033[0m\n", err.Error())
			}
		})
	}

	return client
}

func bail(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

func bailIfError(err error) {
	if err != nil {
		bail("Error: %s\n", err.Error())
	}
}

func main() {
	client := getClient()

	if download != "" {
		dom := proxy.NewDomain(proxy.NewZoneName(download))

		err := dom.Find(client)
		bailIfError(err)

		fmt.Printf("%s", dom.ZoneFile)

		os.Exit(0)
	}

	filename := flag.Arg(0)
	r, err := os.Open(filename)
	bailIfError(err)
	defer r.Close()

	var zoneTokens []*tokenContainer
	var zoneName proxy.ZoneName

	for token := range dns.ParseZone(r, "", "") {
		if token.Error != nil {
			fmt.Printf("%s\n", token.Error.Error())
		} else {
			switch token.RR.(type) {
			case *dns.A:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.CNAME:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.MX:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.NS:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.SOA:
				soa := token.RR.(*dns.SOA)
				trimmed := strings.Trim(soa.Header().Name, ".")
				if trimmed != "" {
					zoneName = proxy.NewZoneName(soa.Header().Name)
				}
			case *dns.TXT:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			default:
				fmt.Printf("%T not supported yet\n", token.RR)
			}
		}
	}

	if zoneName.String("") == "" {
		bail("Could not derive zone name from zone file")
	}

	dom := proxy.NewDomain(zoneName)

	err = dom.FindOrAdd(client)
	bailIfError(err)

	err = dom.RefreshRecords(client)
	bailIfError(err)

	// Match the zone with records retrieved from DO.
	for _, rec := range dom.Records {
		for _, token := range zoneTokens {
			if !token.matched && rec.Matches(token.Token) {
				rec.Matched = true
				token.matched = true
			}
		}
	}

	if !yes {
		// Present work for the user.
		list := []string{}
		for _, rec := range dom.Records {
			if !rec.Matched {
				list = append(list, rec.String())
			}
		}

		if len(list) > 0 {
			fmt.Printf("Records to delete:\n")
			for _, r := range list {
				fmt.Printf("%s\n", r)
			}
			fmt.Printf("\n")
		}

		list = nil
		for _, token := range zoneTokens {
			if !token.matched {
				list = append(list, token.String())
			}
		}
		if len(list) > 0 {
			fmt.Printf("Records to add:\n")
			for _, r := range list {
				fmt.Printf("%s\n", r)
			}
			fmt.Printf("\n")
		}

		fmt.Printf("Continue (y/N)? ")

		bio := bufio.NewReader(os.Stdin)
		line, _, _ := bio.ReadLine()

		// If we get anything but y/Y we abort.
		if !(string(line) == "y" || string(line) == "Y") {
			bail("Aborting")
		}
	}

	// Delete DO records not present in the zone.
	for _, rec := range dom.Records {
		if !rec.Matched {
			fmt.Printf("Deleting %+v\n", rec)
			err = rec.Delete(client)
			bailIfError(err)
		}
	}

	// Add zone records missing from DO.
	for _, token := range zoneTokens {
		if !token.matched {
			req := token.CreateRequest()

			fmt.Printf("Adding %+v\n", req)
			_, _, err := client.Domains.CreateRecord(zoneName.String(""), req)
			bailIfError(err)
		}
	}

	fmt.Printf("Zone synced.\n")
}
<commit_msg>Only ask for confirmation if there's anything to do.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"

	"github.com/digitalocean/godo"
	"github.com/miekg/dns"
	"golang.org/x/oauth2"

	"github.com/abrander/dozone/proxy"
)

const (
	patEnvName = "DIGITALOCEAN_ACCESS_TOKEN"
)

var (
	pat tokenSource

	client    *godo.Client
	debugHTTP = true
	yes       = true
	download  string
)

func init() {
	pat = tokenSource(os.Getenv(patEnvName))

	if pat == "" {
		bail("Please set the environment variable: %s\n", patEnvName)
	}

	flag.BoolVar(&debugHTTP, "debugHTTP", false, "Output Digital Ocean API requests for debugging")
	flag.BoolVar(&yes, "yes", false, "Don't ask before committing")
	flag.StringVar(&download, "download", "", "Download zone")

	flag.Parse()
}

func getClient() *godo.Client {
	if client != nil {
		return client
	}

	oauthClient := oauth2.NewClient(oauth2.NoContext, pat)
	client = godo.NewClient(oauthClient)

	if debugHTTP {
		client.OnRequestCompleted(func(req *http.Request, resp *http.Response) {
			data, err := httputil.DumpRequestOut(req, true)
			if err == nil {
				fmt.Printf("\033[33m%s\033[0m\n", data)
			} else {
				log.Fatalf("\033[33mERROR: %s\033[0m\n", err.Error())
			}

			data, err = httputil.DumpResponse(resp, true)
			if err == nil {
				fmt.Printf("\033[32m%s\033[0m\n", data)
			} else {
				log.Fatalf("\033[32mERROR: %s\033[0m\n", err.Error())
			}
		})
	}

	return client
}

func bail(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

func bailIfError(err error) {
	if err != nil {
		bail("Error: %s\n", err.Error())
	}
}

func main() {
	client := getClient()

	if download != "" {
		dom := proxy.NewDomain(proxy.NewZoneName(download))

		err := dom.Find(client)
		bailIfError(err)

		fmt.Printf("%s", dom.ZoneFile)

		os.Exit(0)
	}

	filename := flag.Arg(0)
	r, err := os.Open(filename)
	bailIfError(err)
	defer r.Close()

	var zoneTokens []*tokenContainer
	var zoneName proxy.ZoneName

	for token := range dns.ParseZone(r, "", "") {
		if token.Error != nil {
			fmt.Printf("%s\n", token.Error.Error())
		} else {
			switch token.RR.(type) {
			case *dns.A:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.CNAME:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.MX:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.NS:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			case *dns.SOA:
				soa := token.RR.(*dns.SOA)
				trimmed := strings.Trim(soa.Header().Name, ".")
				if trimmed != "" {
					zoneName = proxy.NewZoneName(soa.Header().Name)
				}
			case *dns.TXT:
				zoneTokens = append(zoneTokens, &tokenContainer{Token: token})
			default:
				fmt.Printf("%T not supported yet\n", token.RR)
			}
		}
	}

	if zoneName.String("") == "" {
		bail("Could not derive zone name from zone file")
	}

	dom := proxy.NewDomain(zoneName)

	err = dom.FindOrAdd(client)
	bailIfError(err)

	err = dom.RefreshRecords(client)
	bailIfError(err)

	// Match the zone with records retrieved from DO.
	for _, rec := range dom.Records {
		for _, token := range zoneTokens {
			if !token.matched && rec.Matches(token.Token) {
				rec.Matched = true
				token.matched = true
			}
		}
	}

	if !yes {
		// Present work for the user.
		list := []string{}
		for _, rec := range dom.Records {
			if !rec.Matched {
				list = append(list, rec.String())
			}
		}

		if len(list) > 0 {
			fmt.Printf("Records to delete:\n")
			for _, r := range list {
				fmt.Printf("%s\n", r)
			}
			fmt.Printf("\n")
		}

		numChanges := len(list)

		list = nil
		for _, token := range zoneTokens {
			if !token.matched {
				list = append(list, token.String())
			}
		}
		if len(list) > 0 {
			fmt.Printf("Records to add:\n")
			for _, r := range list {
				fmt.Printf("%s\n", r)
			}
			fmt.Printf("\n")
		}

		numChanges += len(list)

		if numChanges > 0 {
			fmt.Printf("%d change(s). Continue (y/N)? ", numChanges)

			bio := bufio.NewReader(os.Stdin)
			line, _, _ := bio.ReadLine()

			// If we get anything but y/Y we abort.
			if !(string(line) == "y" || string(line) == "Y") {
				bail("Aborting")
			}
		}
	}

	// Delete DO records not present in the zone.
	for _, rec := range dom.Records {
		if !rec.Matched {
			fmt.Printf("Deleting %+v\n", rec)
			err = rec.Delete(client)
			bailIfError(err)
		}
	}

	// Add zone records missing from DO.
	for _, token := range zoneTokens {
		if !token.matched {
			req := token.CreateRequest()

			fmt.Printf("Adding %+v\n", req)
			_, _, err := client.Domains.CreateRecord(zoneName.String(""), req)
			bailIfError(err)
		}
	}

	fmt.Printf("Zone synced.\n")
}
<|endoftext|>


<commit_before>package main

import "fmt"

import "os"
import "bufio"
import "strings"
import "time"
import "strconv"
import "flag"

func main() {
	dur := flag.Duration("d", time.Second, "duration unit")
	flag.Parse()

	fmt.Println("bout")
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		// fmt.Println(line) // Println will add back the final '\n'
		if strings.HasPrefix(line, "Benchmark") && strings.HasSuffix(line, "ns/op") {
			fields := strings.Fields(line)
			if len(fields) < 3 {
				fmt.Println(line)
				continue
			}
			nsPerOp, err := strconv.ParseInt(fields[2], 10, 64)
			if err != nil {
				fmt.Println(line)
				continue
			}
			opsPerDur := dur.Nanoseconds() / nsPerOp
			fmt.Printf("%s\t%d ops/%v\n", line, opsPerDur, dur)
			continue
		}
		fmt.Println(line)
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}
<commit_msg>remove shit<commit_after>package main

import "fmt"

import "os"
import "bufio"
import "strings"
import "time"
import "strconv"
import "flag"

func main() {
	dur := flag.Duration("d", time.Second, "duration unit")
	flag.Parse()

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		line := scanner.Text()
		// fmt.Println(line) // Println will add back the final '\n'
		if strings.HasPrefix(line, "Benchmark") && strings.HasSuffix(line, "ns/op") {
			fields := strings.Fields(line)
			if len(fields) < 3 {
				fmt.Println(line)
				continue
			}
			nsPerOp, err := strconv.ParseInt(fields[2], 10, 64)
			if err != nil {
				fmt.Println(line)
				continue
			}
			opsPerDur := dur.Nanoseconds() / nsPerOp
			fmt.Printf("%s\t%d ops/%v\n", line, opsPerDur, dur)
			continue
		}
		fmt.Println(line)
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"sync"

	//"github.com/aws/aws-sdk-go/aws"
	//"github.com/aws/aws-sdk-go/service/s3"
	"launchpad.net/gommap"
)

var runStage = flag.String("stage", "", "specify the stage to run - can be \"mapper\" or \"reducer\"")
var numDelims = flag.Uint("numDelims", 0, "specify the number of times the delimiter is expected to appear in each line")
var filename = flag.String("file", "", "specify the file to open")

func main() {
	flag.Parse()

	if *runStage == "" || *numDelims == 0 || *filename == "" {
		flag.PrintDefaults()
		return
	}

	if _, err := os.Stat(*filename); os.IsNotExist(err) {
		log.Fatalf("no such file or directory: %s", *filename)
	}

	switch *runStage {
	case "mapper":
		runMapper()
	case "reducer":
		runReducer()
	default:
		log.Fatalln("stage must be either \"mapper\" or \"reducer\"")
	}
}

func runMapper() {
	file, err := os.Open(*filename)
	check(err)

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	check(err)

	numLines, err := countLines(bytes.NewReader(mmap))
	check(err)

	lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)

	lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")

	sub := [][][]byte{
		lines[:(numLines / 4)],
		lines[(numLines / 4):(numLines / 2)],
		lines[(numLines / 2) : (numLines/2)+(numLines/4)],
		lines[(numLines/2)+(numLines/4) : numLines],
	}

	jobs := make(chan []byte)
	results := make(chan string)

	wg := new(sync.WaitGroup)
	for w := 0; w <= 3; w++ {
		wg.Add(1)
		go normalizeLines(jobs, results, wg)
	}

	go func() {
		for i := 0; i <= 3; i++ {
			jobs <- bytes.Join(sub[i], []byte{'\n'})
		}
		close(jobs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	for v := range results {
		fmt.Fprintf(os.Stderr, "working with: %s\n", v)
	}
}

func normalizeLines(jobs <-chan []byte, results chan<- string, wg *sync.WaitGroup) {
	defer wg.Done()

	for j := range jobs {
		in := bufio.NewReader(bytes.NewReader(j))

		line, err := in.ReadString('\n')

		if foundEOF(err) {
			break
		}

		//delimCount := uint(strings.Count(line, "|"))

		values := strings.Split(line, "|")

		for _, value := range values {
			trimmedValue := strings.TrimSpace(value)
			if !strings.Contains(value, "\n") {
				fmt.Printf("%s\\|", trimmedValue)
			} else {
				fmt.Printf("%s\n", trimmedValue)
			}
		}
		results <- string(j)
	}
}

func runReducer() {
	//svc := s3.New(&aws.Config{Region: aws.String("us-west-2")})
}

func increment(group string, counter string) {
	//fmt.Fprintf(os.Stderr, "reporter:counter:%s,%s,1\n", group, counter)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func foundEOF(e error) bool {
	if e == io.EOF {
		return true
	} else if e != nil {
		log.Fatal(e)
	}
	return false
}

func countLines(r io.Reader) (int, error) {
	// play with this buffer size to optimize for speed
	buf := make([]byte, 8196)
	lineCount := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return lineCount, err
		}

		lineCount += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return lineCount, nil
}
<commit_msg>use bufio.Scanner instead of reader<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"sync"

	//"github.com/aws/aws-sdk-go/aws"
	//"github.com/aws/aws-sdk-go/service/s3"
	"launchpad.net/gommap"
)

var runStage = flag.String("stage", "", "specify the stage to run - can be \"mapper\" or \"reducer\"")
var numDelims = flag.Uint("numDelims", 0, "specify the number of times the delimiter is expected to appear in each line")
var filename = flag.String("file", "", "specify the file to open")
var destination = flag.String("destination", "", "specify where to write results")
var badRows = flag.String("badRows", "", "specify where to write bad rows")
var bufferSize = flag.Uint("bufferSize", 8196, "specify the buffer size to use why scanning through files")

func main() {
	flag.Parse()

	if *runStage == "" || *numDelims == 0 || *filename == "" || *destination == "" || *badRows == "" {
		flag.PrintDefaults()
		return
	}

	if _, err := os.Stat(*filename); os.IsNotExist(err) {
		log.Fatalf("no such file or directory: %s", *filename)
	}

	switch *runStage {
	case "mapper":
		runMapper()
	case "reducer":
		runReducer()
	default:
		log.Fatalln("stage must be either \"mapper\" or \"reducer\"")
	}
}

func runMapper() {
	file, err := os.Open(*filename)
	check(err)

	mmap, err := gommap.Map(file.Fd(), gommap.PROT_READ, gommap.MAP_PRIVATE)
	check(err)

	numLines, err := countLines(bytes.NewReader(mmap))
	check(err)

	lines := bytes.SplitN(mmap, []byte{'\n'}, numLines)

	lines[numLines-1] = bytes.Trim(lines[numLines-1], "\n")

	sub := [][][]byte{
		lines[:(numLines / 4)],
		lines[(numLines / 4):(numLines / 2)],
		lines[(numLines / 2) : (numLines/2)+(numLines/4)],
		lines[(numLines/2)+(numLines/4) : numLines],
	}

	jobs := make(chan []byte)
	results := make(chan string)
	//bad := make(chan string)

	wg := new(sync.WaitGroup)
	for w := 0; w <= 3; w++ {
		wg.Add(1)
		go normalizeLines(jobs, results, wg)
	}

	go func() {
		for i := 0; i <= 3; i++ {
			jobs <- bytes.Join(sub[i], []byte{'\n'})
		}
		close(jobs)
	}()

	go func() {
		wg.Wait()
		close(results)
	}()

	for v := range results {
		fmt.Fprintf(os.Stdout, "%s\n", v)
	}
}

func normalizeLines(jobs <-chan []byte, results chan<- string, wg *sync.WaitGroup) {
	defer wg.Done()

	j := <-jobs
	scanner := bufio.NewScanner(bytes.NewReader(j))
	for scanner.Scan() {
		line := scanner.Text()

		//delimCount := uint(strings.Count(line, "|"))

		values := strings.Split(line, "|")

		var buf bytes.Buffer

		for _, value := range values {
			trimmedValue := strings.TrimSpace(value)

			buf.WriteString(trimmedValue)

			if !strings.Contains(value, "\n") {
				buf.WriteString("\\|")
			} else {
				buf.WriteString("\n")
			}
		}
		results <- string(buf.String())
	}
}

func runReducer() {
	//svc := s3.New(&aws.Config{Region: aws.String("us-west-2")})
}

func increment(group string, counter string) {
	//fmt.Fprintf(os.Stderr, "reporter:counter:%s,%s,1\n", group, counter)
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func foundEOF(e error) bool {
	if e == io.EOF {
		return true
	} else if e != nil {
		log.Fatal(e)
	}
	return false
}

func countLines(r io.Reader) (int, error) {
	// play with this buffer size to optimize for speed
	buf := make([]byte, *bufferSize)
	lineCount := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return lineCount, err
		}

		lineCount += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return lineCount, nil
}
<|endoftext|>


<commit_before>/*
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
package main

import (
	"github.com/gorilla/websocket"
	"html/template"
	"log"
	"net/http"
	"github.com/rvelhote/dnspropagation"
)

type WebsocketRequest struct {
    Domain     string `json:"domain"`
    RecordType string `json:"type"`
}

func index(w http.ResponseWriter, req *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	t, _ := template.New("index.html").ParseFiles("templates/index.html")
	t.Execute(w, map[string]string{"Title": "Check DNS Propagation Worldwide"})
}

var upgrader = websocket.Upgrader{
	ReadBufferSize:    4096,
	WriteBufferSize:   4096,
	EnableCompression: true,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func query(w http.ResponseWriter, req *http.Request, configuration []dnspropagation.Server) {
	conn, _ := upgrader.Upgrade(w, req, nil)

	websocketreq := WebsocketRequest{}
	conn.ReadJSON(&websocketreq)

	if len(websocketreq.Domain) == 0 {
		log.Print("Empty domain")
	}

	if len(websocketreq.RecordType) == 0 || dnspropagation.RecordTypes[websocketreq.RecordType] == 0 {
		log.Print("Invalid DNS record specified")
	}

	sem := make(chan dnspropagation.Response, len(configuration))

	for _, server := range configuration {
		go func(server dnspropagation.Server, conn *websocket.Conn) {
			request := dnspropagation.DnsQuery{ Domain: websocketreq.Domain, Record: websocketreq.RecordType, Server: server }
			sem <- request.GetResponse()
		}(server, conn)
	}

	for i := 0; i < len(configuration); i++ {
		conn.WriteJSON(<-sem)
	}

	conn.Close()
}

func main() {
	servers, _ := dnspropagation.LoadConfiguration("conf/servers.json")

	fs := http.FileServer(http.Dir("assets"))
	http.Handle("/assets/", http.StripPrefix("/assets/", fs))

	http.HandleFunc("/", index)

	http.HandleFunc("/api/v1/query", func(w http.ResponseWriter, req *http.Request) {
		query(w, req, servers)
	})

	http.ListenAndServe(":8080", nil)
}
<commit_msg>The conn object was not being used in the goroutine<commit_after>/*
 * The MIT License (MIT)
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
package main

import (
	"github.com/gorilla/websocket"
	"html/template"
	"log"
	"net/http"
	"github.com/rvelhote/dnspropagation"
)

type WebsocketRequest struct {
    Domain     string `json:"domain"`
    RecordType string `json:"type"`
}

func index(w http.ResponseWriter, req *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	t, _ := template.New("index.html").ParseFiles("templates/index.html")
	t.Execute(w, map[string]string{"Title": "Check DNS Propagation Worldwide"})
}

var upgrader = websocket.Upgrader{
	ReadBufferSize:    4096,
	WriteBufferSize:   4096,
	EnableCompression: true,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

func query(w http.ResponseWriter, req *http.Request, configuration []dnspropagation.Server) {
	conn, _ := upgrader.Upgrade(w, req, nil)

	websocketreq := WebsocketRequest{}
	conn.ReadJSON(&websocketreq)

	if len(websocketreq.Domain) == 0 {
		log.Print("Empty domain")
	}

	if len(websocketreq.RecordType) == 0 || dnspropagation.RecordTypes[websocketreq.RecordType] == 0 {
		log.Print("Invalid DNS record specified")
	}

	sem := make(chan dnspropagation.Response, len(configuration))

	for _, server := range configuration {
		go func(server dnspropagation.Server) {
			request := dnspropagation.DnsQuery{ Domain: websocketreq.Domain, Record: websocketreq.RecordType, Server: server }
			sem <- request.GetResponse()
		}(server)
	}

	for i := 0; i < len(configuration); i++ {
		conn.WriteJSON(<-sem)
	}

	conn.Close()
}

func main() {
	servers, _ := dnspropagation.LoadConfiguration("conf/servers.json")

	fs := http.FileServer(http.Dir("assets"))
	http.Handle("/assets/", http.StripPrefix("/assets/", fs))

	http.HandleFunc("/", index)

	http.HandleFunc("/api/v1/query", func(w http.ResponseWriter, req *http.Request) {
		query(w, req, servers)
	})

	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/node"
	"github.com/Dataman-Cloud/swan/src/version"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func setupLogger(logLevel string) {
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		level = logrus.DebugLevel
	}
	logrus.SetLevel(level)

	logrus.SetOutput(os.Stderr)

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		FullTimestamp:   true,
	})
}

func main() {
	app := cli.NewApp()
	app.Name = "swan"
	app.Usage = "A general purpose mesos framework"
	app.Version = version.Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen-addr",
			Usage:  "listener address for agent",
			EnvVar: "SWAN_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "advertise-addr",
			Usage:  "advertise address for agent",
			EnvVar: "SWAN_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-listen-addr",
			Usage:  "swan raft serverlistener address",
			EnvVar: "SWAN_RAFT_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-advertise-addr",
			Usage:  "swan raft advertise address",
			EnvVar: "SWAN_RAFT_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "join-addrs",
			Usage:  "the addrs new node join to. Splited by ','",
			EnvVar: "SWAN_JOIN_ADDRS",
		},
		cli.StringFlag{
			Name:   "janitor-advertise-ip",
			Usage:  "janitor proxy advertise ip",
			EnvVar: "SWAN_JANITOR_ADVERTISE_IP",
		},

		cli.StringFlag{
			Name:   "zk-path",
			Usage:  "zookeeper mesos paths. eg. zk://host1:port1,host2:port2,.../path",
			EnvVar: "SWAN_MESOS_ZKPATH",
		},
		cli.StringFlag{
			Name:   "log-level,l",
			Usage:  "customize debug level [debug|info|error]",
			EnvVar: "SWAN_LOG_LEVEL",
		},
		cli.StringFlag{
			Name:   "mode",
			Usage:  "server mode, manager|agent|mixed ",
			EnvVar: "SWAN_MODE",
		},
		cli.StringFlag{
			Name:   "data-dir,d",
			Usage:  "swan data store dir",
			EnvVar: "SWAN_DATA_DIR",
		},
		cli.StringFlag{
			Name:   "domain",
			Usage:  "domain which resolve to proxies. eg. swan.com, which make any task can be access from path likes 0.appname.username.cluster.swan.com",
			EnvVar: "SWAN_DOMAIN",
		},
	}
	app.Action = func(c *cli.Context) error {
		if c.NArg() == 0 {
			return cli.ShowAppHelp(c)
		}

		config, err := config.NewConfig(c)
		if err != nil {
			logrus.Errorf("load config failed. Error: %s", err)
			return err
		}

		setupLogger(config.LogLevel)

		node, err := node.NewNode(config)
		if err != nil {
			logrus.Error("Node initialization failed")
			return err
		}

		if err := node.Start(context.Background()); err != nil {
			logrus.Errorf("start node failed. Error: %s", err.Error())
			return err
		}

		return nil
	}

	if err := app.Run(os.Args); err != nil {
		logrus.Errorf("%s", err.Error())
		os.Exit(1)
	}
}
<commit_msg>fixed bug for #345<commit_after>package main

import (
	"os"

	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/node"
	"github.com/Dataman-Cloud/swan/src/version"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func setupLogger(logLevel string) {
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		level = logrus.DebugLevel
	}
	logrus.SetLevel(level)

	logrus.SetOutput(os.Stderr)

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		FullTimestamp:   true,
	})
}

func main() {
	app := cli.NewApp()
	app.Name = "swan"
	app.Usage = "A general purpose mesos framework"
	app.Version = version.Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen-addr",
			Usage:  "listener address for agent",
			EnvVar: "SWAN_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "advertise-addr",
			Usage:  "advertise address for agent",
			EnvVar: "SWAN_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-listen-addr",
			Usage:  "swan raft serverlistener address",
			EnvVar: "SWAN_RAFT_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-advertise-addr",
			Usage:  "swan raft advertise address",
			EnvVar: "SWAN_RAFT_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "join-addrs",
			Usage:  "the addrs new node join to. Splited by ','",
			EnvVar: "SWAN_JOIN_ADDRS",
		},
		cli.StringFlag{
			Name:   "janitor-advertise-ip",
			Usage:  "janitor proxy advertise ip",
			EnvVar: "SWAN_JANITOR_ADVERTISE_IP",
		},

		cli.StringFlag{
			Name:   "zk-path",
			Usage:  "zookeeper mesos paths. eg. zk://host1:port1,host2:port2,.../path",
			EnvVar: "SWAN_MESOS_ZKPATH",
		},
		cli.StringFlag{
			Name:   "log-level,l",
			Usage:  "customize debug level [debug|info|error]",
			EnvVar: "SWAN_LOG_LEVEL",
		},
		cli.StringFlag{
			Name:   "mode",
			Usage:  "server mode, manager|agent|mixed ",
			EnvVar: "SWAN_MODE",
		},
		cli.StringFlag{
			Name:   "data-dir,d",
			Usage:  "swan data store dir",
			EnvVar: "SWAN_DATA_DIR",
		},
		cli.StringFlag{
			Name:   "domain",
			Usage:  "domain which resolve to proxies. eg. swan.com, which make any task can be access from path likes 0.appname.username.cluster.swan.com",
			EnvVar: "SWAN_DOMAIN",
		},
	}
	app.Action = func(c *cli.Context) error {
		if c.NumFlags() == 0 {
			return cli.ShowAppHelp(c)
		}

		config, err := config.NewConfig(c)
		if err != nil {
			logrus.Errorf("load config failed. Error: %s", err)
			return err
		}

		setupLogger(config.LogLevel)

		node, err := node.NewNode(config)
		if err != nil {
			logrus.Error("Node initialization failed")
			return err
		}

		if err := node.Start(context.Background()); err != nil {
			logrus.Errorf("start node failed. Error: %s", err.Error())
			return err
		}

		return nil
	}

	if err := app.Run(os.Args); err != nil {
		logrus.Errorf("%s", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
  "crypto/sha256"
  "encoding/hex"
  "flag"
  "fmt"
  "runtime"
  "time"
)

var (
  username = flag.String("username", "ristenpart", "What is the username to crack?")
  salt     = flag.String("salt", "134153169", "What is the salt to crack?")
  hash     = flag.String("hash", "37448ba7de7f5b4396697edaeddcd7bc840964e6ce82016915b830a91d69eb2f", "What is the hash to crack?")
  size     = flag.Int("size", 1<<15, "What is the batch size for each worker?")
  max      = flag.Int("max", 1<<31, "What is the max integer password to look for?")
)

func init() {
  flag.Parse()

  // Set runtime GOMAXPROCS
  runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
  batch := make(chan int)

  for i := 0; i < runtime.NumCPU(); i++ {
    go func() {
      for batchIndex := range batch {
        for i := batchIndex; i < (int(batchIndex+1) * (*size)); i++ {
          sha := sha256.New()
          sha.Write([]byte(fmt.Sprintf("%s,%d,%s", *username, i, *salt)))
          test := hex.EncodeToString(sha.Sum(nil))
          if test == *hash {
            fmt.Printf("The integer password: %d\n", i)
            fmt.Printf("Produces this hash: %s\n", test)
            close(batch)
          }
        }
      }
    }()
  }

  defer func() { recover() }()

  timeStart := time.Now()
  batchStart := 0
  for i := 0; i < *max / *size; i++ {
    batch <- i
    if timeEnd := time.Now(); timeEnd.Sub(timeStart) > time.Second {
      fmt.Printf("Hashing at %d/sec\n", int(i-batchStart)*(*size))
      batchStart = i
      timeStart = time.Now()
    }
  }
}
<commit_msg>Rewrite timing to be more accurate<commit_after>package main

import (
  "crypto/sha256"
  "encoding/hex"
  "flag"
  "fmt"
  "runtime"
  "sync/atomic"
  "time"
)

var (
  username = flag.String("username", "ristenpart", "What is the username to crack?")
  salt     = flag.String("salt", "134153169", "What is the salt to crack?")
  hash     = flag.String("hash", "37448ba7de7f5b4396697edaeddcd7bc840964e6ce82016915b830a91d69eb2f", "What is the hash to crack?")
  size     = flag.Int("size", 1<<15, "What is the batch size for each worker?")
  max      = flag.Int("max", 1<<31, "What is the max integer password to look for?")
)

func init() {
  flag.Parse()

  // Set runtime GOMAXPROCS
  runtime.GOMAXPROCS(runtime.NumCPU())
}

func main() {
  ticker := time.NewTicker(time.Second)
  var ops uint64 = 0

  go func() {
    lastOps := atomic.LoadUint64(&ops)
    for _ = range ticker.C {
      recentOps := atomic.LoadUint64(&ops)
      fmt.Printf("Hashing at %d/sec\n", recentOps-lastOps)
      lastOps = recentOps
    }
  }()

  batch := make(chan int)

  for i := 0; i < runtime.NumCPU(); i++ {
    go func() {
      for batchIndex := range batch {
        for i := batchIndex; i < (int(batchIndex+1) * (*size)); i++ {
          sha := sha256.New()
          sha.Write([]byte(fmt.Sprintf("%s,%d,%s", *username, i, *salt)))
          test := hex.EncodeToString(sha.Sum(nil))
          atomic.AddUint64(&ops, 1)
          if test == *hash {
            fmt.Println("The integer password:", i)
            fmt.Println("Produces this hash:", test)
            ticker.Stop()
            close(batch)
          }
        }
      }
    }()
  }

  defer func() { recover() }()

  for i := 0; i < *max / *size; i++ {
    batch <- i
  }
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/perthgophers/puddle/controllers"
	"github.com/perthgophers/puddle/messagerouter"
	"github.com/perthgophers/puddle/responses"
	"net/http"
	"os"
	"os/exec"
)

// SLACKTOKEN is the slack API token
var SLACKTOKEN string

// GITTAG Current Git Tag
var GITTAG string

const CHANNEL = "C32K3QDFE"
const SPAMCHANNEL = "C33C4MJSH"

func init() {
	SLACKTOKEN = os.Getenv("SLACKTOKEN")

	out, err := exec.Command("git", "rev-parse", "HEAD").Output()
	if err != nil {
		GITTAG = "NO TAG"
	} else {
		GITTAG = string(out)
	}
	GITTAG += "/"
	out, err = exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err == nil {
		GITTAG += string(out)
	}
}

func main() {
	http.HandleFunc("/logs/log.html", controllers.ServeTastic)
	go http.ListenAndServe(":8080", nil)

	mr := messagerouter.New(SLACKTOKEN, GITTAG, CHANNEL, SPAMCHANNEL)

	responses.Init(mr)
	mr.Run()
}
<commit_msg>Logging interface, logs formatted html to log.txt<commit_after>package main

import (
	"fmt"
	"github.com/perthgophers/puddle/controllers"
	"github.com/perthgophers/puddle/messagerouter"
	"github.com/perthgophers/puddle/responses"
	"html"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"time"
)

// SLACKTOKEN is the slack API token
var SLACKTOKEN string

// GITTAG Current Git Tag
var GITTAG string

const CHANNEL = "C32K3QDFE"
const SPAMCHANNEL = "C33C4MJSH"

// PuddleLogger is an io.Writer to add HTML to log output
type PuddleLogger struct {
	file *os.File
}

// NewPuddleLogger initialises a PuddleLogger and rotates the log file
func NewPuddleLogger() *PuddleLogger {
	pf := new(PuddleLogger)
	pf.Rotate()

	return pf
}

// Write writes len(b) bytes to the log File.
// It returns the number of bytes written and an error, if any.
// Write returns a non-nil error when n != len(b).
// Wraps logput in an html paragraph tag
func (pl *PuddleLogger) Write(b []byte) (n int, err error) {
	s := `<p class="log_output">` + html.EscapeString(string(b)) + `</p>`

	if pl == nil || pl.file == nil {
		return 0, os.ErrInvalid
	}
	n, e := pl.file.Write([]byte(s))

	if n < 0 {
		n = 0
	}
	if n != len(b) {
		err = io.ErrShortWrite
	}

	return n, e
}

// Close closes the log file
func (pf *PuddleLogger) Close() {
	if pf.file != nil {
		pf.file.Close()
	}
	pf.file = nil
}

// Rotate closes the opened file, and renames it to log_<datestamp>.txt for archival
// It then creates a new log.txt and opens it
func (pf *PuddleLogger) Rotate() {
	pf.Close()
	t := time.Now()
	dtSuffix := t.Format("2006_January__2_15_03_05")
	err := os.Rename("./logs/log.txt", "./logs/log_"+dtSuffix+".txt")
	if err != nil {
		fmt.Println(err)
	}
	pf.file, err = os.OpenFile("./logs/log.txt", os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)

	if err != nil {
		log.Fatalln("Unable to open log")
	}
}

func init() {
	SLACKTOKEN = os.Getenv("SLACKTOKEN")

	out, err := exec.Command("git", "rev-parse", "HEAD").Output()
	if err != nil {
		GITTAG = "NO TAG"
	} else {
		GITTAG = string(out)
	}
	GITTAG += "/"
	out, err = exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD").Output()
	if err == nil {
		GITTAG += string(out)
	}
}

func main() {
	logger := NewPuddleLogger()

	log.SetOutput(logger)
	defer logger.Close()

	http.HandleFunc("/log", controllers.ServeTastic)
	go http.ListenAndServe(":8080", nil)

	mr := messagerouter.New(SLACKTOKEN, GITTAG, CHANNEL, SPAMCHANNEL)

	responses.Init(mr)
	mr.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/urfave/cli"
)

var (
	version = "0.0.0"
	build   = "0"
)

func main() {
	app := cli.NewApp()
	app.Name = "slack plugin"
	app.Usage = "slack plugin"
	app.Action = run
	app.Version = fmt.Sprintf("%s+%s", version, build)
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "webhook",
			Usage:  "slack webhook url",
			EnvVar: "SLACK_WEBHOOK,PLUGIN_WEBHOOK",
		},
		cli.StringFlag{
			Name:   "channel",
			Usage:  "slack channel",
			EnvVar: "PLUGIN_CHANNEL",
		},
		cli.StringFlag{
			Name:   "recipient",
			Usage:  "slack recipient",
			EnvVar: "PLUGIN_RECIPIENT",
		},
		cli.StringFlag{
			Name:   "username",
			Usage:  "slack username",
			EnvVar: "PLUGIN_USERNAME",
		},
		cli.StringFlag{
			Name:   "template",
			Usage:  "slack template",
			EnvVar: "PLUGIN_TEMPLATE",
		},
		cli.BoolFlag{
			Name:   "link-names",
			Usage:  "slack link names",
			EnvVar: "PLUGIN_LINK_NAMES",
		},
		cli.StringFlag{
			Name:   "image",
			Usage:  "slack image url",
			EnvVar: "PLUGIN_IMAGE_URL",
		},
		cli.StringFlag{
			Name:   "icon.url",
			Usage:  "slack icon url",
			EnvVar: "PLUGIN_ICON_URL",
		},
		cli.StringFlag{
			Name:   "icon.emoji",
			Usage:  "slack emoji url",
			EnvVar: "PLUGIN_ICON_EMOJI",
		},
		cli.StringFlag{
			Name:   "repo.owner",
			Usage:  "repository owner",
			EnvVar: "DRONE_REPO_OWNER",
		},
		cli.StringFlag{
			Name:   "repo.name",
			Usage:  "repository name",
			EnvVar: "DRONE_REPO_NAME",
		},
		cli.StringFlag{
			Name:   "commit.sha",
			Usage:  "git commit sha",
			EnvVar: "DRONE_COMMIT_SHA",
			Value:  "00000000",
		},
		cli.StringFlag{
			Name:   "commit.ref",
			Value:  "refs/heads/master",
			Usage:  "git commit ref",
			EnvVar: "DRONE_COMMIT_REF",
		},
		cli.StringFlag{
			Name:   "commit.branch",
			Value:  "master",
			Usage:  "git commit branch",
			EnvVar: "DRONE_COMMIT_BRANCH",
		},
		cli.StringFlag{
			Name:   "commit.author",
			Usage:  "git author username",
			EnvVar: "DRONE_COMMIT_AUTHOR",
		},
		cli.StringFlag{
			Name:   "commit.author.email",
			Usage:  "git author email",
			EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL",
		},
		cli.StringFlag{
			Name:   "commit.author.avatar",
			Usage:  "git author avatar",
			EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR",
		},
		cli.StringFlag{
			Name:   "commit.author.name",
			Usage:  "git author name",
			EnvVar: "DRONE_COMMIT_AUTHOR_NAME",
		},
		cli.StringFlag{
			Name:   "commit.pull",
			Usage:  "git pull request",
			EnvVar: "DRONE_PULL_REQUEST",
		},
		cli.StringFlag{
			Name:   "commit.message",
			Usage:  "commit message",
			EnvVar: "DRONE_COMMIT_MESSAGE",
		},
		cli.StringFlag{
			Name:   "build.event",
			Value:  "push",
			Usage:  "build event",
			EnvVar: "DRONE_BUILD_EVENT",
		},
		cli.IntFlag{
			Name:   "build.number",
			Usage:  "build number",
			EnvVar: "DRONE_BUILD_NUMBER",
		},
		cli.StringFlag{
			Name:   "build.status",
			Usage:  "build status",
			Value:  "success",
			EnvVar: "DRONE_BUILD_STATUS",
		},
		cli.StringFlag{
			Name:   "build.link",
			Usage:  "build link",
			EnvVar: "DRONE_BUILD_LINK",
		},
		cli.Int64Flag{
			Name:   "build.started",
			Usage:  "build started",
			EnvVar: "DRONE_BUILD_STARTED",
		},
		cli.Int64Flag{
			Name:   "build.created",
			Usage:  "build created",
			EnvVar: "DRONE_BUILD_CREATED",
		},
		cli.StringFlag{
			Name:   "build.tag",
			Usage:  "build tag",
			EnvVar: "DRONE_TAG",
		},
		cli.StringFlag{
			Name:   "build.deployTo",
			Usage:  "environment deployed to",
			EnvVar: "DRONE_DEPLOY_TO",
		},
		cli.Int64Flag{
			Name:   "job.started",
			Usage:  "job started",
			EnvVar: "DRONE_JOB_STARTED",
		},
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

func run(c *cli.Context) error {
	msg := c.String("commit.message")
	splitMsg := strings.Split(msg, "\n")

	plugin := Plugin{
		Repo: Repo{
			Owner: c.String("repo.owner"),
			Name:  c.String("repo.name"),
		},
		Build: Build{
			Tag:    c.String("build.tag"),
			Number: c.Int("build.number"),
			Event:  c.String("build.event"),
			Status: c.String("build.status"),
			Commit: c.String("commit.sha"),
			Ref:    c.String("commit.ref"),
			Branch: c.String("commit.branch"),
			Author: Author{
				Username: c.String("commit.author"),
				Name:     c.String("commit.author.name"),
				Email:    c.String("commit.author.email"),
				Avatar:   c.String("commit.author.avatar"),
			},
			Pull:         c.String("commit.pull"),
			Message:      msg,
			MessageTitle: strings.TrimSpace(splitMsg[0]),
			MessageBody:  strings.TrimSpace(strings.Join(splitMsg[1:], "\n")),
			DeployTo:     c.String("build.deployTo"),
			Link:         c.String("build.link"),
			Started:      c.Int64("build.started"),
			Created:      c.Int64("build.created"),
		},
		Job: Job{
			Started: c.Int64("job.started"),
		},
		Config: Config{
			Webhook:   c.String("webhook"),
			Channel:   c.String("channel"),
			Recipient: c.String("recipient"),
			Username:  c.String("username"),
			Template:  c.String("template"),
			ImageURL:  c.String("image"),
			IconURL:   c.String("icon.url"),
			IconEmoji: c.String("icon.emoji"),
			LinkNames: c.Bool("link-names"),
		},
	}

	return plugin.Exec()
}
<commit_msg>doc: add the reason for not checking split msg length<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/urfave/cli"
)

var (
	version = "0.0.0"
	build   = "0"
)

func main() {
	app := cli.NewApp()
	app.Name = "slack plugin"
	app.Usage = "slack plugin"
	app.Action = run
	app.Version = fmt.Sprintf("%s+%s", version, build)
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "webhook",
			Usage:  "slack webhook url",
			EnvVar: "SLACK_WEBHOOK,PLUGIN_WEBHOOK",
		},
		cli.StringFlag{
			Name:   "channel",
			Usage:  "slack channel",
			EnvVar: "PLUGIN_CHANNEL",
		},
		cli.StringFlag{
			Name:   "recipient",
			Usage:  "slack recipient",
			EnvVar: "PLUGIN_RECIPIENT",
		},
		cli.StringFlag{
			Name:   "username",
			Usage:  "slack username",
			EnvVar: "PLUGIN_USERNAME",
		},
		cli.StringFlag{
			Name:   "template",
			Usage:  "slack template",
			EnvVar: "PLUGIN_TEMPLATE",
		},
		cli.BoolFlag{
			Name:   "link-names",
			Usage:  "slack link names",
			EnvVar: "PLUGIN_LINK_NAMES",
		},
		cli.StringFlag{
			Name:   "image",
			Usage:  "slack image url",
			EnvVar: "PLUGIN_IMAGE_URL",
		},
		cli.StringFlag{
			Name:   "icon.url",
			Usage:  "slack icon url",
			EnvVar: "PLUGIN_ICON_URL",
		},
		cli.StringFlag{
			Name:   "icon.emoji",
			Usage:  "slack emoji url",
			EnvVar: "PLUGIN_ICON_EMOJI",
		},
		cli.StringFlag{
			Name:   "repo.owner",
			Usage:  "repository owner",
			EnvVar: "DRONE_REPO_OWNER",
		},
		cli.StringFlag{
			Name:   "repo.name",
			Usage:  "repository name",
			EnvVar: "DRONE_REPO_NAME",
		},
		cli.StringFlag{
			Name:   "commit.sha",
			Usage:  "git commit sha",
			EnvVar: "DRONE_COMMIT_SHA",
			Value:  "00000000",
		},
		cli.StringFlag{
			Name:   "commit.ref",
			Value:  "refs/heads/master",
			Usage:  "git commit ref",
			EnvVar: "DRONE_COMMIT_REF",
		},
		cli.StringFlag{
			Name:   "commit.branch",
			Value:  "master",
			Usage:  "git commit branch",
			EnvVar: "DRONE_COMMIT_BRANCH",
		},
		cli.StringFlag{
			Name:   "commit.author",
			Usage:  "git author username",
			EnvVar: "DRONE_COMMIT_AUTHOR",
		},
		cli.StringFlag{
			Name:   "commit.author.email",
			Usage:  "git author email",
			EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL",
		},
		cli.StringFlag{
			Name:   "commit.author.avatar",
			Usage:  "git author avatar",
			EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR",
		},
		cli.StringFlag{
			Name:   "commit.author.name",
			Usage:  "git author name",
			EnvVar: "DRONE_COMMIT_AUTHOR_NAME",
		},
		cli.StringFlag{
			Name:   "commit.pull",
			Usage:  "git pull request",
			EnvVar: "DRONE_PULL_REQUEST",
		},
		cli.StringFlag{
			Name:   "commit.message",
			Usage:  "commit message",
			EnvVar: "DRONE_COMMIT_MESSAGE",
		},
		cli.StringFlag{
			Name:   "build.event",
			Value:  "push",
			Usage:  "build event",
			EnvVar: "DRONE_BUILD_EVENT",
		},
		cli.IntFlag{
			Name:   "build.number",
			Usage:  "build number",
			EnvVar: "DRONE_BUILD_NUMBER",
		},
		cli.StringFlag{
			Name:   "build.status",
			Usage:  "build status",
			Value:  "success",
			EnvVar: "DRONE_BUILD_STATUS",
		},
		cli.StringFlag{
			Name:   "build.link",
			Usage:  "build link",
			EnvVar: "DRONE_BUILD_LINK",
		},
		cli.Int64Flag{
			Name:   "build.started",
			Usage:  "build started",
			EnvVar: "DRONE_BUILD_STARTED",
		},
		cli.Int64Flag{
			Name:   "build.created",
			Usage:  "build created",
			EnvVar: "DRONE_BUILD_CREATED",
		},
		cli.StringFlag{
			Name:   "build.tag",
			Usage:  "build tag",
			EnvVar: "DRONE_TAG",
		},
		cli.StringFlag{
			Name:   "build.deployTo",
			Usage:  "environment deployed to",
			EnvVar: "DRONE_DEPLOY_TO",
		},
		cli.Int64Flag{
			Name:   "job.started",
			Usage:  "job started",
			EnvVar: "DRONE_JOB_STARTED",
		},
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

func run(c *cli.Context) error {
	msg := c.String("commit.message")
	// not checking the length here
	// as split will always return at least one element
	// check it if using more than the first element
	splitMsg := strings.Split(msg, "\n")

	plugin := Plugin{
		Repo: Repo{
			Owner: c.String("repo.owner"),
			Name:  c.String("repo.name"),
		},
		Build: Build{
			Tag:    c.String("build.tag"),
			Number: c.Int("build.number"),
			Event:  c.String("build.event"),
			Status: c.String("build.status"),
			Commit: c.String("commit.sha"),
			Ref:    c.String("commit.ref"),
			Branch: c.String("commit.branch"),
			Author: Author{
				Username: c.String("commit.author"),
				Name:     c.String("commit.author.name"),
				Email:    c.String("commit.author.email"),
				Avatar:   c.String("commit.author.avatar"),
			},
			Pull:         c.String("commit.pull"),
			Message:      msg,
			MessageTitle: strings.TrimSpace(splitMsg[0]),
			MessageBody:  strings.TrimSpace(strings.Join(splitMsg[1:], "\n")),
			DeployTo:     c.String("build.deployTo"),
			Link:         c.String("build.link"),
			Started:      c.Int64("build.started"),
			Created:      c.Int64("build.created"),
		},
		Job: Job{
			Started: c.Int64("job.started"),
		},
		Config: Config{
			Webhook:   c.String("webhook"),
			Channel:   c.String("channel"),
			Recipient: c.String("recipient"),
			Username:  c.String("username"),
			Template:  c.String("template"),
			ImageURL:  c.String("image"),
			IconURL:   c.String("icon.url"),
			IconEmoji: c.String("icon.emoji"),
			LinkNames: c.Bool("link-names"),
		},
	}

	return plugin.Exec()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

const (
	version = "0.8.8"
)

func init() {
	flag.Usage = func() {
		h := "Usage:\n"
		h += "  bozr [OPTIONS] (DIR|FILE)\n\n"

		h += "Options:\n"
		h += "  -d, --debug		Enable debug mode\n"
		h += "  -H, --host		Server to test\n"
		h += "  -w, --worker		Execute in parallel with specified number of workers\n"
		h += "  -h, --help		Print usage\n"
		h += "  -i, --info		Enable info mode. Print request and response details\n"
		h += "      --junit		Enable junit xml reporter\n"
		h += "      --junit-output	Destination for junit report files\n"
		h += "  -v, --version		Print version information and quit\n\n"

		h += "Examples:\n"
		h += "  bozr ./examples\n"
		h += "  bozr -w 2 ./examples\n"
		h += "  bozr -H http://example.com ./examples \n"

		fmt.Fprintf(os.Stderr, h)
	}
}

var (
	suitesDir       string
	hostFlag        string
	routinesFlag    int
	infoFlag        bool
	debugFlag       bool
	helpFlag        bool
	versionFlag     bool
	junitFlag       bool
	junitOutputFlag string

	info  *log.Logger
	debug *log.Logger
)

const suiteExt = ".suite.json"

func initLogger() {
	infoHandler := ioutil.Discard
	debugHandler := ioutil.Discard

	if infoFlag {
		infoHandler = os.Stdout
	}

	if debugFlag {
		debugHandler = os.Stdout
	}

	info = log.New(infoHandler, "", 0)
	debug = log.New(debugHandler, "DEBUG: ", log.Ltime|log.Lshortfile)
}

func main() {
	flag.BoolVar(&debugFlag, "d", false, "Enable debug mode.")
	flag.BoolVar(&debugFlag, "debug", false, "Enable debug mode")

	flag.BoolVar(&infoFlag, "i", false, "Enable info mode. Print request and response details.")
	flag.BoolVar(&infoFlag, "info", false, "Enable info mode. Print request and response details.")

	flag.StringVar(&hostFlag, "H", "", "Test server address. Example: http://example.com/api.")
	flag.IntVar(&routinesFlag, "w", 1, "Execute test sutes in parallel with provided numer of workers. Default is 1.")

	flag.BoolVar(&helpFlag, "h", false, "Print usage")
	flag.BoolVar(&helpFlag, "help", false, "Print usage")

	flag.BoolVar(&versionFlag, "v", false, "Print version information and quit")
	flag.BoolVar(&versionFlag, "version", false, "Print version information and quit")

	flag.BoolVar(&junitFlag, "junit", false, "Enable junit xml reporter")
	flag.StringVar(&junitOutputFlag, "junit-output", "./report", "Destination for junit report files. Default ")

	flag.Parse()

	initLogger()

	if versionFlag {
		fmt.Println("bozr version " + version)
		return
	}

	if helpFlag {
		flag.Usage()
		return
	}

	if len(hostFlag) > 0 {
		_, err := url.ParseRequestURI(hostFlag)
		if err != nil {
			terminate("Invalid host is specified.")
			return
		}
	}

	if routinesFlag < 1 || routinesFlag > 9 {
		fmt.Println("Invalid routines parameter [", routinesFlag, "]. Setting to default [1]")
		routinesFlag = 1
	}

	suitesDir = flag.Arg(0)

	if suitesDir == "" {
		terminate("You must specify a directory or file with tests.")
		flag.Usage()
		return
	}

	// check specified source dir/file exists
	_, err := os.Lstat(suitesDir)
	if err != nil {
		terminate(err.Error())
		return
	}

	err = ValidateSuites(suitesDir, suiteExt)
	if err != nil {
		terminate("One or more test suites are invalid.", err.Error())
		return
	}

	loader := NewSuiteLoader(suitesDir, suiteExt)
	reporter := createReporter()

	RunParallel(loader, reporter, runSuite, routinesFlag)
}

func runSuite(suite TestSuite) []TestResult {
	results := []TestResult{}

	for _, testCase := range suite.Cases {

		result := TestResult{
			Suite:     suite,
			Case:      testCase,
			ExecFrame: TimeFrame{Start: time.Now(), End: time.Now()},
		}

		if testCase.Ignore != nil {
			result.Skipped = true
			result.SkippedMsg = *testCase.Ignore

			results = append(results, result)
			continue
		}

		rememberedMap := make(map[string]interface{})
		for _, c := range testCase.Calls {
			addAll(c.Args, rememberedMap)
			terr := call(suite.Dir, c, rememberedMap)
			if terr != nil {
				result.Error = terr
				break
			}
		}

		result.ExecFrame.End = time.Now()

		results = append(results, result)
	}

	return results
}

func addAll(src, target map[string]interface{}) {
	for key, val := range src {
		target[key] = val
	}
}

func createReporter() Reporter {
	reporters := []Reporter{NewConsoleReporter()}
	if junitFlag {
		path, _ := filepath.Abs(junitOutputFlag)
		reporters = append(reporters, NewJUnitReporter(path))
	}
	reporter := NewMultiReporter(reporters...)
	reporter.Init()

	return reporter
}

func call(suitePath string, call Call, rememberMap map[string]interface{}) *TError {

	terr := &TError{}

	on := call.On

	dat := []byte(on.Body)
	if on.BodyFile != "" {
		uri, err := toAbsPath(suitePath, on.BodyFile)
		if err != nil {
			terr.Cause = err
			return terr
		}

		if d, err := ioutil.ReadFile(uri); err == nil {
			dat = d
		} else {
			terr.Cause = fmt.Errorf("Can't read body file: %s", err.Error())
			return terr
		}
	}

	req, err := populateRequest(on, string(dat), rememberMap)
	if err != nil {
		terr.Cause = err
		return terr
	}

	printRequestInfo(req, dat)

	client := &http.Client{}

	resp, err := client.Do(req)

	if err != nil {
		debug.Print("Error when sending request", err)
		terr.Cause = err
		return terr
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		debug.Print("Error reading response")
		terr.Cause = err
		return terr
	}

	testResp := Response{http: *resp, body: body}
	terr.Resp = testResp

	info.Println(strings.Repeat("-", 50))
	info.Println(testResp.ToString())
	info.Println("")

	exps, err := expectations(call, suitePath)
	if err != nil {
		terr.Cause = err
		return terr
	}

	for _, exp := range exps {
		checkErr := exp.check(&testResp)
		if checkErr != nil {
			terr.Cause = checkErr
			return terr
		}
	}

	err = rememberBody(&testResp, call.Remember.Body, rememberMap)
	debug.Print("Remember: ", rememberMap)
	if err != nil {
		debug.Print("Error remember")
		terr.Cause = err
		return terr
	}

	rememberHeaders(testResp.http.Header, call.Remember.Headers, rememberMap)

	return nil
}

func populateRequest(on On, body string, rememberMap map[string]interface{}) (*http.Request, error) {

	urlStr, err := urlPrefix(populateRememberedVars(on.URL, rememberMap))
	if err != nil {
		return nil, errors.New("Cannot create request. Invalid url: " + on.URL)
	}

	body = populateRememberedVars(body, rememberMap)
	dat := []byte(body)

	req, err := http.NewRequest(on.Method, urlStr, bytes.NewBuffer(dat))
	if err != nil {
		return nil, err
	}

	for key, value := range on.Headers {
		req.Header.Add(key, populateRememberedVars(value, rememberMap))
	}

	q := req.URL.Query()
	for key, value := range on.Params {
		q.Add(key, populateRememberedVars(value, rememberMap))
	}
	req.URL.RawQuery = q.Encode()

	return req, nil
}

func urlPrefix(p string) (string, error) {
	if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
		return p, nil
	}

	return concatURL(hostFlag, p)
}

func concatURL(base string, p string) (string, error) {
	baseURL, err := url.ParseRequestURI(base)
	if err != nil {
		return "", err
	}
	return baseURL.Scheme + "://" + baseURL.Host + path.Join(baseURL.Path, p), nil
}

func populateRememberedVars(str string, rememberMap map[string]interface{}) string {
	res := str
	for varName, val := range rememberMap {
		placeholder := "{" + varName + "}"
		res = strings.Replace(res, placeholder, toString(val), -1)
	}
	return res
}

// toString returns value suitable to insert as an argument
// if value if a float where decimal part is zero - convert to int
func toString(rw interface{}) string {
	var sv interface{} = rw
	if fv, ok := rw.(float64); ok {
		_, frac := math.Modf(fv)
		if frac == 0 {
			sv = int(fv)
		}
	}

	return fmt.Sprintf("%v", sv)
}

func expectations(call Call, suitePath string) ([]ResponseExpectation, error) {
	var exps []ResponseExpectation
	if call.Expect.StatusCode != 0 {
		exps = append(exps, StatusCodeExpectation{statusCode: call.Expect.StatusCode})
	}

	if call.Expect.hasSchema() {
		var (
			schemeURI string
			err       error
		)

		if call.Expect.BodySchemaFile != "" {
			schemeURI, err = toAbsPath(suitePath, call.Expect.BodySchemaFile)
			if err != nil {
				return nil, err
			}
			schemeURI = "file:///" + schemeURI
		}

		if call.Expect.BodySchemaURI != "" {
			isHTTP := strings.HasPrefix(call.Expect.BodySchemaURI, "http://")
			isHTTPS := strings.HasPrefix(call.Expect.BodySchemaURI, "https://")
			if !(isHTTP || isHTTPS) {
				schemeURI = hostFlag + call.Expect.BodySchemaURI
			} else {
				schemeURI = call.Expect.BodySchemaURI
			}
		}
		exps = append(exps, BodySchemaExpectation{schemaURI: schemeURI})
	}

	if len(call.Expect.Body) > 0 {
		exps = append(exps, BodyExpectation{pathExpectations: call.Expect.Body})
	}

	if len(call.Expect.Absent) > 0 {
		exps = append(exps, AbsentExpectation{paths: call.Expect.Absent})
	}

	if len(call.Expect.Headers) > 0 {
		for k, v := range call.Expect.Headers {
			exps = append(exps, HeaderExpectation{Name: k, Value: v})
		}
	}

	if call.Expect.ContentType != "" {
		exps = append(exps, ContentTypeExpectation{call.Expect.ContentType})
	}

	// and so on
	return exps, nil
}

func toAbsPath(suitePath string, assetPath string) (string, error) {
	debug.Printf("Building absolute path using: suiteDir: %s, srcDir: %s, assetPath: %s", suitesDir, suitePath, assetPath)
	if filepath.IsAbs(assetPath) {
		// ignore srcDir
		return assetPath, nil
	}

	uri, err := filepath.Abs(filepath.Join(suitesDir, suitePath, assetPath))
	if err != nil {
		return "", errors.New("Invalid file path: " + assetPath)
	}

	return filepath.ToSlash(uri), nil
}

func rememberBody(resp *Response, remember map[string]string, rememberedMap map[string]interface{}) (err error) {

	for varName, pathLine := range remember {
		body, err := resp.Body()
		if err != nil {
			debug.Print("Can't parse response body to Map for [remember]")
			return err
		}

		if rememberVar, err := GetByPath(body, pathLine); err == nil {
			rememberedMap[varName] = rememberVar
		} else {
			strErr := fmt.Sprintf("Remembered value not found, path: %v", pathLine)
			err = errors.New(strErr)
		}
		//fmt.Printf("v: %v\n", getByPath(bodyMap, b...))
	}

	return err
}

func rememberHeaders(header http.Header, remember map[string]string, rememberedMap map[string]interface{}) {
	for valueName, headerName := range remember {
		value := header.Get(headerName)
		if value == "" {
			continue
		}

		rememberedMap[valueName] = value
	}
}

func printRequestInfo(req *http.Request, body []byte) {
	info.Println()
	info.Printf("%s %s %s\n", req.Method, req.URL.String(), req.Proto)

	if len(req.Header) > 0 {
		info.Println()
	}

	for k, v := range req.Header {
		info.Printf("%s: %s", k, strings.Join(v, " "))
	}
	info.Println()

	if len(body) > 0 {
		info.Printf(string(body))
	}
}

func terminate(msgLines ...string) {
	for _, line := range msgLines {
		fmt.Fprintln(os.Stderr, line)
	}

	os.Exit(1)
}
<commit_msg>version 0.8.9<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)

const (
	version = "0.8.9"
)

func init() {
	flag.Usage = func() {
		h := "Usage:\n"
		h += "  bozr [OPTIONS] (DIR|FILE)\n\n"

		h += "Options:\n"
		h += "  -d, --debug		Enable debug mode\n"
		h += "  -H, --host		Server to test\n"
		h += "  -w, --worker		Execute in parallel with specified number of workers\n"
		h += "  -h, --help		Print usage\n"
		h += "  -i, --info		Enable info mode. Print request and response details\n"
		h += "      --junit		Enable junit xml reporter\n"
		h += "      --junit-output	Destination for junit report files\n"
		h += "  -v, --version		Print version information and quit\n\n"

		h += "Examples:\n"
		h += "  bozr ./examples\n"
		h += "  bozr -w 2 ./examples\n"
		h += "  bozr -H http://example.com ./examples \n"

		fmt.Fprintf(os.Stderr, h)
	}
}

var (
	suitesDir       string
	hostFlag        string
	routinesFlag    int
	infoFlag        bool
	debugFlag       bool
	helpFlag        bool
	versionFlag     bool
	junitFlag       bool
	junitOutputFlag string

	info  *log.Logger
	debug *log.Logger
)

const suiteExt = ".suite.json"

func initLogger() {
	infoHandler := ioutil.Discard
	debugHandler := ioutil.Discard

	if infoFlag {
		infoHandler = os.Stdout
	}

	if debugFlag {
		debugHandler = os.Stdout
	}

	info = log.New(infoHandler, "", 0)
	debug = log.New(debugHandler, "DEBUG: ", log.Ltime|log.Lshortfile)
}

func main() {
	flag.BoolVar(&debugFlag, "d", false, "Enable debug mode.")
	flag.BoolVar(&debugFlag, "debug", false, "Enable debug mode")

	flag.BoolVar(&infoFlag, "i", false, "Enable info mode. Print request and response details.")
	flag.BoolVar(&infoFlag, "info", false, "Enable info mode. Print request and response details.")

	flag.StringVar(&hostFlag, "H", "", "Test server address. Example: http://example.com/api.")
	flag.IntVar(&routinesFlag, "w", 1, "Execute test sutes in parallel with provided numer of workers. Default is 1.")

	flag.BoolVar(&helpFlag, "h", false, "Print usage")
	flag.BoolVar(&helpFlag, "help", false, "Print usage")

	flag.BoolVar(&versionFlag, "v", false, "Print version information and quit")
	flag.BoolVar(&versionFlag, "version", false, "Print version information and quit")

	flag.BoolVar(&junitFlag, "junit", false, "Enable junit xml reporter")
	flag.StringVar(&junitOutputFlag, "junit-output", "./report", "Destination for junit report files. Default ")

	flag.Parse()

	initLogger()

	if versionFlag {
		fmt.Println("bozr version " + version)
		return
	}

	if helpFlag {
		flag.Usage()
		return
	}

	if len(hostFlag) > 0 {
		_, err := url.ParseRequestURI(hostFlag)
		if err != nil {
			terminate("Invalid host is specified.")
			return
		}
	}

	if routinesFlag < 1 || routinesFlag > 9 {
		fmt.Println("Invalid routines parameter [", routinesFlag, "]. Setting to default [1]")
		routinesFlag = 1
	}

	suitesDir = flag.Arg(0)

	if suitesDir == "" {
		terminate("You must specify a directory or file with tests.")
		flag.Usage()
		return
	}

	// check specified source dir/file exists
	_, err := os.Lstat(suitesDir)
	if err != nil {
		terminate(err.Error())
		return
	}

	err = ValidateSuites(suitesDir, suiteExt)
	if err != nil {
		terminate("One or more test suites are invalid.", err.Error())
		return
	}

	loader := NewSuiteLoader(suitesDir, suiteExt)
	reporter := createReporter()

	RunParallel(loader, reporter, runSuite, routinesFlag)
}

func runSuite(suite TestSuite) []TestResult {
	results := []TestResult{}

	for _, testCase := range suite.Cases {

		result := TestResult{
			Suite:     suite,
			Case:      testCase,
			ExecFrame: TimeFrame{Start: time.Now(), End: time.Now()},
		}

		if testCase.Ignore != nil {
			result.Skipped = true
			result.SkippedMsg = *testCase.Ignore

			results = append(results, result)
			continue
		}

		rememberedMap := make(map[string]interface{})
		for _, c := range testCase.Calls {
			addAll(c.Args, rememberedMap)
			terr := call(suite.Dir, c, rememberedMap)
			if terr != nil {
				result.Error = terr
				break
			}
		}

		result.ExecFrame.End = time.Now()

		results = append(results, result)
	}

	return results
}

func addAll(src, target map[string]interface{}) {
	for key, val := range src {
		target[key] = val
	}
}

func createReporter() Reporter {
	reporters := []Reporter{NewConsoleReporter()}
	if junitFlag {
		path, _ := filepath.Abs(junitOutputFlag)
		reporters = append(reporters, NewJUnitReporter(path))
	}
	reporter := NewMultiReporter(reporters...)
	reporter.Init()

	return reporter
}

func call(suitePath string, call Call, rememberMap map[string]interface{}) *TError {

	terr := &TError{}

	on := call.On

	dat := []byte(on.Body)
	if on.BodyFile != "" {
		uri, err := toAbsPath(suitePath, on.BodyFile)
		if err != nil {
			terr.Cause = err
			return terr
		}

		if d, err := ioutil.ReadFile(uri); err == nil {
			dat = d
		} else {
			terr.Cause = fmt.Errorf("Can't read body file: %s", err.Error())
			return terr
		}
	}

	req, err := populateRequest(on, string(dat), rememberMap)
	if err != nil {
		terr.Cause = err
		return terr
	}

	printRequestInfo(req, dat)

	client := &http.Client{}

	resp, err := client.Do(req)

	if err != nil {
		debug.Print("Error when sending request", err)
		terr.Cause = err
		return terr
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		debug.Print("Error reading response")
		terr.Cause = err
		return terr
	}

	testResp := Response{http: *resp, body: body}
	terr.Resp = testResp

	info.Println(strings.Repeat("-", 50))
	info.Println(testResp.ToString())
	info.Println("")

	exps, err := expectations(call, suitePath)
	if err != nil {
		terr.Cause = err
		return terr
	}

	for _, exp := range exps {
		checkErr := exp.check(&testResp)
		if checkErr != nil {
			terr.Cause = checkErr
			return terr
		}
	}

	err = rememberBody(&testResp, call.Remember.Body, rememberMap)
	debug.Print("Remember: ", rememberMap)
	if err != nil {
		debug.Print("Error remember")
		terr.Cause = err
		return terr
	}

	rememberHeaders(testResp.http.Header, call.Remember.Headers, rememberMap)

	return nil
}

func populateRequest(on On, body string, rememberMap map[string]interface{}) (*http.Request, error) {

	urlStr, err := urlPrefix(populateRememberedVars(on.URL, rememberMap))
	if err != nil {
		return nil, errors.New("Cannot create request. Invalid url: " + on.URL)
	}

	body = populateRememberedVars(body, rememberMap)
	dat := []byte(body)

	req, err := http.NewRequest(on.Method, urlStr, bytes.NewBuffer(dat))
	if err != nil {
		return nil, err
	}

	for key, value := range on.Headers {
		req.Header.Add(key, populateRememberedVars(value, rememberMap))
	}

	q := req.URL.Query()
	for key, value := range on.Params {
		q.Add(key, populateRememberedVars(value, rememberMap))
	}
	req.URL.RawQuery = q.Encode()

	return req, nil
}

func urlPrefix(p string) (string, error) {
	if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
		return p, nil
	}

	return concatURL(hostFlag, p)
}

func concatURL(base string, p string) (string, error) {
	baseURL, err := url.ParseRequestURI(base)
	if err != nil {
		return "", err
	}
	return baseURL.Scheme + "://" + baseURL.Host + path.Join(baseURL.Path, p), nil
}

func populateRememberedVars(str string, rememberMap map[string]interface{}) string {
	res := str
	for varName, val := range rememberMap {
		placeholder := "{" + varName + "}"
		res = strings.Replace(res, placeholder, toString(val), -1)
	}
	return res
}

// toString returns value suitable to insert as an argument
// if value if a float where decimal part is zero - convert to int
func toString(rw interface{}) string {
	var sv interface{} = rw
	if fv, ok := rw.(float64); ok {
		_, frac := math.Modf(fv)
		if frac == 0 {
			sv = int(fv)
		}
	}

	return fmt.Sprintf("%v", sv)
}

func expectations(call Call, suitePath string) ([]ResponseExpectation, error) {
	var exps []ResponseExpectation
	if call.Expect.StatusCode != 0 {
		exps = append(exps, StatusCodeExpectation{statusCode: call.Expect.StatusCode})
	}

	if call.Expect.hasSchema() {
		var (
			schemeURI string
			err       error
		)

		if call.Expect.BodySchemaFile != "" {
			schemeURI, err = toAbsPath(suitePath, call.Expect.BodySchemaFile)
			if err != nil {
				return nil, err
			}
			schemeURI = "file:///" + schemeURI
		}

		if call.Expect.BodySchemaURI != "" {
			isHTTP := strings.HasPrefix(call.Expect.BodySchemaURI, "http://")
			isHTTPS := strings.HasPrefix(call.Expect.BodySchemaURI, "https://")
			if !(isHTTP || isHTTPS) {
				schemeURI = hostFlag + call.Expect.BodySchemaURI
			} else {
				schemeURI = call.Expect.BodySchemaURI
			}
		}
		exps = append(exps, BodySchemaExpectation{schemaURI: schemeURI})
	}

	if len(call.Expect.Body) > 0 {
		exps = append(exps, BodyExpectation{pathExpectations: call.Expect.Body})
	}

	if len(call.Expect.Absent) > 0 {
		exps = append(exps, AbsentExpectation{paths: call.Expect.Absent})
	}

	if len(call.Expect.Headers) > 0 {
		for k, v := range call.Expect.Headers {
			exps = append(exps, HeaderExpectation{Name: k, Value: v})
		}
	}

	if call.Expect.ContentType != "" {
		exps = append(exps, ContentTypeExpectation{call.Expect.ContentType})
	}

	// and so on
	return exps, nil
}

func toAbsPath(suitePath string, assetPath string) (string, error) {
	debug.Printf("Building absolute path using: suiteDir: %s, srcDir: %s, assetPath: %s", suitesDir, suitePath, assetPath)
	if filepath.IsAbs(assetPath) {
		// ignore srcDir
		return assetPath, nil
	}

	uri, err := filepath.Abs(filepath.Join(suitesDir, suitePath, assetPath))
	if err != nil {
		return "", errors.New("Invalid file path: " + assetPath)
	}

	return filepath.ToSlash(uri), nil
}

func rememberBody(resp *Response, remember map[string]string, rememberedMap map[string]interface{}) (err error) {

	for varName, pathLine := range remember {
		body, err := resp.Body()
		if err != nil {
			debug.Print("Can't parse response body to Map for [remember]")
			return err
		}

		if rememberVar, err := GetByPath(body, pathLine); err == nil {
			rememberedMap[varName] = rememberVar
		} else {
			strErr := fmt.Sprintf("Remembered value not found, path: %v", pathLine)
			err = errors.New(strErr)
		}
		//fmt.Printf("v: %v\n", getByPath(bodyMap, b...))
	}

	return err
}

func rememberHeaders(header http.Header, remember map[string]string, rememberedMap map[string]interface{}) {
	for valueName, headerName := range remember {
		value := header.Get(headerName)
		if value == "" {
			continue
		}

		rememberedMap[valueName] = value
	}
}

func printRequestInfo(req *http.Request, body []byte) {
	info.Println()
	info.Printf("%s %s %s\n", req.Method, req.URL.String(), req.Proto)

	if len(req.Header) > 0 {
		info.Println()
	}

	for k, v := range req.Header {
		info.Printf("%s: %s", k, strings.Join(v, " "))
	}
	info.Println()

	if len(body) > 0 {
		info.Printf(string(body))
	}
}

func terminate(msgLines ...string) {
	for _, line := range msgLines {
		fmt.Fprintln(os.Stderr, line)
	}

	os.Exit(1)
}
<|endoftext|>