<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/rzh/utils/go/mc/parser"
)

/*
type Testbed struct {
	Type    string                       `json:type`
	Servers map[string]map[string]string `json:"servers"`
}

type TestDriver struct {
	Version   string `json:version`
	GitSHA    string `json:git_hash`
	BuildDate string `json:build_date`
}

type DateTime struct {
	Date int64 `json:"$date"`
}

// definition of Stat to be reported to dashboard
type Stats struct {
	Harness       string                 `json:"harness"`
	Workload      string                 `json:"workload"`
	ServerVersion string                 `json:"server_version"`
	ServerGitSHA  string                 `json:"server_git_hash"`
	Attributes    map[string]interface{} `json:"attributes"`
	Testbed       Testbed                `json:"test_bed"`
	TestDriver    TestDriver             `json:"test_driver"`
	Summary       parser.StatsSummary    `json:"summary"`

	// TPS string
	// Run_Date   string
	Start_Time DateTime `json:"start_time"` //epoch time, time.Now().Unix()
	End_Time   DateTime `json:"end_time"`
	// ID         string
	// Type         string // hammertime, sysbench, mongo-sim
	History      []string
	Server_Stats map[string]parser.ServerStats `json:"server_stats"`
}
*/

func replaceDot(s string) string {
	return strings.Replace(s, ".", "_", -1)
}

// only used for clean report
type HammerTask_ struct {
	Run_id        string   `json:run_id`
	Cmd           string   `json:cmd`
	Hammer_folder string   `json:hammer_folder`
	Clients       []string `json:clients`
	Servers       []string `json:servers`

	// log files to be collected from client and server
	Client_logs []string `json:client_logs`
	Server_logs []string `json:server_logs`

	Type string `json:type`
}

func (r *TheRun) reportResults(run_id int, log_file string, run_dir string) {
	// this is the place to analyze results.
	t := strings.ToLower(r.Runs[run_id].Type)
	// r.Runs[run_id].Stats.Type = t
	r.Runs[run_id].Stats.Harness = t
	// r.Runs[run_id].Stats.ID = r.Runs[run_id].Run_id
	r.Runs[run_id].Stats.Workload = r.Runs[run_id].Run_id

	// cache run first
	//rr, _ := json.Marshal(r.Runs[run_id])
	rr := r.Runs[run_id]
	r.Runs[run_id].Stats.Attributes["run-by"] = "hammer-mc"
	r.Runs[run_id].Stats.Attributes["hammer-mc-cmd"] = HammerTask_{Run_id: rr.Run_id,
		Cmd: rr.Cmd, Clients: rr.Clients, Servers: rr.Servers,
		Client_logs: rr.Client_logs, Server_logs: rr.Server_logs,
		Type: rr.Type}

	if len(report_url) == 0 {
		report_url = append(report_url, "http://54.68.84.192:8080/api/v1/results")
		report_url = append(report_url, "http://dyno.mongodb.parts/api/v1/results")
	}
	var err error
	switch t {
	case "sysbench":
		log.Println("Process sysbench results")
		cum, history, att := parser.ProcessSysbenchResult(log_file)

		// r.Runs[run_id].Stats.TPS = cum
		r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput, err = strconv.ParseFloat(strings.Replace(cum, ",", "", -1), 64)
		if err != nil {
			log.Panicln("Error parsing op_throughput ", cum, ", error: ", err)
		}

		r.Runs[run_id].Stats.History = history

		// merge attribute into Stats
		for k, v := range att {
			r.Runs[run_id].Stats.Attributes[k] = v
		}

	case "sysbench-insert":
		log.Println("Process sysbench results")
		cum, history, att := parser.ProcessSysbenchInsertResult(log_file)

		// r.Runs[run_id].Stats.TPS = cum
		r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput, err = strconv.ParseFloat(strings.Replace(cum, ",", "", -1), 64)
		if err != nil {
			log.Panicln("Error parsing op_throughput ", cum, ", error: ", err)
		}

		r.Runs[run_id].Stats.History = history
		r.Runs[run_id].Stats.Harness = "sysbench" // still keep the same harness name

		// merge attribute into Stats
		for k, v := range att {
			r.Runs[run_id].Stats.Attributes[k] = v
		}
	case "mongo-sim":
		log.Println("Processing mongo-sim results")
		result_ := parser.ProcessMongoSIMResult(log_file)

		// need merge the two Stats together. Will copy
		//	r.Runs[run_id].Stats.Summary = result_.Summary
		// log.Printf("%# v\n", result_.Summary)
		r.Runs[run_id].Stats.Summary.Nodes = make([]map[string]parser.NodeStats, 10, 10)
		copy(r.Runs[run_id].Stats.Summary.Nodes, result_.Summary.Nodes)
		// r.Runs[run_id].Stats.Testbed = result_.Testbed
		r.Runs[run_id].Stats.TestDriver = result_.TestDriver

		// merge attributes together
		for k, v := range result_.Attributes {
			if val, ok := r.Runs[run_id].Stats.Attributes[k]; ok {
				log.Println("Discard hammer-mc attribute[", k, "] = ", val, " with new value from mongo-sim ", v)
			}
			r.Runs[run_id].Stats.Attributes[k] = v
		}

	case "mongo-perf":
		log.Println("Processing mongo-perf results")
		result_ := parser.ProcessMongoPerfResult(log_file)

		for k, v := range result_ {
			_ = k
			_ = v
			/*
				Name    string
				Thread  int64
				Result  float64
				CV      string
				Version string
				GitSHA  string
			*/
			r.Runs[run_id].Stats.Workload = v.Name
			r.Runs[run_id].Stats.Attributes["nThread"] = v.Thread
			r.Runs[run_id].Stats.Attributes["CV"] = v.CV
			fmt.Println("server version is :", v.Version)
			r.Runs[run_id].Stats.ServerVersion = strings.Fields(v.Version)[2]
			r.Runs[run_id].Stats.ServerGitSHA = v.GitSHA
			r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput = v.Result

			// print
			s, _ := json.MarshalIndent(r.Runs[run_id].Stats, "  ", "    ")
			os.Stdout.Write(s)
			fmt.Println("\n********")

			// report to server
			if len(report_url) != 0 {
				// report to report_url if it is not empty
				for _, rurl_ := range report_url {
					r, err := http.Post(rurl_, "application/json", bytes.NewBuffer(s))

					if err == nil {
						log.Println("Submit results to server succeeded with reponse:\n", r)
					} else {
						log.Panicln("Submit results failed with error: ", err)
					}
				}
			}
		}

		// mongo-perf will not report server stats since it is not meaningful
		return

	case "task":
		// will not report task
		return

	default:
		log.Println("no type infor, ignore results analyzing")

		// not return results
		return
	}

	// report pidstat here
	r.Runs[run_id].Stats.Server_Stats = make(map[string]parser.ServerStats)

	for k := 0; k < len(r.Runs[run_id].Servers); k++ {
		pidfile := run_dir + "/pidstat.log--" + r.Runs[run_id].Servers[k]
		stats := parser.ParsePIDStat(pidfile)

		// r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])] = make(ServerStats)
		// r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk] = make([]parser.DataPoint, len(vv))
		// log.Println("++++> ", copy(r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk], vv))
		//append(r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk], vv)
		log.Println("\n\n++++++++-----+++++++++\n\n")
		//r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])].Cpu = make([]parser.DataPoint, len(stats["cpu"]), len(stats["cpu"]))
		// copy(r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])].Cpu, stats["cpu"])
		r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])] = stats
		// log.Println(r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])])
	}

	r.Runs[run_id].Stats.Throughput = r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput
	s, _ := json.MarshalIndent(r.Runs[run_id].Stats, "  ", "    ")
	if len(report_url) != 0 {
		// report to report_url if it is not empty

		for _, rurl := range report_url {
			r, err := http.Post(rurl, "application/json", bytes.NewBuffer(s))

			if err == nil {
				log.Println("Submit results to server [", rurl, "] succeeded with reponse:\n", r)
			} else {
				log.Panicln("Submit results to server [", rurl, "] failed with error: ", err)
			}
		}
	}

	// print
	os.Stdout.Write(s)
	fmt.Println("\n********")
	//fmt.Printf("%# v", pretty.Formatter(r.Runs[run_id].Stats))
}
<commit_msg>recording run id<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/rzh/utils/go/mc/parser"
)

/*
type Testbed struct {
	Type    string                       `json:type`
	Servers map[string]map[string]string `json:"servers"`
}

type TestDriver struct {
	Version   string `json:version`
	GitSHA    string `json:git_hash`
	BuildDate string `json:build_date`
}

type DateTime struct {
	Date int64 `json:"$date"`
}

// definition of Stat to be reported to dashboard
type Stats struct {
	Harness       string                 `json:"harness"`
	Workload      string                 `json:"workload"`
	ServerVersion string                 `json:"server_version"`
	ServerGitSHA  string                 `json:"server_git_hash"`
	Attributes    map[string]interface{} `json:"attributes"`
	Testbed       Testbed                `json:"test_bed"`
	TestDriver    TestDriver             `json:"test_driver"`
	Summary       parser.StatsSummary    `json:"summary"`

	// TPS string
	// Run_Date   string
	Start_Time DateTime `json:"start_time"` //epoch time, time.Now().Unix()
	End_Time   DateTime `json:"end_time"`
	// ID         string
	// Type         string // hammertime, sysbench, mongo-sim
	History      []string
	Server_Stats map[string]parser.ServerStats `json:"server_stats"`
}
*/

func replaceDot(s string) string {
	return strings.Replace(s, ".", "_", -1)
}

// only used for clean report
type HammerTask_ struct {
	Run           string   `json:"run"`
	Run_id        string   `json:"run_id"`
	Cmd           string   `json:"cmd"`
	Hammer_folder string   `json:"hammer_folder"`
	Clients       []string `json:"clients"`
	Servers       []string `json:"servers"`

	// log files to be collected from client and server
	Client_logs []string `json:"client_logs"`
	Server_logs []string `json:"server_logs"`

	Type string `json:"type"`
}

func (r *TheRun) reportResults(run_id int, log_file string, run_dir string) {
	// this is the place to analyze results.
	t := strings.ToLower(r.Runs[run_id].Type)
	// r.Runs[run_id].Stats.Type = t
	r.Runs[run_id].Stats.Harness = t
	// r.Runs[run_id].Stats.ID = r.Runs[run_id].Run_id
	r.Runs[run_id].Stats.Workload = r.Runs[run_id].Run_id

	// cache run first
	//rr, _ := json.Marshal(r.Runs[run_id])
	rr := r.Runs[run_id]
	r.Runs[run_id].Stats.Attributes["run-by"] = "hammer-mc"
	r.Runs[run_id].Stats.Attributes["hammer-mc-cmd"] = HammerTask_{Run_id: rr.Run_id,
		Cmd: rr.Cmd, Clients: rr.Clients, Servers: rr.Servers,
		Client_logs: rr.Client_logs, Server_logs: rr.Server_logs,
		Type: rr.Type, Run: run}

	if len(report_url) == 0 {
		report_url = append(report_url, "http://54.68.84.192:8080/api/v1/results")
		report_url = append(report_url, "http://dyno.mongodb.parts/api/v1/results")
	}
	var err error
	switch t {
	case "sysbench":
		log.Println("Process sysbench results")
		cum, history, att := parser.ProcessSysbenchResult(log_file)

		// r.Runs[run_id].Stats.TPS = cum
		r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput, err = strconv.ParseFloat(strings.Replace(cum, ",", "", -1), 64)
		if err != nil {
			log.Panicln("Error parsing op_throughput ", cum, ", error: ", err)
		}

		r.Runs[run_id].Stats.History = history

		// merge attribute into Stats
		for k, v := range att {
			r.Runs[run_id].Stats.Attributes[k] = v
		}

	case "sysbench-insert":
		log.Println("Process sysbench results")
		cum, history, att := parser.ProcessSysbenchInsertResult(log_file)

		// r.Runs[run_id].Stats.TPS = cum
		r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput, err = strconv.ParseFloat(strings.Replace(cum, ",", "", -1), 64)
		if err != nil {
			log.Panicln("Error parsing op_throughput ", cum, ", error: ", err)
		}

		r.Runs[run_id].Stats.History = history
		r.Runs[run_id].Stats.Harness = "sysbench" // still keep the same harness name

		// merge attribute into Stats
		for k, v := range att {
			r.Runs[run_id].Stats.Attributes[k] = v
		}
	case "mongo-sim":
		log.Println("Processing mongo-sim results")
		result_ := parser.ProcessMongoSIMResult(log_file)

		// need merge the two Stats together. Will copy
		//	r.Runs[run_id].Stats.Summary = result_.Summary
		// log.Printf("%# v\n", result_.Summary)
		r.Runs[run_id].Stats.Summary.Nodes = make([]map[string]parser.NodeStats, 10, 10)
		copy(r.Runs[run_id].Stats.Summary.Nodes, result_.Summary.Nodes)
		// r.Runs[run_id].Stats.Testbed = result_.Testbed
		r.Runs[run_id].Stats.TestDriver = result_.TestDriver

		// merge attributes together
		for k, v := range result_.Attributes {
			if val, ok := r.Runs[run_id].Stats.Attributes[k]; ok {
				log.Println("Discard hammer-mc attribute[", k, "] = ", val, " with new value from mongo-sim ", v)
			}
			r.Runs[run_id].Stats.Attributes[k] = v
		}

	case "mongo-perf":
		log.Println("Processing mongo-perf results")
		result_ := parser.ProcessMongoPerfResult(log_file)

		for k, v := range result_ {
			_ = k
			_ = v
			/*
				Name    string
				Thread  int64
				Result  float64
				CV      string
				Version string
				GitSHA  string
			*/
			r.Runs[run_id].Stats.Workload = v.Name
			r.Runs[run_id].Stats.Attributes["nThread"] = v.Thread
			r.Runs[run_id].Stats.Attributes["CV"] = v.CV
			fmt.Println("server version is :", v.Version)
			r.Runs[run_id].Stats.ServerVersion = strings.Fields(v.Version)[2]
			r.Runs[run_id].Stats.ServerGitSHA = v.GitSHA
			r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput = v.Result

			// print
			s, _ := json.MarshalIndent(r.Runs[run_id].Stats, "  ", "    ")
			os.Stdout.Write(s)
			fmt.Println("\n********")

			// report to server
			if len(report_url) != 0 {
				// report to report_url if it is not empty
				for _, rurl_ := range report_url {
					r, err := http.Post(rurl_, "application/json", bytes.NewBuffer(s))

					if err == nil {
						log.Println("Submit results to server succeeded with reponse:\n", r)
					} else {
						log.Panicln("Submit results failed with error: ", err)
					}
				}
			}
		}

		// mongo-perf will not report server stats since it is not meaningful
		return

	case "task":
		// will not report task
		return

	default:
		log.Println("no type infor, ignore results analyzing")

		// not return results
		return
	}

	// report pidstat here
	r.Runs[run_id].Stats.Server_Stats = make(map[string]parser.ServerStats)

	for k := 0; k < len(r.Runs[run_id].Servers); k++ {
		pidfile := run_dir + "/pidstat.log--" + r.Runs[run_id].Servers[k]
		stats := parser.ParsePIDStat(pidfile)

		// r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])] = make(ServerStats)
		// r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk] = make([]parser.DataPoint, len(vv))
		// log.Println("++++> ", copy(r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk], vv))
		//append(r.Runs[run_id].Stats.Server_Stats[r.Runs[run_id].Servers[k]][kk], vv)
		log.Println("\n\n++++++++-----+++++++++\n\n")
		//r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])].Cpu = make([]parser.DataPoint, len(stats["cpu"]), len(stats["cpu"]))
		// copy(r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])].Cpu, stats["cpu"])
		r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])] = stats
		// log.Println(r.Runs[run_id].Stats.Server_Stats[replaceDot(r.Runs[run_id].Servers[k])])
	}

	r.Runs[run_id].Stats.Throughput = r.Runs[run_id].Stats.Summary.AllNodes.Op_throughput
	s, _ := json.MarshalIndent(r.Runs[run_id].Stats, "  ", "    ")
	if len(report_url) != 0 {
		// report to report_url if it is not empty

		for _, rurl := range report_url {
			r, err := http.Post(rurl, "application/json", bytes.NewBuffer(s))

			if err == nil {
				log.Println("Submit results to server [", rurl, "] succeeded with reponse:\n", r)
			} else {
				log.Panicln("Submit results to server [", rurl, "] failed with error: ", err)
			}
		}
	}

	// print
	os.Stdout.Write(s)
	fmt.Println("\n********")
	//fmt.Printf("%# v", pretty.Formatter(r.Runs[run_id].Stats))
}
<|endoftext|>


<commit_before>package goat

import (
	"encoding/json"
	"io"
	"os"
	"os/user"
)

// DBConf represents database configuration
type DBConf struct {
	Database string
	Username string
	Password string
}

// Conf represents server configuration
type Conf struct {
	Port      int
	Passkey   bool
	Whitelist bool
	Interval  int
	HTTP      bool
	UDP       bool
	DB        DBConf
}

// LoadConfig loads configuration
func LoadConfig() Conf {
	// Configuration path
	var path string
	config := "config.json"

	// Load current user from OS, to get home directory
	user, err := user.Current()
	if err != nil {
		Static.LogChan <- err.Error()
		path = "."
	} else {
		// Store config in standard location
		path = user.HomeDir + "/.config/goat/"
	}

	Static.LogChan <- "Loading configuration: " + path + config

	// Check file existence
	_, err = os.Stat(path + config)
	if err != nil {
		if os.IsNotExist(err) {
			Static.LogChan <- "Could not find configuration, attempting to create it..."

			err = os.MkdirAll(path, 0775)
			if err != nil {
				Static.LogChan <- "Failed to create directory: " + path
			}

			// Attempt to copy config to home directory
			source, err := os.Open(config)
			if err != nil {
				Static.LogChan <- "Failed to read source file: " + config
			}

			// Open destination file
			dest, err := os.Create(path + config)
			if err != nil {
				Static.LogChan <- "Failed to create destination file: " + path + config
			}

			// Copy contents
			_, err = io.Copy(dest, source)
			if err != nil {
				Static.LogChan <- "Failed to copy to configuration file: " + path + config
			}

			// Close files
			source.Close()
			dest.Close()
		}
	}

	// Load configuration file
	conf := Conf{}
	configFile, err := os.Open(path + config)
	if err == nil {
		// Decode JSON
		err = json.NewDecoder(configFile).Decode(&conf)
		if err != nil {
			Static.LogChan <- "Could not parse config.json"
			return Conf{}
		}
	} else {
		Static.LogChan <- "Failed to open config.json"
		return Conf{}
	}

	return conf
}
<commit_msg>Simplify logic in config<commit_after>package goat

import (
	"encoding/json"
	"io"
	"os"
	"os/user"
)

// DBConf represents database configuration
type DBConf struct {
	Database string
	Username string
	Password string
}

// Conf represents server configuration
type Conf struct {
	Port      int
	Passkey   bool
	Whitelist bool
	Interval  int
	HTTP      bool
	UDP       bool
	DB        DBConf
}

// LoadConfig loads configuration
func LoadConfig() Conf {
	// Configuration path
	var path string
	config := "config.json"

	// Load current user from OS, to get home directory
	user, err := user.Current()
	if err != nil {
		Static.LogChan <- err.Error()
		path = "."
	} else {
		// Store config in standard location
		path = user.HomeDir + "/.config/goat/"
	}

	Static.LogChan <- "Loading configuration: " + path + config

	// Check file existence
	_, err = os.Stat(path + config)
	if err != nil {
		if os.IsNotExist(err) {
			Static.LogChan <- "Could not find configuration, attempting to create it..."

			err = os.MkdirAll(path, 0775)
			if err != nil {
				Static.LogChan <- "Failed to create directory: " + path
			}

			// Attempt to copy config to home directory
			source, err := os.Open(config)
			if err != nil {
				Static.LogChan <- "Failed to read source file: " + config
			}

			// Open destination file
			dest, err := os.Create(path + config)
			if err != nil {
				Static.LogChan <- "Failed to create destination file: " + path + config
			}

			// Copy contents
			_, err = io.Copy(dest, source)
			if err != nil {
				Static.LogChan <- "Failed to copy to configuration file: " + path + config
			}

			// Close files
			source.Close()
			dest.Close()
		}
	}

	// Load configuration file
	conf := Conf{}
	configFile, err := os.Open(path + config)
	if err != nil {
		Static.LogChan <- "Failed to open config.json"
		return Conf{}
	}

	// Decode JSON
	err = json.NewDecoder(configFile).Decode(&conf)
	if err != nil {
		Static.LogChan <- "Could not parse config.json"
		return Conf{}
	}

	return conf
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/urfave/cli.v2"
)

func main() {
	app := &cli.App{
		Name:    "godockerize",
		Usage:   "build Docker images from Go packages",
		Version: "0.0.1",
		Commands: []*cli.Command{
			{
				Name:      "build",
				Usage:     "build a Docker image from a Go package",
				ArgsUsage: "[package]",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "tag",
						Aliases: []string{"t"},
						Usage:   "output Docker image name and optionally a tag in the 'name:tag' format",
					},
					&cli.StringFlag{
						Name:  "base",
						Usage: "base Docker image name",
						Value: "alpine:3.4",
					},
				},
				Action: doBuild,
			},
		},
	}
	app.Run(os.Args)
}

func doBuild(c *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	args := c.Args()
	if args.Len() != 1 {
		return errors.New(`"godockerize build" requires exactly 1 argument`)
	}

	pkg, err := build.Import(args.First(), wd, 0)
	if err != nil {
		return err
	}

	tmpdir, err := ioutil.TempDir("", "godockerize")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpdir)
	binname := path.Base(pkg.ImportPath)

	fset := token.NewFileSet()
	var expose []string
	var install []string
	var run []string
	for _, name := range pkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(pkg.Dir, name), nil, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, cg := range f.Comments {
			for _, c := range cg.List {
				if strings.HasPrefix(c.Text, "//docker:") {
					parts := strings.SplitN(c.Text[9:], " ", 2)
					switch parts[0] {
					case "expose":
						expose = append(expose, strings.Fields(parts[1])...)
					case "install":
						install = append(install, strings.Fields(parts[1])...)
					case "run":
						run = append(run, parts[1])
					default:
						return fmt.Errorf("%s: invalid docker comment: %s", fset.Position(c.Pos()), c.Text)
					}
				}
			}
		}
	}

	var dockerfile bytes.Buffer
	fmt.Fprintf(&dockerfile, "  FROM %s\n", c.String("base"))

	for _, pkg := range install {
		if strings.HasSuffix(pkg, "@edge") {
			fmt.Fprintf(&dockerfile, "  RUN echo -e \"@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\\n@edge http://dl-cdn.alpinelinux.org/alpine/edge/community\" >> /etc/apk/repositories\n")
			break
		}
	}
	if len(install) != 0 {
		fmt.Fprintf(&dockerfile, "  RUN apk add --no-cache %s\n", strings.Join(sortedStringSet(install), " "))
	}

	for _, cmd := range run {
		fmt.Fprintf(&dockerfile, "  RUN %s\n", cmd)
	}
	if len(expose) != 0 {
		fmt.Fprintf(&dockerfile, "  EXPOSE %s\n", strings.Join(sortedStringSet(expose), " "))
	}
	fmt.Fprintf(&dockerfile, "  ENTRYPOINT [\"/usr/local/bin/%s\"]\n", binname)
	fmt.Fprintf(&dockerfile, "  ADD %s /usr/local/bin/\n", binname)

	fmt.Println("godockerize: Generated Dockerfile:")
	fmt.Print(dockerfile.String())

	ioutil.WriteFile(filepath.Join(tmpdir, "Dockerfile"), dockerfile.Bytes(), 0777)
	if err != nil {
		return err
	}

	fmt.Println("godockerize: Building Go binary...")
	cmd := exec.Command("go", "build", "-o", binname, pkg.ImportPath)
	cmd.Dir = tmpdir
	cmd.Env = []string{
		"GOARCH=amd64",
		"GOOS=linux",
		"GOROOT=" + build.Default.GOROOT,
		"GOPATH=" + build.Default.GOPATH,
		"CGO_ENABLED=0",
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	fmt.Println("godockerize: Building Docker image...")
	cmd = exec.Command("docker", "build", "-t", c.String("tag"), ".")
	cmd.Dir = tmpdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func sortedStringSet(in []string) []string {
	set := make(map[string]struct{})
	for _, s := range in {
		set[s] = struct{}{}
	}
	var out []string
	for s := range set {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}
<commit_msg>allow empty tag parameter<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/urfave/cli.v2"
)

func main() {
	app := &cli.App{
		Name:    "godockerize",
		Usage:   "build Docker images from Go packages",
		Version: "0.0.1",
		Commands: []*cli.Command{
			{
				Name:      "build",
				Usage:     "build a Docker image from a Go package",
				ArgsUsage: "[package]",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "tag",
						Aliases: []string{"t"},
						Usage:   "output Docker image name and optionally a tag in the 'name:tag' format",
					},
					&cli.StringFlag{
						Name:  "base",
						Usage: "base Docker image name",
						Value: "alpine:3.4",
					},
				},
				Action: doBuild,
			},
		},
	}
	app.Run(os.Args)
}

func doBuild(c *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	args := c.Args()
	if args.Len() != 1 {
		return errors.New(`"godockerize build" requires exactly 1 argument`)
	}

	pkg, err := build.Import(args.First(), wd, 0)
	if err != nil {
		return err
	}

	tmpdir, err := ioutil.TempDir("", "godockerize")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpdir)
	binname := path.Base(pkg.ImportPath)

	fset := token.NewFileSet()
	var expose []string
	var install []string
	var run []string
	for _, name := range pkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(pkg.Dir, name), nil, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, cg := range f.Comments {
			for _, c := range cg.List {
				if strings.HasPrefix(c.Text, "//docker:") {
					parts := strings.SplitN(c.Text[9:], " ", 2)
					switch parts[0] {
					case "expose":
						expose = append(expose, strings.Fields(parts[1])...)
					case "install":
						install = append(install, strings.Fields(parts[1])...)
					case "run":
						run = append(run, parts[1])
					default:
						return fmt.Errorf("%s: invalid docker comment: %s", fset.Position(c.Pos()), c.Text)
					}
				}
			}
		}
	}

	var dockerfile bytes.Buffer
	fmt.Fprintf(&dockerfile, "  FROM %s\n", c.String("base"))

	for _, pkg := range install {
		if strings.HasSuffix(pkg, "@edge") {
			fmt.Fprintf(&dockerfile, "  RUN echo -e \"@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\\n@edge http://dl-cdn.alpinelinux.org/alpine/edge/community\" >> /etc/apk/repositories\n")
			break
		}
	}
	if len(install) != 0 {
		fmt.Fprintf(&dockerfile, "  RUN apk add --no-cache %s\n", strings.Join(sortedStringSet(install), " "))
	}

	for _, cmd := range run {
		fmt.Fprintf(&dockerfile, "  RUN %s\n", cmd)
	}
	if len(expose) != 0 {
		fmt.Fprintf(&dockerfile, "  EXPOSE %s\n", strings.Join(sortedStringSet(expose), " "))
	}
	fmt.Fprintf(&dockerfile, "  ENTRYPOINT [\"/usr/local/bin/%s\"]\n", binname)
	fmt.Fprintf(&dockerfile, "  ADD %s /usr/local/bin/\n", binname)

	fmt.Println("godockerize: Generated Dockerfile:")
	fmt.Print(dockerfile.String())

	ioutil.WriteFile(filepath.Join(tmpdir, "Dockerfile"), dockerfile.Bytes(), 0777)
	if err != nil {
		return err
	}

	fmt.Println("godockerize: Building Go binary...")
	cmd := exec.Command("go", "build", "-o", binname, pkg.ImportPath)
	cmd.Dir = tmpdir
	cmd.Env = []string{
		"GOARCH=amd64",
		"GOOS=linux",
		"GOROOT=" + build.Default.GOROOT,
		"GOPATH=" + build.Default.GOPATH,
		"CGO_ENABLED=0",
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	fmt.Println("godockerize: Building Docker image...")
	dockerArgs := []string{"build"}
	if tag := c.String("tag"); tag != "" {
		dockerArgs = append(dockerArgs, "-t", tag)
	}
	dockerArgs = append(dockerArgs, ".")
	cmd = exec.Command("docker", dockerArgs...)
	cmd.Dir = tmpdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func sortedStringSet(in []string) []string {
	set := make(map[string]struct{})
	for _, s := range in {
		set[s] = struct{}{}
	}
	var out []string
	for s := range set {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}
<|endoftext|>


<commit_before>package gomodometer

import (
	"errors"
	"fmt"
	"os"
)

const conversionCoefficient float64 = 0.000254

type MouseOdometer struct {
	deviceFile *os.File
	readings   chan []byte
	quit       chan struct{}
	x, y       int
}

func NewMouseOdometer(deviceFileName string) *MouseOdometer {
	f, err := os.Open(deviceFileName)
	if err != nil {
		return nil
	}
	return &MouseOdometer{deviceFile: f}
}

func (o *MouseOdometer) Start() {
	o.setup()
	go o.readMouseDevice()
	go o.sumReadingsUntilQuit()
}

func (o *MouseOdometer) Stop() (float64, float64) {
	if o.quit != nil {
		o.quit <- struct{}{}
	}
	return convertToCentimeters(o.x), convertToCentimeters(o.y)
}

func (o *MouseOdometer) setup() {
	o.x, o.y = 0.0, 0.0
	o.readings = make(chan []byte)
	o.quit = make(chan struct{})
}

func (o *MouseOdometer) readMouseDevice() {
	buffer := make([]byte, 3)
	for {
		num, err := o.deviceFile.Read(buffer)
		if err != nil || num != 3 {
			panic(errors.New("problem reading file"))
		}
		o.readings <- buffer
	}
}

func (o *MouseOdometer) sumReadingsUntilQuit() {
	for {
		select {
		case <-o.quit:
			return
		case reading := <-o.readings:
			o.x += normalizeReading(reading[1])
			o.y += normalizeReading(reading[2])
			fmt.Println(convertToCentimeters(o.x), " ", convertToCentimeters(o.y)) // gnuplot> plot 'mouse.dat' u 1:2 w l
		}
	}
}

func normalizeReading(raw byte) int {
	return int(raw) - 128
}

func convertToCentimeters(raw int) float64 {
	return float64(raw) * conversionCoefficient
}
<commit_msg>commented out printlns<commit_after>package gomodometer

import (
	"errors"
	"os"
)

const conversionCoefficient float64 = 0.000254

type MouseOdometer struct {
	deviceFile *os.File
	readings   chan []byte
	quit       chan struct{}
	x, y       int
}

func NewMouseOdometer(deviceFileName string) *MouseOdometer {
	f, err := os.Open(deviceFileName)
	if err != nil {
		return nil
	}
	return &MouseOdometer{deviceFile: f}
}

func (o *MouseOdometer) Start() {
	o.setup()
	go o.readMouseDevice()
	go o.sumReadingsUntilQuit()
}

func (o *MouseOdometer) Stop() (float64, float64) {
	if o.quit != nil {
		o.quit <- struct{}{}
	}
	return convertToCentimeters(o.x), convertToCentimeters(o.y)
}

func (o *MouseOdometer) setup() {
	o.x, o.y = 0.0, 0.0
	o.readings = make(chan []byte)
	o.quit = make(chan struct{})
}

func (o *MouseOdometer) readMouseDevice() {
	buffer := make([]byte, 3)
	for {
		num, err := o.deviceFile.Read(buffer)
		if err != nil || num != 3 {
			panic(errors.New("problem reading file"))
		}
		o.readings <- buffer
	}
}

func (o *MouseOdometer) sumReadingsUntilQuit() {
	for {
		select {
		case <-o.quit:
			return
		case reading := <-o.readings:
			o.x += normalizeReading(reading[1])
			o.y += normalizeReading(reading[2])
			//fmt.Println(convertToCentimeters(o.x), " ", convertToCentimeters(o.y)) // gnuplot> plot 'mouse.dat' u 1:2 w l
		}
	}
}

func normalizeReading(raw byte) int {
	return int(raw) - 128
}

func convertToCentimeters(raw int) float64 {
	return float64(raw) * conversionCoefficient
}
<|endoftext|>


<commit_before>package plugin

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
)

// If this is true, then the "unexpected EOF" panic will not be
// raised throughout the clients. This is very rarely set, and is set
// by CleanupClients.
var Killed = false

// This is a slice of the "managed" clients which are cleaned up when
// calling Cleanup
var managedClients = make([]*Client, 0, 5)

// Client handles the lifecycle of a plugin application, determining its
// RPC address, and returning various types of interface implementations
// across the multi-process communication layer.
type Client struct {
	config      *ClientConfig
	exited      bool
	doneLogging chan struct{}
	l           sync.Mutex
	address     net.Addr
	client      *RPCClient
}

// ClientConfig is the configuration used to initialize a new
// plugin client. After being used to initialize a plugin client,
// that configuration must not be modified again.
type ClientConfig struct {
	// HandshakeConfig is the configuration that must match servers.
	HandshakeConfig

	// Plugins are the plugins that can be consumed.
	Plugins map[string]Plugin

	// The unstarted subprocess for starting the plugin.
	Cmd *exec.Cmd

	// Managed represents if the client should be managed by the
	// plugin package or not. If true, then by calling CleanupClients,
	// it will automatically be cleaned up. Otherwise, the client
	// user is fully responsible for making sure to Kill all plugin
	// clients. By default the client is _not_ managed.
	Managed bool

	// The minimum and maximum port to use for communicating with
	// the subprocess. If not set, this defaults to 10,000 and 25,000
	// respectively.
	MinPort, MaxPort uint

	// StartTimeout is the timeout to wait for the plugin to say it
	// has started successfully.
	StartTimeout time.Duration

	// If non-nil, then the stderr of the client will be written to here
	// (as well as the log). This is the original os.Stderr of the subprocess.
	// This isn't the output of synced stderr.
	Stderr io.Writer

	// SyncStdout, SyncStderr can be set to override the
	// respective os.Std* values in the plugin. Care should be taken to
	// avoid races here. If these are nil, then this will automatically be
	// hooked up to os.Stdin, Stdout, and Stderr, respectively.
	//
	// If the default values (nil) are used, then this package will not
	// sync any of these streams.
	SyncStdout io.Writer
	SyncStderr io.Writer
}

// This makes sure all the managed subprocesses are killed and properly
// logged. This should be called before the parent process running the
// plugins exits.
//
// This must only be called _once_.
func CleanupClients() {
	// Set the killed to true so that we don't get unexpected panics
	Killed = true

	// Kill all the managed clients in parallel and use a WaitGroup
	// to wait for them all to finish up.
	var wg sync.WaitGroup
	for _, client := range managedClients {
		wg.Add(1)

		go func(client *Client) {
			client.Kill()
			wg.Done()
		}(client)
	}

	log.Println("[DEBUG] waiting for all plugin processes to complete...")
	wg.Wait()
}

// Creates a new plugin client which manages the lifecycle of an external
// plugin and gets the address for the RPC connection.
//
// The client must be cleaned up at some point by calling Kill(). If
// the client is a managed client (created with NewManagedClient) you
// can just call CleanupClients at the end of your program and they will
// be properly cleaned.
func NewClient(config *ClientConfig) (c *Client) {
	if config.MinPort == 0 && config.MaxPort == 0 {
		config.MinPort = 10000
		config.MaxPort = 25000
	}

	if config.StartTimeout == 0 {
		config.StartTimeout = 1 * time.Minute
	}

	if config.Stderr == nil {
		config.Stderr = ioutil.Discard
	}

	if config.SyncStdout == nil {
		config.SyncStdout = ioutil.Discard
	}
	if config.SyncStderr == nil {
		config.SyncStderr = ioutil.Discard
	}

	c = &Client{config: config}
	if config.Managed {
		managedClients = append(managedClients, c)
	}

	return
}

// Client returns an RPC client for the plugin.
//
// Subsequent calls to this will return the same RPC client.
func (c *Client) Client() (*RPCClient, error) {
	addr, err := c.Start()
	if err != nil {
		return nil, err
	}

	c.l.Lock()
	defer c.l.Unlock()

	if c.client != nil {
		return c.client, nil
	}

	// Connect to the client
	conn, err := net.Dial(addr.Network(), addr.String())
	if err != nil {
		return nil, err
	}
	if tcpConn, ok := conn.(*net.TCPConn); ok {
		// Make sure to set keep alive so that the connection doesn't die
		tcpConn.SetKeepAlive(true)
	}

	// Create the actual RPC client
	c.client, err = NewRPCClient(conn, c.config.Plugins)
	if err != nil {
		conn.Close()
		return nil, err
	}

	// Begin the stream syncing so that stdin, out, err work properly
	err = c.client.SyncStreams(
		c.config.SyncStdout,
		c.config.SyncStderr)
	if err != nil {
		c.client.Close()
		c.client = nil
		return nil, err
	}

	return c.client, nil
}

// Tells whether or not the underlying process has exited.
func (c *Client) Exited() bool {
	c.l.Lock()
	defer c.l.Unlock()
	return c.exited
}

// End the executing subprocess (if it is running) and perform any cleanup
// tasks necessary such as capturing any remaining logs and so on.
//
// This method blocks until the process successfully exits.
//
// This method can safely be called multiple times.
func (c *Client) Kill() {
	cmd := c.config.Cmd

	if cmd.Process == nil {
		return
	}

	cmd.Process.Kill()

	// Wait for the client to finish logging so we have a complete log
	<-c.doneLogging
}

// Starts the underlying subprocess, communicating with it to negotiate
// a port for RPC connections, and returning the address to connect via RPC.
//
// This method is safe to call multiple times. Subsequent calls have no effect.
// Once a client has been started once, it cannot be started again, even if
// it was killed.
func (c *Client) Start() (addr net.Addr, err error) {
	c.l.Lock()
	defer c.l.Unlock()

	if c.address != nil {
		return c.address, nil
	}

	c.doneLogging = make(chan struct{})

	env := []string{
		fmt.Sprintf("%s=%s", c.config.MagicCookieKey, c.config.MagicCookieValue),
		fmt.Sprintf("PLUGIN_MIN_PORT=%d", c.config.MinPort),
		fmt.Sprintf("PLUGIN_MAX_PORT=%d", c.config.MaxPort),
	}

	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	cmd := c.config.Cmd
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, env...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = stderr_w
	cmd.Stdout = stdout_w

	log.Printf("[DEBUG] Starting plugin: %s %#v", cmd.Path, cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}

	// Make sure the command is properly cleaned up if there is an error
	defer func() {
		r := recover()

		if err != nil || r != nil {
			cmd.Process.Kill()
		}

		if r != nil {
			panic(r)
		}
	}()

	// Start goroutine to wait for process to exit
	exitCh := make(chan struct{})
	go func() {
		// Make sure we close the write end of our stderr/stdout so
		// that the readers send EOF properly.
		defer stderr_w.Close()
		defer stdout_w.Close()

		// Wait for the command to end.
		cmd.Wait()

		// Log and make sure to flush the logs write away
		log.Printf("[DEBUG] %s: plugin process exited\n", cmd.Path)
		os.Stderr.Sync()

		// Mark that we exited
		close(exitCh)

		// Set that we exited, which takes a lock
		c.l.Lock()
		defer c.l.Unlock()
		c.exited = true
	}()

	// Start goroutine that logs the stderr
	go c.logStderr(stderr_r)

	// Start a goroutine that is going to be reading the lines
	// out of stdout
	linesCh := make(chan []byte)
	go func() {
		defer close(linesCh)

		buf := bufio.NewReader(stdout_r)
		for {
			line, err := buf.ReadBytes('\n')
			if line != nil {
				linesCh <- line
			}

			if err == io.EOF {
				return
			}
		}
	}()

	// Make sure after we exit we read the lines from stdout forever
	// so they don't block since it is an io.Pipe
	defer func() {
		go func() {
			for _ = range linesCh {
			}
		}()
	}()

	// Some channels for the next step
	timeout := time.After(c.config.StartTimeout)

	// Start looking for the address
	log.Printf("[DEBUG] Waiting for RPC address for: %s", cmd.Path)
	select {
	case <-timeout:
		err = errors.New("timeout while waiting for plugin to start")
	case <-exitCh:
		err = errors.New("plugin exited before we could connect")
	case lineBytes := <-linesCh:
		// Trim the line and split by "|" in order to get the parts of
		// the output.
		line := strings.TrimSpace(string(lineBytes))
		parts := strings.SplitN(line, "|", 3)
		if len(parts) < 3 {
			err = fmt.Errorf("Unrecognized remote plugin message: %s", line)
			return
		}

		// Parse the protocol version
		var protocol int64
		protocol, err = strconv.ParseInt(parts[0], 10, 0)
		if err != nil {
			err = fmt.Errorf("Error parsing protocol version: %s", err)
			return
		}

		// Test the API version
		if uint(protocol) != c.config.ProtocolVersion {
			err = fmt.Errorf("Incompatible API version with plugin. "+
				"Plugin version: %s, Ours: %d", parts[0], c.config.ProtocolVersion)
			return
		}

		switch parts[1] {
		case "tcp":
			addr, err = net.ResolveTCPAddr("tcp", parts[2])
		case "unix":
			addr, err = net.ResolveUnixAddr("unix", parts[2])
		default:
			err = fmt.Errorf("Unknown address type: %s", parts[1])
		}
	}

	c.address = addr
	return
}

func (c *Client) logStderr(r io.Reader) {
	bufR := bufio.NewReader(r)
	for {
		line, err := bufR.ReadString('\n')
		if line != "" {
			c.config.Stderr.Write([]byte(line))

			line = strings.TrimRightFunc(line, unicode.IsSpace)
			log.Printf("[DEBUG] %s: %s", filepath.Base(c.config.Cmd.Path), line)
		}

		if err == io.EOF {
			break
		}
	}

	// Flag that we've completed logging for others
	close(c.doneLogging)
}
<commit_msg>Killed should be sync/atomic<commit_after>package plugin

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"unicode"
)

// If this is 1, then we've called CleanupClients. This can be used
// by plugin RPC implementations to change error behavior since you
// can expected network connection errors at this point. This should be
// read by using sync/atomic.
var Killed uint32 = 0

// This is a slice of the "managed" clients which are cleaned up when
// calling Cleanup
var managedClients = make([]*Client, 0, 5)

// Client handles the lifecycle of a plugin application, determining its
// RPC address, and returning various types of interface implementations
// across the multi-process communication layer.
type Client struct {
	config      *ClientConfig
	exited      bool
	doneLogging chan struct{}
	l           sync.Mutex
	address     net.Addr
	client      *RPCClient
}

// ClientConfig is the configuration used to initialize a new
// plugin client. After being used to initialize a plugin client,
// that configuration must not be modified again.
type ClientConfig struct {
	// HandshakeConfig is the configuration that must match servers.
	HandshakeConfig

	// Plugins are the plugins that can be consumed.
	Plugins map[string]Plugin

	// The unstarted subprocess for starting the plugin.
	Cmd *exec.Cmd

	// Managed represents if the client should be managed by the
	// plugin package or not. If true, then by calling CleanupClients,
	// it will automatically be cleaned up. Otherwise, the client
	// user is fully responsible for making sure to Kill all plugin
	// clients. By default the client is _not_ managed.
	Managed bool

	// The minimum and maximum port to use for communicating with
	// the subprocess. If not set, this defaults to 10,000 and 25,000
	// respectively.
	MinPort, MaxPort uint

	// StartTimeout is the timeout to wait for the plugin to say it
	// has started successfully.
	StartTimeout time.Duration

	// If non-nil, then the stderr of the client will be written to here
	// (as well as the log). This is the original os.Stderr of the subprocess.
	// This isn't the output of synced stderr.
	Stderr io.Writer

	// SyncStdout, SyncStderr can be set to override the
	// respective os.Std* values in the plugin. Care should be taken to
	// avoid races here. If these are nil, then this will automatically be
	// hooked up to os.Stdin, Stdout, and Stderr, respectively.
	//
	// If the default values (nil) are used, then this package will not
	// sync any of these streams.
	SyncStdout io.Writer
	SyncStderr io.Writer
}

// This makes sure all the managed subprocesses are killed and properly
// logged. This should be called before the parent process running the
// plugins exits.
//
// This must only be called _once_.
func CleanupClients() {
	// Set the killed to true so that we don't get unexpected panics
	atomic.StoreUint32(&Killed, 1)

	// Kill all the managed clients in parallel and use a WaitGroup
	// to wait for them all to finish up.
	var wg sync.WaitGroup
	for _, client := range managedClients {
		wg.Add(1)

		go func(client *Client) {
			client.Kill()
			wg.Done()
		}(client)
	}

	log.Println("[DEBUG] waiting for all plugin processes to complete...")
	wg.Wait()
}

// Creates a new plugin client which manages the lifecycle of an external
// plugin and gets the address for the RPC connection.
//
// The client must be cleaned up at some point by calling Kill(). If
// the client is a managed client (created with NewManagedClient) you
// can just call CleanupClients at the end of your program and they will
// be properly cleaned.
func NewClient(config *ClientConfig) (c *Client) {
	if config.MinPort == 0 && config.MaxPort == 0 {
		config.MinPort = 10000
		config.MaxPort = 25000
	}

	if config.StartTimeout == 0 {
		config.StartTimeout = 1 * time.Minute
	}

	if config.Stderr == nil {
		config.Stderr = ioutil.Discard
	}

	if config.SyncStdout == nil {
		config.SyncStdout = ioutil.Discard
	}
	if config.SyncStderr == nil {
		config.SyncStderr = ioutil.Discard
	}

	c = &Client{config: config}
	if config.Managed {
		managedClients = append(managedClients, c)
	}

	return
}

// Client returns an RPC client for the plugin.
//
// Subsequent calls to this will return the same RPC client.
func (c *Client) Client() (*RPCClient, error) {
	addr, err := c.Start()
	if err != nil {
		return nil, err
	}

	c.l.Lock()
	defer c.l.Unlock()

	if c.client != nil {
		return c.client, nil
	}

	// Connect to the client
	conn, err := net.Dial(addr.Network(), addr.String())
	if err != nil {
		return nil, err
	}
	if tcpConn, ok := conn.(*net.TCPConn); ok {
		// Make sure to set keep alive so that the connection doesn't die
		tcpConn.SetKeepAlive(true)
	}

	// Create the actual RPC client
	c.client, err = NewRPCClient(conn, c.config.Plugins)
	if err != nil {
		conn.Close()
		return nil, err
	}

	// Begin the stream syncing so that stdin, out, err work properly
	err = c.client.SyncStreams(
		c.config.SyncStdout,
		c.config.SyncStderr)
	if err != nil {
		c.client.Close()
		c.client = nil
		return nil, err
	}

	return c.client, nil
}

// Tells whether or not the underlying process has exited.
func (c *Client) Exited() bool {
	c.l.Lock()
	defer c.l.Unlock()
	return c.exited
}

// End the executing subprocess (if it is running) and perform any cleanup
// tasks necessary such as capturing any remaining logs and so on.
//
// This method blocks until the process successfully exits.
//
// This method can safely be called multiple times.
func (c *Client) Kill() {
	cmd := c.config.Cmd

	if cmd.Process == nil {
		return
	}

	cmd.Process.Kill()

	// Wait for the client to finish logging so we have a complete log
	<-c.doneLogging
}

// Starts the underlying subprocess, communicating with it to negotiate
// a port for RPC connections, and returning the address to connect via RPC.
//
// This method is safe to call multiple times. Subsequent calls have no effect.
// Once a client has been started once, it cannot be started again, even if
// it was killed.
func (c *Client) Start() (addr net.Addr, err error) {
	c.l.Lock()
	defer c.l.Unlock()

	if c.address != nil {
		return c.address, nil
	}

	c.doneLogging = make(chan struct{})

	env := []string{
		fmt.Sprintf("%s=%s", c.config.MagicCookieKey, c.config.MagicCookieValue),
		fmt.Sprintf("PLUGIN_MIN_PORT=%d", c.config.MinPort),
		fmt.Sprintf("PLUGIN_MAX_PORT=%d", c.config.MaxPort),
	}

	stdout_r, stdout_w := io.Pipe()
	stderr_r, stderr_w := io.Pipe()

	cmd := c.config.Cmd
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, env...)
	cmd.Stdin = os.Stdin
	cmd.Stderr = stderr_w
	cmd.Stdout = stdout_w

	log.Printf("[DEBUG] Starting plugin: %s %#v", cmd.Path, cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}

	// Make sure the command is properly cleaned up if there is an error
	defer func() {
		r := recover()

		if err != nil || r != nil {
			cmd.Process.Kill()
		}

		if r != nil {
			panic(r)
		}
	}()

	// Start goroutine to wait for process to exit
	exitCh := make(chan struct{})
	go func() {
		// Make sure we close the write end of our stderr/stdout so
		// that the readers send EOF properly.
		defer stderr_w.Close()
		defer stdout_w.Close()

		// Wait for the command to end.
		cmd.Wait()

		// Log and make sure to flush the logs write away
		log.Printf("[DEBUG] %s: plugin process exited\n", cmd.Path)
		os.Stderr.Sync()

		// Mark that we exited
		close(exitCh)

		// Set that we exited, which takes a lock
		c.l.Lock()
		defer c.l.Unlock()
		c.exited = true
	}()

	// Start goroutine that logs the stderr
	go c.logStderr(stderr_r)

	// Start a goroutine that is going to be reading the lines
	// out of stdout
	linesCh := make(chan []byte)
	go func() {
		defer close(linesCh)

		buf := bufio.NewReader(stdout_r)
		for {
			line, err := buf.ReadBytes('\n')
			if line != nil {
				linesCh <- line
			}

			if err == io.EOF {
				return
			}
		}
	}()

	// Make sure after we exit we read the lines from stdout forever
	// so they don't block since it is an io.Pipe
	defer func() {
		go func() {
			for _ = range linesCh {
			}
		}()
	}()

	// Some channels for the next step
	timeout := time.After(c.config.StartTimeout)

	// Start looking for the address
	log.Printf("[DEBUG] Waiting for RPC address for: %s", cmd.Path)
	select {
	case <-timeout:
		err = errors.New("timeout while waiting for plugin to start")
	case <-exitCh:
		err = errors.New("plugin exited before we could connect")
	case lineBytes := <-linesCh:
		// Trim the line and split by "|" in order to get the parts of
		// the output.
		line := strings.TrimSpace(string(lineBytes))
		parts := strings.SplitN(line, "|", 3)
		if len(parts) < 3 {
			err = fmt.Errorf("Unrecognized remote plugin message: %s", line)
			return
		}

		// Parse the protocol version
		var protocol int64
		protocol, err = strconv.ParseInt(parts[0], 10, 0)
		if err != nil {
			err = fmt.Errorf("Error parsing protocol version: %s", err)
			return
		}

		// Test the API version
		if uint(protocol) != c.config.ProtocolVersion {
			err = fmt.Errorf("Incompatible API version with plugin. "+
				"Plugin version: %s, Ours: %d", parts[0], c.config.ProtocolVersion)
			return
		}

		switch parts[1] {
		case "tcp":
			addr, err = net.ResolveTCPAddr("tcp", parts[2])
		case "unix":
			addr, err = net.ResolveUnixAddr("unix", parts[2])
		default:
			err = fmt.Errorf("Unknown address type: %s", parts[1])
		}
	}

	c.address = addr
	return
}

func (c *Client) logStderr(r io.Reader) {
	bufR := bufio.NewReader(r)
	for {
		line, err := bufR.ReadString('\n')
		if line != "" {
			c.config.Stderr.Write([]byte(line))

			line = strings.TrimRightFunc(line, unicode.IsSpace)
			log.Printf("[DEBUG] %s: %s", filepath.Base(c.config.Cmd.Path), line)
		}

		if err == io.EOF {
			break
		}
	}

	// Flag that we've completed logging for others
	close(c.doneLogging)
}
<|endoftext|>


<commit_before>package gcm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

const (
	gcmEndpoint = "https://gcm-http.googleapis.com/gcm/send"
)

// Request level error codes
const (
	BadRequest          = "BadRequest"
	AuthenticationError = "AuthenticationError"
	InternalServerError = "InternalServerError"
	UnknownError        = "UnknownError"
)

// Message level error codes
const (
	MissingRegistrationError       = "MissingRegistration"
	InvalidRegistrationError       = "InvalidRegistration"
	NotRegisteredError             = "NotRegistered"
	MessageTooBigError             = "MessageTooBig"
	InvalidDataKeyError            = "InvalidDataKey"
	InvalidTTLError                = "InvalidTtl"
	DeviceMessageRateExceededError = "DeviceMessageRateExceeded"
	TopicsMessageRateExceededError = "TopicsMessageRateExceeded"
	MismatchSenderIDError          = "MismatchSenderId"
)

func NewClient(apiKey string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	return &Client{apiKey: apiKey, httpClient: httpClient, Endpoint: gcmEndpoint}
}

type Client struct {
	apiKey     string
	httpClient *http.Client
	Endpoint   string
}

type GCMError struct {
	HTTPCode    int
	Type        string
	Message     string
	ShouldRetry bool
	//@TODO consider retry-after header
}

func (e *GCMError) Error() string {
	return fmt.Sprintf("[%s] %d: %s", e.Type, e.HTTPCode, e.Message)
}

func (c *Client) Send(message *Message) (*Response, error) {
	req, err := c.createHTTPRequest(message)

	if err != nil {
		return nil, err
	}

	res, err := c.httpClient.Do(req)

	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()

	//log.Printf("RESPONSE: %#v\n", res)
	//log.Printf("BODY: %#v\n", string(body))

	switch {
	case res.StatusCode == 400:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: InternalServerError, Message: string(body)}
	case res.StatusCode == 401:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: AuthenticationError}
	case res.StatusCode >= 500:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: InternalServerError, ShouldRetry: true}
	case res.StatusCode != 200:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: UnknownError, ShouldRetry: false, Message: string(body)}
	}

	responseObj := &Response{}
	if err := json.Unmarshal(body, responseObj); err != nil {
		return nil, err
	}

	return responseObj, err
}

func (c *Client) createHTTPRequest(message *Message) (*http.Request, error) {
	body, err := json.Marshal(message)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", c.Endpoint, bytes.NewBuffer(body))

	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", "key="+c.apiKey)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	defer req.Body.Close()

	return req, err
}
<commit_msg>Fix 400 error type<commit_after>package gcm

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

const (
	gcmEndpoint = "https://gcm-http.googleapis.com/gcm/send"
)

// Request level error codes
const (
	BadRequest          = "BadRequest"
	AuthenticationError = "AuthenticationError"
	InternalServerError = "InternalServerError"
	UnknownError        = "UnknownError"
)

// Message level error codes
const (
	MissingRegistrationError       = "MissingRegistration"
	InvalidRegistrationError       = "InvalidRegistration"
	NotRegisteredError             = "NotRegistered"
	MessageTooBigError             = "MessageTooBig"
	InvalidDataKeyError            = "InvalidDataKey"
	InvalidTTLError                = "InvalidTtl"
	DeviceMessageRateExceededError = "DeviceMessageRateExceeded"
	TopicsMessageRateExceededError = "TopicsMessageRateExceeded"
	MismatchSenderIDError          = "MismatchSenderId"
)

func NewClient(apiKey string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	return &Client{apiKey: apiKey, httpClient: httpClient, Endpoint: gcmEndpoint}
}

type Client struct {
	apiKey     string
	httpClient *http.Client
	Endpoint   string
}

type GCMError struct {
	HTTPCode    int
	Type        string
	Message     string
	ShouldRetry bool
	//@TODO consider retry-after header
}

func (e *GCMError) Error() string {
	return fmt.Sprintf("[%s] %d: %s", e.Type, e.HTTPCode, e.Message)
}

func (c *Client) Send(message *Message) (*Response, error) {
	req, err := c.createHTTPRequest(message)

	if err != nil {
		return nil, err
	}

	res, err := c.httpClient.Do(req)

	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()

	//log.Printf("RESPONSE: %#v\n", res)
	//log.Printf("BODY: %#v\n", string(body))

	switch {
	case res.StatusCode == 400:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: BadRequest, Message: string(body)}
	case res.StatusCode == 401:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: AuthenticationError}
	case res.StatusCode >= 500:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: InternalServerError, ShouldRetry: true}
	case res.StatusCode != 200:
		return nil, &GCMError{HTTPCode: res.StatusCode, Type: UnknownError, ShouldRetry: false, Message: string(body)}
	}

	responseObj := &Response{}
	if err := json.Unmarshal(body, responseObj); err != nil {
		return nil, err
	}

	return responseObj, err
}

func (c *Client) createHTTPRequest(message *Message) (*http.Request, error) {
	body, err := json.Marshal(message)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", c.Endpoint, bytes.NewBuffer(body))

	if err != nil {
		return nil, err
	}

	req.Header.Set("Authorization", "key="+c.apiKey)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")

	defer req.Body.Close()

	return req, err
}
<|endoftext|>


<commit_before>package groveclient

import (
	"bytes"
	"encoding/json"
	"strings"

	pc "github.com/t11e/go-pebbleclient"
)

//go:generate go run vendor/github.com/vektra/mockery/cmd/mockery/mockery.go -name=Client -case=underscore

type Client interface {
	Get(uid string, options GetOptions) (*PostItem, error)
	GetMany(uids []string, options GetManyOptions) (*GetManyOutput, error)
	Update(postItem *PostItem) (*PostItem, error)
}

type client struct {
	c pc.Client
}

type GetOptions struct {
	Raw *bool
}

type GetManyOptions struct {
	Limit *int
	Raw   *bool
}

type GetManyOutput struct {
	Posts []PostItem `json:"posts"`
}

// Register registers us in a connector.
func Register(connector *pc.Connector) {
	connector.Register((*Client)(nil), func(client pc.Client) (pc.Service, error) {
		return New(client)
	})
}

func New(pebbleClient pc.Client) (Client, error) {
	return &client{pebbleClient.WithOptions(pc.Options{
		ServiceName: "grove",
		ApiVersion:  1,
	})}, nil
}

func (c *client) Get(uid string, options GetOptions) (*PostItem, error) {
	params := pc.Params{
		"raw": options.Raw != nil && *options.Raw,
		"uid": uid,
	}

	var out PostItem
	err := c.c.Get("/posts/:uid", &pc.RequestOptions{
		Params: params,
	}, &out)
	if err != nil {
		return nil, err
	}
	return &out, err
}

func (c *client) GetMany(uids []string, options GetManyOptions) (*GetManyOutput, error) {
	uidList := strings.Join(uids, ",")
	if len(uids) > 1 {
		uidList = uidList + ","
	}

	params := pc.Params{
		"raw":  options.Raw != nil && *options.Raw,
		"uids": uidList,
	}
	if options.Limit != nil {
		params["limit"] = *options.Limit
	}

	var out GetManyOutput
	err := c.c.Get("/posts/:uids", &pc.RequestOptions{
		Params: params,
	}, &out)
	if err != nil {
		return nil, err
	}
	return &out, err
}

func (c *client) Update(postItem *PostItem) (*PostItem, error) {
	params := pc.Params{
		"uid": postItem.Post.Uid,
	}
	payload, err := json.Marshal(postItem)
	if err != nil {
		return nil, err
	}
	var out PostItem
	err = c.c.Put("/posts/:uid", &pc.RequestOptions{
		Params: params,
	}, bytes.NewReader(payload), &out)
	if err != nil {
		return nil, err
	}
	return &out, err
}
<commit_msg>Improves Client.Update() to take a different struct so it can better handle the optional field overrides that grove supports. Productizes a simple ConflictError.<commit_after>package groveclient

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	pc "github.com/t11e/go-pebbleclient"
)

//go:generate go run vendor/github.com/vektra/mockery/cmd/mockery/mockery.go -name=Client -case=underscore

type Client interface {
	Get(uid string, options GetOptions) (*PostItem, error)
	GetMany(uids []string, options GetManyOptions) (*GetManyOutput, error)
	Update(uid string, pu PostUpdate, options UpdateOptions) (*PostItem, error)
}

type PostUpdate struct {
	CreatedAt        *time.Time      `json:"created_at,omitempty"`
	UpdatedAt        *time.Time      `json:"updated_at,omitempty"`
	Document         json.RawMessage `json:"document,omitempty"`
	ExternalDocument json.RawMessage `json:"external_document,omitempty"`
	Sensitive        json.RawMessage `json:"sensitive,omitempty"`
	Protected        json.RawMessage `json:"protected,omitempty"`
	Tags             *[]string       `json:"tags,omitempty",`
	Deleted          *bool           `json:"deleted,omitempty"`
	Published        *bool           `json:"published,omitempty"`
	ExternalId       string          `json:"external_id,omitempty"`
	Version          int             `json:"version,omitempty"`
}

type client struct {
	c pc.Client
}

type GetOptions struct {
	Raw *bool
}

type GetManyOptions struct {
	Limit *int
	Raw   *bool
}

type UpdateOptions struct {
	Merge      *bool   `json:"merge,omitempty"`
	ExternalID *string `json:"external_id,omitempty"`
}

type GetManyOutput struct {
	Posts []PostItem `json:"posts"`
}

// Register registers us in a connector.
func Register(connector *pc.Connector) {
	connector.Register((*Client)(nil), func(client pc.Client) (pc.Service, error) {
		return New(client)
	})
}

func New(pebbleClient pc.Client) (Client, error) {
	return &client{pebbleClient.WithOptions(pc.Options{
		ServiceName: "grove",
		ApiVersion:  1,
	})}, nil
}

func (c *client) Get(uid string, options GetOptions) (*PostItem, error) {
	params := pc.Params{
		"raw": options.Raw != nil && *options.Raw,
		"uid": uid,
	}

	var out PostItem
	err := c.c.Get("/posts/:uid", &pc.RequestOptions{
		Params: params,
	}, &out)
	if err != nil {
		return nil, err
	}
	return &out, err
}

func (c *client) GetMany(uids []string, options GetManyOptions) (*GetManyOutput, error) {
	uidList := strings.Join(uids, ",")
	if len(uids) > 1 {
		uidList = uidList + ","
	}

	params := pc.Params{
		"raw":  options.Raw != nil && *options.Raw,
		"uids": uidList,
	}
	if options.Limit != nil {
		params["limit"] = *options.Limit
	}

	var out GetManyOutput
	err := c.c.Get("/posts/:uids", &pc.RequestOptions{
		Params: params,
	}, &out)
	if err != nil {
		return nil, err
	}
	return &out, err
}

func (c *client) Update(uid string, pu PostUpdate, options UpdateOptions) (*PostItem, error) {
	params := pc.Params{
		"uid": uid,
	}
	if options.Merge != nil {
		params["merge"] = *options.Merge
	}
	if options.ExternalID != nil {
		params["external_id"] = *options.ExternalID
	}
	payload, err := json.Marshal(&struct {
		Post PostUpdate `json:"post"`
	}{pu})
	if err != nil {
		return nil, err
	}
	result := PostItem{}
	err = c.c.Put("/posts/:uid", &pc.RequestOptions{
		Params: params,
	}, bytes.NewReader(payload), &result)
	if err == nil {
		return &result, nil
	}
	if reqErr, ok := err.(*pc.RequestError); ok && reqErr.Resp.StatusCode == 409 {
		return &result, ConflictError{uid}
	}
	return &result, err
}

type ConflictError struct {
	uid string
}

func (e ConflictError) Error() string {
	return fmt.Sprintf("grove post failed to update due to version conflict: %s", e.uid)
}
<|endoftext|>


<commit_before>package mqttclient

import (
	"crypto/tls"
	"errors"
	"fmt"
	mqtt "github.com/clearblade/mqtt_parsing"
	"io"
	mrand "math/rand"
	"net"
	"sync"
	"time"
)

type Client struct {

	//this is all clearblade specific
	SystemKey    string
	SystemSecret string
	AuthToken    string
	Clientid     string
	//this will usually be occupied
	//by a net.Conn
	C                   io.ReadWriteCloser
	internalOutgoingBuf chan []byte
	Timeout             time.Duration
	//thou shalt type channels consumed by others
	ClientErrorBuffer chan error

	internalErrorBuffer chan *errWrap
	shutdown_reader     chan struct{}
	shutdown_writer     chan struct{}
	//introduce a sync write mode?
	last_timeout_reccd time.Time

	shutting_down bool

	msg_store                *storage
	subscriptions            *outgoing_topics
	waiting_for_subscription *subscription_store
	//TODO:redesign around a thread-local
	//rng
	rando    *mrand.Rand
	randomut *sync.RWMutex
}

var (
	Verbose bool
)

//Start connects to the mqtt broker. It does not send the connect packet. Use the SendConnect function for that.
func (c *Client) Start(addr string, ssl *tls.Config) error {
	var con net.Conn
	var err error
	if ssl != nil {
		con, err = tls.Dial("tcp", addr, ssl)
	} else {
		con, err = net.Dial("tcp", addr)
	}
	if err != nil {
		return err
	}
	c.C = con

	go c.connectionWriter()
	go c.connectionListener()
	go c.errorTree()
	return nil
}

//NewClient allocates a new client. It is supplied with the (in order of appearance)
//Token, SystemKey,SystemSecret,Clientid, and the mqtt timeout
//Note that the following combinations of (Token|SystemKey|SystemSecret) are allowed
//(Token && SystemKey), (SystemKey && SystemSecret)
func NewClient(tok, sk, ss, cid string, timeout int) *Client {
	client := &Client{
		msg_store:                newStorage(),
		waiting_for_subscription: newSubscriptionStore(),
		subscriptions:            newOutgoingTopics(),
		SystemSecret:             ss,
		SystemKey:                sk,
		AuthToken:                tok,
		Clientid:                 cid,
		Timeout:                  time.Duration(timeout) * time.Second,
		//internalOutgoingBuf:      make(chan []byte, 30),
		internalOutgoingBuf: make(chan []byte),
		ClientErrorBuffer:   make(chan error, 10),
		internalErrorBuffer: make(chan *errWrap, 2),
		shutdown_reader:     make(chan struct{}, 1),
		shutdown_writer:     make(chan struct{}, 1),
		rando:               mrand.New(mrand.NewSource(time.Now().UnixNano())),
		randomut:            new(sync.RWMutex),
	}
	return client
}

//sendMessage is an internal function that acts as a central point
//of failure for all of the message sending channels
//sort of like a fan-in, except this simply allows us to do
// all of the error handling logic in one place
func (c *Client) sendMessage(m mqtt.Message) error {
	_, err := c.C.Write(m.Encode())
	if err != nil {
		c.internalErrorBuffer <- &errWrap{
			err:      err,
			reciever: _CON_WRITER,
		}
		return err
	}
	return nil
}

//connectionWriter is an internal function. it essentially sits in a goroutine and writes to the connection
//whenever it recieves a message over the channel
func (c *Client) connectionWriter() {
	if c.C == nil {
		return
	}
	for {
		select {
		case out := <-c.internalOutgoingBuf:
			_, err := c.C.Write(out)
			if err != nil {
				c.internalErrorBuffer <- &errWrap{
					err:      err,
					reciever: _CON_WRITER,
				}
				return
			}
		case <-c.shutdown_writer:
			return
		}
	}
}

//connectionListener is another function that sits on a goroutine.
//DecodePacket blocks until it reads a complete mqtt packet
func (c *Client) connectionListener() {
	if c.C == nil {
		return
	}
	mch, ech, shutdown := make(chan mqtt.Message, 10), make(chan error, 1), false
	//we have to establish an internal chain of goroutines here
	//otherwise we couldn't shutdown the listener on demand
	//since it's really hard to coordinate all the shutting down
	//when a connection drops
	//we're waiting for the connection listener to simply fail
	//this allows us to handle it a bit more gracefully
	//in order to shut down via channels directly we'd have to
	//wait for the read to fail anyway.
	go func(m chan mqtt.Message, e chan error) {
		for {
			msg, err := mqtt.DecodePacket(c.C)
			if err != nil {
				shutdown = true
				e <- err
				return
			}
			m <- msg
		}
	}(mch, ech)
	sentPing := false
	for {
		select {

		case msg := <-mch:
			//dispatch the internal functions
			c.dispatch(msg)
			sentPing = false
		case e := <-ech:
			//an error was recieved from the listenr
			if !shutdown {
				c.internalErrorBuffer <- &errWrap{
					err:      e,
					reciever: _CON_READER,
				}
			}
		case <-time.After(c.Timeout):
			//keepalive
			if !sentPing {
				c.sendMessage(&mqtt.Pingreq{})
				sentPing = true
			} else if time.Since(c.last_timeout_reccd) > c.Timeout && sentPing {
				//there was a timeout
				c.Shutdown(true)
			}

		case <-c.shutdown_reader:
			shutdown = true
			return
		}
	}
}

//this is the "business logic" of the client. it decides how each packet mutates the
//internal state of the client
func (c *Client) dispatch(msg mqtt.Message) {
	//for example we make a decision if the client sees this request or not
	//or if we have to send another message in a flow
	c.last_timeout_reccd = time.Now()
	switch msg.Type() {
	case mqtt.CONNECT:
		//shouldn't happen?
	case mqtt.CONNACK:
	case mqtt.PUBLISH:
		pubMsg := msg.(*mqtt.Publish)
		c.subscriptions.relay_message(pubMsg, pubMsg.Topic.Whole)
		switch msg.(*mqtt.Publish).Header.QOS {
		case 1:
			c.sendMessage(&mqtt.Puback{
				MessageId: pubMsg.MessageId,
			})
		case 2:
			c.sendMessage(&mqtt.Pubrec{
				MessageId: pubMsg.MessageId,
			})
		}
	case mqtt.PUBACK:
		//TODO:handle resend
	case mqtt.PUBREC:
		//discard, store the fact that it was recieved
		c.sendMessage(&mqtt.Pubrel{
			MessageId: msg.(*mqtt.Pubrec).MessageId,
			Header: &mqtt.StaticHeader{
				DUP:    false,
				Retain: false,
				QOS:    1,
			}})
	case mqtt.PUBREL:
		//this shouldn't have happened
	case mqtt.SUBSCRIBE:
		//this is not supposed to happen
	case mqtt.SUBACK:
		//the subscribe call blocks, so we need to forward the message
		//along that the subscribe was acknowleged so we can return
		//control flow to the parent program
		//of course, the problem is that a suback does not have
		//the subscriptions in it by name
		//but it does have the same message id
		//so we have to retrieve that and then match up the subscribe
		//TODO:NOTE THAT WE ARE ONLY USING ONE TOPIC PER SUBSCRIBE MESSSAGE
		//THIS LOGIC WILL NEED TWEAKING IF THAT CHANGES
		msg := c.msg_store.getEntry(msg.(*mqtt.Suback).MessageId)
		if msg == nil {
			//this is a bad thing to happen
			return
		}
		sub, ok := msg.(*mqtt.Subscribe)
		if !ok {
			//this is a worse thing to happen
			return
		}
		//TODO: BUG:: IF WE MAKE MULTISUBSCRIPTION, THIS WILL BREAK
		//we've now released control flow in that channel
		//also it'll allocate the userside channel and all that good stuff
		c.waiting_for_subscription.relay_message(msg, sub.Subscriptions[0].Topic.Whole)

	case mqtt.UNSUBSCRIBE:
		//this shouldn't happen
	case mqtt.UNSUBACK:
		//not gorgeous, but it do the thing
		msg := c.msg_store.getEntry(msg.(*mqtt.Unsuback).MessageId)
		unsub, ok := msg.(*mqtt.Unsubscribe)
		if !ok {
			return
		}
		c.subscriptions.remove_subscription(unsub.Topics[0].Topic.Whole)
		//TODO: do something besides prepend UNSUBSCRIBE# (the hash makes it an invalid mqtt topic, which should prevent collisions) to the front of the topic
		//but also doesn't allocate another big-ole map wrapper
		err := c.waiting_for_subscription.relay_message(msg, "UNSUBSCRIBE#"+unsub.Topics[0].Topic.Whole)
		if err != nil {
			c.ClientErrorBuffer <- err
		}
	case mqtt.PINGREQ:
		//shouldn't happen
	case mqtt.PINGRESP:
		//pingresp will reset the counter elsewhere
	case mqtt.DISCONNECT:
		//shouldn't happen
	default:
		c.ClientErrorBuffer <- fmt.Errorf("Invalid mqtt type recieved %+v", msg)
	}
}

//errorTree is the goroutine that sits on it's own goroutine and waits for
//a message to be recieved on c.internalErrorBuffer. It's our "in case of emergency break glass"
//way of reporting an error, and shutting the entire thing down
func (c *Client) errorTree() {
	//we still need to shutdown the listeners anyway
	//at least write will probably not error out

	//since you're reading the text of this fn, prepare your face for some exposition on how this mechanism works
	//so, we've spread reading and writing to the conn (or whatever) across goroutines, this is great
	//high speed low drag
	//but what happens if one goroutine encounters an error? the goroutines don't know about each other, so what do we do?
	//well, writing, and reading from to a closed connection is an error condition. so if one goroutine dies, then the other
	//will be taken down with it
	//we also use this mechanism for a regular shutdown of the client's connection, simply crashing them both and releasing the resources
	e := <-c.internalErrorBuffer
	if e.reciever != _CON_READER {
		c.shutdown_reader <- struct{}{}
	}
	if e.reciever != _CON_WRITER {
		c.shutdown_writer <- struct{}{}
	}
	if c.C != nil {
		c.C.Close()
	}
	if e.reciever != _REGULAR_SHUTDOWN {
		c.ClientErrorBuffer <- fmt.Errorf("Shutting down: Recieved error %v\n", e.err.Error())
	}
}

//Shutdown sends a disconnect packet (if asked), and then disconnects from the broker after a set time limit
func (c *Client) Shutdown(sendDisconnect bool) error {
	var err error
	if sendDisconnect {
		e := SendDisconnect(c)
		if e != nil {
			//don't return here, wait to finish the flow
			err = errors.New("While sending disconnect: " + e.Error() + "\nNote:connection was shut down anyway")
		}
		<-time.After(time.Second)
	}
	c.internalErrorBuffer <- &errWrap{reciever: _REGULAR_SHUTDOWN}
	return err
}

func (c *Client) randoPerm(i int) []int {
	c.randomut.RLock()
	order := c.rando.Perm(i)
	c.randomut.RUnlock()
	return order
}

func (c *Client) getInt() int {
	c.randomut.RLock()
	num := c.rando.Int()
	c.randomut.RUnlock()
	return num
}
<commit_msg>broke out keep alive ping into it's own goroutine<commit_after>package mqttclient

import (
	"crypto/tls"
	"errors"
	"fmt"
	mqtt "github.com/clearblade/mqtt_parsing"
	"io"
	mrand "math/rand"
	"net"
	"sync"
	"time"
)

type Client struct {

	//this is all clearblade specific
	SystemKey    string
	SystemSecret string
	AuthToken    string
	Clientid     string
	//this will usually be occupied
	//by a net.Conn
	C                   io.ReadWriteCloser
	internalOutgoingBuf chan []byte
	Timeout             time.Duration
	//thou shalt type channels consumed by others
	ClientErrorBuffer chan error

	internalErrorBuffer chan *errWrap
	shutdown_reader     chan struct{}
	shutdown_writer     chan struct{}
	//introduce a sync write mode?
	last_timeout_reccd time.Time

	shutting_down bool

	msg_store                *storage
	subscriptions            *outgoing_topics
	waiting_for_subscription *subscription_store
	//TODO:redesign around a thread-local
	//rng
	rando    *mrand.Rand
	randomut *sync.RWMutex
}

var (
	Verbose bool
)

//Start connects to the mqtt broker. It does not send the connect packet. Use the SendConnect function for that.
func (c *Client) Start(addr string, ssl *tls.Config) error {
	var con net.Conn
	var err error
	if ssl != nil {
		con, err = tls.Dial("tcp", addr, ssl)
	} else {
		con, err = net.Dial("tcp", addr)
	}
	if err != nil {
		return err
	}
	c.C = con

	go c.connectionWriter()
	go c.connectionListener()
	go c.errorTree()
	return nil
}

//NewClient allocates a new client. It is supplied with the (in order of appearance)
//Token, SystemKey,SystemSecret,Clientid, and the mqtt timeout
//Note that the following combinations of (Token|SystemKey|SystemSecret) are allowed
//(Token && SystemKey), (SystemKey && SystemSecret)
func NewClient(tok, sk, ss, cid string, timeout int) *Client {
	client := &Client{
		msg_store:                newStorage(),
		waiting_for_subscription: newSubscriptionStore(),
		subscriptions:            newOutgoingTopics(),
		SystemSecret:             ss,
		SystemKey:                sk,
		AuthToken:                tok,
		Clientid:                 cid,
		Timeout:                  time.Duration(timeout) * time.Second,
		//internalOutgoingBuf:      make(chan []byte, 30),
		internalOutgoingBuf: make(chan []byte),
		ClientErrorBuffer:   make(chan error, 10),
		internalErrorBuffer: make(chan *errWrap, 2),
		shutdown_reader:     make(chan struct{}, 1),
		shutdown_writer:     make(chan struct{}, 1),
		rando:               mrand.New(mrand.NewSource(time.Now().UnixNano())),
		randomut:            new(sync.RWMutex),
	}
	return client
}

//sendMessage is an internal function that acts as a central point
//of failure for all of the message sending channels
//sort of like a fan-in, except this simply allows us to do
// all of the error handling logic in one place
func (c *Client) sendMessage(m mqtt.Message) error {
	_, err := c.C.Write(m.Encode())
	if err != nil {
		c.internalErrorBuffer <- &errWrap{
			err:      err,
			reciever: _CON_WRITER,
		}
		return err
	}
	return nil
}

//connectionWriter is an internal function. it essentially sits in a goroutine and writes to the connection
//whenever it recieves a message over the channel
func (c *Client) connectionWriter() {
	if c.C == nil {
		return
	}
	for {
		select {
		case out := <-c.internalOutgoingBuf:
			_, err := c.C.Write(out)
			if err != nil {
				c.internalErrorBuffer <- &errWrap{
					err:      err,
					reciever: _CON_WRITER,
				}
				return
			}
		case <-c.shutdown_writer:
			return
		}
	}
}

//connectionListener is another function that sits on a goroutine.
//DecodePacket blocks until it reads a complete mqtt packet
func (c *Client) connectionListener() {
	if c.C == nil {
		return
	}
	mch, ech, shutdown := make(chan mqtt.Message, 10), make(chan error, 1), false
	//we have to establish an internal chain of goroutines here
	//otherwise we couldn't shutdown the listener on demand
	//since it's really hard to coordinate all the shutting down
	//when a connection drops
	//we're waiting for the connection listener to simply fail
	//this allows us to handle it a bit more gracefully
	//in order to shut down via channels directly we'd have to
	//wait for the read to fail anyway.
	go func(m chan mqtt.Message, e chan error) {
		for {
			msg, err := mqtt.DecodePacket(c.C)
			if err != nil {
				shutdown = true
				e <- err
				return
			}
			m <- msg
		}
	}(mch, ech)

	heardFromServer := true
	go func(cli *Client) {
		for {
			time.Sleep(cli.Timeout)
			if heardFromServer {
				cli.sendMessage(&mqtt.Pingreq{})
				heardFromServer = false
			} else if time.Since(cli.last_timeout_reccd) > cli.Timeout && !heardFromServer {
				//there was a timeout
				cli.Shutdown(true)
			}
		}
	}(c)
	for {
		select {

		case msg := <-mch:
			//dispatch the internal functions
			c.dispatch(msg)
			heardFromServer = true
		case e := <-ech:
			//an error was recieved from the listenr
			if !shutdown {
				c.internalErrorBuffer <- &errWrap{
					err:      e,
					reciever: _CON_READER,
				}
			}
		case <-c.shutdown_reader:
			shutdown = true
			return
		}
	}
}

//this is the "business logic" of the client. it decides how each packet mutates the
//internal state of the client
func (c *Client) dispatch(msg mqtt.Message) {
	//for example we make a decision if the client sees this request or not
	//or if we have to send another message in a flow
	c.last_timeout_reccd = time.Now()
	switch msg.Type() {
	case mqtt.CONNECT:
		//shouldn't happen?
	case mqtt.CONNACK:
	case mqtt.PUBLISH:
		pubMsg := msg.(*mqtt.Publish)
		c.subscriptions.relay_message(pubMsg, pubMsg.Topic.Whole)
		switch msg.(*mqtt.Publish).Header.QOS {
		case 1:
			c.sendMessage(&mqtt.Puback{
				MessageId: pubMsg.MessageId,
			})
		case 2:
			c.sendMessage(&mqtt.Pubrec{
				MessageId: pubMsg.MessageId,
			})
		}
	case mqtt.PUBACK:
		//TODO:handle resend
	case mqtt.PUBREC:
		//discard, store the fact that it was recieved
		c.sendMessage(&mqtt.Pubrel{
			MessageId: msg.(*mqtt.Pubrec).MessageId,
			Header: &mqtt.StaticHeader{
				DUP:    false,
				Retain: false,
				QOS:    1,
			}})
	case mqtt.PUBREL:
		//this shouldn't have happened
	case mqtt.SUBSCRIBE:
		//this is not supposed to happen
	case mqtt.SUBACK:
		//the subscribe call blocks, so we need to forward the message
		//along that the subscribe was acknowleged so we can return
		//control flow to the parent program
		//of course, the problem is that a suback does not have
		//the subscriptions in it by name
		//but it does have the same message id
		//so we have to retrieve that and then match up the subscribe
		//TODO:NOTE THAT WE ARE ONLY USING ONE TOPIC PER SUBSCRIBE MESSSAGE
		//THIS LOGIC WILL NEED TWEAKING IF THAT CHANGES
		msg := c.msg_store.getEntry(msg.(*mqtt.Suback).MessageId)
		if msg == nil {
			//this is a bad thing to happen
			return
		}
		sub, ok := msg.(*mqtt.Subscribe)
		if !ok {
			//this is a worse thing to happen
			return
		}
		//TODO: BUG:: IF WE MAKE MULTISUBSCRIPTION, THIS WILL BREAK
		//we've now released control flow in that channel
		//also it'll allocate the userside channel and all that good stuff
		c.waiting_for_subscription.relay_message(msg, sub.Subscriptions[0].Topic.Whole)

	case mqtt.UNSUBSCRIBE:
		//this shouldn't happen
	case mqtt.UNSUBACK:
		//not gorgeous, but it do the thing
		msg := c.msg_store.getEntry(msg.(*mqtt.Unsuback).MessageId)
		unsub, ok := msg.(*mqtt.Unsubscribe)
		if !ok {
			return
		}
		c.subscriptions.remove_subscription(unsub.Topics[0].Topic.Whole)
		//TODO: do something besides prepend UNSUBSCRIBE# (the hash makes it an invalid mqtt topic, which should prevent collisions) to the front of the topic
		//but also doesn't allocate another big-ole map wrapper
		err := c.waiting_for_subscription.relay_message(msg, "UNSUBSCRIBE#"+unsub.Topics[0].Topic.Whole)
		if err != nil {
			c.ClientErrorBuffer <- err
		}
	case mqtt.PINGREQ:
		//shouldn't happen
	case mqtt.PINGRESP:
		//pingresp will reset the counter elsewhere
	case mqtt.DISCONNECT:
		//shouldn't happen
	default:
		c.ClientErrorBuffer <- fmt.Errorf("Invalid mqtt type recieved %+v", msg)
	}
}

//errorTree is the goroutine that sits on it's own goroutine and waits for
//a message to be recieved on c.internalErrorBuffer. It's our "in case of emergency break glass"
//way of reporting an error, and shutting the entire thing down
func (c *Client) errorTree() {
	//we still need to shutdown the listeners anyway
	//at least write will probably not error out

	//since you're reading the text of this fn, prepare your face for some exposition on how this mechanism works
	//so, we've spread reading and writing to the conn (or whatever) across goroutines, this is great
	//high speed low drag
	//but what happens if one goroutine encounters an error? the goroutines don't know about each other, so what do we do?
	//well, writing, and reading from to a closed connection is an error condition. so if one goroutine dies, then the other
	//will be taken down with it
	//we also use this mechanism for a regular shutdown of the client's connection, simply crashing them both and releasing the resources
	e := <-c.internalErrorBuffer
	if e.reciever != _CON_READER {
		c.shutdown_reader <- struct{}{}
	}
	if e.reciever != _CON_WRITER {
		c.shutdown_writer <- struct{}{}
	}
	if c.C != nil {
		c.C.Close()
	}
	if e.reciever != _REGULAR_SHUTDOWN {
		c.ClientErrorBuffer <- fmt.Errorf("Shutting down: Recieved error %v\n", e.err.Error())
	}
}

//Shutdown sends a disconnect packet (if asked), and then disconnects from the broker after a set time limit
func (c *Client) Shutdown(sendDisconnect bool) error {
	var err error
	if sendDisconnect {
		e := SendDisconnect(c)
		if e != nil {
			//don't return here, wait to finish the flow
			err = errors.New("While sending disconnect: " + e.Error() + "\nNote:connection was shut down anyway")
		}
		<-time.After(time.Second)
	}
	c.internalErrorBuffer <- &errWrap{reciever: _REGULAR_SHUTDOWN}
	return err
}

func (c *Client) randoPerm(i int) []int {
	c.randomut.RLock()
	order := c.rando.Perm(i)
	c.randomut.RUnlock()
	return order
}

func (c *Client) getInt() int {
	c.randomut.RLock()
	num := c.rando.Int()
	c.randomut.RUnlock()
	return num
}
<|endoftext|>


<commit_before>package support

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	neturl "net/url"
	"strings"

	docker "github.com/giantswarm/hijack-stream-support/docker"
)

type HijackHttpOptions struct {
	Method             string
	Url                string
	Host               string // If set, this will be passed as `Host` header to the request.
	DockerTermProtocol bool
	InputStream        io.Reader
	ErrorStream        io.Writer
	OutputStream       io.Writer
	Data               interface{}
	Header             http.Header
	Log                docker.Logger
}

var (
	ErrMissingMethod = errors.New("Method not set")
	ErrMissingUrl    = errors.New("Url not set")
)

// HijackHttpRequest performs an HTTP  request with given method, url and data and hijacks the request (after a successful connection) to stream
// data from/to the given input, output and error streams.
func HijackHttpRequest(options HijackHttpOptions) error {
	if options.Log == nil {
		// Make sure there is always a logger
		options.Log = &logIgnore{}
	}
	if options.Method == "" {
		return ErrMissingMethod
	}
	if options.Url == "" {
		return ErrMissingUrl
	}

	req, err := createHijackHttpRequest(options)
	if err != nil {
		return err
	}

	// Parse URL for endpoint data
	ep, err := neturl.Parse(options.Url)
	if err != nil {
		return err
	}

	protocol := ep.Scheme
	address := ep.Path
	if protocol != "unix" {
		protocol = "tcp"
		address = ep.Host
		if !strings.Contains(address, ":") {
			if ep.Scheme == "https" {
				address = address + ":443"
			} else {
				address = address + ":80"
			}
		}
	}

	// Dial the server
	var dial net.Conn
	//fmt.Printf("Dialing %s %s\n", protocol, address)
	if ep.Scheme == "https" {
		config := &tls.Config{}
		dial, err = docker.TLSDial(protocol, address, config)
		if err != nil {
			fmt.Printf("TLS Dialing %s %s failed %#v\n", protocol, address, err)
			return err
		}
	} else {
		dial, err = net.Dial(protocol, address)
		if err != nil {
			fmt.Printf("Dialing %s %s failed %#v\n", protocol, address, err)
			return err
		}
	}

	// Start initial HTTP connection
	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()

	clientconn.Do(req)

	// Hijack HTTP connection
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	// Stream data
	return streamData(rwc, br, options)
}

// createHijackHttpRequest creates an upgradable HTTP request according to the given options
func createHijackHttpRequest(options HijackHttpOptions) (*http.Request, error) {
	var params io.Reader
	if options.Data != nil {
		buf, err := json.Marshal(options.Data)
		if err != nil {
			return nil, err
		}
		params = bytes.NewBuffer(buf)
	}

	req, err := http.NewRequest(options.Method, options.Url, params)
	if err != nil {
		return nil, err
	}
	if options.Header != nil {
		for k, values := range options.Header {
			req.Header.Del(k)
			for _, v := range values {
				req.Header.Set(k, v)
			}
		}
	}
	req.Header.Set("Content-Type", "text/plain")
	//req.Header.Set("Connection", "Upgrade")
	//req.Header.Set("Upgrade", "tcp")
	if options.Host != "" {
		req.Host = options.Host
	}
	return req, nil
}

// streamData copies both input/output/error streams to/from the hijacked streams
func streamData(rwc io.Writer, br io.Reader, options HijackHttpOptions) error {
	errsIn := make(chan error, 1)
	errsOut := make(chan error, 1)
	exit := make(chan bool)

	go func() {
		defer close(exit)
		defer close(errsOut)
		var err error
		stdout := options.OutputStream
		if stdout == nil {
			stdout = ioutil.Discard
		}
		stderr := options.ErrorStream
		if stderr == nil {
			stderr = ioutil.Discard
		}
		if !options.DockerTermProtocol {
			// When TTY is ON, use regular copy
			_, err = io.Copy(stdout, br)
		} else {
			_, err = docker.StdCopy(stdout, stderr, br, options.Log)
		}
		errsOut <- err
	}()
	go func() {
		defer close(errsIn)
		var err error
		in := options.InputStream
		if in != nil {
			_, err = io.Copy(rwc, in)
		}
		if err := rwc.(closeWriter).CloseWrite(); err != nil {
			options.Log.Debugf("CloseWrite failed %#v", err)
		}
		errsIn <- err
	}()
	<-exit
	select {
	case err := <-errsOut:
		return err
	case err := <-errsIn:
		return err
	}
}

// ----------------------------------------------
// private interface supporting CloseWrite calls.

type closeWriter interface {
	CloseWrite() error
}

// ----------------------------------------------
// Helper to ignore debug los in case we got no logger

type logIgnore struct {
}

func (this *logIgnore) Debugf(msg string, args ...interface{}) {
	// Ignore the log message
}
<commit_msg>Enabled upgrade header<commit_after>package support

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httputil"
	neturl "net/url"
	"strings"

	docker "github.com/giantswarm/hijack-stream-support/docker"
)

type HijackHttpOptions struct {
	Method             string
	Url                string
	Host               string // If set, this will be passed as `Host` header to the request.
	DockerTermProtocol bool
	InputStream        io.Reader
	ErrorStream        io.Writer
	OutputStream       io.Writer
	Data               interface{}
	Header             http.Header
	Log                docker.Logger
}

var (
	ErrMissingMethod = errors.New("Method not set")
	ErrMissingUrl    = errors.New("Url not set")
)

// HijackHttpRequest performs an HTTP  request with given method, url and data and hijacks the request (after a successful connection) to stream
// data from/to the given input, output and error streams.
func HijackHttpRequest(options HijackHttpOptions) error {
	if options.Log == nil {
		// Make sure there is always a logger
		options.Log = &logIgnore{}
	}
	if options.Method == "" {
		return ErrMissingMethod
	}
	if options.Url == "" {
		return ErrMissingUrl
	}

	req, err := createHijackHttpRequest(options)
	if err != nil {
		return err
	}

	// Parse URL for endpoint data
	ep, err := neturl.Parse(options.Url)
	if err != nil {
		return err
	}

	protocol := ep.Scheme
	address := ep.Path
	if protocol != "unix" {
		protocol = "tcp"
		address = ep.Host
		if !strings.Contains(address, ":") {
			if ep.Scheme == "https" {
				address = address + ":443"
			} else {
				address = address + ":80"
			}
		}
	}

	// Dial the server
	var dial net.Conn
	//fmt.Printf("Dialing %s %s\n", protocol, address)
	if ep.Scheme == "https" {
		config := &tls.Config{}
		dial, err = docker.TLSDial(protocol, address, config)
		if err != nil {
			fmt.Printf("TLS Dialing %s %s failed %#v\n", protocol, address, err)
			return err
		}
	} else {
		dial, err = net.Dial(protocol, address)
		if err != nil {
			fmt.Printf("Dialing %s %s failed %#v\n", protocol, address, err)
			return err
		}
	}

	// Start initial HTTP connection
	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()

	clientconn.Do(req)

	// Hijack HTTP connection
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	// Stream data
	return streamData(rwc, br, options)
}

// createHijackHttpRequest creates an upgradable HTTP request according to the given options
func createHijackHttpRequest(options HijackHttpOptions) (*http.Request, error) {
	var params io.Reader
	if options.Data != nil {
		buf, err := json.Marshal(options.Data)
		if err != nil {
			return nil, err
		}
		params = bytes.NewBuffer(buf)
	}

	req, err := http.NewRequest(options.Method, options.Url, params)
	if err != nil {
		return nil, err
	}
	if options.Header != nil {
		for k, values := range options.Header {
			req.Header.Del(k)
			for _, v := range values {
				req.Header.Set(k, v)
			}
		}
	}
	req.Header.Set("Content-Type", "text/plain")
	req.Header.Set("Connection", "Upgrade")
	req.Header.Set("Upgrade", "tcp")
	if options.Host != "" {
		req.Host = options.Host
	}
	return req, nil
}

// streamData copies both input/output/error streams to/from the hijacked streams
func streamData(rwc io.Writer, br io.Reader, options HijackHttpOptions) error {
	errsIn := make(chan error, 1)
	errsOut := make(chan error, 1)
	exit := make(chan bool)

	go func() {
		defer close(exit)
		defer close(errsOut)
		var err error
		stdout := options.OutputStream
		if stdout == nil {
			stdout = ioutil.Discard
		}
		stderr := options.ErrorStream
		if stderr == nil {
			stderr = ioutil.Discard
		}
		if !options.DockerTermProtocol {
			// When TTY is ON, use regular copy
			_, err = io.Copy(stdout, br)
		} else {
			_, err = docker.StdCopy(stdout, stderr, br, options.Log)
		}
		errsOut <- err
	}()
	go func() {
		defer close(errsIn)
		var err error
		in := options.InputStream
		if in != nil {
			_, err = io.Copy(rwc, in)
		}
		if err := rwc.(closeWriter).CloseWrite(); err != nil {
			options.Log.Debugf("CloseWrite failed %#v", err)
		}
		errsIn <- err
	}()
	<-exit
	select {
	case err := <-errsOut:
		return err
	case err := <-errsIn:
		return err
	}
}

// ----------------------------------------------
// private interface supporting CloseWrite calls.

type closeWriter interface {
	CloseWrite() error
}

// ----------------------------------------------
// Helper to ignore debug los in case we got no logger

type logIgnore struct {
}

func (this *logIgnore) Debugf(msg string, args ...interface{}) {
	// Ignore the log message
}
<|endoftext|>