<commit_before>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sfconnection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 1

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`

		Reconnect uint `long:"reconnect" default:"30" description:"Reconnect period, seconds"`

		Debug       []bool `short:"D" long:"debug" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("amlistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Printf("Argument parser error: %s\n", err)
		os.Exit(1)
	}

	host, port, err := sfconnection.ParseSfConnectionString(opts.Positional.ConnectionString)
	if err != nil {
		fmt.Printf("ERROR: %s\n", err)
		os.Exit(1)
	}

	dsp := sfconnection.NewMessageDispatcher(new(sfconnection.Message))
	receive := make(chan sfconnection.Packet)
	dsp.RegisterMessageSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	// Configure logging
	logformat := log.Ldate | log.Ltime | log.Lmicroseconds
	var logger *log.Logger
	if len(opts.Debug) > 0 {
		if len(opts.Debug) > 1 {
			logformat = logformat | log.Lshortfile
		}
		logger = log.New(os.Stdout, "INFO:  ", logformat)
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", logformat))
		sfc.SetInfoLogger(logger)
	} else {
		logger = log.New(os.Stdout, "", logformat)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", logformat))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", logformat))

	// Connect to the host
	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	// Set up signals to close nicely on Control+C
	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<commit_msg>SF interfaces changed a bit.<commit_after>// Author  Raido Pahtma
// License MIT

package main

import "fmt"
import "os"
import "os/signal"
import "log"
import "time"

import "github.com/jessevdk/go-flags"
import "github.com/proactivity-lab/go-sfconnection"

const ApplicationVersionMajor = 0
const ApplicationVersionMinor = 1
const ApplicationVersionPatch = 2

var ApplicationBuildDate string
var ApplicationBuildDistro string

func main() {

	var opts struct {
		Positional struct {
			ConnectionString string `description:"Connectionstring sf@HOST:PORT"`
		} `positional-args:"yes"`

		Reconnect uint `long:"reconnect" default:"30" description:"Reconnect period, seconds"`

		Debug       []bool `short:"D" long:"debug" description:"Debug mode, print raw packets"`
		ShowVersion func() `short:"V" long:"version" description:"Show application version"`
	}

	opts.ShowVersion = func() {
		if ApplicationBuildDate == "" {
			ApplicationBuildDate = "YYYY-mm-dd_HH:MM:SS"
		}
		if ApplicationBuildDistro == "" {
			ApplicationBuildDistro = "unknown"
		}
		fmt.Printf("amlistener %d.%d.%d (%s %s)\n", ApplicationVersionMajor, ApplicationVersionMinor, ApplicationVersionPatch, ApplicationBuildDate, ApplicationBuildDistro)
		os.Exit(0)
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Printf("Argument parser error: %s\n", err)
		os.Exit(1)
	}

	host, port, err := sfconnection.ParseSfConnectionString(opts.Positional.ConnectionString)
	if err != nil {
		fmt.Printf("ERROR: %s\n", err)
		os.Exit(1)
	}

	dsp := sfconnection.NewMessageDispatcher(sfconnection.NewMessage(0, 0))
	receive := make(chan *sfconnection.Message)
	dsp.RegisterMessageSnooper(receive)

	sfc := sfconnection.NewSfConnection()
	sfc.AddDispatcher(dsp)

	// Configure logging
	logformat := log.Ldate | log.Ltime | log.Lmicroseconds
	var logger *log.Logger
	if len(opts.Debug) > 0 {
		if len(opts.Debug) > 1 {
			logformat = logformat | log.Lshortfile
		}
		logger = log.New(os.Stdout, "INFO:  ", logformat)
		sfc.SetDebugLogger(log.New(os.Stdout, "DEBUG: ", logformat))
		sfc.SetInfoLogger(logger)
	} else {
		logger = log.New(os.Stdout, "", logformat)
	}
	sfc.SetWarningLogger(log.New(os.Stdout, "WARN:  ", logformat))
	sfc.SetErrorLogger(log.New(os.Stdout, "ERROR: ", logformat))

	// Connect to the host
	sfc.Autoconnect(host, port, time.Duration(opts.Reconnect)*time.Second)

	// Set up signals to close nicely on Control+C
	signals := make(chan os.Signal)
	signal.Notify(signals, os.Interrupt, os.Kill)

	for interrupted := false; interrupted == false; {
		select {
		case msg := <-receive:
			logger.Printf("%s\n", msg)
		case sig := <-signals:
			signal.Stop(signals)
			logger.Printf("signal %s\n", sig)
			sfc.Disconnect()
			interrupted = true
		}
	}
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"gopkg.in/urfave/cli.v1"
	"os"
)

func main() {
	// This won't be needed in cli v2
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help"
	cli.HelpFlag.Hidden = true

	app := cli.NewApp()
	app.Name = "k6"
	app.Usage = "a next generation load generator"
	app.Version = "0.4.2"
	app.Commands = []cli.Command{
		commandRun,
		commandInspect,
		commandStatus,
		commandStats,
		commandScale,
		commandStart,
		commandPause,
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "show debug messages",
		},
		cli.StringFlag{
			Name:  "address, a",
			Usage: "address for the API",
			Value: "127.0.0.1:6565",
		},
	}
	app.Before = func(cc *cli.Context) error {
		gin.SetMode(gin.ReleaseMode)

		if cc.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		return nil
	}
	if err := app.Run(os.Args); err != nil {
		os.Exit(1)
	}
}
<commit_msg>[release] v0.4.3<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"gopkg.in/urfave/cli.v1"
	"os"
)

func main() {
	// This won't be needed in cli v2
	cli.VersionFlag.Name = "version"
	cli.HelpFlag.Name = "help"
	cli.HelpFlag.Hidden = true

	app := cli.NewApp()
	app.Name = "k6"
	app.Usage = "a next generation load generator"
	app.Version = "0.4.3"
	app.Commands = []cli.Command{
		commandRun,
		commandInspect,
		commandStatus,
		commandStats,
		commandScale,
		commandStart,
		commandPause,
	}
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "show debug messages",
		},
		cli.StringFlag{
			Name:  "address, a",
			Usage: "address for the API",
			Value: "127.0.0.1:6565",
		},
	}
	app.Before = func(cc *cli.Context) error {
		gin.SetMode(gin.ReleaseMode)

		if cc.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		return nil
	}
	if err := app.Run(os.Args); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"math/rand"
	"time"
	"html/template"
	//"io/ioutil"

)

var (
	mandrillApiUrl string
	mandrillKey    string
	casgoDestination string
	casgoAPIKey string
)

func main() {

	if os.Getenv("CASGO_API_KEY") == "" {
		log.Println("Generating Random API Key...")
		casgoAPIKey = GenerateAPIKey(20)
		log.Println("CASGO_API_KEY:",getKey())
	}else{
		casgoAPIKey = os.Getenv("CASGO_API_KEY")
	}
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()

	mandrillApiUrl = "https://mandrillapp.com/api/1.0/"
	mandrillKey = os.Getenv("MANDRILL_KEY")
	if mandrillKey == "" {
		log.Fatal("MANDRILL_KEY is Crucial. Type: export MANDRILL_KEY=123456789")
		os.Exit(1)
	}

	casgoDestination = os.Getenv("CASGO_DESTINATION")
	if casgoDestination == "" {
		log.Fatal("CASGO_DESTINATION is Crucial. Type: export CASGO_DESTINATION=\"your@email.com\"")
		os.Exit(1)
	}

	log.Printf("Starting Server on http://127.0.0.1:%s", *port)

	r := mux.NewRouter()

	r.NotFoundHandler = http.HandlerFunc(RedirectHomeHandler)
	r.HandleFunc("/", HomeHandler)

	r.HandleFunc("/contact", ContactHandler)
	r.HandleFunc("/contact/", ContactHandler)

	r.HandleFunc("/{whatever}", LoveHandler)

	r.HandleFunc("/" + casgoAPIKey + "/send", EmailHandler)
	http.Handle("/", r)
	//http.NotFound() NotFoundHandler()
	log.Fatal(http.ListenAndServe(":"+*port, r))
	log.Println("Switching Logs to debug.log")
	OpenLog()
	log.Println("info: Listening on", *port)
}

// RANDOM STUFF

func init() {
		rand.Seed(time.Now().UnixNano())
}

var runes = []rune("____ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890123456789012345678901234567890")

func GenerateAPIKey(n int) string {
		b := make([]rune, n)
		for i := range b {
				b[i] = runes[rand.Intn(len(runes))]
		}
return string(b)

}

func getKey() string {

return casgoAPIKey

}

// This function opens a log file. "debug.log"

func OpenLog(){
f, err := os.OpenFile("./debug.log", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
if err != nil {
    log.Fatal("error opening file: %v", err)
		log.Fatal("MANDRILL_KEY is Crucial.")
		os.Exit(1)
}

log.SetOutput(f)
}

// This is the home page it is blank. "This server is broken"

func HomeHandler(w http.ResponseWriter, r *http.Request) {
	 fmt.Fprint(w, "Fatal Error")
	// http.ServeFile("./templates/form.html")
}

// I love lamp. This displays affection for r.URL.Path[1:]

func LoveHandler(w http.ResponseWriter, r *http.Request) {
	 fmt.Fprintf(w, "I love %s!", r.URL.Path[1:])
	 log.Printf("I love %s says %s at %s", r.URL.Path[1:], r.UserAgent(), r.RemoteAddr)
}

// Display contact form with CSRF and a Cookie. And maybe a captcha and drawbridge.
func ContactHandler(w http.ResponseWriter, r *http.Request) {

		var key string
		//var err string
		key = getKey()
		t, err := template.New("Contact").ParseFiles("templates/form.html")
		log.Println(err)

  //  t, _ := template.ParseFiles("templates/form.html")
    // t.Execute(w, p)
		log.Println(t.ExecuteTemplate(w, "Contact", key,))
		//t.Execute(w, template.HTML(`<b>World</b>`))
	//	err, response = http.FileServer(http.Dir("/usr/share/doc"))
    //fmt.Fprint(w, "Contact form here")
	 log.Printf("pre-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
}

// Redirect everything /
func RedirectHomeHandler(rw http.ResponseWriter, r *http.Request) {
	http.Redirect(rw, r, "/", 301)
}

// Uses mandrillapp.com default sender address.
func EmailHandler(rw http.ResponseWriter, r *http.Request) {
	destination := casgoDestination
	var query url.Values
	if r.Method == "GET" {
		query = r.URL.Query()
	} else if r.Method == "POST" {
		r.ParseForm()
		query = r.Form
	} else {
		fmt.Fprintln(rw, "Please submit via GET or POST.")
	}
	EmailSender(rw, r, destination, query)

}

func EmailSender(rw http.ResponseWriter, r *http.Request, destination string, query url.Values) {
	form := ParseQuery(query)
	if form.Email == "" {
		http.Redirect(rw, r, "/", 301)
		return
	}
	if sendEmail(destination, form) {
		fmt.Fprintln(rw, "0 Success! Your message has been delivered.")
		log.Printf("SUCCESS-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
	} else {
		log.Printf("debug: %s at %s", form, destination)
		fmt.Fprintln(rw, "1 Uh-oh! We were unable to deliver your message. Please confirm that you entered a valid email address.")
		log.Printf("FAIL-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
	}
}

func ParseQuery(query url.Values) *Form {
	form := new(Form)
	additionalFields := ""
	for k, v := range query {
		k = strings.ToLower(k)
		if (k == "email") {
			form.Email = v[0]
		//} else if (k == "name") {
		//	form.Name = v[0]
		} else if (k == "subject") {
			form.Subject = v[0]
		} else if (k == "message") {
			form.Message = k + ": " + v[0] + "
\n"
		} else {
			additionalFields = additionalFields + k + ": " + v[0] + "
\n"
		}
	}
	if form.Subject == "" {
		form.Subject = "You have mail!"
	}
	if additionalFields != "" {
		if form.Message == "" {
			form.Message = form.Message + "Message:\n
" + additionalFields
		} else {
			form.Message = form.Message + "\n
Additional:\n
" + additionalFields
		}
	}
	return form
}
<commit_msg>About to add CSRF token to form<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/gorilla/csrf"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"math/rand"
	"time"
	"html/template"
	//"io/ioutil"

)

var (
	mandrillApiUrl string
	mandrillKey    string
	casgoDestination string
	casgoAPIKey string
)

func main() {

	if os.Getenv("CASGO_API_KEY") == "" {
		log.Println("Generating Random API Key...")
		casgoAPIKey = GenerateAPIKey(20)
		log.Println("CASGO_API_KEY:",getKey())
	}else{
		casgoAPIKey = os.Getenv("CASGO_API_KEY")
	}
	port := flag.String("port", "8080", "HTTP Port to listen on")
	flag.Parse()

	mandrillApiUrl = "https://mandrillapp.com/api/1.0/"
	mandrillKey = os.Getenv("MANDRILL_KEY")
	if mandrillKey == "" {
		log.Fatal("MANDRILL_KEY is Crucial. Type: export MANDRILL_KEY=123456789")
		os.Exit(1)
	}

	casgoDestination = os.Getenv("CASGO_DESTINATION")
	if casgoDestination == "" {
		log.Fatal("CASGO_DESTINATION is Crucial. Type: export CASGO_DESTINATION=\"your@email.com\"")
		os.Exit(1)
	}

	log.Printf("Starting Server on http://127.0.0.1:%s", *port)

	r := mux.NewRouter()

	r.NotFoundHandler = http.HandlerFunc(RedirectHomeHandler)
	r.HandleFunc("/", HomeHandler)

	r.HandleFunc("/contact", ContactHandler)
	r.HandleFunc("/contact/", ContactHandler)

	r.HandleFunc("/{whatever}", LoveHandler)

	r.HandleFunc("/" + casgoAPIKey + "/send", EmailHandler)
	http.Handle("/", r)
	//http.NotFound() NotFoundHandler()
	log.Fatal(http.ListenAndServe(":"+*port, csrf.Protect([]byte("32-byte-long-auth-key"))(r)))
	log.Println("Switching Logs to debug.log")
	OpenLog()
	log.Println("info: Listening on", *port)
}

// RANDOM STUFF

func init() {
		rand.Seed(time.Now().UnixNano())
}

var runes = []rune("____ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890123456789012345678901234567890")

func GenerateAPIKey(n int) string {
		b := make([]rune, n)
		for i := range b {
				b[i] = runes[rand.Intn(len(runes))]
		}
return string(b)

}

func getKey() string {

return casgoAPIKey

}

// This function opens a log file. "debug.log"

func OpenLog(){
f, err := os.OpenFile("./debug.log", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
if err != nil {
    log.Fatal("error opening file: %v", err)
		log.Fatal("MANDRILL_KEY is Crucial.")
		os.Exit(1)
}

log.SetOutput(f)
}

// This is the home page it is blank. "This server is broken"

func HomeHandler(w http.ResponseWriter, r *http.Request) {
	 fmt.Fprint(w, "Fatal Error")
	// http.ServeFile("./templates/form.html")
}

// I love lamp. This displays affection for r.URL.Path[1:]

func LoveHandler(w http.ResponseWriter, r *http.Request) {
	 fmt.Fprintf(w, "I love %s!", r.URL.Path[1:])
	 log.Printf("I love %s says %s at %s", r.URL.Path[1:], r.UserAgent(), r.RemoteAddr)
}

// Display contact form with CSRF and a Cookie. And maybe a captcha and drawbridge.
func ContactHandler(w http.ResponseWriter, r *http.Request) {
		//w.Header().Set("X-CSRF-Token", csrf.Token(r))
		var key string
		//var err string
		key = getKey()
		t, err := template.New("Contact").ParseFiles("templates/form.html")
		log.Println(err)

  //  t, _ := template.ParseFiles("templates/form.html")
    // t.Execute(w, p)
		log.Println(t.ExecuteTemplate(w, "Contact", key,))
		//t.Execute(w, template.HTML(`<b>World</b>`))
	//	err, response = http.FileServer(http.Dir("/usr/share/doc"))
    //fmt.Fprint(w, "Contact form here")
	 log.Printf("pre-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
}

// Redirect everything /
func RedirectHomeHandler(rw http.ResponseWriter, r *http.Request) {
	http.Redirect(rw, r, "/", 301)
}

// Uses mandrillapp.com default sender address.
func EmailHandler(rw http.ResponseWriter, r *http.Request) {
	destination := casgoDestination
	var query url.Values
	if r.Method == "GET" {
		query = r.URL.Query()
	} else if r.Method == "POST" {
		r.ParseForm()
		query = r.Form
	} else {
		fmt.Fprintln(rw, "Please submit via GET or POST.")
	}
	EmailSender(rw, r, destination, query)

}

func EmailSender(rw http.ResponseWriter, r *http.Request, destination string, query url.Values) {
	form := ParseQuery(query)
	if form.Email == "" {
		http.Redirect(rw, r, "/", 301)
		return
	}
	if sendEmail(destination, form) {
		fmt.Fprintln(rw, "0 Success! Your message has been delivered.")
		log.Printf("SUCCESS-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
	} else {
		log.Printf("debug: %s at %s", form, destination)
		fmt.Fprintln(rw, "1 Uh-oh! We were unable to deliver your message. Please confirm that you entered a valid email address.")
		log.Printf("FAIL-contact: %s at %s", r.UserAgent(), r.RemoteAddr)
	}
}

func ParseQuery(query url.Values) *Form {
	form := new(Form)
	additionalFields := ""
	for k, v := range query {
		k = strings.ToLower(k)
		if (k == "email") {
			form.Email = v[0]
		//} else if (k == "name") {
		//	form.Name = v[0]
		} else if (k == "subject") {
			form.Subject = v[0]
		} else if (k == "message") {
			form.Message = k + ": " + v[0] + "
\n"
		} else {
			additionalFields = additionalFields + k + ": " + v[0] + "
\n"
		}
	}
	if form.Subject == "" {
		form.Subject = "You have mail!"
	}
	if additionalFields != "" {
		if form.Message == "" {
			form.Message = form.Message + "Message:\n
" + additionalFields
		} else {
			form.Message = form.Message + "\n
Additional:\n
" + additionalFields
		}
	}
	return form
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/rebuy-de/aws-nuke/v2/cmd"
)

type NukeParameters struct {
	ConfigPath string

	Profile         string
	AccessKeyID     string
	SecretAccessKey string

	NoDryRun   bool
	Force      bool
	ForceSleep int
	Quiet      bool

	MaxWaitRetries int
}

func main() {
	if err := cmd.NewRootCommand().Execute(); err != nil {
		os.Exit(-1)
	}
}
<commit_msg>Remove unused type NukeParameters in main.go (#822)<commit_after>package main

import (
	"os"

	"github.com/rebuy-de/aws-nuke/v2/cmd"
)

func main() {
	if err := cmd.NewRootCommand().Execute(); err != nil {
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
	"unicode"
)

func main() {
	var configFile string
	var debug bool
	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")

	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.Handle("/websocket", websocket.Handler(socketHandler))
	//	log.Fatal(http.ListenAndServe(net.JoinHostPort(config.Server, srvPort), nil))
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

//used for web samaritan robot
func socketHandler(ws *websocket.Conn) {
	for {
		var in, response string
		var ret []string
		sf := func(c rune) bool {
			return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
		}
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		fmt.Printf("Received: %s\n", in)
		zh := false
		for _, r := range in {
			if unicode.Is(unicode.Scripts["Han"], r) {
				log.Printf(in)
				zh = true
				break
			}
		}
		if zh {
			response = tlAI(in)
			// Separate into fields with func.
			ret = strings.FieldsFunc(response, sf)

		} else {
			response = mitAI(in)
			ret = strings.FieldsFunc(response, sf)
		}
		for i := range ret {
			websocket.Message.Send(ws, ret[i])
			time.Sleep(time.Second)
		}
		websocket.Message.Send(ws, "")
	}
}
func ajax(w http.ResponseWriter, r *http.Request) {
	var messages = make(chan string)
	w.Header().Add("Access-Control-Allow-Origin", "*")
	body := r.FormValue("text")
	if body != "" {
		go func(string) {
			ret := receive(body)
			for i := range ret {
				messages <- ret[i]
			}

		}(body)
	}
	io.WriteString(w, <-messages)
}

func receive(in string) (ret []string) {
	if in == "" {
		return
	}
	fmt.Printf("Received: %s\n", in)
	var response string
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	zh := false
	for _, r := range in {
		if unicode.Is(unicode.Scripts["Han"], r) {
			log.Printf(in)
			zh = true
			break
		}
	}
	if zh {
		response = tlAI(in)
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<commit_msg>GET or POST<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"io"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"
	"unicode"
)

var messages = make(chan string)

func main() {
	var configFile string
	var debug bool
	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.BoolVar(&debug, "d", false, "debug mode")

	flag.Parse()
	config, err := ParseConfig(configFile)
	if err != nil {
		log.Fatal("a vailid json config file must exist")
	}

	redisPort := strconv.Itoa(config.RedisPort)
	redisServer := net.JoinHostPort(config.RedisAddress, redisPort)
	if !conn.Ping(redisServer, config.RedisPassword) {
		log.Fatal("connect to redis server failed")
	}
	conn.Pool = conn.NewPool(redisServer, config.RedisPassword, config.RedisDB)
	robot := newRobot(config.RobotToken, config.RobotName, config.WebHookUrl)
	robot.bot.Debug = debug
	go robot.run()
	srvPort := strconv.Itoa(config.Port)
	http.HandleFunc("/ajax", ajax)
	http.Handle("/websocket", websocket.Handler(socketHandler))
	//	log.Fatal(http.ListenAndServe(net.JoinHostPort(config.Server, srvPort), nil))
	log.Fatal(http.ListenAndServeTLS(net.JoinHostPort(config.Server, srvPort), config.Cert, config.CertKey, nil))

}

//used for web samaritan robot
func socketHandler(ws *websocket.Conn) {
	for {
		var in, response string
		var ret []string
		sf := func(c rune) bool {
			return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
		}
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		fmt.Printf("Received: %s\n", in)
		zh := false
		for _, r := range in {
			if unicode.Is(unicode.Scripts["Han"], r) {
				log.Printf(in)
				zh = true
				break
			}
		}
		if zh {
			response = tlAI(in)
			// Separate into fields with func.
			ret = strings.FieldsFunc(response, sf)

		} else {
			response = mitAI(in)
			ret = strings.FieldsFunc(response, sf)
		}
		for i := range ret {
			websocket.Message.Send(ws, ret[i])
			time.Sleep(time.Second)
		}
		websocket.Message.Send(ws, "")
	}
}
func ajax(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Access-Control-Allow-Origin", "*")

	if r.Method == "GET" {
		io.WriteString(w, <-messages)

	} else {
		body := r.FormValue("text")
		if body != "" {
			go func(string) {
				ret := receive(body)
				for i := range ret {
					messages <- ret[i]
				}

			}(body)
		}
	}
}

func receive(in string) (ret []string) {
	fmt.Printf("Received: %s\n", in)
	var response string
	sf := func(c rune) bool {
		return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
	}
	zh := false
	for _, r := range in {
		if unicode.Is(unicode.Scripts["Han"], r) {
			log.Printf(in)
			zh = true
			break
		}
	}
	if zh {
		response = tlAI(in)
		// Separate into fields with func.
		ret = strings.FieldsFunc(response, sf)

	} else {
		response = mitAI(in)
		ret = strings.FieldsFunc(response, sf)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/nishanths/fullstory"

	"github.com/fullstorydev/hauser/config"
	"github.com/fullstorydev/hauser/warehouse"
)

var (
	conf               *config.Config
	currentBackoffStep = uint(0)
	bundleFieldsMap    = warehouse.BundleFields()
)

// Record represents a single export row in the export file
type Record map[string]interface{}

type ExportProcessor func(warehouse.Warehouse, *fullstory.Client, []fullstory.ExportMeta) (int, error)

// TransformExportJSONRecord transforms the record map (extracted from the API response json) to a
// slice of strings. The slice of strings contains values in the same order as the existing export table.
// For existing export table fields that do not exist in the json record, an empty string is populated.
func TransformExportJSONRecord(wh warehouse.Warehouse, rec map[string]interface{}) ([]string, error) {
	var line []string
	// Change all record keys to lower case. We do this because columns are case insensitive for most warehouse solutions.
	rec = getRecordWithLowerCaseKeys(rec)

	// Map of CustomVars
	customVarsMap := make(map[string]interface{})
	for key, val := range rec {
		if field, ok := bundleFieldsMap[key]; !ok {
			customVarsMap[field.Name] = val
		}
	}

	// Fetch the table columns so can build the csv with a column order that matches the export table
	tableColumns := wh.GetExportTableColumns()
	for _, col := range tableColumns {
		field, isPartOfExportBundle := bundleFieldsMap[col]

		// These are columns in the export table that we are not going to populate
		if !isPartOfExportBundle {
			line = append(line, "")
			continue;
		}

		if field.IsCustomVar {
			customVars, err := json.Marshal(customVarsMap)
			if err != nil {
				return nil, err
			}
			line = append(line, string(customVars))
		} else {
			if val, valExists := rec[col]; valExists {
				line = append(line, wh.ValueToString(val, field.IsTime))
			} else {
				line = append(line, "")
			}
		}
	}
	return line, nil
}

func ProcessExportsSince(wh warehouse.Warehouse, since time.Time, exportProcessor ExportProcessor) (int, error) {
	log.Printf("Checking for new export files since %s", since)

	fs := fullstory.NewClient(conf.FsApiToken)
	if conf.ExportURL != "" {
		fs.Config.BaseURL = conf.ExportURL
	}
	exports, err := fs.ExportList(since)
	if err != nil {
		log.Printf("Failed to fetch export list: %s", err)
		return 0, err
	}

	return exportProcessor(wh, fs, exports)
}

// ProcessFilesIndividually iterates over the list of available export files and processes them one by one, until an error
// occurs, or until they are all processed.
func ProcessFilesIndividually(wh warehouse.Warehouse, fs *fullstory.Client, exports []fullstory.ExportMeta) (int, error) {
	for _, e := range exports {
		log.Printf("Processing bundle %d (start: %s, end: %s)", e.ID, e.Start.UTC(), e.Stop.UTC())
		filename := filepath.Join(conf.TmpDir, fmt.Sprintf("%d.csv", e.ID))
		mark := time.Now()
		outfile, err := os.Create(filename)
		if err != nil {
			log.Printf("Failed to create tmp file: %s", err)
			return 0, err
		}
		defer os.Remove(filename)
		defer outfile.Close()
		csvOut := csv.NewWriter(outfile)

		recordCount, err := WriteBundleToCSV(fs, e.ID, csvOut, wh)
		if err != nil {
			return 0, err
		}

		if err := LoadBundles(wh, filename, e); err != nil {
			return 0, err
		}

		log.Printf("Processing of bundle %d (%d records) took %s", e.ID, recordCount,
			time.Since(mark))
	}

	// return how many files were processed
	return len(exports), nil
}

// ProcessFilesByDay creates a single intermediate CSV file for all the export bundles on a given day.  It assumes the
// day to be processed is the day from the first export bundle's Start value.  When all the bundles with that same day
// have been written to the CSV file, it is loaded to the warehouse, and the function quits without attempting to
// process remaining bundles (they'll get picked up on the next call to ProcessExportsSince)
func ProcessFilesByDay(wh warehouse.Warehouse, fs *fullstory.Client, exports []fullstory.ExportMeta) (int, error) {
	if len(exports) == 0 {
		return 0, nil
	}

	log.Printf("Creating group file starting with bundle %d (start: %s)", exports[0].ID, exports[0].Start.UTC())
	filename := filepath.Join(conf.TmpDir, fmt.Sprintf("%d-%s.csv", exports[0].ID, exports[0].Start.UTC().Format("20060102")))
	mark := time.Now()
	outfile, err := os.Create(filename)
	if err != nil {
		log.Printf("Failed to create tmp file: %s", err)
		return 0, err
	}
	defer os.Remove(filename)
	defer outfile.Close()
	csvOut := csv.NewWriter(outfile)

	var processedBundles []fullstory.ExportMeta
	var totalRecords int
	groupDay := exports[0].Start.UTC().Truncate(24 * time.Hour)
	for _, e := range exports {
		if !groupDay.Equal(e.Start.UTC().Truncate(24 * time.Hour)) {
			break
		}

		recordCount, err := WriteBundleToCSV(fs, e.ID, csvOut, wh)
		if err != nil {
			return 0, err
		}

		log.Printf("Wrote bundle %d (%d records, start: %s, stop: %s)", e.ID, recordCount, e.Start.UTC(), e.Stop.UTC())
		totalRecords += recordCount
		processedBundles = append(processedBundles, e)
	}

	if err := LoadBundles(wh, filename, processedBundles...); err != nil {
		return 0, err
	}

	log.Printf("Processing of %d bundles (%d records) took %s", len(processedBundles), totalRecords,
		time.Since(mark))

	// return how many files were processed
	return len(processedBundles), nil
}

func LoadBundles (wh warehouse.Warehouse, filename string, bundles ...fullstory.ExportMeta) error {
	var objPath string
	var err error
	if objPath, err = wh.UploadFile(filename); err != nil {
		log.Printf(wh.GetUploadFailedMsg(filename, err))
		return err
	}

	if wh.IsUploadOnly() {
		return nil
	}

	defer wh.DeleteFile(objPath)

	if err := wh.LoadToWarehouse(objPath, bundles...); err != nil {
		log.Printf("Failed to load file '%s' to warehouse: %s", filename, err)
		return err
	}

	// If we've already copied in the data but fail to save the sync point, we're
	// still okay - the next call to LastSyncPoint() will see that there are export
	// records beyond the sync point and remove them - ie, we will reprocess the
	// current export file
	if err := wh.SaveSyncPoints(bundles...); err != nil {
		log.Printf("Failed to save sync points for bundles ending with %d: %s", bundles[len(bundles)].ID, err)
		return err
	}
	return nil
}

// WriteBundleToCSV writes the bundle corresponding to the given bundleID to the csv Writer
func WriteBundleToCSV(fs *fullstory.Client, bundleID int, csvOut *csv.Writer, wh warehouse.Warehouse) (numRecords int, err error) {
	stream, err := fs.ExportData(bundleID)
	if err != nil {
		log.Printf("Failed to fetch bundle %d: %s", bundleID, err)
		return 0, err
	}
	defer stream.Close()

	gzstream, err := gzip.NewReader(stream)
	if err != nil {
		log.Printf("Failed gzip reader: %s", err)
		return 0, err
	}

	decoder := json.NewDecoder(gzstream)
	decoder.UseNumber()

	// skip array open delimiter
	if _, err := decoder.Token(); err != nil {
		log.Printf("Failed json decode of array open token: %s", err)
		return 0, err
	}

	var recordCount int
	for decoder.More() {
		var r Record
		if err := decoder.Decode(&r); err != nil {
			log.Printf("failed json decode of record: %s", err)
			return recordCount, err
		}
		line, err := TransformExportJSONRecord(wh, r)
		if err != nil {
			log.Printf("Failed object transform, bundle %d; skipping record. %s", bundleID, err)
			continue
		}
		csvOut.Write(line)
		recordCount++
	}

	if _, err := decoder.Token(); err != nil {
		log.Printf("Failed json decode of array close token: %s", err)
		return recordCount, err
	}

	csvOut.Flush()
	return recordCount, nil
}

func BackoffOnError(err error) bool {
	if err != nil {
		if currentBackoffStep == uint(conf.BackoffStepsMax) {
			log.Fatalf("Reached max retries; exiting")
		}
		dur := conf.Backoff.Duration * (1 << currentBackoffStep)
		log.Printf("Pausing; will retry operation in %s", dur)
		time.Sleep(dur)
		currentBackoffStep++
		return true
	}
	currentBackoffStep = 0
	return false
}

func getRecordWithLowerCaseKeys(rec map[string]interface{}) map[string]interface{} {
	m := make(map[string]interface{})
	for k, v := range rec {
		m[strings.ToLower(k)] = v
	}
	return m
}

func main() {
	conffile := flag.String("c", "config.toml", "configuration file")
	flag.Parse()

	var err error
	if conf, err = config.Load(*conffile); err != nil {
		log.Fatal(err)
	}

	exportProcessor := ProcessFilesIndividually
	if conf.GroupFilesByDay {
		exportProcessor = ProcessFilesByDay
	}

	var wh warehouse.Warehouse
	switch conf.Warehouse {
	case "redshift":
		wh = warehouse.NewRedshift(conf)
	case "bigquery":
		wh = warehouse.NewBigQuery(conf)
	default:
		if len(conf.Warehouse) == 0 {
			log.Fatal("Warehouse type must be specified in configuration")
		} else {
			log.Fatalf("Warehouse type '%s' unrecognized", conf.Warehouse)
		}
	}

	if err := wh.EnsureCompatibleExportTable(); err != nil {
		log.Fatal(err)
	}

	for {
		lastSyncedRecord, err := wh.LastSyncPoint()
		if BackoffOnError(err) {
			continue
		}

		numBundles, err := ProcessExportsSince(wh, lastSyncedRecord, exportProcessor)
		if BackoffOnError(err) {
			continue
		}

		// if we processed any bundles, there may be more - check until nothing comes back
		if numBundles > 0 {
			continue
		}

		log.Printf("No exports pending; sleeping %s", conf.CheckInterval.Duration)
		time.Sleep(conf.CheckInterval.Duration)
	}
}
<commit_msg>Reduce warehouse DB calls by figuring out the export table columns once<commit_after>package main

import (
	"compress/gzip"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/nishanths/fullstory"

	"github.com/fullstorydev/hauser/config"
	"github.com/fullstorydev/hauser/warehouse"
)

var (
	conf               *config.Config
	currentBackoffStep = uint(0)
	bundleFieldsMap    = warehouse.BundleFields()
)

// Record represents a single export row in the export file
type Record map[string]interface{}

type ExportProcessor func(warehouse.Warehouse, []string, *fullstory.Client, []fullstory.ExportMeta) (int, error)

// TransformExportJSONRecord transforms the record map (extracted from the API response json) to a
// slice of strings. The slice of strings contains values in the same order as the existing export table.
// For existing export table fields that do not exist in the json record, an empty string is populated.
func TransformExportJSONRecord(wh warehouse.Warehouse, tableColumns []string, rec map[string]interface{}) ([]string, error) {
	var line []string
	// Change all record keys to lower case. We do this because columns are case insensitive for most warehouse solutions.
	rec = getRecordWithLowerCaseKeys(rec)

	// Map of CustomVars
	customVarsMap := make(map[string]interface{})
	for key, val := range rec {
		if field, ok := bundleFieldsMap[key]; !ok {
			customVarsMap[field.Name] = val
		}
	}

	// Fetch the table columns so can build the csv with a column order that matches the export table
	for _, col := range tableColumns {
		field, isPartOfExportBundle := bundleFieldsMap[col]

		// These are columns in the export table that we are not going to populate
		if !isPartOfExportBundle {
			line = append(line, "")
			continue;
		}

		if field.IsCustomVar {
			customVars, err := json.Marshal(customVarsMap)
			if err != nil {
				return nil, err
			}
			line = append(line, string(customVars))
		} else {
			if val, valExists := rec[col]; valExists {
				line = append(line, wh.ValueToString(val, field.IsTime))
			} else {
				line = append(line, "")
			}
		}
	}
	return line, nil
}

func ProcessExportsSince(wh warehouse.Warehouse, tableColumns []string, since time.Time, exportProcessor ExportProcessor) (int, error) {
	log.Printf("Checking for new export files since %s", since)

	fs := fullstory.NewClient(conf.FsApiToken)
	if conf.ExportURL != "" {
		fs.Config.BaseURL = conf.ExportURL
	}
	exports, err := fs.ExportList(since)
	if err != nil {
		log.Printf("Failed to fetch export list: %s", err)
		return 0, err
	}

	return exportProcessor(wh, tableColumns, fs, exports)
}

// ProcessFilesIndividually iterates over the list of available export files and processes them one by one, until an error
// occurs, or until they are all processed.
func ProcessFilesIndividually(wh warehouse.Warehouse, tableColumns []string, fs *fullstory.Client, exports []fullstory.ExportMeta) (int, error) {
	for _, e := range exports {
		log.Printf("Processing bundle %d (start: %s, end: %s)", e.ID, e.Start.UTC(), e.Stop.UTC())
		filename := filepath.Join(conf.TmpDir, fmt.Sprintf("%d.csv", e.ID))
		mark := time.Now()
		outfile, err := os.Create(filename)
		if err != nil {
			log.Printf("Failed to create tmp file: %s", err)
			return 0, err
		}
		defer os.Remove(filename)
		defer outfile.Close()
		csvOut := csv.NewWriter(outfile)

		recordCount, err := WriteBundleToCSV(fs, e.ID, tableColumns, csvOut, wh)
		if err != nil {
			return 0, err
		}

		if err := LoadBundles(wh, filename, e); err != nil {
			return 0, err
		}

		log.Printf("Processing of bundle %d (%d records) took %s", e.ID, recordCount,
			time.Since(mark))
	}

	// return how many files were processed
	return len(exports), nil
}

// ProcessFilesByDay creates a single intermediate CSV file for all the export bundles on a given day.  It assumes the
// day to be processed is the day from the first export bundle's Start value.  When all the bundles with that same day
// have been written to the CSV file, it is loaded to the warehouse, and the function quits without attempting to
// process remaining bundles (they'll get picked up on the next call to ProcessExportsSince)
func ProcessFilesByDay(wh warehouse.Warehouse, tableColumns []string, fs *fullstory.Client, exports []fullstory.ExportMeta) (int, error) {
	if len(exports) == 0 {
		return 0, nil
	}

	log.Printf("Creating group file starting with bundle %d (start: %s)", exports[0].ID, exports[0].Start.UTC())
	filename := filepath.Join(conf.TmpDir, fmt.Sprintf("%d-%s.csv", exports[0].ID, exports[0].Start.UTC().Format("20060102")))
	mark := time.Now()
	outfile, err := os.Create(filename)
	if err != nil {
		log.Printf("Failed to create tmp file: %s", err)
		return 0, err
	}
	defer os.Remove(filename)
	defer outfile.Close()
	csvOut := csv.NewWriter(outfile)

	var processedBundles []fullstory.ExportMeta
	var totalRecords int
	groupDay := exports[0].Start.UTC().Truncate(24 * time.Hour)
	for _, e := range exports {
		if !groupDay.Equal(e.Start.UTC().Truncate(24 * time.Hour)) {
			break
		}

		recordCount, err := WriteBundleToCSV(fs, e.ID, tableColumns, csvOut, wh)
		if err != nil {
			return 0, err
		}

		log.Printf("Wrote bundle %d (%d records, start: %s, stop: %s)", e.ID, recordCount, e.Start.UTC(), e.Stop.UTC())
		totalRecords += recordCount
		processedBundles = append(processedBundles, e)
	}

	if err := LoadBundles(wh, filename, processedBundles...); err != nil {
		return 0, err
	}

	log.Printf("Processing of %d bundles (%d records) took %s", len(processedBundles), totalRecords,
		time.Since(mark))

	// return how many files were processed
	return len(processedBundles), nil
}

func LoadBundles (wh warehouse.Warehouse, filename string, bundles ...fullstory.ExportMeta) error {
	var objPath string
	var err error
	if objPath, err = wh.UploadFile(filename); err != nil {
		log.Printf(wh.GetUploadFailedMsg(filename, err))
		return err
	}

	if wh.IsUploadOnly() {
		return nil
	}

	defer wh.DeleteFile(objPath)

	if err := wh.LoadToWarehouse(objPath, bundles...); err != nil {
		log.Printf("Failed to load file '%s' to warehouse: %s", filename, err)
		return err
	}

	// If we've already copied in the data but fail to save the sync point, we're
	// still okay - the next call to LastSyncPoint() will see that there are export
	// records beyond the sync point and remove them - ie, we will reprocess the
	// current export file
	if err := wh.SaveSyncPoints(bundles...); err != nil {
		log.Printf("Failed to save sync points for bundles ending with %d: %s", bundles[len(bundles)].ID, err)
		return err
	}
	return nil
}

// WriteBundleToCSV writes the bundle corresponding to the given bundleID to the csv Writer
func WriteBundleToCSV(fs *fullstory.Client, bundleID int, tableColumns []string, csvOut *csv.Writer, wh warehouse.Warehouse) (numRecords int, err error) {
	stream, err := fs.ExportData(bundleID)
	if err != nil {
		log.Printf("Failed to fetch bundle %d: %s", bundleID, err)
		return 0, err
	}
	defer stream.Close()

	gzstream, err := gzip.NewReader(stream)
	if err != nil {
		log.Printf("Failed gzip reader: %s", err)
		return 0, err
	}

	decoder := json.NewDecoder(gzstream)
	decoder.UseNumber()

	// skip array open delimiter
	if _, err := decoder.Token(); err != nil {
		log.Printf("Failed json decode of array open token: %s", err)
		return 0, err
	}

	var recordCount int
	for decoder.More() {
		var r Record
		if err := decoder.Decode(&r); err != nil {
			log.Printf("failed json decode of record: %s", err)
			return recordCount, err
		}
		line, err := TransformExportJSONRecord(wh, tableColumns, r)
		if err != nil {
			log.Printf("Failed object transform, bundle %d; skipping record. %s", bundleID, err)
			continue
		}
		csvOut.Write(line)
		recordCount++
	}

	if _, err := decoder.Token(); err != nil {
		log.Printf("Failed json decode of array close token: %s", err)
		return recordCount, err
	}

	csvOut.Flush()
	return recordCount, nil
}

func BackoffOnError(err error) bool {
	if err != nil {
		if currentBackoffStep == uint(conf.BackoffStepsMax) {
			log.Fatalf("Reached max retries; exiting")
		}
		dur := conf.Backoff.Duration * (1 << currentBackoffStep)
		log.Printf("Pausing; will retry operation in %s", dur)
		time.Sleep(dur)
		currentBackoffStep++
		return true
	}
	currentBackoffStep = 0
	return false
}

func getRecordWithLowerCaseKeys(rec map[string]interface{}) map[string]interface{} {
	m := make(map[string]interface{})
	for k, v := range rec {
		m[strings.ToLower(k)] = v
	}
	return m
}

func main() {
	conffile := flag.String("c", "config.toml", "configuration file")
	flag.Parse()

	var err error
	if conf, err = config.Load(*conffile); err != nil {
		log.Fatal(err)
	}

	exportProcessor := ProcessFilesIndividually
	if conf.GroupFilesByDay {
		exportProcessor = ProcessFilesByDay
	}

	var wh warehouse.Warehouse
	switch conf.Warehouse {
	case "redshift":
		wh = warehouse.NewRedshift(conf)
	case "bigquery":
		wh = warehouse.NewBigQuery(conf)
	default:
		if len(conf.Warehouse) == 0 {
			log.Fatal("Warehouse type must be specified in configuration")
		} else {
			log.Fatalf("Warehouse type '%s' unrecognized", conf.Warehouse)
		}
	}

	if err := wh.EnsureCompatibleExportTable(); err != nil {
		log.Fatal(err)
	}

	// NB: We SHOULD fetch the table columns ONLY after the call to EnsureCompatibleExportTable.
	// The EnsureCompatibleExportTable function potentially alters the schema of the export table in the client warehouse.
	tableColumns := wh.GetExportTableColumns()
	for {
		lastSyncedRecord, err := wh.LastSyncPoint()
		if BackoffOnError(err) {
			continue
		}

		numBundles, err := ProcessExportsSince(wh, tableColumns, lastSyncedRecord, exportProcessor)
		if BackoffOnError(err) {
			continue
		}

		// if we processed any bundles, there may be more - check until nothing comes back
		if numBundles > 0 {
			continue
		}

		log.Printf("No exports pending; sleeping %s", conf.CheckInterval.Duration)
		time.Sleep(conf.CheckInterval.Duration)
	}
}
<|endoftext|>


<commit_before>package main

import (
   "bufio"
   "fmt"
   "os"
)

func cat(c chan string, e chan error, quit chan struct{}) {
   scanner := bufio.NewScanner(os.Stdin)
   for scanner.Scan() {
      c <- scanner.Text()
   }
   if err := scanner.Err(); err != nil {
      e <- err
   }
   quit <- struct{}{}
}

func main() {
   c := make(chan string)
   quit := make(chan struct{})
   err := make(chan error)
   go cat(c, err, quit)
   for {
      select {
         case x := <-c:
            fmt.Println(x)
         case x := <-err:
            fmt.Fprintln(os.Stderr, "reading standard input:", x)
         case <-quit:
            return
      }
   }
}
<commit_msg>format<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
)

func cat(c chan string, e chan error, quit chan struct{}) {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		c <- scanner.Text()
	}
	if err := scanner.Err(); err != nil {
		e <- err
	}
	quit <- struct{}{}
}

func main() {
	c := make(chan string)
	quit := make(chan struct{})
	err := make(chan error)
	go cat(c, err, quit)
	for {
		select {
		case x := <-c:
			fmt.Println(x)
		case x := <-err:
			fmt.Fprintln(os.Stderr, "reading standard input:", x)
		case <-quit:
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.crypto/ssh"
	"code.google.com/p/gopass"
	"flag"
	"github.com/sudharsh/henchman/henchman"
	"log"
	"os"
	"os/user"
	"path"
)

func currentUsername() *user.User {
	u, err := user.Current()
	if err != nil {
		panic("Couldn't get current username: " + err.Error())
	}
	return u
}

func defaultKeyFile() string {
	u := currentUsername()
	return path.Join(u.HomeDir, ".ssh", "id_rsa")
}

func main() {
	username := flag.String("user", currentUsername().Username, "User to run as")
	usePassword := flag.Bool("password", false, "Use password authentication")
	keyfile := flag.String("private-keyfile", defaultKeyFile(), "Path to the keyfile")

	flag.Parse()
	planFile := flag.Arg(0)
	if *username == "" {
		os.Exit(1)
	}

	var sshAuth ssh.ClientAuth
	var err error
	if *usePassword {
		var password string
		if password, err = gopass.GetPass("Password:"); err != nil {
			log.Fatalf("Couldn't get password: " + err.Error())
			os.Exit(1)
		}
		sshAuth, err = henchman.PasswordAuth(password)
	} else {
		sshAuth, err = henchman.ClientKeyAuth(*keyfile)
	}
	if err != nil {
		log.Fatalf("SSH Auth prep failed: " + err.Error())
	}
	config := &ssh.ClientConfig{
		User: *username,
		Auth: []ssh.ClientAuth{sshAuth},
	}

	plan, err := henchman.ParsePlan(planFile)
	if err != nil {
		log.Fatalf("Couldn't read the plan: %s", err)
		os.Exit(1)
	}

	sem := make(chan int, 100)
	for _, hostname := range plan.Hosts {
		go func() {
			machine := henchman.Machine{hostname, config}
			for _, task := range plan.Tasks {
				machine.RunTask(&task)
			}
			sem <- 1
		}()
		<-sem
	}

}
<commit_msg>Get all tasks first and then run each task on the machine<commit_after>package main

import (
	"code.google.com/p/go.crypto/ssh"
	"code.google.com/p/gopass"
	"flag"
	"github.com/sudharsh/henchman/henchman"
	"log"
	"os"
	"os/user"
	"path"
)

func currentUsername() *user.User {
	u, err := user.Current()
	if err != nil {
		panic("Couldn't get current username: " + err.Error())
	}
	return u
}

func defaultKeyFile() string {
	u := currentUsername()
	return path.Join(u.HomeDir, ".ssh", "id_rsa")
}

func main() {
	username := flag.String("user", currentUsername().Username, "User to run as")
	usePassword := flag.Bool("password", false, "Use password authentication")
	keyfile := flag.String("private-keyfile", defaultKeyFile(), "Path to the keyfile")

	flag.Parse()
	planFile := flag.Arg(0)
	if *username == "" {
		os.Exit(1)
	}

	var sshAuth ssh.ClientAuth
	var err error
	if *usePassword {
		var password string
		if password, err = gopass.GetPass("Password:"); err != nil {
			log.Fatalf("Couldn't get password: " + err.Error())
			os.Exit(1)
		}
		sshAuth, err = henchman.PasswordAuth(password)
	} else {
		sshAuth, err = henchman.ClientKeyAuth(*keyfile)
	}
	if err != nil {
		log.Fatalf("SSH Auth prep failed: " + err.Error())
	}
	config := &ssh.ClientConfig{
		User: *username,
		Auth: []ssh.ClientAuth{sshAuth},
	}

	plan, err := henchman.ParsePlan(planFile)
	if err != nil {
		log.Fatalf("Couldn't read the plan: %s", err)
		os.Exit(1)
	}

	sem := make(chan int, 100)
	for _, task := range plan.Tasks {
		for _, hostname := range plan.Hosts {
			go func() {
				machine := henchman.Machine{hostname, config}
				machine.RunTask(&task)
				sem <- 1
			}()
			<- sem
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/jessevdk/go-flags"

	"github.com/ivanilves/lstags/auth"
	"github.com/ivanilves/lstags/docker/jsonconfig"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/registry"
)

type options struct {
	DefaultRegistry    string `short:"r" long:"default-registry" default:"registry.hub.docker.com" description:"Docker registry to use by default" env:"DEFAULT_REGISTRY"`
	Username           string `short:"u" long:"username" default:"" description:"Docker registry username" env:"USERNAME"`
	Password           string `short:"p" long:"password" default:"" description:"Docker registry password" env:"PASSWORD"`
	DockerJSON         string `shord:"j" long:"docker-json" default:"~/.docker/config.json" env:"DOCKER_JSON"`
	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
	Pull               bool   `short:"P" long:"pull" description:"Pull images matched by filter" env:"PULL"`
	InsecureRegistry   bool   `short:"i" long:"insecure-registry" description:"Use insecure plain-HTTP registriy" env:"INSECURE_REGISTRY"`
	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace registry HTTP requests" env:"TRACE_REQUESTS"`
	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
	Positional         struct {
		Repositories []string `positional-arg-name:"REPO1 REPO2" description:"Docker repositories to operate on"`
	} `positional-args:"yes"`
}

func suicide(err error) {
	fmt.Printf("%s\n", err.Error())
	os.Exit(1)
}

func getVersion() string {
	return VERSION
}

func trimFilter(repoWithFilter string) (string, string, error) {
	parts := strings.Split(repoWithFilter, "~")

	repository := parts[0]

	if len(parts) < 2 {
		return repository, ".*", nil
	}

	if len(parts) > 2 {
		return "", "", errors.New("Unable to trim filter from repository (too many '~'!): " + repoWithFilter)
	}

	f := parts[1]

	if !strings.HasPrefix(f, "/") || !strings.HasSuffix(f, "/") {
		return "", "", errors.New("Filter should be passed in a form: /REGEXP/")
	}

	filter := f[1 : len(f)-1]

	return repository, filter, nil
}

func matchesFilter(s, filter string) bool {
	matched, err := regexp.MatchString(filter, s)
	if err != nil {
		return false
	}

	return matched
}

func isHostname(s string) bool {
	if strings.Contains(s, ".") {
		return true
	}

	if strings.Contains(s, ":") {
		return true
	}

	if s == "localhost" {
		return true
	}

	return false
}

func getRegistryName(repository, defaultRegistry string) string {
	r := strings.Split(repository, "/")[0]

	if isHostname(r) {
		return r
	}

	return defaultRegistry
}

func assignCredentials(registry, passedUsername, passedPassword, dockerJSON string) (string, string, error) {
	useDefaultDockerJSON := dockerJSON == "~/.docker/config.json"
	areCredentialsPassed := passedUsername != "" && passedPassword != ""

	c, err := jsonconfig.Load(dockerJSON)
	if err != nil {
		if useDefaultDockerJSON {
			return passedUsername, passedPassword, nil
		}

		return "", "", err
	}

	username, password, defined := c.GetCredentials(registry)
	if !defined || areCredentialsPassed {
		return passedUsername, passedPassword, nil
	}

	return username, password, nil
}

func getAuthorization(t auth.TokenResponse) string {
	return t.Method() + " " + t.Token()
}

func main() {
	o := options{}

	_, err := flags.Parse(&o)
	if err != nil {
		suicide(err)
	}
	if o.Version {
		println(getVersion())
		os.Exit(0)
	}
	if len(o.Positional.Repositories) == 0 {
		suicide(errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'"))
	}

	if o.InsecureRegistry {
		auth.WebSchema = "http://"
		registry.WebSchema = "http://"
	}

	registry.TraceRequests = o.TraceRequests

	const format = "%-12s %-45s %-15s %-25s %s\n"
	fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")

	repoCount := len(o.Positional.Repositories)

	type tagResult struct {
		Tags []*tag.Tag
		Repo string
	}

	trc := make(chan tagResult, repoCount)

	for _, r := range o.Positional.Repositories {
		go func(r string, o options, trc chan tagResult) {
			repository, filter, err := trimFilter(r)
			if err != nil {
				suicide(err)
			}

			registryName := getRegistryName(repository, o.DefaultRegistry)

			repoRegistryName := registry.FormatRepoName(repository, registryName)
			repoLocalName := local.FormatRepoName(repository, registryName)

			username, password, err := assignCredentials(registryName, o.Username, o.Password, o.DockerJSON)
			if err != nil {
				suicide(err)
			}

			tresp, err := auth.NewToken(registryName, repoRegistryName, username, password)
			if err != nil {
				suicide(err)
			}

			authorization := getAuthorization(tresp)

			registryTags, err := registry.FetchTags(registryName, repoRegistryName, authorization, o.ConcurrentRequests)
			if err != nil {
				suicide(err)
			}
			localTags, err := local.FetchTags(repoLocalName)
			if err != nil {
				suicide(err)
			}

			sortedKeys, names, joinedTags := tag.Join(registryTags, localTags)

			tags := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := names[key]

				tg := joinedTags[name]

				if !matchesFilter(tg.GetName(), filter) {
					continue
				}

				tags = append(tags, tg)
			}

			trc <- tagResult{Tags: tags, Repo: repoLocalName}
		}(r, o, trc)
	}

	tagResults := make([]tagResult, repoCount)
	repoNumber := 0
	for tr := range trc {
		repoNumber++
		tagResults = append(tagResults, tr)
		if repoNumber >= repoCount {
			close(trc)
		}
	}

	for _, tr := range tagResults {
		for _, tg := range tr.Tags {
			fmt.Printf(
				format,
				tg.GetState(),
				tg.GetShortDigest(),
				tg.GetImageID(),
				tg.GetCreatedString(),
				tr.Repo+":"+tg.GetName(),
			)
		}
	}

	if o.Pull {
		done := make(chan bool, repoCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo string, done chan bool) {
				for _, tg := range tags {
					if tg.NeedsPull() {
						ref := repo + ":" + tg.GetName()

						fmt.Printf("PULLING %s\n", ref)
						err := local.Pull(ref)
						if err != nil {
							suicide(err)
						}
					}

					done <- true
				}
			}(tr.Tags, tr.Repo, done)
		}

		repoNumber := 0
		for range done {
			repoNumber++

			if repoNumber >= repoCount {
				close(done)
			}
		}
	}

}
<commit_msg>Fix fuzzy CLI help<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/jessevdk/go-flags"

	"github.com/ivanilves/lstags/auth"
	"github.com/ivanilves/lstags/docker/jsonconfig"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/registry"
)

type options struct {
	DefaultRegistry    string `short:"r" long:"default-registry" default:"registry.hub.docker.com" description:"Default Docker registry to use" env:"DEFAULT_REGISTRY"`
	DockerJSON         string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials (use it, please <3)" env:"DOCKER_JSON"`
	Username           string `short:"u" long:"username" default:"" description:"Override Docker registry username (not recommended, please use JSON file)" env:"USERNAME"`
	Password           string `short:"p" long:"password" default:"" description:"Override Docker registry password (not recommended, please use JSON file)" env:"PASSWORD"`
	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
	Pull               bool   `short:"P" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"`
	InsecureRegistry   bool   `short:"i" long:"insecure-registry" description:"Use insecure plain-HTTP connection to registries (not recommended!)" env:"INSECURE_REGISTRY"`
	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
	Positional         struct {
		Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"`
	} `positional-args:"yes" required:"yes"`
}

func suicide(err error) {
	fmt.Printf("%s\n", err.Error())
	os.Exit(1)
}

func getVersion() string {
	return VERSION
}

func trimFilter(repoWithFilter string) (string, string, error) {
	parts := strings.Split(repoWithFilter, "~")

	repository := parts[0]

	if len(parts) < 2 {
		return repository, ".*", nil
	}

	if len(parts) > 2 {
		return "", "", errors.New("Unable to trim filter from repository (too many '~'!): " + repoWithFilter)
	}

	f := parts[1]

	if !strings.HasPrefix(f, "/") || !strings.HasSuffix(f, "/") {
		return "", "", errors.New("Filter should be passed in a form: /REGEXP/")
	}

	filter := f[1 : len(f)-1]

	return repository, filter, nil
}

func matchesFilter(s, filter string) bool {
	matched, err := regexp.MatchString(filter, s)
	if err != nil {
		return false
	}

	return matched
}

func isHostname(s string) bool {
	if strings.Contains(s, ".") {
		return true
	}

	if strings.Contains(s, ":") {
		return true
	}

	if s == "localhost" {
		return true
	}

	return false
}

func getRegistryName(repository, defaultRegistry string) string {
	r := strings.Split(repository, "/")[0]

	if isHostname(r) {
		return r
	}

	return defaultRegistry
}

func assignCredentials(registry, passedUsername, passedPassword, dockerJSON string) (string, string, error) {
	useDefaultDockerJSON := dockerJSON == "~/.docker/config.json"
	areCredentialsPassed := passedUsername != "" && passedPassword != ""

	c, err := jsonconfig.Load(dockerJSON)
	if err != nil {
		if useDefaultDockerJSON {
			return passedUsername, passedPassword, nil
		}

		return "", "", err
	}

	username, password, defined := c.GetCredentials(registry)
	if !defined || areCredentialsPassed {
		return passedUsername, passedPassword, nil
	}

	return username, password, nil
}

func getAuthorization(t auth.TokenResponse) string {
	return t.Method() + " " + t.Token()
}

func main() {
	o := options{}

	_, err := flags.Parse(&o)
	if err != nil {
		os.Exit(1)
	}
	if o.Version {
		println(getVersion())
		os.Exit(0)
	}
	if len(o.Positional.Repositories) == 0 {
		suicide(errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'"))
	}

	if o.InsecureRegistry {
		auth.WebSchema = "http://"
		registry.WebSchema = "http://"
	}

	registry.TraceRequests = o.TraceRequests

	const format = "%-12s %-45s %-15s %-25s %s\n"
	fmt.Printf(format, "<STATE>", "<DIGEST>", "<(local) ID>", "<Created At>", "<TAG>")

	repoCount := len(o.Positional.Repositories)

	type tagResult struct {
		Tags []*tag.Tag
		Repo string
	}

	trc := make(chan tagResult, repoCount)

	for _, r := range o.Positional.Repositories {
		go func(r string, o options, trc chan tagResult) {
			repository, filter, err := trimFilter(r)
			if err != nil {
				suicide(err)
			}

			registryName := getRegistryName(repository, o.DefaultRegistry)

			repoRegistryName := registry.FormatRepoName(repository, registryName)
			repoLocalName := local.FormatRepoName(repository, registryName)

			username, password, err := assignCredentials(registryName, o.Username, o.Password, o.DockerJSON)
			if err != nil {
				suicide(err)
			}

			tresp, err := auth.NewToken(registryName, repoRegistryName, username, password)
			if err != nil {
				suicide(err)
			}

			authorization := getAuthorization(tresp)

			registryTags, err := registry.FetchTags(registryName, repoRegistryName, authorization, o.ConcurrentRequests)
			if err != nil {
				suicide(err)
			}
			localTags, err := local.FetchTags(repoLocalName)
			if err != nil {
				suicide(err)
			}

			sortedKeys, names, joinedTags := tag.Join(registryTags, localTags)

			tags := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := names[key]

				tg := joinedTags[name]

				if !matchesFilter(tg.GetName(), filter) {
					continue
				}

				tags = append(tags, tg)
			}

			trc <- tagResult{Tags: tags, Repo: repoLocalName}
		}(r, o, trc)
	}

	tagResults := make([]tagResult, repoCount)
	repoNumber := 0
	for tr := range trc {
		repoNumber++
		tagResults = append(tagResults, tr)
		if repoNumber >= repoCount {
			close(trc)
		}
	}

	for _, tr := range tagResults {
		for _, tg := range tr.Tags {
			fmt.Printf(
				format,
				tg.GetState(),
				tg.GetShortDigest(),
				tg.GetImageID(),
				tg.GetCreatedString(),
				tr.Repo+":"+tg.GetName(),
			)
		}
	}

	if o.Pull {
		done := make(chan bool, repoCount)

		for _, tr := range tagResults {
			go func(tags []*tag.Tag, repo string, done chan bool) {
				for _, tg := range tags {
					if tg.NeedsPull() {
						ref := repo + ":" + tg.GetName()

						fmt.Printf("PULLING %s\n", ref)
						err := local.Pull(ref)
						if err != nil {
							suicide(err)
						}
					}

					done <- true
				}
			}(tr.Tags, tr.Repo, done)
		}

		repoNumber := 0
		for range done {
			repoNumber++

			if repoNumber >= repoCount {
				close(done)
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/jrperritt/rack/commands/blockstoragecommands"
	"github.com/jrperritt/rack/commands/filescommands"
	"github.com/jrperritt/rack/commands/networkscommands"
	"github.com/jrperritt/rack/commands/serverscommands"
	"github.com/jrperritt/rack/setup"
	"github.com/jrperritt/rack/util"

	"github.com/jrperritt/rack/internal/github.com/codegangsta/cli"
)

func main() {
	cli.HelpPrinter = printHelp
	cli.CommandHelpTemplate = `NAME: {{.Name}} - {{.Usage}}{{if .Description}}

DESCRIPTION: {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
{{range .Flags}}{{flag .}}
{{end}}{{ end }}
`
	app := cli.NewApp()
	app.Name = "rack"
	app.Usage = Usage()
	app.HideVersion = true
	app.EnableBashCompletion = true
	app.Commands = Cmds()
	app.Before = func(c *cli.Context) error {
		//fmt.Printf("c.Args: %+v\n", c.Args())
		return nil
	}
	app.CommandNotFound = commandNotFound
	app.Run(os.Args)
}

// Usage returns, you guessed it, the usage information
func Usage() string {
	return "An opinionated CLI for the Rackspace cloud"
}

// Desc returns, you guessed it, the description
func Desc() string {
	return `Rack is an opinionated command-line tool that allows Rackspace users
to accomplish tasks in a simple, idiomatic way. It seeks to provide
flexibility through common Unix practices like piping and composability. All
commands have been tested against Rackspace's live API.`
}

// Cmds returns a list of commands supported by the tool
func Cmds() []cli.Command {
	return []cli.Command{
		{
			Name:  "init",
			Usage: "[Linux/OS X only] Creates the rack man page and sets up command completion for the Bash shell.",
			Action: func(c *cli.Context) {
				setup.Init(c)
				man()
			},
		},
		{
			Name:   "configure",
			Usage:  "Interactively create a config file for Rackspace authentication.",
			Action: configure,
		},
		{
			Name:   "version",
			Usage:  "Print the version of this binary.",
			Action: version,
		},
		{
			Name:        "servers",
			Usage:       "Operations on cloud servers, both virtual and bare metal.",
			Subcommands: serverscommands.Get(),
		},
		{
			Name:        "files",
			Usage:       "Object storage for files and media.",
			Subcommands: filescommands.Get(),
		},
		{
			Name:        "networks",
			Usage:       "Software-defined networking.",
			Subcommands: networkscommands.Get(),
		},
		{
			Name:        "block-storage",
			Usage:       "Block-level storage, exposed as volumes to mount to host servers. Work with volumes and their associated snapshots.",
			Subcommands: blockstoragecommands.Get(),
		},
	}
}

func printHelp(out io.Writer, templ string, data interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
		"flag": flag,
	}

	w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func flag(flag cli.Flag) string {
	switch flag.(type) {
	case cli.StringFlag:
		flagType := flag.(cli.StringFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.IntFlag:
		flagType := flag.(cli.IntFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.BoolFlag:
		flagType := flag.(cli.BoolFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.StringSliceFlag:
		flagType := flag.(cli.StringSliceFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	}
	return ""
}

func version(c *cli.Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\ncommit: %v", c.App.Name, util.Version, util.Commit)
}
<commit_msg>add new line after 'rack version'<commit_after>package main

import (
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/jrperritt/rack/commands/blockstoragecommands"
	"github.com/jrperritt/rack/commands/filescommands"
	"github.com/jrperritt/rack/commands/networkscommands"
	"github.com/jrperritt/rack/commands/serverscommands"
	"github.com/jrperritt/rack/setup"
	"github.com/jrperritt/rack/util"

	"github.com/jrperritt/rack/internal/github.com/codegangsta/cli"
)

func main() {
	cli.HelpPrinter = printHelp
	cli.CommandHelpTemplate = `NAME: {{.Name}} - {{.Usage}}{{if .Description}}

DESCRIPTION: {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
{{range .Flags}}{{flag .}}
{{end}}{{ end }}
`
	app := cli.NewApp()
	app.Name = "rack"
	app.Usage = Usage()
	app.HideVersion = true
	app.EnableBashCompletion = true
	app.Commands = Cmds()
	app.Before = func(c *cli.Context) error {
		//fmt.Printf("c.Args: %+v\n", c.Args())
		return nil
	}
	app.CommandNotFound = commandNotFound
	app.Run(os.Args)
}

// Usage returns, you guessed it, the usage information
func Usage() string {
	return "An opinionated CLI for the Rackspace cloud"
}

// Desc returns, you guessed it, the description
func Desc() string {
	return `Rack is an opinionated command-line tool that allows Rackspace users
to accomplish tasks in a simple, idiomatic way. It seeks to provide
flexibility through common Unix practices like piping and composability. All
commands have been tested against Rackspace's live API.`
}

// Cmds returns a list of commands supported by the tool
func Cmds() []cli.Command {
	return []cli.Command{
		{
			Name:  "init",
			Usage: "[Linux/OS X only] Creates the rack man page and sets up command completion for the Bash shell.",
			Action: func(c *cli.Context) {
				setup.Init(c)
				man()
			},
		},
		{
			Name:   "configure",
			Usage:  "Interactively create a config file for Rackspace authentication.",
			Action: configure,
		},
		{
			Name:   "version",
			Usage:  "Print the version of this binary.",
			Action: version,
		},
		{
			Name:        "servers",
			Usage:       "Operations on cloud servers, both virtual and bare metal.",
			Subcommands: serverscommands.Get(),
		},
		{
			Name:        "files",
			Usage:       "Object storage for files and media.",
			Subcommands: filescommands.Get(),
		},
		{
			Name:        "networks",
			Usage:       "Software-defined networking.",
			Subcommands: networkscommands.Get(),
		},
		{
			Name:        "block-storage",
			Usage:       "Block-level storage, exposed as volumes to mount to host servers. Work with volumes and their associated snapshots.",
			Subcommands: blockstoragecommands.Get(),
		},
	}
}

func printHelp(out io.Writer, templ string, data interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
		"flag": flag,
	}

	w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func flag(flag cli.Flag) string {
	switch flag.(type) {
	case cli.StringFlag:
		flagType := flag.(cli.StringFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.IntFlag:
		flagType := flag.(cli.IntFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.BoolFlag:
		flagType := flag.(cli.BoolFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	case cli.StringSliceFlag:
		flagType := flag.(cli.StringSliceFlag)
		return fmt.Sprintf("%s\t%s", flagType.Name, flagType.Usage)
	}
	return ""
}

func version(c *cli.Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\ncommit: %v\n", c.App.Name, util.Version, util.Commit)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"runtime"

	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"

	stdlog "log"
)

var (
	version   string
	branch    string
	revision  string
	buildDate string
	goVersion = runtime.Version()
)

func main() {

	// bootstrap
	envvarHelper := NewEnvvarHelper("ESTAFETTE_")
	whenEvaluator := NewWhenEvaluator(envvarHelper)
	dockerRunner := NewDockerRunner(envvarHelper)
	pipelineRunner := NewPipelineRunner(envvarHelper, whenEvaluator, dockerRunner)
	endOfLifeHelper := NewEndOfLifeHelper(envvarHelper)

	// detect controlling server
	ciServer := envvarHelper.getEstafetteEnv("ESTAFETTE_CI_SERVER")

	if ciServer == "gocd" {

		// pretty print for go.cd integration
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msg("Starting estafette-ci-builder...")

		// read yaml
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Getting current working directory failed")
		}

		log.Info().Msgf("Running %v pipelines", len(manifest.Pipelines))

		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Setting global environment variables failed")
		}

		envvars := envvarHelper.collectEstafetteEnvvars(manifest)

		result, err := pipelineRunner.runPipelines(manifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing pipelines from manifest failed")
		}

		renderStats(result)

		handleExit(result)

	} else if ciServer == "estafette" {

		// log as severity for stackdriver logging to recognize the level
		zerolog.LevelFieldName = "severity"

		gitName := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_NAME")
		gitBranch := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_BRANCH")
		gitRevision := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_REVISION")
		jobName := envvarHelper.getEstafetteEnv("ESTAFETTE_BUILD_JOB_NAME")
		builderTrack := envvarHelper.getEstafetteEnv("ESTAFETTE_CI_BUILDER_TRACK")
		if builderTrack == "" {
			builderTrack = "stable"
		}

		// set some default fields added to all logs
		log.Logger = zerolog.New(os.Stdout).With().
			Timestamp().
			Str("app", "estafette-ci-builder").
			Str("version", version).
			Str("jobName", jobName).
			Str("gitName", gitName).
			Str("gitBranch", gitBranch).
			Str("gitRevision", gitRevision).
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msg("Starting estafette-ci-builder...")

		// start docker daemon
		err := dockerRunner.startDockerDaemon()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Error starting docker daemon")
		}

		// wait for docker daemon to be ready for usage
		dockerRunner.waitForDockerDaemon()

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Getting current working directory failed")
		}

		// set some envvars
		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Setting global environment variables failed")
		}

		// run git clone via pipeline runner
		estafetteGitCloneManifest := manifest.EstafetteManifest{
			Pipelines: []*manifest.EstafettePipeline{
				&manifest.EstafettePipeline{
					Name:             "git-clone",
					ContainerImage:   fmt.Sprintf("extensions/git-clone:%v", builderTrack),
					Shell:            "/bin/sh",
					WorkingDirectory: "/estafette-work",
					When:             "status == 'succeeded'",
				},
			},
		}

		// collect estafette envvars and run the git clone step
		envvars := envvarHelper.collectEstafetteEnvvars(estafetteGitCloneManifest)
		gitCloneResult, err := pipelineRunner.runPipelines(estafetteGitCloneManifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing git clone step failed")
		}

		// check if manifest exists
		if !manifest.Exists(".estafette.yaml") {
			log.Info().Msg(".estafette.yaml file does not exist, exiting...")
			endOfLifeHelper.sendBuildFinishedEvent("builder:nomanifest")
			os.Exit(0)
		}

		// read .estafette.yaml manifest
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// collect estafette envvars and run pipelines from manifest
		log.Info().Msgf("Running %v pipelines", len(manifest.Pipelines))
		envvars = envvarHelper.collectEstafetteEnvvars(manifest)
		result, err := pipelineRunner.runPipelines(manifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing pipelines from manifest failed")
		}

		// merge git clone and manifest result
		result.PipelineResults = append(gitCloneResult.PipelineResults, result.PipelineResults...)

		// send result to ci-api
		log.Info().Interface("result", result).Msg("Finished running pipelines")
		endOfLifeHelper.sendBuildFinishedEvent("builder:succeeded")
		os.Exit(0)
	}
}
}
<commit_msg>fix vs code save error<commit_after>package main

import (
	"fmt"
	"os"
	"runtime"

	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"

	stdlog "log"
)

var (
	version   string
	branch    string
	revision  string
	buildDate string
	goVersion = runtime.Version()
)

func main() {

	// bootstrap
	envvarHelper := NewEnvvarHelper("ESTAFETTE_")
	whenEvaluator := NewWhenEvaluator(envvarHelper)
	dockerRunner := NewDockerRunner(envvarHelper)
	pipelineRunner := NewPipelineRunner(envvarHelper, whenEvaluator, dockerRunner)
	endOfLifeHelper := NewEndOfLifeHelper(envvarHelper)

	// detect controlling server
	ciServer := envvarHelper.getEstafetteEnv("ESTAFETTE_CI_SERVER")

	if ciServer == "gocd" {

		// pretty print for go.cd integration
		log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().
			Timestamp().
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msg("Starting estafette-ci-builder...")

		// read yaml
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Getting current working directory failed")
		}

		log.Info().Msgf("Running %v pipelines", len(manifest.Pipelines))

		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Setting global environment variables failed")
		}

		envvars := envvarHelper.collectEstafetteEnvvars(manifest)

		result, err := pipelineRunner.runPipelines(manifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing pipelines from manifest failed")
		}

		renderStats(result)

		handleExit(result)

	} else if ciServer == "estafette" {

		// log as severity for stackdriver logging to recognize the level
		zerolog.LevelFieldName = "severity"

		gitName := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_NAME")
		gitBranch := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_BRANCH")
		gitRevision := envvarHelper.getEstafetteEnv("ESTAFETTE_GIT_REVISION")
		jobName := envvarHelper.getEstafetteEnv("ESTAFETTE_BUILD_JOB_NAME")
		builderTrack := envvarHelper.getEstafetteEnv("ESTAFETTE_CI_BUILDER_TRACK")
		if builderTrack == "" {
			builderTrack = "stable"
		}

		// set some default fields added to all logs
		log.Logger = zerolog.New(os.Stdout).With().
			Timestamp().
			Str("app", "estafette-ci-builder").
			Str("version", version).
			Str("jobName", jobName).
			Str("gitName", gitName).
			Str("gitBranch", gitBranch).
			Str("gitRevision", gitRevision).
			Logger()

		stdlog.SetFlags(0)
		stdlog.SetOutput(log.Logger)

		// log startup message
		log.Info().
			Str("branch", branch).
			Str("revision", revision).
			Str("buildDate", buildDate).
			Str("goVersion", goVersion).
			Msg("Starting estafette-ci-builder...")

		// start docker daemon
		err := dockerRunner.startDockerDaemon()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Error starting docker daemon")
		}

		// wait for docker daemon to be ready for usage
		dockerRunner.waitForDockerDaemon()

		// get current working directory
		dir, err := os.Getwd()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Getting current working directory failed")
		}

		// set some envvars
		err = envvarHelper.setEstafetteGlobalEnvvars()
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Setting global environment variables failed")
		}

		// run git clone via pipeline runner
		estafetteGitCloneManifest := manifest.EstafetteManifest{
			Pipelines: []*manifest.EstafettePipeline{
				&manifest.EstafettePipeline{
					Name:             "git-clone",
					ContainerImage:   fmt.Sprintf("extensions/git-clone:%v", builderTrack),
					Shell:            "/bin/sh",
					WorkingDirectory: "/estafette-work",
					When:             "status == 'succeeded'",
				},
			},
		}

		// collect estafette envvars and run the git clone step
		envvars := envvarHelper.collectEstafetteEnvvars(estafetteGitCloneManifest)
		gitCloneResult, err := pipelineRunner.runPipelines(estafetteGitCloneManifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing git clone step failed")
		}

		// check if manifest exists
		if !manifest.Exists(".estafette.yaml") {
			log.Info().Msg(".estafette.yaml file does not exist, exiting...")
			endOfLifeHelper.sendBuildFinishedEvent("builder:nomanifest")
			os.Exit(0)
		}

		// read .estafette.yaml manifest
		manifest, err := manifest.ReadManifestFromFile(".estafette.yaml")
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Reading .estafette.yaml manifest failed")
		}

		// collect estafette envvars and run pipelines from manifest
		log.Info().Msgf("Running %v pipelines", len(manifest.Pipelines))
		envvars = envvarHelper.collectEstafetteEnvvars(manifest)
		result, err := pipelineRunner.runPipelines(manifest, dir, envvars)
		if err != nil {
			endOfLifeHelper.handleFatal(err, "Executing pipelines from manifest failed")
		}

		// merge git clone and manifest result
		result.PipelineResults = append(gitCloneResult.PipelineResults, result.PipelineResults...)

		// send result to ci-api
		log.Info().Interface("result", result).Msg("Finished running pipelines")
		endOfLifeHelper.sendBuildFinishedEvent("builder:succeeded")
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var remoteAddr = flag.String("r", "", "remote addr")
var payloadSize = flag.Int("s", 64, "payload size")
var interval = flag.Int64("i", 100, "interval in milliseconds")
var count = flag.Int("c", 50, "send count")
var number = flag.Int("n", 1, "how many connections")
var protocol = flag.String("p", "tcp", "protocol: tcp or udp")
var genCharts = flag.Bool("g", false, "generate charts")

var payload string

const headSize = 15

type Result struct {
	max, min, avg int
	data          []int
}

func init() {
	payload = randString(*payloadSize)
}

func encodePacket() []byte {
	sendTime, _ := time.Now().MarshalBinary()
	sendTime = append(sendTime, payload...)
	return sendTime
}

func decodeLatency(bs []byte) int {
	before := time.Time{}
	err := before.UnmarshalBinary(bs)
	if err != nil {
		log.Fatal(err)
	}
	t := time.Since(before) / time.Millisecond
	return int(t)
}

func handleReceive(conn net.Conn, notify chan Result) {
	bs := make([]byte, headSize)
	ps := make([]byte, *payloadSize)
	all := int64(0)
	max := 0
	min := 0xFFFFFFFFFFFFFFF
	avg := (0)
	data := make([]int, *count)
	for i := 0; i < *count; i++ {
		_, err := io.ReadFull(conn, bs)
		if err != nil {
			log.Println(err)
			break
		}
		elapsed := decodeLatency(bs)
		if elapsed > max {
			max = elapsed
		}
		if elapsed < min {
			min = elapsed
		}
		log.Printf("[%d] packet RTT: [%d] ms", i, elapsed)
		data[i] = elapsed
		_, err = io.ReadFull(conn, ps)
		if err != nil {
			log.Println(err)
			break
		}
		all = all + int64(elapsed)
		avg = int(all / int64(i+1))
	}
	notify <- Result{max, min, avg, data}
}

func runOne(remoteAddr string, notify chan Result) {
	conn, err := net.Dial(*protocol, remoteAddr)
	if err != nil {
		log.Panicln(err)
	}
	defer conn.Close()
	go handleReceive(conn, notify)
	for i := 0; i < *count; i++ {
		buf := strings.NewReader(string(encodePacket()))
		_, err := io.CopyN(conn, buf, int64(buf.Len()))
		if err != nil {
			log.Println(err)
			break
		}
		time.Sleep(time.Duration(*interval) * time.Millisecond)
	}
}

func makeCharts(idx int, data []int) {
	t := time.Now()
	now := fmt.Sprintf("%d-%d-%dH%dM%dS%d-%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), idx)
	const header = `
# The chart type , option : spline/line/bar/column/area
ChartType = spline
Title = packet RTT latency (ms)
SubTitle = %s
ValueSuffix = ms

# The x Axis numbers. The count this numbers MUST be the same with the data series
XAxisNumbers = %s

# The y Axis text
YAxisText = Latency (ms)

# The data and the name of the lines
Data|Latency = %s
`
	x := []string{}
	d := []string{}
	for i := 0; i < len(data); i++ {
		x = append(x, strconv.Itoa(i))
		d = append(d, strconv.Itoa(data[i]))
	}
	sx := strings.Join(x, ", ")
	sd := strings.Join(d, ", ")

	all := fmt.Sprintf(header, now, sx, sd)
	ioutil.WriteFile(now+".chart", []byte(all), os.ModePerm)
}

func main() {
	flag.Parse()
	results := make(chan Result, *number)
	wg := &sync.WaitGroup{}
	for i := 0; i < *number; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			runOne(*remoteAddr, results)
		}()
	}
	wg.Wait()
	for i := 0; i < *number; i++ {
		ret := <-results
		if *genCharts {
			makeCharts(i, ret.data)
		}
		log.Printf("RTT min: [%d] ms, RTT max: [%d] ms, RTT avg: [%d] ms\n", ret.min, ret.max, ret.avg)
	}
}

var src = rand.NewSource(time.Now().UnixNano())

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

func randString(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}
<commit_msg>log with file&line<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var remoteAddr = flag.String("r", "", "remote addr")
var payloadSize = flag.Int("s", 64, "payload size")
var interval = flag.Int64("i", 100, "interval in milliseconds")
var count = flag.Int("c", 50, "send count")
var number = flag.Int("n", 1, "how many connections")
var protocol = flag.String("p", "tcp", "protocol: tcp or udp")
var genCharts = flag.Bool("g", false, "generate charts")

var payload string

const headSize = 15

type Result struct {
	max, min, avg int
	data          []int
}

func init() {
	payload = randString(*payloadSize)
}

func encodePacket() []byte {
	sendTime, _ := time.Now().MarshalBinary()
	sendTime = append(sendTime, payload...)
	return sendTime
}

func decodeLatency(bs []byte) int {
	before := time.Time{}
	err := before.UnmarshalBinary(bs)
	if err != nil {
		log.Fatal(err)
	}
	t := time.Since(before) / time.Millisecond
	return int(t)
}

func handleReceive(conn net.Conn, notify chan Result) {
	bs := make([]byte, headSize)
	ps := make([]byte, *payloadSize)
	all := int64(0)
	max := 0
	min := 0xFFFFFFFFFFFFFFF
	avg := (0)
	data := make([]int, *count)
	for i := 0; i < *count; i++ {
		_, err := io.ReadFull(conn, bs)
		if err != nil {
			log.Println(err)
			break
		}
		elapsed := decodeLatency(bs)
		if elapsed > max {
			max = elapsed
		}
		if elapsed < min {
			min = elapsed
		}
		log.Printf("[%d] packet RTT: [%d] ms", i, elapsed)
		data[i] = elapsed
		_, err = io.ReadFull(conn, ps)
		if err != nil {
			log.Println(err)
			break
		}
		all = all + int64(elapsed)
		avg = int(all / int64(i+1))
	}
	notify <- Result{max, min, avg, data}
}

func runOne(remoteAddr string, notify chan Result) {
	conn, err := net.Dial(*protocol, remoteAddr)
	if err != nil {
		log.Panicln(err)
	}
	defer conn.Close()
	go handleReceive(conn, notify)
	for i := 0; i < *count; i++ {
		buf := strings.NewReader(string(encodePacket()))
		_, err := io.CopyN(conn, buf, int64(buf.Len()))
		if err != nil {
			log.Println(err)
			break
		}
		time.Sleep(time.Duration(*interval) * time.Millisecond)
	}
}

func makeCharts(idx int, data []int) {
	t := time.Now()
	now := fmt.Sprintf("%d-%d-%dH%dM%dS%d-%d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), idx)
	const header = `
# The chart type , option : spline/line/bar/column/area
ChartType = spline
Title = packet RTT latency (ms)
SubTitle = %s
ValueSuffix = ms

# The x Axis numbers. The count this numbers MUST be the same with the data series
XAxisNumbers = %s

# The y Axis text
YAxisText = Latency (ms)

# The data and the name of the lines
Data|Latency = %s
`
	x := []string{}
	d := []string{}
	for i := 0; i < len(data); i++ {
		x = append(x, strconv.Itoa(i))
		d = append(d, strconv.Itoa(data[i]))
	}
	sx := strings.Join(x, ", ")
	sd := strings.Join(d, ", ")

	all := fmt.Sprintf(header, now, sx, sd)
	ioutil.WriteFile(now+".chart", []byte(all), os.ModePerm)
}

func main() {
	flag.Parse()
	log.SetFlags(log.Flags() | log.Lshortfile)
	results := make(chan Result, *number)
	wg := &sync.WaitGroup{}
	for i := 0; i < *number; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			runOne(*remoteAddr, results)
		}()
	}
	wg.Wait()
	for i := 0; i < *number; i++ {
		ret := <-results
		if *genCharts {
			makeCharts(i, ret.data)
		}
		log.Printf("RTT min: [%d] ms, RTT max: [%d] ms, RTT avg: [%d] ms\n", ret.min, ret.max, ret.avg)
	}
}

var src = rand.NewSource(time.Now().UnixNano())

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
	letterIdxBits = 6                    // 6 bits to represent a letter index
	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
)

func randString(n int) string {
	b := make([]byte, n)
	// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
		if remain == 0 {
			cache, remain = src.Int63(), letterIdxMax
		}
		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
			b[i] = letterBytes[idx]
			i--
		}
		cache >>= letterIdxBits
		remain--
	}

	return string(b)
}
<|endoftext|>