<commit_before>package main

import (
	"crypto/md5"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/apex/httplog"
	"github.com/apex/log"
	"github.com/apex/log/handlers/logfmt"
	"github.com/blacktop/scifgif/ascii"
	"github.com/blacktop/scifgif/database"
	"github.com/blacktop/scifgif/giphy"
	"github.com/blacktop/scifgif/xkcd"
	"github.com/gorilla/mux"
	"github.com/urfave/cli"
)

const (
	xkcdFolder    = "images/xkcd"
	giphyFolder   = "images/giphy"
	dilbertFolder = "images/dilbert"
	contribFolder = "images/contrib"
)

var (
	// Version stores the plugin's version
	Version string
	// BuildTime stores the plugin's build time
	BuildTime string
	// Token stores the webhook api token
	Token string
	// Host microservice host
	Host string
	// Port microservice port
	Port string
	// APIkey stores Giphy's API key
	APIkey string

	db *database.Database
)

// WebHookResponse mattermost webhook response struct
type WebHookResponse struct {
	Text         string `json:"text,omitempty"`
	Username     string `json:"username,omitempty"`
	IconURL      string `json:"icon_url,omitempty"`
	ResponseType string `json:"response_type,omitempty"`
}

var appHelpTemplate = `Usage: {{.Name}} {{if .Flags}}[OPTIONS] {{end}}COMMAND [arg...]

{{.Usage}}

Version: {{.Version}}{{if or .Author .Email}}

Author:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}
{{if .Flags}}
Options:
  {{range .Flags}}{{.}}
  {{end}}{{end}}
Commands:
  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Run '{{.Name}} COMMAND --help' for more information on a command.
`

func init() {
	log.SetHandler(logfmt.Default)
}

func main() {

	cli.AppHelpTemplate = appHelpTemplate
	app := cli.NewApp()

	app.Name = "scifgif"
	app.Author = "blacktop"
	app.Email = "https://github.com/blacktop"
	app.Version = Version + ", BuildTime: " + BuildTime
	app.Compiled, _ = time.Parse("20060102", BuildTime)
	app.Usage = "Humorous Image Micro-Service"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, V",
			Usage: "verbose output",
		},
		cli.IntFlag{
			Name:   "timeout",
			Value:  60,
			Usage:  "database timeout (in seconds)",
			EnvVar: "TIMEOUT",
		},
		cli.IntFlag{
			Name:   "number, N",
			Value:  500,
			Usage:  "number of gifs to download",
			EnvVar: "IMAGE_NUMBER",
		},
		cli.IntFlag{
			Name:   "xkcd-count",
			Value:  -1,
			Usage:  "number of xkcd comics to download",
			EnvVar: "IMAGE_XKCD_COUNT",
		},
		cli.StringFlag{
			Name:   "date",
			Value:  "",
			Usage:  "dilbert comic start-from date",
			EnvVar: "IMAGE_DILBERT_DATE",
		},
		cli.StringFlag{
			Name:        "host",
			Value:       "",
			Usage:       "microservice host",
			EnvVar:      "SCIFGIF_HOST",
			Destination: &Host,
		},
		cli.StringFlag{
			Name:        "port",
			Value:       "3993",
			Usage:       "microservice port",
			EnvVar:      "SCIFGIF_PORT",
			Destination: &Port,
		},
		cli.StringFlag{
			Name:        "token",
			Value:       "",
			Usage:       "webhook token",
			EnvVar:      "SCIFGIF_TOKEN",
			Destination: &Token,
		},
	}
	app.Commands = []cli.Command{
		{
			Name:    "update",
			Aliases: []string{"u"},
			Usage:   "Update images",
			Action: func(c *cli.Context) error {

				log.WithFields(log.Fields{
					"search_for": "reactions",
					"number":     c.GlobalInt("number"),
				}).Info("download reactions Giphy gifs and ingest metadata into database")
				err := giphy.GetAllGiphy(giphyFolder, []string{"reactions"}, c.GlobalInt("number"))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "star wars",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download star wars Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"star", "wars"}, min(c.GlobalInt("number"), 500))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "futurama",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download futurama Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"futurama"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "rick and morty",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download rick and morty Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"rick", "and", "morty"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "office space",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download office space Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"office", "space"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "it crowd",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download it crowd Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"it", "crowd"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "the office",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download the office Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"the", "office"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "hacker",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download futurama Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"hacker"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"number": c.GlobalInt("number"),
				}).Info("download xkcd comics and ingest metadata into database")
				err = xkcd.GetAllXkcd(xkcdFolder, c.GlobalInt("number"))
				// err = xkcd.GetAllXkcd(xkcdFolder, c.GlobalInt("xkcd-count"))
				if err != nil {
					return err
				}

				log.Info("load all ascii-emojis into database")
				err = ascii.GetAllASCIIEmoji()
				if err != nil {
					return err
				}

				// log.WithFields(log.Fields{
				// 	"date": c.GlobalString("date"),
				// }).Info("download dilbert comics and ingest metadata into database")
				// err = dilbert.GetAllDilbert(dilbertFolder, c.GlobalString("date"))
				// if err != nil {
				// 	return err
				// }

				log.Info("* finalize database db")
				err = database.Finalize()
				if err != nil {
					return err
				}

				return nil
			},
		},
	}
	app.Action = func(c *cli.Context) error {
		var err error

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if len(c.String("token")) == 0 {
			log.Warn("no webhook token set: --token")
		}

		db, err = database.Open()
		if err != nil {
			return err
		}
		defer db.Close()

		// create http routes
		router := mux.NewRouter().StrictSlash(true)
		router.HandleFunc("/icon/xkcd", getXkcdIcon).Methods("GET")
		router.HandleFunc("/icon/giphy", getGiphyIcon).Methods("GET")
		router.HandleFunc("/icon/dilbert", getDilbertIcon).Methods("GET")
		router.HandleFunc("/images", addImage).Methods("PUT")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", updateImageKeywords).Methods("PATCH")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", getImage).Methods("GET")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", deleteImage).Methods("DELETE")
		// xkcd routes
		router.HandleFunc("/xkcd", getRandomXKCD).Methods("GET")
		router.HandleFunc("/xkcd/number/{number}", getXkcdByNumber).Methods("GET")
		router.HandleFunc("/xkcd/search", getSearchXKCD).Methods("GET")
		router.HandleFunc("/xkcd/new_post", postXkcdMattermost).Methods("POST")
		router.HandleFunc("/xkcd/slash", postXkcdMattermostSlash).Methods("POST")
		// Giphy routes
		router.HandleFunc("/giphy", getRandomGiphy).Methods("GET")
		router.HandleFunc("/giphy/search", getSearchGiphy).Methods("GET")
		router.HandleFunc("/giphy/new_post", postGiphyMattermost).Methods("POST")
		router.HandleFunc("/giphy/slash", postGiphyMattermostSlash).Methods("POST")
		// Ascii-emoji routes
		router.HandleFunc("/ascii", getRandomASCII).Methods("GET")
		router.HandleFunc("/ascii/search", getSearchASCII).Methods("GET")
		// router.HandleFunc("/ascii/new_post", postASCIIMattermost).Methods("POST")
		router.HandleFunc("/ascii/slash", postASCIIMattermostSlash).Methods("POST")
		// Dilbert routes
		router.HandleFunc("/dilbert", getRandomDilbert).Methods("GET")
		router.HandleFunc("/dilbert/date/{date}", getDilbertByDate).Methods("GET")
		router.HandleFunc("/dilbert/search", getSearchDilbert).Methods("GET")
		router.HandleFunc("/dilbert/new_post", postDilbertMattermost).Methods("POST")
		router.HandleFunc("/dilbert/slash", postDilbertMattermostSlash).Methods("POST")
		// reactJS Web App routes
		router.HandleFunc("/web/search", getWebSearch).Methods("GET")
		router.PathPrefix("/").Handler(http.FileServer(http.Dir("./web/build")))

		// start microservice
		log.WithFields(log.Fields{
			"host":  Host,
			"port":  Port,
			"token": Token,
		}).Info("web service listening")
		err = http.ListenAndServe(":"+Port, httplog.New(router))
		log.WithError(err).Fatal("error listening")

		return nil
	}

	err := app.Run(os.Args)
	if err != nil {
		log.WithError(err).Fatal("running app failed")
	}
}

func isNumeric(s string) bool {
	_, err := strconv.ParseFloat(s, 64)
	return err == nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func getMD5(f io.Reader) (string, error) {
	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}
	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

// this is overkill to protext against malicious URL from being created
func makeURL(scheme, host, port, path string) string {
	var u string
	var base *url.URL

	p, err := url.Parse(path)
	if err != nil {
		log.WithError(err).Error("url parse failed")
	}

	// don't show port if it is the HTTP/HTTPS port
	if strings.EqualFold(port, "80") || strings.EqualFold(port, "443") {
		u = fmt.Sprintf("%s://%s", scheme, host)
		base, err = url.Parse(u)
		if err != nil {
			log.WithError(err).Error("url parse failed")
		}
		return base.ResolveReference(p).String()
	}

	u = fmt.Sprintf("%s://%s:%s", scheme, host, port)
	base, err = url.Parse(u)
	if err != nil {
		log.WithError(err).Error("url parse failed")
	}

	return base.ResolveReference(p).String()
}
<commit_msg>get all of xkcd<commit_after>package main

import (
	"crypto/md5"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/apex/httplog"
	"github.com/apex/log"
	"github.com/apex/log/handlers/logfmt"
	"github.com/blacktop/scifgif/ascii"
	"github.com/blacktop/scifgif/database"
	"github.com/blacktop/scifgif/giphy"
	"github.com/blacktop/scifgif/xkcd"
	"github.com/gorilla/mux"
	"github.com/urfave/cli"
)

const (
	xkcdFolder    = "images/xkcd"
	giphyFolder   = "images/giphy"
	dilbertFolder = "images/dilbert"
	contribFolder = "images/contrib"
)

var (
	// Version stores the plugin's version
	Version string
	// BuildTime stores the plugin's build time
	BuildTime string
	// Token stores the webhook api token
	Token string
	// Host microservice host
	Host string
	// Port microservice port
	Port string
	// APIkey stores Giphy's API key
	APIkey string

	db *database.Database
)

// WebHookResponse mattermost webhook response struct
type WebHookResponse struct {
	Text         string `json:"text,omitempty"`
	Username     string `json:"username,omitempty"`
	IconURL      string `json:"icon_url,omitempty"`
	ResponseType string `json:"response_type,omitempty"`
}

var appHelpTemplate = `Usage: {{.Name}} {{if .Flags}}[OPTIONS] {{end}}COMMAND [arg...]

{{.Usage}}

Version: {{.Version}}{{if or .Author .Email}}

Author:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}
{{if .Flags}}
Options:
  {{range .Flags}}{{.}}
  {{end}}{{end}}
Commands:
  {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
  {{end}}
Run '{{.Name}} COMMAND --help' for more information on a command.
`

func init() {
	log.SetHandler(logfmt.Default)
}

func main() {

	cli.AppHelpTemplate = appHelpTemplate
	app := cli.NewApp()

	app.Name = "scifgif"
	app.Author = "blacktop"
	app.Email = "https://github.com/blacktop"
	app.Version = Version + ", BuildTime: " + BuildTime
	app.Compiled, _ = time.Parse("20060102", BuildTime)
	app.Usage = "Humorous Image Micro-Service"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, V",
			Usage: "verbose output",
		},
		cli.IntFlag{
			Name:   "timeout",
			Value:  60,
			Usage:  "database timeout (in seconds)",
			EnvVar: "TIMEOUT",
		},
		cli.IntFlag{
			Name:   "number, N",
			Value:  500,
			Usage:  "number of gifs to download",
			EnvVar: "IMAGE_NUMBER",
		},
		cli.IntFlag{
			Name:   "xkcd-count",
			Value:  -1,
			Usage:  "number of xkcd comics to download",
			EnvVar: "IMAGE_XKCD_COUNT",
		},
		cli.StringFlag{
			Name:   "date",
			Value:  "",
			Usage:  "dilbert comic start-from date",
			EnvVar: "IMAGE_DILBERT_DATE",
		},
		cli.StringFlag{
			Name:        "host",
			Value:       "",
			Usage:       "microservice host",
			EnvVar:      "SCIFGIF_HOST",
			Destination: &Host,
		},
		cli.StringFlag{
			Name:        "port",
			Value:       "3993",
			Usage:       "microservice port",
			EnvVar:      "SCIFGIF_PORT",
			Destination: &Port,
		},
		cli.StringFlag{
			Name:        "token",
			Value:       "",
			Usage:       "webhook token",
			EnvVar:      "SCIFGIF_TOKEN",
			Destination: &Token,
		},
	}
	app.Commands = []cli.Command{
		{
			Name:    "update",
			Aliases: []string{"u"},
			Usage:   "Update images",
			Action: func(c *cli.Context) error {

				log.WithFields(log.Fields{
					"search_for": "reactions",
					"number":     c.GlobalInt("number"),
				}).Info("download reactions Giphy gifs and ingest metadata into database")
				err := giphy.GetAllGiphy(giphyFolder, []string{"reactions"}, c.GlobalInt("number"))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "star wars",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download star wars Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"star", "wars"}, min(c.GlobalInt("number"), 500))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "futurama",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download futurama Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"futurama"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "rick and morty",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download rick and morty Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"rick", "and", "morty"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "office space",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download office space Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"office", "space"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "it crowd",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download it crowd Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"it", "crowd"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "the office",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download the office Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"the", "office"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"search_for": "hacker",
					"number":     min(c.GlobalInt("number"), 250),
				}).Info("download futurama Giphy gifs and ingest metadata into database")
				err = giphy.GetAllGiphy(giphyFolder, []string{"hacker"}, min(c.GlobalInt("number"), 250))
				if err != nil {
					return err
				}

				log.WithFields(log.Fields{
					"number": c.GlobalInt("xkcd-count"),
				}).Info("download xkcd comics and ingest metadata into database")
				err = xkcd.GetAllXkcd(xkcdFolder, c.GlobalInt("xkcd-count"))
				if err != nil {
					return err
				}

				log.Info("load all ascii-emojis into database")
				err = ascii.GetAllASCIIEmoji()
				if err != nil {
					return err
				}

				// log.WithFields(log.Fields{
				// 	"date": c.GlobalString("date"),
				// }).Info("download dilbert comics and ingest metadata into database")
				// err = dilbert.GetAllDilbert(dilbertFolder, c.GlobalString("date"))
				// if err != nil {
				// 	return err
				// }

				log.Info("* finalize database db")
				err = database.Finalize()
				if err != nil {
					return err
				}

				return nil
			},
		},
	}
	app.Action = func(c *cli.Context) error {
		var err error

		if c.Bool("verbose") {
			log.SetLevel(log.DebugLevel)
		}

		if len(c.String("token")) == 0 {
			log.Warn("no webhook token set: --token")
		}

		db, err = database.Open()
		if err != nil {
			return err
		}
		defer db.Close()

		// create http routes
		router := mux.NewRouter().StrictSlash(true)
		router.HandleFunc("/icon/xkcd", getXkcdIcon).Methods("GET")
		router.HandleFunc("/icon/giphy", getGiphyIcon).Methods("GET")
		router.HandleFunc("/icon/dilbert", getDilbertIcon).Methods("GET")
		router.HandleFunc("/images", addImage).Methods("PUT")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", updateImageKeywords).Methods("PATCH")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", getImage).Methods("GET")
		router.HandleFunc("/images/{source:(?:giphy|xkcd|dilbert|default|contrib)}/{file}", deleteImage).Methods("DELETE")
		// xkcd routes
		router.HandleFunc("/xkcd", getRandomXKCD).Methods("GET")
		router.HandleFunc("/xkcd/number/{number}", getXkcdByNumber).Methods("GET")
		router.HandleFunc("/xkcd/search", getSearchXKCD).Methods("GET")
		router.HandleFunc("/xkcd/new_post", postXkcdMattermost).Methods("POST")
		router.HandleFunc("/xkcd/slash", postXkcdMattermostSlash).Methods("POST")
		// Giphy routes
		router.HandleFunc("/giphy", getRandomGiphy).Methods("GET")
		router.HandleFunc("/giphy/search", getSearchGiphy).Methods("GET")
		router.HandleFunc("/giphy/new_post", postGiphyMattermost).Methods("POST")
		router.HandleFunc("/giphy/slash", postGiphyMattermostSlash).Methods("POST")
		// Ascii-emoji routes
		router.HandleFunc("/ascii", getRandomASCII).Methods("GET")
		router.HandleFunc("/ascii/search", getSearchASCII).Methods("GET")
		// router.HandleFunc("/ascii/new_post", postASCIIMattermost).Methods("POST")
		router.HandleFunc("/ascii/slash", postASCIIMattermostSlash).Methods("POST")
		// Dilbert routes
		router.HandleFunc("/dilbert", getRandomDilbert).Methods("GET")
		router.HandleFunc("/dilbert/date/{date}", getDilbertByDate).Methods("GET")
		router.HandleFunc("/dilbert/search", getSearchDilbert).Methods("GET")
		router.HandleFunc("/dilbert/new_post", postDilbertMattermost).Methods("POST")
		router.HandleFunc("/dilbert/slash", postDilbertMattermostSlash).Methods("POST")
		// reactJS Web App routes
		router.HandleFunc("/web/search", getWebSearch).Methods("GET")
		router.PathPrefix("/").Handler(http.FileServer(http.Dir("./web/build")))

		// start microservice
		log.WithFields(log.Fields{
			"host":  Host,
			"port":  Port,
			"token": Token,
		}).Info("web service listening")
		err = http.ListenAndServe(":"+Port, httplog.New(router))
		log.WithError(err).Fatal("error listening")

		return nil
	}

	err := app.Run(os.Args)
	if err != nil {
		log.WithError(err).Fatal("running app failed")
	}
}

func isNumeric(s string) bool {
	_, err := strconv.ParseFloat(s, 64)
	return err == nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func getMD5(f io.Reader) (string, error) {
	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}
	return fmt.Sprintf("%x", h.Sum(nil)), nil
}

// this is overkill to protext against malicious URL from being created
func makeURL(scheme, host, port, path string) string {
	var u string
	var base *url.URL

	p, err := url.Parse(path)
	if err != nil {
		log.WithError(err).Error("url parse failed")
	}

	// don't show port if it is the HTTP/HTTPS port
	if strings.EqualFold(port, "80") || strings.EqualFold(port, "443") {
		u = fmt.Sprintf("%s://%s", scheme, host)
		base, err = url.Parse(u)
		if err != nil {
			log.WithError(err).Error("url parse failed")
		}
		return base.ResolveReference(p).String()
	}

	u = fmt.Sprintf("%s://%s:%s", scheme, host, port)
	base, err = url.Parse(u)
	if err != nil {
		log.WithError(err).Error("url parse failed")
	}

	return base.ResolveReference(p).String()
}
<|endoftext|>


<commit_before>package cmd

import (
	"strings"

	ccli "github.com/micro/cli"
	"github.com/micro/go-micro/cmd"
	"github.com/micro/micro/api"
	"github.com/micro/micro/bot"
	"github.com/micro/micro/cli"
	"github.com/micro/micro/new"
	"github.com/micro/micro/plugin"
	"github.com/micro/micro/proxy"
	"github.com/micro/micro/run"
	"github.com/micro/micro/web"
)

var (
	name        = "micro"
	description = "A cloud-native toolkit"
	version     = "0.7.1"
)

func setup(app *ccli.App) {
	app.Flags = append(app.Flags,
		ccli.BoolFlag{
			Name:   "enable_acme",
			Usage:  "Enables ACME support via Let's Encrypt. ACME hosts should also be specified.",
			EnvVar: "MICRO_ENABLE_ACME",
		},
		ccli.StringFlag{
			Name:   "acme_hosts",
			Usage:  "Comma separated list of hostnames to manage ACME certs for",
			EnvVar: "MICRO_ACME_HOSTS",
		},
		ccli.BoolFlag{
			Name:   "enable_tls",
			Usage:  "Enable TLS support. Expects cert and key file to be specified",
			EnvVar: "MICRO_ENABLE_TLS",
		},
		ccli.StringFlag{
			Name:   "tls_cert_file",
			Usage:  "Path to the TLS Certificate file",
			EnvVar: "MICRO_TLS_CERT_FILE",
		},
		ccli.StringFlag{
			Name:   "tls_key_file",
			Usage:  "Path to the TLS Key file",
			EnvVar: "MICRO_TLS_KEY_FILE",
		},
		ccli.StringFlag{
			Name:   "tls_client_ca_file",
			Usage:  "Path to the TLS CA file to verify clients against",
			EnvVar: "MICRO_TLS_CLIENT_CA_FILE",
		},
		ccli.StringFlag{
			Name:   "api_address",
			Usage:  "Set the api address e.g 0.0.0.0:8080",
			EnvVar: "MICRO_API_ADDRESS",
		},
		ccli.StringFlag{
			Name:   "proxy_address",
			Usage:  "Proxy requests via the HTTP address specified",
			EnvVar: "MICRO_PROXY_ADDRESS",
		},
		ccli.StringFlag{
			Name:   "web_address",
			Usage:  "Set the web UI address e.g 0.0.0.0:8082",
			EnvVar: "MICRO_WEB_ADDRESS",
		},
		ccli.IntFlag{
			Name:   "register_ttl",
			EnvVar: "MICRO_REGISTER_TTL",
			Usage:  "Register TTL in seconds",
		},
		ccli.IntFlag{
			Name:   "register_interval",
			EnvVar: "MICRO_REGISTER_INTERVAL",
			Usage:  "Register interval in seconds",
		},
		ccli.StringFlag{
			Name:   "api_handler",
			Usage:  "Specify the request handler to be used for mapping HTTP requests to services; {api, proxy, rpc}",
			EnvVar: "MICRO_API_HANDLER",
		},
		ccli.StringFlag{
			Name:   "api_namespace",
			Usage:  "Set the namespace used by the API e.g. com.example.api",
			EnvVar: "MICRO_API_NAMESPACE",
		},
		ccli.StringFlag{
			Name:   "web_namespace",
			Usage:  "Set the namespace used by the Web proxy e.g. com.example.web",
			EnvVar: "MICRO_WEB_NAMESPACE",
		},
		ccli.StringFlag{
			Name:   "api_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_API_CORS",
		},
		ccli.StringFlag{
			Name:   "web_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_WEB_CORS",
		},
		ccli.StringFlag{
			Name:   "proxy_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_PROXY_CORS",
		},
		ccli.BoolFlag{
			Name:   "enable_stats",
			Usage:  "Enable stats",
			EnvVar: "MICRO_ENABLE_STATS",
		},
	)

	plugins := plugin.Plugins()

	for _, p := range plugins {
		if flags := p.Flags(); len(flags) > 0 {
			app.Flags = append(app.Flags, flags...)
		}

		if cmds := p.Commands(); len(cmds) > 0 {
			app.Commands = append(app.Commands, cmds...)
		}
	}

	before := app.Before

	app.Before = func(ctx *ccli.Context) error {
		if len(ctx.String("api_handler")) > 0 {
			api.Handler = ctx.String("api_handler")
		}
		if len(ctx.String("api_address")) > 0 {
			api.Address = ctx.String("api_address")
		}
		if len(ctx.String("proxy_address")) > 0 {
			proxy.Address = ctx.String("proxy_address")
		}
		if len(ctx.String("web_address")) > 0 {
			web.Address = ctx.String("web_address")
		}
		if len(ctx.String("api_namespace")) > 0 {
			api.Namespace = ctx.String("api_namespace")
		}
		if len(ctx.String("web_namespace")) > 0 {
			web.Namespace = ctx.String("web_namespace")
		}

		// origin comma separated string to map
		fn := func(s string) map[string]bool {
			origins := make(map[string]bool)
			for _, origin := range strings.Split(s, ",") {
				origins[origin] = true
			}
			return origins
		}

		if len(ctx.String("api_cors")) > 0 {
			api.CORS = fn(ctx.String("api_cors"))
		}
		if len(ctx.String("proxy_cors")) > 0 {
			proxy.CORS = fn(ctx.String("proxy_cors"))
		}
		if len(ctx.String("web_cors")) > 0 {
			web.CORS = fn(ctx.String("web_cors"))
		}

		for _, p := range plugins {
			p.Init(ctx)
		}

		return before(ctx)
	}
}

func Init() {
	app := cmd.App()
	app.Commands = append(app.Commands, api.Commands()...)
	app.Commands = append(app.Commands, bot.Commands()...)
	app.Commands = append(app.Commands, cli.Commands()...)
	app.Commands = append(app.Commands, proxy.Commands()...)
	app.Commands = append(app.Commands, new.Commands()...)
	app.Commands = append(app.Commands, run.Commands()...)
	app.Commands = append(app.Commands, web.Commands()...)
	app.Action = func(context *ccli.Context) { ccli.ShowAppHelp(context) }

	setup(app)

	cmd.Init(
		cmd.Name(name),
		cmd.Description(description),
		cmd.Version(version),
	)
}
<commit_msg>update to 0.8.0 release<commit_after>package cmd

import (
	"strings"

	ccli "github.com/micro/cli"
	"github.com/micro/go-micro/cmd"
	"github.com/micro/micro/api"
	"github.com/micro/micro/bot"
	"github.com/micro/micro/cli"
	"github.com/micro/micro/new"
	"github.com/micro/micro/plugin"
	"github.com/micro/micro/proxy"
	"github.com/micro/micro/run"
	"github.com/micro/micro/web"
)

var (
	name        = "micro"
	description = "A cloud-native toolkit"
	version     = "0.8.0"
)

func setup(app *ccli.App) {
	app.Flags = append(app.Flags,
		ccli.BoolFlag{
			Name:   "enable_acme",
			Usage:  "Enables ACME support via Let's Encrypt. ACME hosts should also be specified.",
			EnvVar: "MICRO_ENABLE_ACME",
		},
		ccli.StringFlag{
			Name:   "acme_hosts",
			Usage:  "Comma separated list of hostnames to manage ACME certs for",
			EnvVar: "MICRO_ACME_HOSTS",
		},
		ccli.BoolFlag{
			Name:   "enable_tls",
			Usage:  "Enable TLS support. Expects cert and key file to be specified",
			EnvVar: "MICRO_ENABLE_TLS",
		},
		ccli.StringFlag{
			Name:   "tls_cert_file",
			Usage:  "Path to the TLS Certificate file",
			EnvVar: "MICRO_TLS_CERT_FILE",
		},
		ccli.StringFlag{
			Name:   "tls_key_file",
			Usage:  "Path to the TLS Key file",
			EnvVar: "MICRO_TLS_KEY_FILE",
		},
		ccli.StringFlag{
			Name:   "tls_client_ca_file",
			Usage:  "Path to the TLS CA file to verify clients against",
			EnvVar: "MICRO_TLS_CLIENT_CA_FILE",
		},
		ccli.StringFlag{
			Name:   "api_address",
			Usage:  "Set the api address e.g 0.0.0.0:8080",
			EnvVar: "MICRO_API_ADDRESS",
		},
		ccli.StringFlag{
			Name:   "proxy_address",
			Usage:  "Proxy requests via the HTTP address specified",
			EnvVar: "MICRO_PROXY_ADDRESS",
		},
		ccli.StringFlag{
			Name:   "web_address",
			Usage:  "Set the web UI address e.g 0.0.0.0:8082",
			EnvVar: "MICRO_WEB_ADDRESS",
		},
		ccli.IntFlag{
			Name:   "register_ttl",
			EnvVar: "MICRO_REGISTER_TTL",
			Usage:  "Register TTL in seconds",
		},
		ccli.IntFlag{
			Name:   "register_interval",
			EnvVar: "MICRO_REGISTER_INTERVAL",
			Usage:  "Register interval in seconds",
		},
		ccli.StringFlag{
			Name:   "api_handler",
			Usage:  "Specify the request handler to be used for mapping HTTP requests to services; {api, proxy, rpc}",
			EnvVar: "MICRO_API_HANDLER",
		},
		ccli.StringFlag{
			Name:   "api_namespace",
			Usage:  "Set the namespace used by the API e.g. com.example.api",
			EnvVar: "MICRO_API_NAMESPACE",
		},
		ccli.StringFlag{
			Name:   "web_namespace",
			Usage:  "Set the namespace used by the Web proxy e.g. com.example.web",
			EnvVar: "MICRO_WEB_NAMESPACE",
		},
		ccli.StringFlag{
			Name:   "api_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_API_CORS",
		},
		ccli.StringFlag{
			Name:   "web_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_WEB_CORS",
		},
		ccli.StringFlag{
			Name:   "proxy_cors",
			Usage:  "Comma separated whitelist of allowed origins for CORS",
			EnvVar: "MICRO_PROXY_CORS",
		},
		ccli.BoolFlag{
			Name:   "enable_stats",
			Usage:  "Enable stats",
			EnvVar: "MICRO_ENABLE_STATS",
		},
	)

	plugins := plugin.Plugins()

	for _, p := range plugins {
		if flags := p.Flags(); len(flags) > 0 {
			app.Flags = append(app.Flags, flags...)
		}

		if cmds := p.Commands(); len(cmds) > 0 {
			app.Commands = append(app.Commands, cmds...)
		}
	}

	before := app.Before

	app.Before = func(ctx *ccli.Context) error {
		if len(ctx.String("api_handler")) > 0 {
			api.Handler = ctx.String("api_handler")
		}
		if len(ctx.String("api_address")) > 0 {
			api.Address = ctx.String("api_address")
		}
		if len(ctx.String("proxy_address")) > 0 {
			proxy.Address = ctx.String("proxy_address")
		}
		if len(ctx.String("web_address")) > 0 {
			web.Address = ctx.String("web_address")
		}
		if len(ctx.String("api_namespace")) > 0 {
			api.Namespace = ctx.String("api_namespace")
		}
		if len(ctx.String("web_namespace")) > 0 {
			web.Namespace = ctx.String("web_namespace")
		}

		// origin comma separated string to map
		fn := func(s string) map[string]bool {
			origins := make(map[string]bool)
			for _, origin := range strings.Split(s, ",") {
				origins[origin] = true
			}
			return origins
		}

		if len(ctx.String("api_cors")) > 0 {
			api.CORS = fn(ctx.String("api_cors"))
		}
		if len(ctx.String("proxy_cors")) > 0 {
			proxy.CORS = fn(ctx.String("proxy_cors"))
		}
		if len(ctx.String("web_cors")) > 0 {
			web.CORS = fn(ctx.String("web_cors"))
		}

		for _, p := range plugins {
			p.Init(ctx)
		}

		return before(ctx)
	}
}

func Init() {
	app := cmd.App()
	app.Commands = append(app.Commands, api.Commands()...)
	app.Commands = append(app.Commands, bot.Commands()...)
	app.Commands = append(app.Commands, cli.Commands()...)
	app.Commands = append(app.Commands, proxy.Commands()...)
	app.Commands = append(app.Commands, new.Commands()...)
	app.Commands = append(app.Commands, run.Commands()...)
	app.Commands = append(app.Commands, web.Commands()...)
	app.Action = func(context *ccli.Context) { ccli.ShowAppHelp(context) }

	setup(app)

	cmd.Init(
		cmd.Name(name),
		cmd.Description(description),
		cmd.Version(version),
	)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"strings"

	"github.com/evandroflores/claimr/model"
	"github.com/shomali11/slacker"
)

var commands []model.Command

const (
	directMessagePrefix = "D"
	channelPrefix       = "<#C"
	userPrefix          = "<@U"
)

// Register add a command to commands list an prepare to register to slacker
func Register(usage string, description string, handler func(request *slacker.Request, response slacker.ResponseWriter)) {
	commands = append(commands, model.Command{Usage: usage, Description: description, Handler: handler})
}

// CommandList returns the list of registered commands
func CommandList() []model.Command {
	return commands
}

func notImplemented(request *slacker.Request, response slacker.ResponseWriter) {
	response.Reply(Messages["not-implemented"])
}

func isDirect(channelID string) (bool, error) {
	if strings.HasPrefix(strings.ToUpper(channelID), directMessagePrefix) {
		return true, fmt.Errorf(Messages["direct-not-allowed"])
	}
	return false, nil
}

func hasUserOnText(message string) (bool, error) {
	if strings.Contains(strings.ToUpper(message), userPrefix) {
		return true, fmt.Errorf(Messages["shouldnt-mention-user"])
	}
	return false, nil
}

func hasChannelOnText(message string) (bool, error) {
	if strings.Contains(strings.ToUpper(message), channelPrefix) {
		return true, fmt.Errorf(Messages["shouldnt-mention-channel"])
	}
	return false, nil
}

func getEvent(request *slacker.Request) ClaimrEvent {
	if request == nil {
		return ClaimrEvent{}
	}
	return ClaimrEvent{
		Team:    request.Event.Team,
		Channel: request.Event.Channel,
		User:    request.Event.User,
	}
}

// ClaimrEvent is a struct to simplify the usage of request.Event (and help testing)
type ClaimrEvent struct {
	Team    string
	Channel string
	User    string
}

// GetEventText exists to help testing event message
func GetEventText(request *slacker.Request) string {
	return request.Event.Msg.Text
}

func isAdmin(userName string) bool {
	if strings.ToUpper(userName) == strings.ToUpper(os.Getenv("CLAIMR_SUPERUSER")) {
		return true
	}

	for _, admin := range model.Admins {
		if strings.ToUpper(userName) == strings.ToUpper(admin.ID) {
			return true
		}
	}

	return false
}

// Check is a struct to help grouping checks before command execution
type Check struct {
	isPositive bool
	message    string
}

// RunChecks runs a list of checks and returns for the first error
func RunChecks(checks []Check) error {
	for _, check := range checks {
		if check.isPositive {
			return fmt.Errorf(check.message)
		}
	}
	return nil
}
<commit_msg>Agregate hasUserOnText and hasChannelOnText into one function<commit_after>package cmd

import (
	"fmt"
	"os"
	"strings"

	"github.com/evandroflores/claimr/model"
	"github.com/shomali11/slacker"
)

var commands []model.Command

const (
	directMessagePrefix = "D"
	channelPrefix       = "<#C"
	userPrefix          = "<@U"
)

// Register add a command to commands list an prepare to register to slacker
func Register(usage string, description string, handler func(request *slacker.Request, response slacker.ResponseWriter)) {
	commands = append(commands, model.Command{Usage: usage, Description: description, Handler: handler})
}

// CommandList returns the list of registered commands
func CommandList() []model.Command {
	return commands
}

func notImplemented(request *slacker.Request, response slacker.ResponseWriter) {
	response.Reply(Messages["not-implemented"])
}

func isDirect(channelID string) (bool, error) {
	if strings.HasPrefix(strings.ToUpper(channelID), directMessagePrefix) {
		return true, fmt.Errorf(Messages["direct-not-allowed"])
	}
	return false, nil
}

func hasUserOnText(message string) (bool, error) {
	if strings.Contains(strings.ToUpper(message), userPrefix) {
		return true, fmt.Errorf(Messages["shouldnt-mention-user"])
	}
	return false, nil
}

func hasChannelOnText(message string) (bool, error) {
	if strings.Contains(strings.ToUpper(message), channelPrefix) {
		return true, fmt.Errorf(Messages["shouldnt-mention-channel"])
	}
	return false, nil
}

func hasUserOrChannelOnText(message string) (bool, error) {
	has, err := hasUserOnText(message)

	if err != nil {
		return has, err
	}
	return hasChannelOnText(message)
}

func getEvent(request *slacker.Request) ClaimrEvent {
	if request == nil {
		return ClaimrEvent{}
	}
	return ClaimrEvent{
		Team:    request.Event.Team,
		Channel: request.Event.Channel,
		User:    request.Event.User,
	}
}

// ClaimrEvent is a struct to simplify the usage of request.Event (and help testing)
type ClaimrEvent struct {
	Team    string
	Channel string
	User    string
}

// GetEventText exists to help testing event message
func GetEventText(request *slacker.Request) string {
	return request.Event.Msg.Text
}

func isAdmin(userName string) bool {
	if strings.ToUpper(userName) == strings.ToUpper(os.Getenv("CLAIMR_SUPERUSER")) {
		return true
	}

	for _, admin := range model.Admins {
		if strings.ToUpper(userName) == strings.ToUpper(admin.ID) {
			return true
		}
	}

	return false
}

// Check is a struct to help grouping checks before command execution
type Check struct {
	isPositive bool
	message    string
}

// RunChecks runs a list of checks and returns for the first error
func RunChecks(checks []Check) error {
	for _, check := range checks {
		if check.isPositive {
			return fmt.Errorf(check.message)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package raidman

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"os"
	"reflect"
	"sync"

	pb "code.google.com/p/goprotobuf/proto"
	"github.com/amir/raidman/proto"
)

type network interface {
	Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error)
}

type tcp struct{}

type udp struct{}

// Client represents a connection to a Riemann server
type Client struct {
	sync.Mutex
	net        network
	connection net.Conn
}

// An Event represents a single Riemann event
type Event struct {
	Ttl         float32
	Time        int64
	Tags        []string
	Host        string // Defaults to os.Hostname()
	State       string
	Service     string
	Metric      interface{} // Could be Int, Float32, Float64
	Description string
}

// Dial establishes a connection to a Riemann server at addr, on the network
// netwrk.
//
// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
func Dial(netwrk, addr string) (c *Client, err error) {
	c = new(Client)

	var cnet network
	switch netwrk {
	case "tcp", "tcp4", "tcp6":
		cnet = new(tcp)
	case "udp", "udp4", "udp6":
		cnet = new(udp)
	default:
		return nil, fmt.Errorf("dial %q: unsupported network %q", netwrk, netwrk)
	}

	c.net = cnet
	c.connection, err = net.Dial(netwrk, addr)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func (network *tcp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	msg := &proto.Msg{}
	data, err := pb.Marshal(message)
	if err != nil {
		return msg, err
	}
	b := new(bytes.Buffer)
	if err = binary.Write(b, binary.BigEndian, uint32(len(data))); err != nil {
		return msg, err
	}
	if _, err = conn.Write(b.Bytes()); err != nil {
		return msg, err
	}
	if _, err = conn.Write(data); err != nil {
		return msg, err
	}
	var header uint32
	if err = binary.Read(conn, binary.BigEndian, &header); err != nil {
		return msg, err
	}
	response := make([]byte, header)
	if _, err = conn.Read(response); err != nil {
		return msg, err
	}
	if err = pb.Unmarshal(response, msg); err != nil {
		return msg, err
	}
	if msg.GetOk() != true {
		return msg, errors.New(msg.GetError())
	}
	return msg, nil
}

func (network *udp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	data, err := pb.Marshal(message)
	if err != nil {
		return nil, err
	}
	if _, err = conn.Write(data); err != nil {
		return nil, err
	}

	return nil, nil
}

func eventToPbEvent(event *Event) (*proto.Event, error) {
	var e proto.Event

	if event.Host == "" {
		event.Host, _ = os.Hostname()
	}
	t := reflect.ValueOf(&e).Elem()
	s := reflect.ValueOf(event).Elem()
	typeOfEvent := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		value := reflect.ValueOf(f.Interface())
		if reflect.Zero(f.Type()) != value && f.Interface() != nil {
			name := typeOfEvent.Field(i).Name
			switch name {
			case "State", "Service", "Host", "Description":
				tmp := reflect.ValueOf(pb.String(value.String()))
				t.FieldByName(name).Set(tmp)
			case "Ttl":
				tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
				t.FieldByName(name).Set(tmp)
			case "Time":
				tmp := reflect.ValueOf(pb.Int64(value.Int()))
				t.FieldByName(name).Set(tmp)
			case "Tags":
				tmp := reflect.ValueOf(value.Interface().([]string))
				t.FieldByName(name).Set(tmp)
			case "Metric":
				switch reflect.TypeOf(f.Interface()).Kind() {
				case reflect.Int:
					tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
					t.FieldByName("MetricSint64").Set(tmp)
				case reflect.Float32:
					tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
					t.FieldByName("MetricF").Set(tmp)
				case reflect.Float64:
					tmp := reflect.ValueOf(pb.Float64(value.Float()))
					t.FieldByName("MetricD").Set(tmp)
				default:
					return nil, fmt.Errorf("Metric of invalid type (type %v)",
						reflect.TypeOf(f.Interface()).Kind())
				}
			}
		}
	}

	return &e, nil
}

func pbEventsToEvents(pbEvents []*proto.Event) []Event {
	var events []Event

	for _, event := range pbEvents {
		e := Event{
			State:       event.GetState(),
			Service:     event.GetService(),
			Host:        event.GetHost(),
			Description: event.GetDescription(),
			Ttl:         event.GetTtl(),
			Time:        event.GetTime(),
			Tags:        event.GetTags(),
		}
		if event.MetricF != nil {
			e.Metric = event.GetMetricF()
		} else if event.MetricD != nil {
			e.Metric = event.GetMetricD()
		} else {
			e.Metric = event.GetMetricSint64()
		}

		events = append(events, e)
	}

	return events
}

// Send sends an event to Riemann
func (c *Client) Send(event *Event) error {
	e, err := eventToPbEvent(event)
	if err != nil {
		return err
	}
	message := &proto.Msg{}
	message.Events = append(message.Events, e)
	c.Lock()
	defer c.Unlock()
	_, err = c.net.Send(message, c.connection)
	if err != nil {
		return err
	}

	return nil
}

// Query returns a list of events matched by query
func (c *Client) Query(q string) ([]Event, error) {
	switch c.net.(type) {
	case *udp:
		return nil, errors.New("Querying over UDP is not supported")
	}
	query := &proto.Query{}
	query.String_ = pb.String(q)
	message := &proto.Msg{}
	message.Query = query
	c.Lock()
	defer c.Unlock()
	response, err := c.net.Send(message, c.connection)
	if err != nil {
		return nil, err
	}
	return pbEventsToEvents(response.GetEvents()), nil
}

// Close closes the connection to Riemann
func (c *Client) Close() {
	c.Lock()
	c.connection.Close()
	c.Unlock()
}
<commit_msg>Package summary for GoDoc<commit_after>// Go Riemann client
package raidman

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"os"
	"reflect"
	"sync"

	pb "code.google.com/p/goprotobuf/proto"
	"github.com/amir/raidman/proto"
)

type network interface {
	Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error)
}

type tcp struct{}

type udp struct{}

// Client represents a connection to a Riemann server
type Client struct {
	sync.Mutex
	net        network
	connection net.Conn
}

// An Event represents a single Riemann event
type Event struct {
	Ttl         float32
	Time        int64
	Tags        []string
	Host        string // Defaults to os.Hostname()
	State       string
	Service     string
	Metric      interface{} // Could be Int, Float32, Float64
	Description string
}

// Dial establishes a connection to a Riemann server at addr, on the network
// netwrk.
//
// Known networks are "tcp", "tcp4", "tcp6", "udp", "udp4", and "udp6".
func Dial(netwrk, addr string) (c *Client, err error) {
	c = new(Client)

	var cnet network
	switch netwrk {
	case "tcp", "tcp4", "tcp6":
		cnet = new(tcp)
	case "udp", "udp4", "udp6":
		cnet = new(udp)
	default:
		return nil, fmt.Errorf("dial %q: unsupported network %q", netwrk, netwrk)
	}

	c.net = cnet
	c.connection, err = net.Dial(netwrk, addr)
	if err != nil {
		return nil, err
	}

	return c, nil
}

func (network *tcp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	msg := &proto.Msg{}
	data, err := pb.Marshal(message)
	if err != nil {
		return msg, err
	}
	b := new(bytes.Buffer)
	if err = binary.Write(b, binary.BigEndian, uint32(len(data))); err != nil {
		return msg, err
	}
	if _, err = conn.Write(b.Bytes()); err != nil {
		return msg, err
	}
	if _, err = conn.Write(data); err != nil {
		return msg, err
	}
	var header uint32
	if err = binary.Read(conn, binary.BigEndian, &header); err != nil {
		return msg, err
	}
	response := make([]byte, header)
	if _, err = conn.Read(response); err != nil {
		return msg, err
	}
	if err = pb.Unmarshal(response, msg); err != nil {
		return msg, err
	}
	if msg.GetOk() != true {
		return msg, errors.New(msg.GetError())
	}
	return msg, nil
}

func (network *udp) Send(message *proto.Msg, conn net.Conn) (*proto.Msg, error) {
	data, err := pb.Marshal(message)
	if err != nil {
		return nil, err
	}
	if _, err = conn.Write(data); err != nil {
		return nil, err
	}

	return nil, nil
}

func eventToPbEvent(event *Event) (*proto.Event, error) {
	var e proto.Event

	if event.Host == "" {
		event.Host, _ = os.Hostname()
	}
	t := reflect.ValueOf(&e).Elem()
	s := reflect.ValueOf(event).Elem()
	typeOfEvent := s.Type()
	for i := 0; i < s.NumField(); i++ {
		f := s.Field(i)
		value := reflect.ValueOf(f.Interface())
		if reflect.Zero(f.Type()) != value && f.Interface() != nil {
			name := typeOfEvent.Field(i).Name
			switch name {
			case "State", "Service", "Host", "Description":
				tmp := reflect.ValueOf(pb.String(value.String()))
				t.FieldByName(name).Set(tmp)
			case "Ttl":
				tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
				t.FieldByName(name).Set(tmp)
			case "Time":
				tmp := reflect.ValueOf(pb.Int64(value.Int()))
				t.FieldByName(name).Set(tmp)
			case "Tags":
				tmp := reflect.ValueOf(value.Interface().([]string))
				t.FieldByName(name).Set(tmp)
			case "Metric":
				switch reflect.TypeOf(f.Interface()).Kind() {
				case reflect.Int:
					tmp := reflect.ValueOf(pb.Int64(int64(value.Int())))
					t.FieldByName("MetricSint64").Set(tmp)
				case reflect.Float32:
					tmp := reflect.ValueOf(pb.Float32(float32(value.Float())))
					t.FieldByName("MetricF").Set(tmp)
				case reflect.Float64:
					tmp := reflect.ValueOf(pb.Float64(value.Float()))
					t.FieldByName("MetricD").Set(tmp)
				default:
					return nil, fmt.Errorf("Metric of invalid type (type %v)",
						reflect.TypeOf(f.Interface()).Kind())
				}
			}
		}
	}

	return &e, nil
}

func pbEventsToEvents(pbEvents []*proto.Event) []Event {
	var events []Event

	for _, event := range pbEvents {
		e := Event{
			State:       event.GetState(),
			Service:     event.GetService(),
			Host:        event.GetHost(),
			Description: event.GetDescription(),
			Ttl:         event.GetTtl(),
			Time:        event.GetTime(),
			Tags:        event.GetTags(),
		}
		if event.MetricF != nil {
			e.Metric = event.GetMetricF()
		} else if event.MetricD != nil {
			e.Metric = event.GetMetricD()
		} else {
			e.Metric = event.GetMetricSint64()
		}

		events = append(events, e)
	}

	return events
}

// Send sends an event to Riemann
func (c *Client) Send(event *Event) error {
	e, err := eventToPbEvent(event)
	if err != nil {
		return err
	}
	message := &proto.Msg{}
	message.Events = append(message.Events, e)
	c.Lock()
	defer c.Unlock()
	_, err = c.net.Send(message, c.connection)
	if err != nil {
		return err
	}

	return nil
}

// Query returns a list of events matched by query
func (c *Client) Query(q string) ([]Event, error) {
	switch c.net.(type) {
	case *udp:
		return nil, errors.New("Querying over UDP is not supported")
	}
	query := &proto.Query{}
	query.String_ = pb.String(q)
	message := &proto.Msg{}
	message.Query = query
	c.Lock()
	defer c.Unlock()
	response, err := c.net.Send(message, c.connection)
	if err != nil {
		return nil, err
	}
	return pbEventsToEvents(response.GetEvents()), nil
}

// Close closes the connection to Riemann
func (c *Client) Close() {
	c.Lock()
	c.connection.Close()
	c.Unlock()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	"github.com/lhcb-org/lbx/lbenv"
)

func lbx_make_cmd_run() *commander.Command {
	cmd := &commander.Command{
		Run:       lbx_run_cmd_run,
		UsageLine: "run",
		Short:     "run a command with the proper runtime environment",
		Long: `
run runs a command with the proper runtime environment.

ex:
 $ lbx run echo "hello"
 hello

 $ lbx run gaudi.py some/jobo.py
`,
		Flag: *flag.NewFlagSet("lbx-run", flag.ExitOnError),
	}
	add_output_level(cmd)
	cmd.Flag.Bool("use-grid", false, "enable auto selection of LHCbGrid project")
	cmd.Flag.String("runtime-projects", "", "comma-separated list of runtime projects to add to the environment (e.g.: \"Foo:v1r2,Bar,Baz:v42\"")
	cmd.Flag.String("overriding-projects", "", "comma-separated list of projects to override packages (e.g: \"Foo:v1r2,Bar,Baz:v42\")")
	return cmd
}

func lbx_run_cmd_run(cmd *commander.Command, args []string) error {
	var err error

	switch len(args) {
	case 0:
		g_ctx.Errorf("lbx-run: needs at least one arg (prog-name). got=%d\n", len(args))
		return fmt.Errorf("lbx-run: invalid number of arguments")
	default:
	}

	type pair struct {
		Project string
		Version string
	}
	projects := make([]pair, 0, 2)
	if cmd.Flag.Lookup("use-grid").Value.Get().(bool) {
		projects = append(projects, pair{
			Project: "LHCbGrid",
			Version: "latest",
		})
	}
	for _, p := range strings.Split(cmd.Flag.Lookup("overriding-projects").Value.Get().(string), ",") {
		if p == "" {
			continue
		}
		str := strings.Split(p, ":")
		name := str[0]
		vers := "latest"
		if len(str) > 1 {
			vers = str[1]
		}
		projects = append(projects, pair{
			Project: name,
			Version: vers,
		})
	}

	projects = append(projects, pair{
		Project: g_ctx.Project,
		Version: g_ctx.Version,
	})

	for _, p := range strings.Split(cmd.Flag.Lookup("runtime-projects").Value.Get().(string), ",") {
		if p == "" {
			continue
		}
		str := strings.Split(p, ":")
		name := str[0]
		vers := "latest"
		if len(str) > 1 {
			vers = str[1]
		}
		projects = append(projects, pair{
			Project: name,
			Version: vers,
		})
	}

	// FIXME: add special search-path

	// set the environment XML search path
	xmlenvpath := make([]string, 0, len(projects))
	for _, p := range projects {
		// FIXME: use ExpandVersionAlias
		proj := p.Project
		vers := p.Version
		paths, err := g_ctx.EnvXMLPath(proj, vers, g_ctx.Platform)
		if err != nil {
			g_ctx.Errorf("lbx-run: error looking up ENVXMLPATH: %v\n", err)
			return err
		}
		xmlenvpath = append(xmlenvpath, paths...)
	}
	fmt.Printf("xml: %v\n", xmlenvpath)

	env := lbenv.New()
	env.SearchPath = xmlenvpath

	// FIXME: handle the extra data packages

	// load the xml files
	for _, p := range projects {
		name := p.Project + "Environment.xml"
		err = env.LoadXMLByName(name)
		if err != nil {
			g_ctx.Errorf("lbx-run: problem loading [%s]: %v\n", name, err)
			return err
		}
	}

	// set the library search path correctly for the non-Linux platforms
	if env.Has("LD_LIBRARY_PATH") {
		k := ""
		// replace LD_LIBRARY_PATH with the corresponding one
		switch runtime.GOOS {
		case "windows":
			k = "PATH"
		case "darwin":
			k = "DYLD_LIBRARY_PATH"
		}
		if k != "" {
			vv := env.Get("LD_LIBRARY_PATH")
			if env.Has(k) {
				v := env.Get(k)
				err = env.Set(k, v.Value+string(os.PathListSeparator)+vv.Value)
			} else {
				err = env.Set(k, vv.Value)
			}

			if err != nil {
				return err
			}
			err = env.Unset("LD_LIBRARY_PATH")
			if err != nil {
				return err
			}
		}
	}
	// extend the prompt variable
	ps1 := os.Getenv("PS1")
	err = env.Set("PS1", fmt.Sprintf("[lbx] %s", ps1))
	if err != nil {
		return err
	}

	bin := exec.Command(args[0], args[1:]...)
	bin.Env = env.Env()
	bin.Stdin = os.Stdin
	bin.Stdout = os.Stdout
	bin.Stderr = os.Stderr

	fmt.Printf("sub-command: %v\n", bin.Args)
	err = bin.Run()
	return err
}
<commit_msg>cmd-run: initialize env. from system<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/gonuts/commander"
	"github.com/gonuts/flag"
	"github.com/lhcb-org/lbx/lbenv"
)

func lbx_make_cmd_run() *commander.Command {
	cmd := &commander.Command{
		Run:       lbx_run_cmd_run,
		UsageLine: "run",
		Short:     "run a command with the proper runtime environment",
		Long: `
run runs a command with the proper runtime environment.

ex:
 $ lbx run echo "hello"
 hello

 $ lbx run gaudi.py some/jobo.py
`,
		Flag: *flag.NewFlagSet("lbx-run", flag.ExitOnError),
	}
	add_output_level(cmd)
	cmd.Flag.Bool("use-grid", false, "enable auto selection of LHCbGrid project")
	cmd.Flag.String("runtime-projects", "", "comma-separated list of runtime projects to add to the environment (e.g.: \"Foo:v1r2,Bar,Baz:v42\"")
	cmd.Flag.String("overriding-projects", "", "comma-separated list of projects to override packages (e.g: \"Foo:v1r2,Bar,Baz:v42\")")
	return cmd
}

func lbx_run_cmd_run(cmd *commander.Command, args []string) error {
	var err error

	switch len(args) {
	case 0:
		g_ctx.Errorf("lbx-run: needs at least one arg (prog-name). got=%d\n", len(args))
		return fmt.Errorf("lbx-run: invalid number of arguments")
	default:
	}

	type pair struct {
		Project string
		Version string
	}
	projects := make([]pair, 0, 2)
	if cmd.Flag.Lookup("use-grid").Value.Get().(bool) {
		projects = append(projects, pair{
			Project: "LHCbGrid",
			Version: "latest",
		})
	}
	for _, p := range strings.Split(cmd.Flag.Lookup("overriding-projects").Value.Get().(string), ",") {
		if p == "" {
			continue
		}
		str := strings.Split(p, ":")
		name := str[0]
		vers := "latest"
		if len(str) > 1 {
			vers = str[1]
		}
		projects = append(projects, pair{
			Project: name,
			Version: vers,
		})
	}

	projects = append(projects, pair{
		Project: g_ctx.Project,
		Version: g_ctx.Version,
	})

	for _, p := range strings.Split(cmd.Flag.Lookup("runtime-projects").Value.Get().(string), ",") {
		if p == "" {
			continue
		}
		str := strings.Split(p, ":")
		name := str[0]
		vers := "latest"
		if len(str) > 1 {
			vers = str[1]
		}
		projects = append(projects, pair{
			Project: name,
			Version: vers,
		})
	}

	// FIXME: add special search-path

	// set the environment XML search path
	xmlenvpath := make([]string, 0, len(projects))
	for _, p := range projects {
		// FIXME: use ExpandVersionAlias
		proj := p.Project
		vers := p.Version
		paths, err := g_ctx.EnvXMLPath(proj, vers, g_ctx.Platform)
		if err != nil {
			g_ctx.Errorf("lbx-run: error looking up ENVXMLPATH: %v\n", err)
			return err
		}
		xmlenvpath = append(xmlenvpath, paths...)
	}
	fmt.Printf("xml: %v\n", xmlenvpath)

	env := lbenv.New()
	env.SearchPath = xmlenvpath
	env.LoadFromSystem = true

	// load from environment
	for _, val := range os.Environ() {
		kv := strings.Split(val, "=")
		k := kv[0]
		v := os.Getenv(k)
		err = env.Set(k, v)
		if err != nil {
			g_ctx.Errorf("lbx-run: problem initializing env.var %q: %v\n", k, err)
			return err
		}
	}

	// FIXME: handle the extra data packages

	// load the xml files
	for _, p := range projects {
		name := p.Project + "Environment.xml"
		err = env.LoadXMLByName(name)
		if err != nil {
			g_ctx.Errorf("lbx-run: problem loading [%s]: %v\n", name, err)
			return err
		}
	}

	// set the library search path correctly for the non-Linux platforms
	if env.Has("LD_LIBRARY_PATH") {
		k := ""
		// replace LD_LIBRARY_PATH with the corresponding one
		switch runtime.GOOS {
		case "windows":
			k = "PATH"
		case "darwin":
			k = "DYLD_LIBRARY_PATH"
		}
		if k != "" {
			vv := env.Get("LD_LIBRARY_PATH")
			if env.Has(k) {
				v := env.Get(k)
				err = env.Set(k, v.Value+string(os.PathListSeparator)+vv.Value)
			} else {
				err = env.Set(k, vv.Value)
			}

			if err != nil {
				return err
			}
			err = env.Unset("LD_LIBRARY_PATH")
			if err != nil {
				return err
			}
		}
	}
	// extend the prompt variable
	ps1 := os.Getenv("PS1")
	err = env.Set("PS1", fmt.Sprintf("[lbx] %s", ps1))
	if err != nil {
		return err
	}

	bin := exec.Command(args[0], args[1:]...)
	bin.Env = env.Env()
	bin.Stdin = os.Stdin
	bin.Stdout = os.Stdout
	bin.Stderr = os.Stderr

	fmt.Printf("sub-command: %v\n", bin.Args)
	err = bin.Run()
	return err
}
<|endoftext|>


<commit_before>package postgres

import (
	"net/http"
	"testing"

	"github.com/nuveo/prest/api"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)
		where := WhereByRequest(r)
		So(where, ShouldContainSubstring, "dbname='prest'")
		So(where, ShouldContainSubstring, "test='cool'")
		So(where, ShouldContainSubstring, "and")
	})
}

func TestConnection(t *testing.T) {
	Convey("Verify database connection", t, func() {
		sqlx := Conn()
		So(sqlx, ShouldNotBeNil)
		err := sqlx.Ping()
		So(err, ShouldBeNil)
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where := PaginateIfPossible(r)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Insert("prest", "public", "test", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name='nuveo'")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Update("prest", "public", "test", "name='prest'", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}
<commit_msg>jsonb support test<commit_after>package postgres

import (
	"net/http"
	"testing"

	"github.com/nuveo/prest/api"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)
		where := WhereByRequest(r)
		So(where, ShouldContainSubstring, "dbname='prest'")
		So(where, ShouldContainSubstring, "test='cool'")
		So(where, ShouldContainSubstring, "and")
	})

	Convey("Where by request with jsonb field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)
		where := WhereByRequest(r)
		So(where, ShouldContainSubstring, "name='nuveo'")
		So(where, ShouldContainSubstring, "data->>'description'='bla'")
		So(where, ShouldContainSubstring, "and")
	})
}

func TestConnection(t *testing.T) {
	Convey("Verify database connection", t, func() {
		sqlx := Conn()
		So(sqlx, ShouldNotBeNil)
		err := sqlx.Ping()
		So(err, ShouldBeNil)
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where := PaginateIfPossible(r)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Insert("prest", "public", "test", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name='nuveo'")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {
		r := api.Request{
			Data: map[string]string{
				"name": "prest",
			},
		}
		json, err := Update("prest", "public", "test", "name='prest'", r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}
<|endoftext|>


<commit_before><commit_msg>bug fix<commit_after><|endoftext|>


<commit_before>// Package geo provides a small set of geometrical types
// and operations on them.
package geo
<commit_msg>update doc comment<commit_after>// Package geo provides a small set of geometrical types
// and operations on them.
// This package aims to be simple and very high quality.
package geo
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/vcs"
	"github.com/ericchiang/godl/internal/forked/glideutil"
)

var errNoManifest = errors.New("manifest file not found, run 'godl init' to create one")

type downloader struct {
	dir   string
	cache cache
}

func newRepo(typ vcs.Type, remote, local string) (vcs.Repo, error) {
	switch typ {
	case vcs.Git:
		return vcs.NewGitRepo(remote, local)
	case vcs.Svn:
		return vcs.NewSvnRepo(remote, local)
	case vcs.Bzr:
		return vcs.NewBzrRepo(remote, local)
	case vcs.Hg:
		return vcs.NewHgRepo(remote, local)
	default:
		return vcs.NewRepo(remote, local)
	}
}

// get downloads a package to a project's "vendor" directory.
//
// Remote and version are both optional.
func (d *downloader) get(pkgName, version, remote string) error {
	m, ok, err := loadManifest(d.dir)
	if err != nil {
		return fmt.Errorf("loading manifest: %v", err)
	}
	if !ok {
		return errNoManifest
	}

	rootPkg, err := glideutil.GetRootFromPackage(pkgName)
	if err != nil {
		return fmt.Errorf("failed to determine root package: %v", err)
	}
	if rootPkg != pkgName {
		return fmt.Errorf("package %q is not the root package import, try downloading %q instead", pkgName, rootPkg)
	}

	// If a remote wasn't specified, but was previously, use that.
	if remote == "" {
		for _, pkg := range m.Import {
			if pkg.Package == pkgName {
				remote = pkg.Remote
				break
			}
		}
	}

	v, err := d.vendorRepo(vcs.NoVCS, pkgName, version, remote)
	if err != nil {
		return err
	}

	return updateManifest(d.dir, func(m *manifest) error {
		p := pkg{
			Package: pkgName,
			Version: v,
			Remote:  remote,
		}

		for i, pkg := range m.Import {
			if pkg.Package == pkgName {
				m.Import[i] = p
				return nil
			}
		}

		m.Import = append(m.Import, p)
		return nil
	})
}

// vendorRepo is like vendor but allows specifying a VSC type. This is to allow
// tests to reference local git repos.
func (d *downloader) vendorRepo(typ vcs.Type, pkgName, version, remote string) (gotVersion string, err error) {
	if u, err := url.Parse(pkgName); err == nil && u.Scheme != "" {
		return "", fmt.Errorf("%q not allowed in import path", u.Scheme)
	}

	if remote == "" {
		remote = "https://" + pkgName
	}

	dest := filepath.Join(d.dir, "vendor", filepath.FromSlash(pkgName))
	err = d.cache.withLock(remote, func(path string) error {
		repo, err := newRepo(typ, remote, path)
		if err != nil {
			return fmt.Errorf("setting up remote: %v", err)
		}

		gotVersion, err = d.downloadRepo(repo, version)
		if err != nil {
			return fmt.Errorf("downloading repo: %v", err)
		}

		if err := os.RemoveAll(dest); err != nil {
			return fmt.Errorf("clearing existing path in vendor directory: %v", err)
		}

		if err := os.MkdirAll(dest, 0755); err != nil {
			return fmt.Errorf("creating target directory: %v", err)
		}

		if err := copyDir(dest, path); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return "", err
	}
	return
}

// downloadRepo attempts to download a repo at a given version. The repo may already be
// cloned, or not.
func (d *downloader) downloadRepo(repo vcs.Repo, version string) (string, error) {
	if !repo.CheckLocal() {
		if err := repo.Get(); err != nil {
			if e, ok := err.(*vcs.RemoteError); ok {
				return "", fmt.Errorf("%s: %s %v", e.Error(), e.Out(), e.Original())
			}
			return "", fmt.Errorf("getting repo: %v", err)
		}
	}

	if version != "" {
		if err := repo.UpdateVersion(version); err == nil {
			return version, nil
		}
	}
	if err := repo.Update(); err != nil {
		return "", fmt.Errorf("updaing repo: %v", err)
	}

	if version == "" {
		return repo.Version()
	}
	if err := repo.UpdateVersion(version); err != nil {
		return "", fmt.Errorf("failed to update to version %s of repo: %v", version, err)
	}
	return version, nil
}

func ignore(info os.FileInfo) bool {
	if info.IsDir() {
		switch info.Name() {
		case "testdata", "vendor":
			return true
		}
		return strings.HasPrefix(info.Name(), ".")
	}

	// Ignore non-normal files (e.g. symlink).
	if info.Mode()&os.ModeType != 0 {
		return true
	}

	switch filepath.Ext(info.Name()) {
	case ".go":
		// Ignore test files.
		if strings.HasSuffix(info.Name(), "_test.go") {
			return true
		}
	case ".s", ".c":
		// Retain assembly and C files.
	default:
		return info.Name() != "LICENSE" && info.Name() != "LICENSE.txt"
	}
	return false
}

func copyDir(dest, src string) error {
	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if err != nil || src == path {
			return err
		}

		if ignore(info) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		if info.IsDir() {
			return nil
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		target := filepath.Join(dest, rel)
		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
			return err
		}

		destF, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
		if err != nil {
			return err
		}
		defer destF.Close()

		f, err := os.OpenFile(path, os.O_RDONLY, info.Mode())
		if err != nil {
			return err
		}
		defer f.Close()

		_, err = io.Copy(destF, f)
		return err
	})
}
<commit_msg>get: only attempt to guess a repo's root if it doesn't provide a remote<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/Masterminds/vcs"
	"github.com/ericchiang/godl/internal/forked/glideutil"
)

var errNoManifest = errors.New("manifest file not found, run 'godl init' to create one")

type downloader struct {
	dir   string
	cache cache
}

func newRepo(typ vcs.Type, remote, local string) (vcs.Repo, error) {
	switch typ {
	case vcs.Git:
		return vcs.NewGitRepo(remote, local)
	case vcs.Svn:
		return vcs.NewSvnRepo(remote, local)
	case vcs.Bzr:
		return vcs.NewBzrRepo(remote, local)
	case vcs.Hg:
		return vcs.NewHgRepo(remote, local)
	default:
		return vcs.NewRepo(remote, local)
	}
}

// get downloads a package to a project's "vendor" directory.
//
// Remote and version are both optional.
func (d *downloader) get(pkgName, version, remote string) error {
	m, ok, err := loadManifest(d.dir)
	if err != nil {
		return fmt.Errorf("loading manifest: %v", err)
	}
	if !ok {
		return errNoManifest
	}

	found := false
	for _, pkg := range m.Import {
		if pkg.Package == pkgName {
			// If a remote wasn't specified, but was previously, use that.
			if remote == "" {
				remote = pkg.Remote
			}
			found = true
			break
		}
	}

	if !found && remote != "" {
		rootPkg, err := glideutil.GetRootFromPackage(pkgName)
		if err != nil {
			return fmt.Errorf("failed to determine root package: %v", err)
		}
		if rootPkg != pkgName {
			return fmt.Errorf("package %q is not the root package import, try downloading %q instead", pkgName, rootPkg)
		}
	}

	v, err := d.vendorRepo(vcs.NoVCS, pkgName, version, remote)
	if err != nil {
		return err
	}

	return updateManifest(d.dir, func(m *manifest) error {
		p := pkg{
			Package: pkgName,
			Version: v,
			Remote:  remote,
		}

		for i, pkg := range m.Import {
			if pkg.Package == pkgName {
				m.Import[i] = p
				return nil
			}
		}

		m.Import = append(m.Import, p)
		return nil
	})
}

// vendorRepo is like vendor but allows specifying a VSC type. This is to allow
// tests to reference local git repos.
func (d *downloader) vendorRepo(typ vcs.Type, pkgName, version, remote string) (gotVersion string, err error) {
	if u, err := url.Parse(pkgName); err == nil && u.Scheme != "" {
		return "", fmt.Errorf("%q not allowed in import path", u.Scheme)
	}

	if remote == "" {
		remote = "https://" + pkgName
	}

	dest := filepath.Join(d.dir, "vendor", filepath.FromSlash(pkgName))
	err = d.cache.withLock(remote, func(path string) error {
		repo, err := newRepo(typ, remote, path)
		if err != nil {
			return fmt.Errorf("setting up remote: %v", err)
		}

		gotVersion, err = d.downloadRepo(repo, version)
		if err != nil {
			return fmt.Errorf("downloading repo: %v", err)
		}

		if err := os.RemoveAll(dest); err != nil {
			return fmt.Errorf("clearing existing path in vendor directory: %v", err)
		}

		if err := os.MkdirAll(dest, 0755); err != nil {
			return fmt.Errorf("creating target directory: %v", err)
		}

		if err := copyDir(dest, path); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return "", err
	}
	return
}

// downloadRepo attempts to download a repo at a given version. The repo may already be
// cloned, or not.
func (d *downloader) downloadRepo(repo vcs.Repo, version string) (string, error) {
	if !repo.CheckLocal() {
		if err := repo.Get(); err != nil {
			if e, ok := err.(*vcs.RemoteError); ok {
				return "", fmt.Errorf("%s: %s %v", e.Error(), e.Out(), e.Original())
			}
			return "", fmt.Errorf("getting repo: %v", err)
		}
	}

	if version != "" {
		if err := repo.UpdateVersion(version); err == nil {
			return version, nil
		}
	}
	if err := repo.Update(); err != nil {
		return "", fmt.Errorf("updaing repo: %v", err)
	}

	if version == "" {
		return repo.Version()
	}
	if err := repo.UpdateVersion(version); err != nil {
		return "", fmt.Errorf("failed to update to version %s of repo: %v", version, err)
	}
	return version, nil
}

func ignore(info os.FileInfo) bool {
	if info.IsDir() {
		switch info.Name() {
		case "testdata", "vendor":
			return true
		}
		return strings.HasPrefix(info.Name(), ".")
	}

	// Ignore non-normal files (e.g. symlink).
	if info.Mode()&os.ModeType != 0 {
		return true
	}

	switch filepath.Ext(info.Name()) {
	case ".go":
		// Ignore test files.
		if strings.HasSuffix(info.Name(), "_test.go") {
			return true
		}
	case ".s", ".c":
		// Retain assembly and C files.
	default:
		return info.Name() != "LICENSE" && info.Name() != "LICENSE.txt"
	}
	return false
}

func copyDir(dest, src string) error {
	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
		if err != nil || src == path {
			return err
		}

		if ignore(info) {
			if info.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}
		if info.IsDir() {
			return nil
		}

		rel, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}

		target := filepath.Join(dest, rel)
		if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
			return err
		}

		destF, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, info.Mode())
		if err != nil {
			return err
		}
		defer destF.Close()

		f, err := os.OpenFile(path, os.O_RDONLY, info.Mode())
		if err != nil {
			return err
		}
		defer f.Close()

		_, err = io.Copy(destF, f)
		return err
	})
}
<|endoftext|>


<commit_before>package rdb

import "gopkg.in/dancannon/gorethink.v2"

// OpenSession initializes the rethink db session
func OpenSession(address string, database string) (session *gorethink.Session, err error) {
	gorethink.SetTags("gorethink", "json")

	session, err = gorethink.Connect(gorethink.ConnectOpts{
		Address:  address,
		Database: database,
	})
	if err != nil {
		return
	}

	// Create database if not already exists
	exists, err := DatabaseExists(database, session)
	if err != nil {
		session.Close()
		return
	}

	if !exists {
		if err = gorethink.DBCreate(database).Exec(session); err != nil {
			session.Close()
			return
		}
	}

	session.Use(database)
	return
}

// DatabaseExists checks if a database exists
func DatabaseExists(name string, s *gorethink.Session) (exists bool, err error) {
	res, err := gorethink.DBList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var dbs []string
	if err = res.All(&dbs); err != nil {
		return
	}

	for _, t := range dbs {
		if t == name {
			exists = true
			break
		}
	}
	return
}

// TableExists checks if a table exists
func TableExists(name string, s *gorethink.Session) (exists bool, err error) {
	res, err := gorethink.TableList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var tables []string
	if err = res.All(&tables); err != nil {
		return
	}
	for _, t := range tables {
		if t == name {
			exists = true
			return
		}
	}
	return
}

// EnsureTable creates a table if it does not already exist
func EnsureTable(name string, s *gorethink.Session) error {
	if exists, err := TableExists(name, s); err != nil || exists {
		return err
	}

	return gorethink.TableCreate(name).Exec(s)
}

// IndexExists checks if a table index exists
func IndexExists(table gorethink.Term, index string, s *gorethink.Session) (exists bool, err error) {
	res, err := table.IndexList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var indices []string
	if err = res.All(&indices); err != nil {
		return
	}
	for _, t := range indices {
		if t == index {
			exists = true
			return
		}
	}
	return
}

// EnsureIndex creates an index if not already exist
func EnsureIndex(table gorethink.Term, index string, s *gorethink.Session) error {
	if exists, err := IndexExists(table, index, s); err != nil || exists {
		return err
	}

	return table.IndexCreate(index).Exec(s)
}

// EnsureIndexFunc creates an index if not already exist
func EnsureIndexFunc(table gorethink.Term, index string, indexFunction interface{}, s *gorethink.Session) error {
	if exists, err := IndexExists(table, index, s); err != nil || exists {
		return err
	}

	return table.IndexCreateFunc(index, indexFunction).Exec(s)
}
<commit_msg>gorethink.ConnectOpts<commit_after>package rdb

import "gopkg.in/dancannon/gorethink.v2"

// OpenSession initializes the rethink db session
func OpenSession(options gorethink.ConnectOpts) (session *gorethink.Session, err error) {
	gorethink.SetTags("gorethink", "json")

	session, err = gorethink.Connect(options)
	if err != nil {
		return
	}

	// Create database if not already exists
	exists, err := DatabaseExists(options.Database, session)
	if err != nil {
		session.Close()
		return
	}

	if !exists {
		if err = gorethink.DBCreate(options.Database).Exec(session); err != nil {
			session.Close()
			return
		}
	}

	session.Use(options.Database)
	return
}

// DatabaseExists checks if a database exists
func DatabaseExists(name string, s *gorethink.Session) (exists bool, err error) {
	res, err := gorethink.DBList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var dbs []string
	if err = res.All(&dbs); err != nil {
		return
	}

	for _, t := range dbs {
		if t == name {
			exists = true
			break
		}
	}
	return
}

// TableExists checks if a table exists
func TableExists(name string, s *gorethink.Session) (exists bool, err error) {
	res, err := gorethink.TableList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var tables []string
	if err = res.All(&tables); err != nil {
		return
	}
	for _, t := range tables {
		if t == name {
			exists = true
			return
		}
	}
	return
}

// EnsureTable creates a table if it does not already exist
func EnsureTable(name string, s *gorethink.Session) error {
	if exists, err := TableExists(name, s); err != nil || exists {
		return err
	}

	return gorethink.TableCreate(name).Exec(s)
}

// IndexExists checks if a table index exists
func IndexExists(table gorethink.Term, index string, s *gorethink.Session) (exists bool, err error) {
	res, err := table.IndexList().Run(s)
	if err != nil {
		return
	}
	defer res.Close()
	var indices []string
	if err = res.All(&indices); err != nil {
		return
	}
	for _, t := range indices {
		if t == index {
			exists = true
			return
		}
	}
	return
}

// EnsureIndex creates an index if not already exist
func EnsureIndex(table gorethink.Term, index string, s *gorethink.Session) error {
	if exists, err := IndexExists(table, index, s); err != nil || exists {
		return err
	}

	return table.IndexCreate(index).Exec(s)
}

// EnsureIndexFunc creates an index if not already exist
func EnsureIndexFunc(table gorethink.Term, index string, indexFunction interface{}, s *gorethink.Session) error {
	if exists, err := IndexExists(table, index, s); err != nil || exists {
		return err
	}

	return table.IndexCreateFunc(index, indexFunction).Exec(s)
}
<|endoftext|>


<commit_before>package merkletree

import (
	"hash"
	"io"
)

// ReaderRoot returns the Merkle root of the data read from the reader, where
// each leaf is 'segmentSize' long and 'h' is used as the hashing function. All
// leaves will be 'segmentSize' bytes, the last leaf may have extra zeros.
func ReaderRoot(r io.Reader, h hash.Hash, segmentSize int) (root []byte, err error) {
	tree := New(h)
	for {
		segment := make([]byte, segmentSize)
		n, readErr := io.ReadFull(r, segment)
		if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
			err = readErr
			return
		}
		if readErr == io.EOF {
			break
		}
		if n < segmentSize {
			segment = segment[:n]
		}
		tree.Push(segment)
	}
	root = tree.Root()
	return
}

// BuildReaderProof returns a proof that certain data is in the merkle tree
// created by the data in the reader. The merkle root, set of proofs, and the
// number of leaves in the Merkle tree are all returned.
func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) (root []byte, proofSet [][]byte, numLeaves uint64, err error) {
	tree := New(h)
	tree.SetIndex(index)
	for {
		segment := make([]byte, segmentSize)
		n, readErr := io.ReadFull(r, segment)
		if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
			err = readErr
			return
		}
		if readErr == io.EOF {
			break
		}
		if n < segmentSize {
			segment = segment[:n]
		}
		tree.Push(segment)
	}
	root, proofSet, _, numLeaves = tree.Prove()
	return
}
<commit_msg>simplify padding code<commit_after>package merkletree

import (
	"hash"
	"io"
)

// readIntoTree will read segments of size 'segmentSize' and push them into the
// tree until EOF is reached.
func readIntoTree(t *Tree, r io.Reader, segmentSize int) error {
	for {
		segment := make([]byte, segmentSize)
		n, readErr := io.ReadFull(r, segment)
		if readErr == io.EOF {
			// All data has been read.
			break
		} else if readErr == io.ErrUnexpectedEOF {
			// This is the last segment, and there aren't enough bytes to fill
			// the entire segment. Note that the next call will return io.EOF.
			segment = segment[:n]
		} else if readErr != nil {
			return readErr
		}
		t.Push(segment)
	}
	return nil
}

// ReaderRoot returns the Merkle root of the data read from the reader, where
// each leaf is 'segmentSize' long and 'h' is used as the hashing function. All
// leaves will be 'segmentSize' bytes, the last leaf may have extra zeros.
func ReaderRoot(r io.Reader, h hash.Hash, segmentSize int) (root []byte, err error) {
	tree := New(h)
	err = readIntoTree(tree, r, segmentSize)
	if err != nil {
		return
	}
	root = tree.Root()
	return
}

// BuildReaderProof returns a proof that certain data is in the merkle tree
// created by the data in the reader. The merkle root, set of proofs, and the
// number of leaves in the Merkle tree are all returned.
func BuildReaderProof(r io.Reader, h hash.Hash, segmentSize int, index uint64) (root []byte, proofSet [][]byte, numLeaves uint64, err error) {
	tree := New(h)
	tree.SetIndex(index)
	err = readIntoTree(tree, r, segmentSize)
	if err != nil {
		return
	}
	root, proofSet, _, numLeaves = tree.Prove()
	return
}
<|endoftext|>


<commit_before>package main

import (
    "net/http"
)

func main() {
    http.HandleFunc("/", home)

    http.ListenAndServe(":8080",nil)
}

func home(res http.ResponseWriter, req *http.Request) {
    http.ServeFile(res,req, "../public/index.html")
}

<commit_msg>Read from and Write to websocket app.go<commit_after>package main

import (
    "fmt"
    "net/http"
    "time"

    "github.com/op/go-logging"
    "github.com/gorilla/websocket"
)

var upgrader = websocket.Upgrader {}
var log = logging.MustGetLogger("Twitter Tweeds Log")

func main() {
    http.HandleFunc("/", home)
    http.HandleFunc("/twitter/stream", twitterStream)

    http.ListenAndServe(":8080",nil)
}

func home(res http.ResponseWriter, req *http.Request) {
    http.ServeFile(res,req, "../public/index.html")
}

func twitterStream(res http.ResponseWriter, req *http.Request) {
    log.Info("WS Twitter Request: ",req.URL.Query())
    filter := req.URL.Query()["filter"]

    if filter != nil {

        // Upgrades the http server connection to the websocket protocol
        conn, _ := upgrader.Upgrade(res, req, nil)

        go wsReader(conn,filter[0])
        go wsWriter(conn,filter[0])

    } else {
        fmt.Fprintf(res, "Warning: Filter param is required to stream the data from Twitter")
    }
}

// Read messages from the websocket
func wsReader(conn *websocket.Conn,filter string) {
    for {
        _, _, err := conn.ReadMessage()

        //Close the websocket connection
        if err != nil {
            conn.Close()
            log.Info("WS Connection Closed: "+filter)
            // fmt.Println("WS Connection Closed:",filter)
            break
        } else {
            log.Error(err)
        }
    }
}
// Write messages to the websocket
func wsWriter(conn *websocket.Conn,filter string) {
    for {
        ch := time.Tick(5 *time.Second)

        for range ch {
            conn.WriteJSON(myStruct{
                Name: filter,
            })
        }
    }
}

type myStruct struct {
    Name string
}<|endoftext|>