<commit_before>package main

import (
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	_ "github.com/lib/pq"
)

var (
	keyFile    = getEnvDefault("UR_KEY_FILE", "key.pem")
	certFile   = getEnvDefault("UR_CRT_FILE", "crt.pem")
	dbConn     = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
	listenAddr = getEnvDefault("UR_LISTEN", "0.0.0.0:8443")
	tpl        *template.Template
)

var funcs = map[string]interface{}{
	"commatize": commatize,
	"number":    number,
}

func getEnvDefault(key, def string) string {
	if val := os.Getenv(key); val != "" {
		return val
	}
	return def
}

type report struct {
	Received time.Time // Only from DB

	UniqueID       string
	Version        string
	LongVersion    string
	Platform       string
	NumFolders     int
	NumDevices     int
	TotFiles       int
	FolderMaxFiles int
	TotMiB         int
	FolderMaxMiB   int
	MemoryUsageMiB int
	SHA256Perf     float64
	MemorySize     int

	Date string
}

func (r *report) Validate() error {
	if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
		return fmt.Errorf("missing required field")
	}
	if len(r.Date) != 8 {
		return fmt.Errorf("date not initialized")
	}
	return nil
}

func setupDB(db *sql.DB) error {
	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
		Received TIMESTAMP NOT NULL,
		UniqueID VARCHAR(32) NOT NULL,
		Version VARCHAR(32) NOT NULL,
		LongVersion VARCHAR(256) NOT NULL,
		Platform VARCHAR(32) NOT NULL,
		NumFolders INTEGER NOT NULL,
		NumDevices INTEGER NOT NULL,
		TotFiles INTEGER NOT NULL,
		FolderMaxFiles INTEGER NOT NULL,
		TotMiB INTEGER NOT NULL,
		FolderMaxMiB INTEGER NOT NULL,
		MemoryUsageMiB INTEGER NOT NULL,
		SHA256Perf DOUBLE PRECISION NOT NULL,
		MemorySize INTEGER NOT NULL,
		Date VARCHAR(8) NOT NULL
	)`)
	if err != nil {
		return err
	}

	row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`)
	if err := row.Scan(nil); err != nil {
		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`)
	}

	row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`)
	if err := row.Scan(nil); err != nil {
		_, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`)
	}

	return err
}

func insertReport(db *sql.DB, r report) error {
	_, err := db.Exec(`INSERT INTO Reports VALUES (now(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
		r.UniqueID, r.Version, r.LongVersion, r.Platform, r.NumFolders,
		r.NumDevices, r.TotFiles, r.FolderMaxFiles, r.TotMiB, r.FolderMaxMiB,
		r.MemoryUsageMiB, r.SHA256Perf, r.MemorySize, r.Date)

	return err
}

type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request)

func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		f(db, w, r)
	})
}

func main() {
	log.SetFlags(log.Ltime | log.Ldate)
	log.SetOutput(os.Stdout)

	// Template

	fd, err := os.Open("static/index.html")
	if err != nil {
		log.Fatalln("template:", err)
	}
	bs, err := ioutil.ReadAll(fd)
	if err != nil {
		log.Fatalln("template:", err)
	}
	fd.Close()
	tpl = template.Must(template.New("index.html").Funcs(funcs).Parse(string(bs)))

	// DB

	db, err := sql.Open("postgres", dbConn)
	if err != nil {
		log.Fatalln("database:", err)
	}
	err = setupDB(db)
	if err != nil {
		log.Fatalln("database:", err)
	}

	// TLS

	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		log.Fatalln("tls:", err)
	}

	cfg := &tls.Config{
		Certificates:           []tls.Certificate{cert},
		SessionTicketsDisabled: true,
	}

	// HTTPS

	listener, err := tls.Listen("tcp", listenAddr, cfg)
	if err != nil {
		log.Fatalln("https:", err)
	}

	srv := http.Server{
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}

	http.HandleFunc("/", withDB(db, rootHandler))
	http.HandleFunc("/newdata", withDB(db, newDataHandler))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	err = srv.Serve(listener)
	if err != nil {
		log.Fatalln("https:", err)
	}
}

func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" || r.URL.Path == "/index.html" {
		k := timestamp()
		rep := getReport(db, k)

		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		err := tpl.Execute(w, rep)
		if err != nil {
			log.Println(err)
		}
	} else {
		http.Error(w, "Not found", 404)
	}
}

func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	var rep report
	rep.Date = time.Now().UTC().Format("20060102")

	lr := &io.LimitedReader{R: r.Body, N: 10240}
	if err := json.NewDecoder(lr).Decode(&rep); err != nil {
		log.Println("json decode:", err)
		http.Error(w, "JSON Decode Error", http.StatusInternalServerError)
		return
	}

	if err := rep.Validate(); err != nil {
		log.Println("validate:", err)
		log.Printf("%#v", rep)
		http.Error(w, "Validation Error", http.StatusInternalServerError)
		return
	}

	if err := insertReport(db, rep); err != nil {
		log.Println("insert:", err)
		log.Printf("%#v", rep)
		http.Error(w, "Database Error", http.StatusInternalServerError)
		return
	}
}

type category struct {
	Values [4]float64
	Key    string
	Descr  string
	Unit   string
	Binary bool
}

var reportCache map[string]interface{}
var reportMutex sync.Mutex

func getReport(db *sql.DB, key string) map[string]interface{} {
	reportMutex.Lock()
	defer reportMutex.Unlock()

	if k := reportCache["key"]; k == key {
		return reportCache
	}

	nodes := 0
	var versions []string
	var platforms []string
	var oses []string
	var numFolders []int
	var numDevices []int
	var totFiles []int
	var maxFiles []int
	var totMiB []int
	var maxMiB []int
	var memoryUsage []int
	var sha256Perf []float64
	var memorySize []int

	rows, err := db.Query(`SELECT * FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
	if err != nil {
		log.Println("sql:", err)
		return nil
	}
	defer rows.Close()

	for rows.Next() {

		var rep report
		err := rows.Scan(&rep.Received, &rep.UniqueID, &rep.Version,
			&rep.LongVersion, &rep.Platform, &rep.NumFolders, &rep.NumDevices,
			&rep.TotFiles, &rep.FolderMaxFiles, &rep.TotMiB, &rep.FolderMaxMiB,
			&rep.MemoryUsageMiB, &rep.SHA256Perf, &rep.MemorySize, &rep.Date)

		if err != nil {
			log.Println("sql:", err)
			return nil
		}

		nodes++
		versions = append(versions, transformVersion(rep.Version))
		platforms = append(platforms, rep.Platform)
		ps := strings.Split(rep.Platform, "-")
		oses = append(oses, ps[0])
		if rep.NumFolders > 0 {
			numFolders = append(numFolders, rep.NumFolders)
		}
		if rep.NumDevices > 0 {
			numDevices = append(numDevices, rep.NumDevices)
		}
		if rep.TotFiles > 0 {
			totFiles = append(totFiles, rep.TotFiles)
		}
		if rep.FolderMaxFiles > 0 {
			maxFiles = append(maxFiles, rep.FolderMaxFiles)
		}
		if rep.TotMiB > 0 {
			totMiB = append(totMiB, rep.TotMiB*(1<<20))
		}
		if rep.FolderMaxMiB > 0 {
			maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
		}
		if rep.MemoryUsageMiB > 0 {
			memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
		}
		if rep.SHA256Perf > 0 {
			sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20))
		}
		if rep.MemorySize > 0 {
			memorySize = append(memorySize, rep.MemorySize*(1<<20))
		}
	}

	var categories []category
	categories = append(categories, category{
		Values: statsForInts(totFiles),
		Descr:  "Files Managed per Device",
	})

	categories = append(categories, category{
		Values: statsForInts(maxFiles),
		Descr:  "Files in Largest Folder",
	})

	categories = append(categories, category{
		Values: statsForInts(totMiB),
		Descr:  "Data Managed per Device",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(maxMiB),
		Descr:  "Data in Largest Folder",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(numDevices),
		Descr:  "Number of Devices in Cluster",
	})

	categories = append(categories, category{
		Values: statsForInts(numFolders),
		Descr:  "Number of Folders Configured",
	})

	categories = append(categories, category{
		Values: statsForInts(memoryUsage),
		Descr:  "Memory Usage",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(memorySize),
		Descr:  "System Memory",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForFloats(sha256Perf),
		Descr:  "SHA-256 Hashing Performance",
		Unit:   "B/s",
		Binary: true,
	})

	r := make(map[string]interface{})
	r["key"] = key
	r["nodes"] = nodes
	r["categories"] = categories
	r["versions"] = analyticsFor(versions, 10)
	r["platforms"] = analyticsFor(platforms, 0)
	r["os"] = analyticsFor(oses, 0)

	reportCache = r

	return r
}

func ensureDir(dir string, mode int) {
	fi, err := os.Stat(dir)
	if os.IsNotExist(err) {
		os.MkdirAll(dir, 0700)
	} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
		os.Chmod(dir, os.FileMode(mode))
	}
}

var vRe = regexp.MustCompile(`^(v\d+\.\d+\.\d+(?:-[a-z]\w+)?)[+\.-]`)

// transformVersion returns a version number formatted correctly, with all
// development versions aggregated into one.
func transformVersion(v string) string {
	if v == "unknown-dev" {
		return v
	}
	if !strings.HasPrefix(v, "v") {
		v = "v" + v
	}
	if m := vRe.FindStringSubmatch(v); len(m) > 0 {
		return m[1] + " (+dev)"
	}

	// Truncate old versions to just the generation part
	if strings.HasPrefix(v, "v0.7") {
		return "v0.7.x"
	}
	if strings.HasPrefix(v, "v0.8") {
		return "v0.8.x"
	}
	if strings.HasPrefix(v, "v0.9") {
		return "v0.9.x"
	}

	return v
}

// timestamp returns a time stamp for the current hour, to be used as a cache key
func timestamp() string {
	return time.Now().Format("20060102T15")
}
<commit_msg>Cache template result, not report<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	_ "github.com/lib/pq"
)

var (
	keyFile    = getEnvDefault("UR_KEY_FILE", "key.pem")
	certFile   = getEnvDefault("UR_CRT_FILE", "crt.pem")
	dbConn     = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
	listenAddr = getEnvDefault("UR_LISTEN", "0.0.0.0:8443")
	tpl        *template.Template
)

var funcs = map[string]interface{}{
	"commatize": commatize,
	"number":    number,
}

func getEnvDefault(key, def string) string {
	if val := os.Getenv(key); val != "" {
		return val
	}
	return def
}

type report struct {
	Received time.Time // Only from DB

	UniqueID       string
	Version        string
	LongVersion    string
	Platform       string
	NumFolders     int
	NumDevices     int
	TotFiles       int
	FolderMaxFiles int
	TotMiB         int
	FolderMaxMiB   int
	MemoryUsageMiB int
	SHA256Perf     float64
	MemorySize     int

	Date string
}

func (r *report) Validate() error {
	if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
		return fmt.Errorf("missing required field")
	}
	if len(r.Date) != 8 {
		return fmt.Errorf("date not initialized")
	}
	return nil
}

func setupDB(db *sql.DB) error {
	_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
		Received TIMESTAMP NOT NULL,
		UniqueID VARCHAR(32) NOT NULL,
		Version VARCHAR(32) NOT NULL,
		LongVersion VARCHAR(256) NOT NULL,
		Platform VARCHAR(32) NOT NULL,
		NumFolders INTEGER NOT NULL,
		NumDevices INTEGER NOT NULL,
		TotFiles INTEGER NOT NULL,
		FolderMaxFiles INTEGER NOT NULL,
		TotMiB INTEGER NOT NULL,
		FolderMaxMiB INTEGER NOT NULL,
		MemoryUsageMiB INTEGER NOT NULL,
		SHA256Perf DOUBLE PRECISION NOT NULL,
		MemorySize INTEGER NOT NULL,
		Date VARCHAR(8) NOT NULL
	)`)
	if err != nil {
		return err
	}

	row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`)
	if err := row.Scan(nil); err != nil {
		_, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`)
	}

	row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`)
	if err := row.Scan(nil); err != nil {
		_, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`)
	}

	return err
}

func insertReport(db *sql.DB, r report) error {
	_, err := db.Exec(`INSERT INTO Reports VALUES (now(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
		r.UniqueID, r.Version, r.LongVersion, r.Platform, r.NumFolders,
		r.NumDevices, r.TotFiles, r.FolderMaxFiles, r.TotMiB, r.FolderMaxMiB,
		r.MemoryUsageMiB, r.SHA256Perf, r.MemorySize, r.Date)

	return err
}

type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request)

func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		f(db, w, r)
	})
}

func main() {
	log.SetFlags(log.Ltime | log.Ldate)
	log.SetOutput(os.Stdout)

	// Template

	fd, err := os.Open("static/index.html")
	if err != nil {
		log.Fatalln("template:", err)
	}
	bs, err := ioutil.ReadAll(fd)
	if err != nil {
		log.Fatalln("template:", err)
	}
	fd.Close()
	tpl = template.Must(template.New("index.html").Funcs(funcs).Parse(string(bs)))

	// DB

	db, err := sql.Open("postgres", dbConn)
	if err != nil {
		log.Fatalln("database:", err)
	}
	err = setupDB(db)
	if err != nil {
		log.Fatalln("database:", err)
	}

	// TLS

	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		log.Fatalln("tls:", err)
	}

	cfg := &tls.Config{
		Certificates:           []tls.Certificate{cert},
		SessionTicketsDisabled: true,
	}

	// HTTPS

	listener, err := tls.Listen("tcp", listenAddr, cfg)
	if err != nil {
		log.Fatalln("https:", err)
	}

	srv := http.Server{
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 5 * time.Second,
	}

	http.HandleFunc("/", withDB(db, rootHandler))
	http.HandleFunc("/newdata", withDB(db, newDataHandler))
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	err = srv.Serve(listener)
	if err != nil {
		log.Fatalln("https:", err)
	}
}

var (
	cacheData []byte
	cacheTime time.Time
	cacheMut  sync.Mutex
)

const maxCacheTime = 5 * 60 * time.Second

func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" || r.URL.Path == "/index.html" {
		cacheMut.Lock()
		defer cacheMut.Unlock()

		if time.Since(cacheTime) > maxCacheTime {
			rep := getReport(db)
			buf := new(bytes.Buffer)
			err := tpl.Execute(buf, rep)
			if err != nil {
				log.Println(err)
				http.Error(w, "Template Error", http.StatusInternalServerError)
				return
			}
			cacheData = buf.Bytes()
			cacheTime = time.Now()
		}

		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write(cacheData)
	} else {
		http.Error(w, "Not found", 404)
		return
	}
}

func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	var rep report
	rep.Date = time.Now().UTC().Format("20060102")

	lr := &io.LimitedReader{R: r.Body, N: 10240}
	if err := json.NewDecoder(lr).Decode(&rep); err != nil {
		log.Println("json decode:", err)
		http.Error(w, "JSON Decode Error", http.StatusInternalServerError)
		return
	}

	if err := rep.Validate(); err != nil {
		log.Println("validate:", err)
		log.Printf("%#v", rep)
		http.Error(w, "Validation Error", http.StatusInternalServerError)
		return
	}

	if err := insertReport(db, rep); err != nil {
		log.Println("insert:", err)
		log.Printf("%#v", rep)
		http.Error(w, "Database Error", http.StatusInternalServerError)
		return
	}
}

type category struct {
	Values [4]float64
	Key    string
	Descr  string
	Unit   string
	Binary bool
}

func getReport(db *sql.DB) map[string]interface{} {
	nodes := 0
	var versions []string
	var platforms []string
	var oses []string
	var numFolders []int
	var numDevices []int
	var totFiles []int
	var maxFiles []int
	var totMiB []int
	var maxMiB []int
	var memoryUsage []int
	var sha256Perf []float64
	var memorySize []int

	rows, err := db.Query(`SELECT * FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
	if err != nil {
		log.Println("sql:", err)
		return nil
	}
	defer rows.Close()

	for rows.Next() {

		var rep report
		err := rows.Scan(&rep.Received, &rep.UniqueID, &rep.Version,
			&rep.LongVersion, &rep.Platform, &rep.NumFolders, &rep.NumDevices,
			&rep.TotFiles, &rep.FolderMaxFiles, &rep.TotMiB, &rep.FolderMaxMiB,
			&rep.MemoryUsageMiB, &rep.SHA256Perf, &rep.MemorySize, &rep.Date)

		if err != nil {
			log.Println("sql:", err)
			return nil
		}

		nodes++
		versions = append(versions, transformVersion(rep.Version))
		platforms = append(platforms, rep.Platform)
		ps := strings.Split(rep.Platform, "-")
		oses = append(oses, ps[0])
		if rep.NumFolders > 0 {
			numFolders = append(numFolders, rep.NumFolders)
		}
		if rep.NumDevices > 0 {
			numDevices = append(numDevices, rep.NumDevices)
		}
		if rep.TotFiles > 0 {
			totFiles = append(totFiles, rep.TotFiles)
		}
		if rep.FolderMaxFiles > 0 {
			maxFiles = append(maxFiles, rep.FolderMaxFiles)
		}
		if rep.TotMiB > 0 {
			totMiB = append(totMiB, rep.TotMiB*(1<<20))
		}
		if rep.FolderMaxMiB > 0 {
			maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
		}
		if rep.MemoryUsageMiB > 0 {
			memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
		}
		if rep.SHA256Perf > 0 {
			sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20))
		}
		if rep.MemorySize > 0 {
			memorySize = append(memorySize, rep.MemorySize*(1<<20))
		}
	}

	var categories []category
	categories = append(categories, category{
		Values: statsForInts(totFiles),
		Descr:  "Files Managed per Device",
	})

	categories = append(categories, category{
		Values: statsForInts(maxFiles),
		Descr:  "Files in Largest Folder",
	})

	categories = append(categories, category{
		Values: statsForInts(totMiB),
		Descr:  "Data Managed per Device",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(maxMiB),
		Descr:  "Data in Largest Folder",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(numDevices),
		Descr:  "Number of Devices in Cluster",
	})

	categories = append(categories, category{
		Values: statsForInts(numFolders),
		Descr:  "Number of Folders Configured",
	})

	categories = append(categories, category{
		Values: statsForInts(memoryUsage),
		Descr:  "Memory Usage",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForInts(memorySize),
		Descr:  "System Memory",
		Unit:   "B",
		Binary: true,
	})

	categories = append(categories, category{
		Values: statsForFloats(sha256Perf),
		Descr:  "SHA-256 Hashing Performance",
		Unit:   "B/s",
		Binary: true,
	})

	r := make(map[string]interface{})
	r["nodes"] = nodes
	r["categories"] = categories
	r["versions"] = analyticsFor(versions, 10)
	r["platforms"] = analyticsFor(platforms, 0)
	r["os"] = analyticsFor(oses, 0)

	return r
}

func ensureDir(dir string, mode int) {
	fi, err := os.Stat(dir)
	if os.IsNotExist(err) {
		os.MkdirAll(dir, 0700)
	} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
		os.Chmod(dir, os.FileMode(mode))
	}
}

var vRe = regexp.MustCompile(`^(v\d+\.\d+\.\d+(?:-[a-z]\w+)?)[+\.-]`)

// transformVersion returns a version number formatted correctly, with all
// development versions aggregated into one.
func transformVersion(v string) string {
	if v == "unknown-dev" {
		return v
	}
	if !strings.HasPrefix(v, "v") {
		v = "v" + v
	}
	if m := vRe.FindStringSubmatch(v); len(m) > 0 {
		return m[1] + " (+dev)"
	}

	// Truncate old versions to just the generation part
	if strings.HasPrefix(v, "v0.7") {
		return "v0.7.x"
	}
	if strings.HasPrefix(v, "v0.8") {
		return "v0.8.x"
	}
	if strings.HasPrefix(v, "v0.9") {
		return "v0.9.x"
	}

	return v
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io/ioutil"
import "log"
import "net/url"
import "os"
import "os/exec"
import "path/filepath"
import "strings"

func cleanWcRoot(wc_path string) (err error) {
	infos, err := ioutil.ReadDir(wc_path)
	if nil != err {
		return
	}
	for _, inf := range infos {
		fmt.Println(inf.Name())
	}
	return nil
}

func execPiped(name string, arg ...string) error {
	fmt.Println(name + " " + strings.Join(arg, " "))
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin  = os.Stdin
	return cmd.Run()
}

func svnDiffCommit(repos *url.URL, wc_path string) (err error) {
	err = execPiped("svn", "checkout", repos.String(), wc_path)
	if nil != err {
		return
	}
	err = cleanWcRoot(wc_path)
	if nil != err {
		return
	}
	// copy all files and folders from source to working copy
	// svn status
	// svn remove all missing files
	// svn commit
	return nil
}

func createRepos(repos_path string) (repos *url.URL, err error) {
	err = execPiped("svnadmin", "create", repos_path)
	if nil != err {
		return
	}
	abs_repos_path, err := filepath.Abs(repos_path)
	if nil != err {
		return
	}
	abs_repos_path = "file://" + abs_repos_path
	repos, err = url.Parse(abs_repos_path)
	return
}

func testSelf() (err error) {
	fmt.Println("Self test --> Start...")
	test_path := "./self_test/"
	repos_path := test_path + "repos/"
	wc_path := test_path + "wc/"
	err = os.Mkdir(test_path, 0755)
	if nil != err {
		return
	}
	defer func() {
		inner_err := os.RemoveAll(test_path)
		if nil != inner_err {
			err = inner_err
		}
	}()
	repos_url, err := createRepos(repos_path)
	err = svnDiffCommit(repos_url, wc_path)
	if nil != err {
		return
	}
	fmt.Println("Self test --> Done.")
	return nil
}

func main() {
	err := testSelf()
	if nil != err {
		log.Fatal(err)
	}
}

<commit_msg>Improved output readability.<commit_after>package main

import "fmt"
import "io/ioutil"
import "log"
import "net/url"
import "os"
import "os/exec"
import "path/filepath"
import "strings"

func cleanWcRoot(wc_path string) (err error) {
	infos, err := ioutil.ReadDir(wc_path)
	if nil != err {
		return
	}
	for _, inf := range infos {
		fmt.Println(inf.Name())
	}
	return nil
}

func execPiped(name string, arg ...string) error {
	fmt.Println(name + " " + strings.Join(arg, " "))
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin  = os.Stdin
	return cmd.Run()
}

func svnDiffCommit(repos *url.URL, wc_path string) (err error) {
	err = execPiped("svn", "checkout", repos.String(), wc_path)
	if nil != err {
		return
	}
	err = cleanWcRoot(wc_path)
	if nil != err {
		return
	}
	// copy all files and folders from source to working copy
	// svn status
	// svn remove all missing files
	// svn commit
	return nil
}

func createRepos(repos_path string) (repos *url.URL, err error) {
	err = execPiped("svnadmin", "create", repos_path)
	if nil != err {
		return
	}
	abs_repos_path, err := filepath.Abs(repos_path)
	if nil != err {
		return
	}
	abs_repos_path = "file://" + abs_repos_path
	repos, err = url.Parse(abs_repos_path)
		return
}

func testSelf() (err error) {
	fmt.Print("\n\nSelf test --> Start...\n\n\n")
	test_path := "./self_test/"
	repos_path := test_path + "repos/"
	wc_path := test_path + "wc/"
	err = os.Mkdir(test_path, 0755)
	if nil != err {
		return
	}
	defer func() {
		inner_err := os.RemoveAll(test_path)
		if nil != inner_err {
			err = inner_err
		}
	}()
	repos_url, err := createRepos(repos_path)
	err = svnDiffCommit(repos_url, wc_path)
	if nil != err {
		return
	}
	fmt.Print("\n\nSelf test --> Success.\n\n\n")
	return nil
}

func main() {
	err := testSelf()
	if nil != err {
		log.Fatal(err)
	}
}

<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/csv"
	"flag"
	"fmt"

	_ "github.com/mattn/go-sqlite3"

	"bytes"
	"crypto/rand"
	"encoding/hex"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	// Parse command line opts
	commands := flag.String("sql", "", "SQL Command(s) to run on the data")
	source_text := flag.String("source", "stdin", "Source file to load, or defaults to stdin")
	delimiter := flag.String("dlm", ",", "Delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	header := flag.Bool("header", false, "Treat file as having the first row as a header row")
	tableName := flag.String("table-name", "tbl", "Override the default table name (tbl)")
	save_to := flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	console := flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	verbose := flag.Bool("verbose", false, "Enable verbose logging")
	flag.Parse()

	if *console && (*source_text == "stdin") {
		log.Fatalln("Can not open console with pipe input, read a file instead")
	}

	seperator := determineSeperator(delimiter)

	// Open db, in memory if possible
	db, openPath := openDB(save_to, console)

	// Open the input source
	var fp *os.File
	fp = openFileOrStdin(source_text)
	defer fp.Close()

	// Init a structured text reader
	reader := csv.NewReader(fp)
	reader.FieldsPerRecord = 0
	reader.Comma = seperator

	// Read the first row
	first_row, read_err := reader.Read()

	if read_err != nil {
		log.Fatalln(read_err)
	}

	var headerRow []string

	if *header {
		headerRow = first_row
		first_row, read_err = reader.Read()

		if read_err != nil {
			log.Fatalln(read_err)
		}
	} else {
		headerRow = make([]string, len(first_row))
		for i := 0; i < len(first_row); i++ {
			headerRow[i] = "c" + strconv.Itoa(i)
		}
	}

	// Create the table to load to
	createTable(tableName, &headerRow, db)

	// Start the clock for importing
	t0 := time.Now()

	// Create transaction
	tx, tx_err := db.Begin()

	if tx_err != nil {
		log.Fatalln(tx_err)
	}

	// Load first row
	stmt := createLoadStmt(tableName, &headerRow, tx)
	loadRow(tableName, &first_row, tx, stmt, verbose)

	// Read the data
	for {
		row, file_err := reader.Read()
		if file_err == io.EOF {
			break
		} else if file_err != nil {
			log.Fatalln(file_err)
		} else {
			loadRow(tableName, &row, tx, stmt, verbose)
		}
	}
	tx.Commit()

	t1 := time.Now()

	if *verbose {
		fmt.Printf("Data loaded in: %v\n", t1.Sub(t0))
	}

	// Determine what sql to execute
	sqls_to_execute := strings.Split(*commands, ";")

	// Execute given SQL
	for _, sql_cmd := range sqls_to_execute {
		if strings.Trim(sql_cmd, " ") != "" {
			result, err := db.Query(sql_cmd)
			if err != nil {
				log.Fatalln(err)
			}
			displayResult(result)
		}
	}

	// Open console
	if *console {
		db.Close()

		cmd := exec.Command("sqlite3", *openPath)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd_err := cmd.Run()

		if len(*save_to) == 0 {
			os.Remove(*openPath)
		}

		if cmd_err != nil {
			log.Fatalln(cmd_err)
		}
	} else if len(*save_to) == 0 {
		db.Close()
		os.Remove(*openPath)
	} else {
		db.Close()
	}
}

func createTable(tableName *string, columnNames *[]string, db *sql.DB) error {
	var buffer bytes.Buffer
	buffer.WriteString("CREATE TABLE IF NOT EXISTS " + (*tableName) + " (")
	for i, col := range *columnNames {
		buffer.WriteString(col + " TEXT")
		if i != len(*columnNames)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	_, err := db.Exec(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return err
}

func createLoadStmt(tableName *string, values *[]string, db *sql.Tx) *sql.Stmt {
	if len(*values) == 0 {
		log.Fatalln("Nothing to build insert with!")
	}
	var buffer bytes.Buffer

	buffer.WriteString("INSERT INTO " + (*tableName) + " VALUES (")
	for i := range *values {
		buffer.WriteString("?")
		if i != len(*values)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	stmt, err := db.Prepare(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return stmt
}

func loadRow(tableName *string, values *[]string, db *sql.Tx, stmt *sql.Stmt, verbose *bool) error {
	if len(*values) == 0 {
		return nil
	}
	vals := make([]interface{}, 0)
	for _, val := range *values {
		vals = append(vals, val)
	}
	_, err := stmt.Exec(vals...)
	if err != nil && *verbose {
		log.Println("Bad row: ", err)
	}
	return err
}

func displayResult(rows *sql.Rows) {
	cols, cols_err := rows.Columns()

	if cols_err != nil {
		log.Fatalln(cols_err)
	}

	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols))
	for i, _ := range cols {
		dest[i] = &rawResult[i]
	}

	for rows.Next() {
		rows.Scan(dest...)

		for i, raw := range rawResult {
			result[i] = string(raw)
		}

		for j, v := range result {
			fmt.Printf("%s", v)
			if j != len(result)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}
}

func openFileOrStdin(path *string) *os.File {
	var fp *os.File
	var err error
	if (*path) == "stdin" {
		fp = os.Stdin
		err = nil
	} else {
		fp, err = os.Open(*cleanPath(path))
	}

	if err != nil {
		log.Fatalln(err)
	}

	return fp
}

func cleanPath(path *string) *string {
	var result string
	usr, err := user.Current()
	if err != nil {
		log.Fatalln(err)
	}

	if (*path)[:2] == "~/" {
		dir := usr.HomeDir + "/"
		result = strings.Replace(*path, "~/", dir, 1)
	} else {
		result = (*path)
	}

	abs_result, abs_err := filepath.Abs(result)
	if abs_err != nil {
		log.Fatalln(err)
	}

	clean_result := filepath.Clean(abs_result)

	return &clean_result
}

func openDB(path *string, no_memory *bool) (*sql.DB, *string) {
	openPath := ":memory:"
	if len(*path) != 0 {
		openPath = *cleanPath(path)
	} else if *no_memory {
		b := make([]byte, 10)
		io.ReadFull(rand.Reader, b)
		tmpPath := filepath.Join(os.TempDir(), "dankbase_"+fmt.Sprintf("%x", b)+".db")
		openPath = tmpPath
	}

	db, err := sql.Open("sqlite3", openPath)

	if err != nil {
		log.Fatalln(err)
	}
	return db, &openPath
}

func determineSeperator(delimiter *string) rune {
	var seperator rune

	if (*delimiter) == "tab" {
		seperator = '\t'
	} else if strings.Index((*delimiter), "0x") == 0 {
		dlm, hex_err := hex.DecodeString((*delimiter)[2:])

		if hex_err != nil {
			log.Fatalln(hex_err)
		}

		seperator, _ = utf8.DecodeRuneInString(string(dlm))
	} else {
		seperator, _ = utf8.DecodeRuneInString(*delimiter)
	}
	return seperator
}
<commit_msg>Use ioutil.TempDir<commit_after>package main

import (
	"database/sql"
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"

	_ "github.com/mattn/go-sqlite3"

	"bytes"
	"encoding/hex"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

func main() {
	// Parse command line opts
	commands := flag.String("sql", "", "SQL Command(s) to run on the data")
	source_text := flag.String("source", "stdin", "Source file to load, or defaults to stdin")
	delimiter := flag.String("dlm", ",", "Delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	header := flag.Bool("header", false, "Treat file as having the first row as a header row")
	tableName := flag.String("table-name", "tbl", "Override the default table name (tbl)")
	save_to := flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	console := flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	verbose := flag.Bool("verbose", false, "Enable verbose logging")
	flag.Parse()

	if *console && (*source_text == "stdin") {
		log.Fatalln("Can not open console with pipe input, read a file instead")
	}

	seperator := determineSeperator(delimiter)

	// Open db, in memory if possible
	db, openPath := openDB(save_to, console)

	// Open the input source
	var fp *os.File
	fp = openFileOrStdin(source_text)
	defer fp.Close()

	// Init a structured text reader
	reader := csv.NewReader(fp)
	reader.FieldsPerRecord = 0
	reader.Comma = seperator

	// Read the first row
	first_row, read_err := reader.Read()

	if read_err != nil {
		log.Fatalln(read_err)
	}

	var headerRow []string

	if *header {
		headerRow = first_row
		first_row, read_err = reader.Read()

		if read_err != nil {
			log.Fatalln(read_err)
		}
	} else {
		headerRow = make([]string, len(first_row))
		for i := 0; i < len(first_row); i++ {
			headerRow[i] = "c" + strconv.Itoa(i)
		}
	}

	// Create the table to load to
	createTable(tableName, &headerRow, db)

	// Start the clock for importing
	t0 := time.Now()

	// Create transaction
	tx, tx_err := db.Begin()

	if tx_err != nil {
		log.Fatalln(tx_err)
	}

	// Load first row
	stmt := createLoadStmt(tableName, &headerRow, tx)
	loadRow(tableName, &first_row, tx, stmt, verbose)

	// Read the data
	for {
		row, file_err := reader.Read()
		if file_err == io.EOF {
			break
		} else if file_err != nil {
			log.Fatalln(file_err)
		} else {
			loadRow(tableName, &row, tx, stmt, verbose)
		}
	}
	stmt.Close()
	tx.Commit()

	t1 := time.Now()

	if *verbose {
		fmt.Printf("Data loaded in: %v\n", t1.Sub(t0))
	}

	// Determine what sql to execute
	sqls_to_execute := strings.Split(*commands, ";")

	// Execute given SQL
	for _, sql_cmd := range sqls_to_execute {
		if strings.Trim(sql_cmd, " ") != "" {
			result, err := db.Query(sql_cmd)
			if err != nil {
				log.Fatalln(err)
			}
			displayResult(result)
		}
	}

	// Open console
	if *console {
		db.Close()

		cmd := exec.Command("sqlite3", *openPath)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd_err := cmd.Run()
		if cmd.Process != nil {
			cmd.Process.Release()
		}

		if len(*save_to) == 0 {
			os.RemoveAll(filepath.Dir(*openPath))
		}

		if cmd_err != nil {
			log.Fatalln(cmd_err)
		}
	} else if len(*save_to) == 0 {
		db.Close()
		os.Remove(*openPath)
	} else {
		db.Close()
	}
}

func createTable(tableName *string, columnNames *[]string, db *sql.DB) error {
	var buffer bytes.Buffer
	buffer.WriteString("CREATE TABLE IF NOT EXISTS " + (*tableName) + " (")
	for i, col := range *columnNames {
		buffer.WriteString(col + " TEXT")
		if i != len(*columnNames)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	_, err := db.Exec(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return err
}

func createLoadStmt(tableName *string, values *[]string, db *sql.Tx) *sql.Stmt {
	if len(*values) == 0 {
		log.Fatalln("Nothing to build insert with!")
	}
	var buffer bytes.Buffer

	buffer.WriteString("INSERT INTO " + (*tableName) + " VALUES (")
	for i := range *values {
		buffer.WriteString("?")
		if i != len(*values)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString(");")
	stmt, err := db.Prepare(buffer.String())
	if err != nil {
		log.Fatalln(err)
	}
	return stmt
}

func loadRow(tableName *string, values *[]string, db *sql.Tx, stmt *sql.Stmt, verbose *bool) error {
	if len(*values) == 0 {
		return nil
	}
	vals := make([]interface{}, 0)
	for _, val := range *values {
		vals = append(vals, val)
	}
	_, err := stmt.Exec(vals...)
	if err != nil && *verbose {
		log.Println("Bad row: ", err)
	}
	return err
}

func displayResult(rows *sql.Rows) {
	cols, cols_err := rows.Columns()

	if cols_err != nil {
		log.Fatalln(cols_err)
	}

	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols))
	for i, _ := range cols {
		dest[i] = &rawResult[i]
	}

	for rows.Next() {
		rows.Scan(dest...)

		for i, raw := range rawResult {
			result[i] = string(raw)
		}

		for j, v := range result {
			fmt.Printf("%s", v)
			if j != len(result)-1 {
				fmt.Printf(", ")
			}
		}
		fmt.Printf("\n")
	}
}

func openFileOrStdin(path *string) *os.File {
	var fp *os.File
	var err error
	if (*path) == "stdin" {
		fp = os.Stdin
		err = nil
	} else {
		fp, err = os.Open(*cleanPath(path))
	}

	if err != nil {
		log.Fatalln(err)
	}

	return fp
}

func cleanPath(path *string) *string {
	var result string
	usr, err := user.Current()
	if err != nil {
		log.Fatalln(err)
	}

	if (*path)[:2] == "~/" {
		dir := usr.HomeDir + "/"
		result = strings.Replace(*path, "~/", dir, 1)
	} else {
		result = (*path)
	}

	abs_result, abs_err := filepath.Abs(result)
	if abs_err != nil {
		log.Fatalln(err)
	}

	clean_result := filepath.Clean(abs_result)

	return &clean_result
}

func openDB(path *string, no_memory *bool) (*sql.DB, *string) {
	openPath := ":memory:"
	if len(*path) != 0 {
		openPath = *cleanPath(path)
	} else if *no_memory {
		outDir, err := ioutil.TempDir(os.TempDir(), "textql")
		if err != nil {
			log.Fatalln(err)
		}
		openPath = filepath.Join(outDir, "textql.db")
	}

	db, err := sql.Open("sqlite3", openPath)

	if err != nil {
		log.Fatalln(err)
	}
	return db, &openPath
}

func determineSeperator(delimiter *string) rune {
	var seperator rune

	if (*delimiter) == "tab" {
		seperator = '\t'
	} else if strings.Index((*delimiter), "0x") == 0 {
		dlm, hex_err := hex.DecodeString((*delimiter)[2:])

		if hex_err != nil {
			log.Fatalln(hex_err)
		}

		seperator, _ = utf8.DecodeRuneInString(string(dlm))
	} else {
		seperator, _ = utf8.DecodeRuneInString(*delimiter)
	}
	return seperator
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
)

const (
	// Frequency of the save tick. Any planes last seen older than this are saved and removed from active
	savePeriod time.Duration = time.Minute * 1
)

var (
	// Command line flags
	addr        string
	port        uint
	verbose     bool
	veryVerbose bool
)

var (
	planeCache = make(map[uint]*Plane)
)

func init() {
	flag.StringVar(&addr, "a", "localhost:30003", "Address and port to connect to for input.")
	flag.UintVar(&port, "p", 8888, "Port to bind output webserver.")
	flag.BoolVar(&verbose, "v", false, "Enable verbose message logging. This will list contents of received messages.")
	flag.BoolVar(&veryVerbose, "vv", false, "Enable very verbose message logging. This will list raw received messages. Requires verbose flag")
}

func main() {
	flag.Parse()

	msgs := make(chan *message, 50)
	cmds := make(chan *BoardCmd)
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Interrupt)

	if verbose {
		fmt.Println("Initalizing databases")
	}
	err := init_db()
	if err != nil {
		fmt.Fprintf(os.Stderr, "errors setting up database: %v", err)
		os.Exit(1)
	}

	json := StartServer(cmds)
	tick := time.NewTicker(savePeriod)

	go connect(msgs)

	for {
		select {
		case m := <-msgs:
			pl, _ := getPlaneByIcao(m.icao)
			if _, ok := planeCache[m.icao]; !ok {
				planeCache[m.icao] = pl
			}
			updatePlane(m, pl)
		case cmd := <-cmds:
			json <- handleCommand(cmd)
		case t := <-tick.C:
			saveData(t)
		case <-sigint:
			saveData(time.Time{})
			err = close_db()
			tick.Stop()
			close(cmds)
			cmds = nil
			close(json)
			close(msgs)
			msgs = nil
			os.Exit(0)
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, "error closing database: %#v", err)
	}
	os.Exit(0)
}

func getPlaneByIcao(icao uint) (*Plane, error) {
	pl, ok := planeCache[icao]
	var err error
	if !ok {
		pl, err = LoadPlane(icao)
		if err != nil && err != planeNotFound {
			fmt.Fprintf(os.Stderr, "error loading from database: %v\n", err)
			pl = &Plane{Icao: icao}
		}
		if err != nil {
			return pl, planeNotFound
		}
	}

	return pl, nil
}

func handleCommand(cmd *BoardCmd) string {
	switch cmd.Cmd {
	case GetCurrent:
		return currentPlanes(cmd.Since)
	case GetAll:
		return getAllPlanes(cmd.Since)
	case GetPlane:
		return detailedPlane(cmd.Icao)
	case GetLocations:
		return getPlaneLocations(cmd.Icao, cmd.Since)
	default:
		fmt.Fprintf(os.Stderr, "unknown board command: %v", cmd.Cmd)
		return ""
	}
}

func saveData(t time.Time) {
	if len(planeCache) == 0 {
		return
	}

	var toSave []*Plane

	var zero time.Time
	if t == zero { // Save all planes if time 0. Probably an interrupt received.
		i := 0
		toSave = make([]*Plane, len(planeCache))
		for icao, pl := range planeCache {
			toSave[i] = pl
			delete(planeCache, icao)
			i++
		}
	} else {
		period := t.Add(-savePeriod)

		for icao, pl := range planeCache {
			if period.After(pl.LastSeen) {
				toSave = append(toSave, pl)
				delete(planeCache, icao)
			}
		}
	}

	if len(toSave) == 0 {
		return
	}

	if t != zeroTime {
		go SavePlanes(toSave)
	}
	err := SavePlanes(toSave)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error saving planes to database: %v\n", err)
	}
}
<commit_msg>Fix issue causing double saves.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
)

const (
	// Frequency of the save tick. Any planes last seen older than this are saved and removed from active
	savePeriod time.Duration = time.Minute * 1
)

var (
	// Command line flags
	addr        string
	port        uint
	verbose     bool
	veryVerbose bool
)

var (
	planeCache = make(map[uint]*Plane)
)

func init() {
	flag.StringVar(&addr, "a", "localhost:30003", "Address and port to connect to for input.")
	flag.UintVar(&port, "p", 8888, "Port to bind output webserver.")
	flag.BoolVar(&verbose, "v", false, "Enable verbose message logging. This will list contents of received messages.")
	flag.BoolVar(&veryVerbose, "vv", false, "Enable very verbose message logging. This will list raw received messages. Requires verbose flag")
}

func main() {
	flag.Parse()

	msgs := make(chan *message, 50)
	cmds := make(chan *BoardCmd)
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint, os.Interrupt)

	if verbose {
		fmt.Println("Initalizing databases")
	}
	err := init_db()
	if err != nil {
		fmt.Fprintf(os.Stderr, "errors setting up database: %v", err)
		os.Exit(1)
	}

	json := StartServer(cmds)
	tick := time.NewTicker(savePeriod)

	go connect(msgs)

	for {
		select {
		case m := <-msgs:
			pl, _ := getPlaneByIcao(m.icao)
			if _, ok := planeCache[m.icao]; !ok {
				planeCache[m.icao] = pl
			}
			updatePlane(m, pl)
		case cmd := <-cmds:
			json <- handleCommand(cmd)
		case t := <-tick.C:
			saveData(t)
		case <-sigint:
			saveData(time.Time{})
			err = close_db()
			tick.Stop()
			close(cmds)
			cmds = nil
			close(json)
			close(msgs)
			msgs = nil
			os.Exit(0)
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, "error closing database: %#v", err)
	}
	os.Exit(0)
}

func getPlaneByIcao(icao uint) (*Plane, error) {
	pl, ok := planeCache[icao]
	var err error
	if !ok {
		pl, err = LoadPlane(icao)
		if err != nil && err != planeNotFound {
			fmt.Fprintf(os.Stderr, "error loading from database: %v\n", err)
			pl = &Plane{Icao: icao}
		}
		if err != nil {
			return pl, planeNotFound
		}
	}

	return pl, nil
}

func handleCommand(cmd *BoardCmd) string {
	switch cmd.Cmd {
	case GetCurrent:
		return currentPlanes(cmd.Since)
	case GetAll:
		return getAllPlanes(cmd.Since)
	case GetPlane:
		return detailedPlane(cmd.Icao)
	case GetLocations:
		return getPlaneLocations(cmd.Icao, cmd.Since)
	default:
		fmt.Fprintf(os.Stderr, "unknown board command: %v", cmd.Cmd)
		return ""
	}
}

func saveData(t time.Time) {
	if len(planeCache) == 0 {
		return
	}

	var toSave []*Plane

	var zero time.Time
	if t == zero { // Save all planes if time 0. Probably an interrupt received.
		i := 0
		toSave = make([]*Plane, len(planeCache))
		for icao, pl := range planeCache {
			toSave[i] = pl
			delete(planeCache, icao)
			i++
		}
	} else {
		period := t.Add(-savePeriod)

		for icao, pl := range planeCache {
			if period.After(pl.LastSeen) {
				toSave = append(toSave, pl)
				delete(planeCache, icao)
			}
		}
	}

	if len(toSave) == 0 {
		return
	}

	if t != zeroTime {
		go SavePlanes(toSave)
	} else {
		err := SavePlanes(toSave)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error saving planes to database: %v\n", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"

	"bufio"

	"github.com/gorilla/mux"
	"gopkg.in/alecthomas/kingpin.v2"
)

const (
	defaultPayloadFilename = "test_payload"
	defaultTimeout         = "5s"
)

var (
	payloadFile       *os.File
	payloadFileLength int64
	outgoingClient    = &http.Client{}
	protocol          = "http"

	//COMMAND LINE STUFF
	cmdline         = kingpin.New("cf-http-payload-tester", "Test your HTTP requests")
	timeout         = cmdline.Flag("timeout", "Time in seconds to wait for response to check calls").Short('t').Default(defaultTimeout).Duration()
	useHTTPS        = cmdline.Flag("https-out", "Use https in outbound URL instead of http").Short('s').Bool()
	payloadFilename = cmdline.Flag("payload", "Target payload file").Short('p').Default(defaultPayloadFilename).String()
)

func main() {
	cmdline.HelpFlag.Short('h')
	kingpin.MustParse(cmdline.Parse(os.Args[1:]))

	err := setup()
	if err != nil {
		log.Fatal(err.Error())
	}

	err = launchAPIServer()
	log.Fatal(err.Error())
}

func setup() (err error) {
	//Get the file to send over HTTP
	payloadFile, err = os.Open(fmt.Sprintf("%s", *payloadFilename))
	if err != nil {
		return fmt.Errorf("Could not open payload file: %s", err.Error())
	}

	//Get length of file for reporting
	stats, err := payloadFile.Stat()
	if err != nil {
		return fmt.Errorf("error stat-ing file: %s", err.Error())
	}

	payloadFileLength = stats.Size()

	//Make sure the PORT env var is set
	if os.Getenv("PORT") == "" {
		return fmt.Errorf("Please set PORT environment variable with port for server to listen on")
	}

	//Make sure that PORT is numeric
	_, err = strconv.Atoi(os.Getenv("PORT"))
	if err != nil {
		return fmt.Errorf("PORT environment variable was not numeric")
	}

	log.Printf("Setting HTTP client timeout to %s", *timeout)
	outgoingClient.Timeout = *timeout

	if *useHTTPS {
		protocol = "https"
	}
	log.Printf("Setting protocol to %s", protocol)

	return nil
}

func launchAPIServer() error {
	router := mux.NewRouter()
	router.HandleFunc("/check/{route}", checkHandler).Methods("GET")
	router.HandleFunc("/listen", listenHandler).Methods("POST")

	return http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), router)
}

type responseJSON struct {
	Status       *int   `json:"status,omitempty"`
	ErrorMessage string `json:"error,omitempty"`
	Bytes        *int64 `json:"bytes"`
}

func responsify(r *responseJSON) []byte {
	ret, err := json.Marshal(r)
	if err != nil {
		panic("Couldn't marshal JSON")
	}
	return ret
}

func checkHandler(w http.ResponseWriter, r *http.Request) {
	outgoingResp := &responseJSON{Bytes: &payloadFileLength}

	route := mux.Vars(r)["route"]
	resp, err := outgoingClient.Post(fmt.Sprintf("%s://%s/listen", protocol, route), "text/plain", bufio.NewReader(payloadFile))

	if err != nil {
		outgoingResp.ErrorMessage = fmt.Sprintf("Error while sending request: %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(responsify(outgoingResp))
		return
	}

	//Not sure this can even happen... but...
	if resp.StatusCode != 200 {
		outgoingResp.ErrorMessage = fmt.Sprintf("Non 200-code returned from request to listening server: %d", resp.StatusCode)
	}

	w.WriteHeader(resp.StatusCode)
	outgoingResp.Status = &resp.StatusCode
	w.Write(responsify(outgoingResp))
}

func listenHandler(w http.ResponseWriter, r *http.Request) {
	//I mean... TCP guarantees that if we're this far, the body is correct
	// So.... if we got this far, the payload was successfully sent
	w.WriteHeader(http.StatusOK)
}
<commit_msg>File seek position wasn't being reset across reads. Now it is.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"

	"bufio"

	"io"

	"github.com/gorilla/mux"
	"gopkg.in/alecthomas/kingpin.v2"
)

const (
	defaultPayloadFilename = "test_payload"
	defaultTimeout         = "5s"
)

var (
	payloadFile       *os.File
	payloadFileLength int64
	outgoingClient    = &http.Client{}
	protocol          = "http"

	//COMMAND LINE STUFF
	cmdline         = kingpin.New("cf-http-payload-tester", "Test your HTTP requests")
	timeout         = cmdline.Flag("timeout", "Time in seconds to wait for response to check calls").Short('t').Default(defaultTimeout).Duration()
	useHTTPS        = cmdline.Flag("https-out", "Use https in outbound URL instead of http").Short('s').Bool()
	payloadFilename = cmdline.Flag("payload", "Target payload file").Short('p').Default(defaultPayloadFilename).String()
)

func main() {
	cmdline.HelpFlag.Short('h')
	kingpin.MustParse(cmdline.Parse(os.Args[1:]))

	err := setup()
	if err != nil {
		log.Fatal(err.Error())
	}

	err = launchAPIServer()
	log.Fatal(err.Error())
}

func setup() (err error) {
	//Get the file to send over HTTP
	payloadFile, err = os.Open(fmt.Sprintf("%s", *payloadFilename))
	if err != nil {
		return fmt.Errorf("Could not open payload file: %s", err.Error())
	}

	//Get length of file for reporting
	stats, err := payloadFile.Stat()
	if err != nil {
		return fmt.Errorf("error stat-ing file: %s", err.Error())
	}

	payloadFileLength = stats.Size()

	//Make sure the PORT env var is set
	if os.Getenv("PORT") == "" {
		return fmt.Errorf("Please set PORT environment variable with port for server to listen on")
	}

	//Make sure that PORT is numeric
	_, err = strconv.Atoi(os.Getenv("PORT"))
	if err != nil {
		return fmt.Errorf("PORT environment variable was not numeric")
	}

	log.Printf("Setting HTTP client timeout to %s", *timeout)
	outgoingClient.Timeout = *timeout

	if *useHTTPS {
		protocol = "https"
	}
	log.Printf("Setting protocol to %s", protocol)

	return nil
}

func launchAPIServer() error {
	router := mux.NewRouter()
	router.HandleFunc("/check/{route}", checkHandler).Methods("GET")
	router.HandleFunc("/listen", listenHandler).Methods("POST")

	return http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), router)
}

type responseJSON struct {
	Status       *int   `json:"status,omitempty"`
	ErrorMessage string `json:"error,omitempty"`
	Bytes        *int64 `json:"bytes"`
}

func responsify(r *responseJSON) []byte {
	ret, err := json.Marshal(r)
	if err != nil {
		panic("Couldn't marshal JSON")
	}
	return ret
}

func checkHandler(w http.ResponseWriter, r *http.Request) {
	outgoingResp := &responseJSON{Bytes: &payloadFileLength}
	route := mux.Vars(r)["route"]
	resp, err := outgoingClient.Post(fmt.Sprintf("%s://%s/listen", protocol, route), "text/plain", bufio.NewReader(payloadFile))

	//Reset payload file seek position to the start of the file
	defer func() {
		_, err = payloadFile.Seek(0, io.SeekStart)
		if err != nil {
			panic("Could not reset payload file seek position")
		}
	}()

	if err != nil {
		outgoingResp.ErrorMessage = fmt.Sprintf("Error while sending request: %s", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write(responsify(outgoingResp))
		return
	}

	//Not sure this can even happen... but...
	if resp.StatusCode != 200 {
		outgoingResp.ErrorMessage = fmt.Sprintf("Non 200-code returned from request to listening server: %d", resp.StatusCode)
	}

	w.WriteHeader(resp.StatusCode)
	outgoingResp.Status = &resp.StatusCode
	w.Write(responsify(outgoingResp))
}

func listenHandler(w http.ResponseWriter, r *http.Request) {
	//I mean... TCP guarantees that if we're this far, the body is correct
	// So.... if we got this far, the payload was successfully sent
	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"reflect"
)

var wordList []string

// TODO: A bug exists in that we're only doing wildcard detection on the root domain
// If a subdomain contains a wildcard, it will not be detected during recursive scanning
var wildcard []string
var wildcardDetected bool

func main() {
	// Parse cmdline
	flag_domain := flag.String("domain", "", "The target domain")
	flag_wordlist := flag.String("wordlist", "wordlist.txt", "Path to the wordlist")
	flag_threads := flag.Int("threads", 20, "Number of concurrent threads")

	flag.Parse()

	if *flag_domain == "" {
		fmt.Println("You must specify a domain")
		flag.PrintDefaults()
		os.Exit(1)
	}
	file, err := os.Open(*flag_wordlist)
	if err != nil {
		panic(err)
	}

	// Check for wildcard record(s)
	randomString := randomString(10)
	wildcard, _ = net.LookupHost(randomString + "." + *flag_domain)
	if len(wildcard) > 0 {
		fmt.Println("Detected wildcard record")
		wildcardDetected = true
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		wordList = append(wordList, scanner.Text())
	}

	queue := make(chan string, *flag_threads)
	done := resolveList(queue, *flag_domain)
	<-done
}

func resolveList(queue chan string, apex string) chan bool {
	doneChan := make(chan bool)
	go func() {
		for i := range wordList {
			domainName := fmt.Sprintf("%s.%s", wordList[i], apex)

			// wait for free worker thread
			queue <- domainName
			go func() {
				ips, err := net.LookupHost(domainName)
				// we have looked up the host, so we can remove this item from the queue
				// so that another go routine can give it a go
				<-queue

				// did not resolve
				if err != nil {
					return
				}

				// Check if it's a wildcard
				if wildcardDetected && reflect.DeepEqual(ips, wildcard) {
					// Not a real finding -- see note about the bug at wildcard definition
					return
				}

				// we found a non-wildcard sub domain, recurse
				fmt.Printf("%s %v\n", domainName, ips)
				childDone := resolveList(queue, domainName)
				// wait for child to finish
				<-childDone
			}()
		}
		doneChan <- true
	}()
	return doneChan
}

func randomString(length int) string {
	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	b := make([]rune, length)
	for i := range b {
		b[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(b)
}
<commit_msg>Report unexpected errors in DNS resolution<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"math/rand"
	"net"
	"os"
	"reflect"
)

var wordList []string

// TODO: A bug exists in that we're only doing wildcard detection on the root domain
// If a subdomain contains a wildcard, it will not be detected during recursive scanning
var wildcard []string
var wildcardDetected bool

func main() {
	// Parse cmdline
	flag_domain := flag.String("domain", "", "The target domain")
	flag_wordlist := flag.String("wordlist", "wordlist.txt", "Path to the wordlist")
	flag_threads := flag.Int("threads", 20, "Number of concurrent threads")

	flag.Parse()

	if *flag_domain == "" {
		fmt.Println("You must specify a domain")
		flag.PrintDefaults()
		os.Exit(1)
	}
	file, err := os.Open(*flag_wordlist)
	if err != nil {
		panic(err)
	}

	// Check for wildcard record(s)
	randomString := randomString(10)
	wildcard, _ = net.LookupHost(randomString + "." + *flag_domain)
	if len(wildcard) > 0 {
		fmt.Println("Detected wildcard record")
		wildcardDetected = true
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		wordList = append(wordList, scanner.Text())
	}

	queue := make(chan string, *flag_threads)
	done := resolveList(queue, *flag_domain)
	<-done
}

func resolveList(queue chan string, apex string) chan bool {
	doneChan := make(chan bool)
	go func() {
		for i := range wordList {
			domainName := fmt.Sprintf("%s.%s", wordList[i], apex)

			// wait for free worker thread
			queue <- domainName
			go func() {
				ips, err := net.LookupHost(domainName)
				// we have looked up the host, so we can remove this item from the queue
				// so that another go routine can give it a go
				<-queue

				// did not resolve
				if err != nil {
					// TODO: find a nicer way of writing this
					errstr := err.Error()
					nsh := "no such host"
					if errstr[len(errstr)-len(nsh):] != nsh {
						fmt.Printf("Unexpected error: %v\n", err)
					}
					return
				}

				// Check if it's a wildcard
				if wildcardDetected && reflect.DeepEqual(ips, wildcard) {
					// Not a real finding -- see note about the bug at wildcard definition
					return
				}

				// we found a non-wildcard sub domain, recurse
				fmt.Printf("%s %v\n", domainName, ips)
				childDone := resolveList(queue, domainName)
				// wait for child to finish
				<-childDone
			}()
		}
		doneChan <- true
	}()
	return doneChan
}

func randomString(length int) string {
	letterRunes := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
	b := make([]rune, length)
	for i := range b {
		b[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(b)
}
<|endoftext|>


<commit_before>// Copyright 2017 John Scherff
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	`fmt`
	`log`
	`os`
	`strings`

	`github.com/google/gousb`
	`github.com/jscherff/gocmdb/usbci`
)

const (
	defaultConfig = `config.json`
	legacyExec = `magtek_inventory`
)

var (
	conf *Config
	slog, clog, elog *log.Logger
)

func main() {

	var err error

	// Build system-wide configuration from config file.

	if conf, err = newConfig(defaultConfig); err != nil {
		log.Fatal(err)
	}

	// Initialize loggers.

	slog, clog, elog = newLoggers()

	// Check for legacy mode.

	if strings.Contains(os.Args[0], legacyExec) {
		legacyMode()
	}

	// Instantiate context to enumerate devices.

	ctx := gousb.NewContext()
	ctx.Debug(conf.DebugLevel)
	defer ctx.Close()

	// Process command-line actions and options.

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, `You must specify an action.`)
		fsAction.Usage()
		os.Exit(1)
	}

	// Parse action flag.

	fsAction.Parse(os.Args[1:2])

	// Parse option flags associated with selected action flag.

	switch {

	case *fActionVersion:
		displayVersion()
		os.Exit(0)

	case *fActionReport:
		if fsReport.Parse(os.Args[2:]); fsReport.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsReport.Usage()
			os.Exit(1)
		}

	case *fActionSerial:
		if fsSerial.Parse(os.Args[2:]); fsSerial.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsSerial.Usage()
			os.Exit(1)
		}

	case *fActionAudit:
		if fsAudit.Parse(os.Args[2:]); fsAudit.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsAudit.Usage()
			os.Exit(1)
		}
	}

	// Open devices that match selection criteria in the Include.ProductID
	// and Include.VendorID maps from the configuration file.

	var openFunc = func(desc *gousb.DeviceDesc) bool {

		vid, pid := desc.Vendor.String(), desc.Product.String()

		if val, ok := conf.Include.ProductID[vid][pid]; ok {
			return val
		}
		if val, ok := conf.Include.VendorID[vid]; ok {
			return val
		}

		return conf.Include.Default
	}

	devs, err := ctx.OpenDevices(openFunc)

	// Log and exit if no relevant devices found.

	if err != nil && conf.DebugLevel > 0 {
		elog.Print(err)
	}

	if len(devs) == 0 {
		elog.Fatalf(`no devices found`)
	}

	// Pass devices to relevant device handlers.

	for _, dev := range devs {

		defer dev.Close()

		slog.Printf(`found USB device: VID %s PID %s`,
			dev.Desc.Vendor.String(),
			dev.Desc.Product.String(),
		)

		switch uint16(dev.Desc.Vendor) {

		case usbci.MagtekVID:

			if d, err := usbci.NewMagtek(dev); err != nil {
				elog.Print(err)
			} else {
				slog.Printf(`identified USB device as %s`, d.Type())
				magtekRouter(d)
			}

		default:

			if d, err := usbci.NewGeneric(dev); err != nil {
				elog.Print(err)
			} else {
				slog.Printf(`identified USB device as %s`, d.Type())
				genericRouter(d)
			}
		}
	}
}

// Find first device matching magtek VID and PID, run legacy report, then exit.
func legacyMode() {

	ctx := gousb.NewContext()
	defer ctx.Close()

	d, err := ctx.OpenDeviceWithVIDPID(
		gousb.ID(usbci.MagtekVID),
		gousb.ID(usbci.MagtekPID),
	)

	if err != nil {
		elog.Fatal(err)
	}

	md, err := usbci.NewMagtek(d)

	if err != nil {
		elog.Fatal(err)
	}

	if err = writeFile(md.Legacy(), conf.Files.Legacy); err != nil {
		elog.Fatal(err)
	}

	os.Exit(0)
}
<commit_msg>Compiled binaries<commit_after>// Copyright 2017 John Scherff
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	`fmt`
	`log`
	`os`
	`strings`

	`github.com/google/gousb`
	`github.com/jscherff/gocmdb/usbci`
)

const (
	defaultConfig = `config.json`
	legacyExec = `magtek_inventory`
)

var (
	conf *Config
	slog, clog, elog *log.Logger
)

func main() {

	var err error

	// Process command-line action flag.

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, `You must specify an action.`)
		fsAction.Usage()
		os.Exit(1)
	}

	fsAction.Parse(os.Args[1:2])

	// If version flag present, display version and exit.

	if *fActionVersion {
		displayVersion()
		os.Exit(0)
	}

	// Build system-wide configuration from config file.

	if conf, err = newConfig(defaultConfig); err != nil {
		log.Fatal(err)
	}

	// Initialize loggers.

	slog, clog, elog = newLoggers()

	// Check for legacy mode.

	if strings.Contains(os.Args[0], legacyExec) {
		legacyMode()
	}

	// Instantiate context to enumerate devices.

	ctx := gousb.NewContext()
	ctx.Debug(conf.DebugLevel)
	defer ctx.Close()

	// Parse option flags associated with selected action flag.

	switch {

	case *fActionReport:
		if fsReport.Parse(os.Args[2:]); fsReport.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsReport.Usage()
			os.Exit(1)
		}

	case *fActionSerial:
		if fsSerial.Parse(os.Args[2:]); fsSerial.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsSerial.Usage()
			os.Exit(1)
		}

	case *fActionAudit:
		if fsAudit.Parse(os.Args[2:]); fsAudit.NFlag() == 0 {
			fmt.Fprintln(os.Stderr, `You must specify an option.`)
			fsAudit.Usage()
			os.Exit(1)
		}
	}

	// Open devices that match selection criteria in the Include.ProductID
	// and Include.VendorID maps from the configuration file.

	var openFunc = func(desc *gousb.DeviceDesc) bool {

		vid, pid := desc.Vendor.String(), desc.Product.String()

		if val, ok := conf.Include.ProductID[vid][pid]; ok {
			return val
		}
		if val, ok := conf.Include.VendorID[vid]; ok {
			return val
		}

		return conf.Include.Default
	}

	devs, err := ctx.OpenDevices(openFunc)

	// Log and exit if no relevant devices found.

	if err != nil && conf.DebugLevel > 0 {
		elog.Print(err)
	}

	if len(devs) == 0 {
		elog.Fatalf(`no devices found`)
	}

	// Pass devices to relevant device handlers.

	for _, dev := range devs {

		defer dev.Close()

		slog.Printf(`found USB device: VID %s PID %s`,
			dev.Desc.Vendor.String(),
			dev.Desc.Product.String(),
		)

		switch uint16(dev.Desc.Vendor) {

		case usbci.MagtekVID:

			if d, err := usbci.NewMagtek(dev); err != nil {
				elog.Print(err)
			} else {
				slog.Printf(`identified USB device as %s`, d.Type())
				magtekRouter(d)
			}

		default:

			if d, err := usbci.NewGeneric(dev); err != nil {
				elog.Print(err)
			} else {
				slog.Printf(`identified USB device as %s`, d.Type())
				genericRouter(d)
			}
		}
	}
}

// Find first device matching magtek VID and PID, run legacy report, then exit.
func legacyMode() {

	ctx := gousb.NewContext()
	defer ctx.Close()

	d, err := ctx.OpenDeviceWithVIDPID(
		gousb.ID(usbci.MagtekVID),
		gousb.ID(usbci.MagtekPID),
	)

	if err != nil {
		elog.Fatal(err)
	}

	md, err := usbci.NewMagtek(d)

	if err != nil {
		elog.Fatal(err)
	}

	if err = writeFile(md.Legacy(), conf.Files.Legacy); err != nil {
		elog.Fatal(err)
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/cookiejar"
	"strconv"
	"time"

	"code.google.com/p/go.net/publicsuffix"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/golang/groupcache"
	"gopkg.in/yaml.v1"
)

type Config struct {
	Listen           string   // API Listen Address
	Peers            []string // as a list of ip:port
	CacheSize        int64    `yaml:"cache_size"`          // in MB
	MaxItemSize      int64    `yaml:"max_item_size"`       // in KB
	ImageCacheSize   int64    `yaml:"image_cache_size"`    // in MB
	ImageMaxItemSize int64    `yaml:"image_max_item_size"` // in KB
}

var (
	flagConfigFile = flag.String("config", "config.yml", "Config file to use.")
)

var (
	defaultHTTPClient *http.Client
)

// http client
func init() {
	timeout := 30 * time.Second
	timeoutDialer := func(netw, addr string) (net.Conn, error) {
		start := time.Now()
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(start.Add(timeout))
		return conn, nil
	}
	jar, err := cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		panic(err)
	}
	defaultHTTPClient = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  timeoutDialer,
		},
		Jar: jar,
	}
}

func main() {
	flag.Parse()

	bytes, err := ioutil.ReadFile(*flagConfigFile)
	if err != nil {
		panic(err)
	}
	config := Config{}
	yaml.Unmarshal(bytes, &config)

	// Setup groupcache peers
	peers := NewPeersPool("http://" + config.Listen)
	peersList := []string{}
	for _, peer := range config.Peers {
		peersList = append(peersList, "http://"+peer)
	}
	peers.Set(peersList...)

	// Setup GGFetch
	htmlFetcher := NewHTMLFetcher("fetch", config.CacheSize<<20, config.MaxItemSize<<10, defaultHTTPClient)
	imageFetcher := NewImageFetcher("image", config.ImageCacheSize<<20, config.ImageMaxItemSize<<10, defaultHTTPClient)

	// Setup
	http.HandleFunc("/fetch", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)
		log.Println("Fetching HTML:", url)

		realUrl, buf, err := htmlFetcher.Fetch(url, ttl)
		if err != nil {
			log.Println("Error while fetching HTML:", err)
		}
		response.Header().Set("X-Real-URL", realUrl)
		_, err = response.Write(buf)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/resize", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)
		width, _ := strconv.Atoi(request.FormValue("width"))
		log.Println("Fetching image:", url, width)

		bytes, err := imageFetcher.Fetch(url, ttl, width)
		if err != nil {
			log.Println("Error while fetching image:", err)
		}
		_, err = response.Write(bytes)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/dimension", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		log.Println("Fetching dimension:", url)

		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)

		config, err := imageFetcher.FetchDimension(url, ttl)
		if err != nil {
			log.Println("Error while fetching image config:", err)
		}
		err = json.NewEncoder(response).Encode(config)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/stats", func(response http.ResponseWriter, request *http.Request) {
		var stats struct {
			HTML, Image struct {
				Main, Hot groupcache.CacheStats
			}
		}
		stats.HTML.Main = htmlFetcher.CacheStats(groupcache.MainCache)
		stats.HTML.Hot = htmlFetcher.CacheStats(groupcache.HotCache)
		stats.Image.Main = htmlFetcher.CacheStats(groupcache.MainCache)
		stats.Image.Hot = htmlFetcher.CacheStats(groupcache.HotCache)
		json.NewEncoder(response).Encode(stats)
	})

	server := &http.Server{
		Addr:         config.Listen,
		Handler:      nil,
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 60 * time.Second,
	}
	gracehttp.Serve(server)
}
<commit_msg>report active goroutines<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/cookiejar"
	"runtime"
	"strconv"
	"time"

	"code.google.com/p/go.net/publicsuffix"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/golang/groupcache"
	"gopkg.in/yaml.v1"
)

type Config struct {
	Listen           string   // API Listen Address
	Peers            []string // as a list of ip:port
	CacheSize        int64    `yaml:"cache_size"`          // in MB
	MaxItemSize      int64    `yaml:"max_item_size"`       // in KB
	ImageCacheSize   int64    `yaml:"image_cache_size"`    // in MB
	ImageMaxItemSize int64    `yaml:"image_max_item_size"` // in KB
}

var (
	flagConfigFile = flag.String("config", "config.yml", "Config file to use.")
)

var (
	defaultHTTPClient *http.Client
)

// http client
func init() {
	timeout := 30 * time.Second
	timeoutDialer := func(netw, addr string) (net.Conn, error) {
		start := time.Now()
		conn, err := net.DialTimeout(netw, addr, timeout)
		if err != nil {
			return nil, err
		}
		conn.SetDeadline(start.Add(timeout))
		return conn, nil
	}
	jar, err := cookiejar.New(&cookiejar.Options{
		PublicSuffixList: publicsuffix.List,
	})
	if err != nil {
		panic(err)
	}
	defaultHTTPClient = &http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial:  timeoutDialer,
		},
		Jar: jar,
	}
}

func main() {
	flag.Parse()

	bytes, err := ioutil.ReadFile(*flagConfigFile)
	if err != nil {
		panic(err)
	}
	config := Config{}
	yaml.Unmarshal(bytes, &config)

	// Setup groupcache peers
	peers := NewPeersPool("http://" + config.Listen)
	peersList := []string{}
	for _, peer := range config.Peers {
		peersList = append(peersList, "http://"+peer)
	}
	peers.Set(peersList...)

	// Setup GGFetch
	htmlFetcher := NewHTMLFetcher("fetch", config.CacheSize<<20, config.MaxItemSize<<10, defaultHTTPClient)
	imageFetcher := NewImageFetcher("image", config.ImageCacheSize<<20, config.ImageMaxItemSize<<10, defaultHTTPClient)

	// Setup
	http.HandleFunc("/fetch", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)
		log.Println("Fetching HTML:", url)

		realUrl, buf, err := htmlFetcher.Fetch(url, ttl)
		if err != nil {
			log.Println("Error while fetching HTML:", err)
		}
		response.Header().Set("X-Real-URL", realUrl)
		_, err = response.Write(buf)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/resize", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)
		width, _ := strconv.Atoi(request.FormValue("width"))
		log.Println("Fetching image:", url, width)

		bytes, err := imageFetcher.Fetch(url, ttl, width)
		if err != nil {
			log.Println("Error while fetching image:", err)
		}
		_, err = response.Write(bytes)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/dimension", func(response http.ResponseWriter, request *http.Request) {
		url := request.FormValue("url")
		log.Println("Fetching dimension:", url)

		ttl, _ := strconv.ParseInt(request.FormValue("ttl"), 10, 64)

		config, err := imageFetcher.FetchDimension(url, ttl)
		if err != nil {
			log.Println("Error while fetching image config:", err)
		}
		err = json.NewEncoder(response).Encode(config)
		if err != nil {
			log.Println("Error while writing response:", err)
		}
	})

	http.HandleFunc("/stats", func(response http.ResponseWriter, request *http.Request) {
		var stats struct {
			Goroutines  int
			HTML, Image struct {
				Main, Hot groupcache.CacheStats
			}
		}
		stats.Goroutines = runtime.NumGoroutine()
		stats.HTML.Main = htmlFetcher.CacheStats(groupcache.MainCache)
		stats.HTML.Hot = htmlFetcher.CacheStats(groupcache.HotCache)
		stats.Image.Main = htmlFetcher.CacheStats(groupcache.MainCache)
		stats.Image.Hot = htmlFetcher.CacheStats(groupcache.HotCache)
		json.NewEncoder(response).Encode(stats)
	})

	server := &http.Server{
		Addr:         config.Listen,
		Handler:      nil,
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 60 * time.Second,
	}
	gracehttp.Serve(server)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"

	"github.com/jmoiron/sqlx"
	"github.com/julienschmidt/httprouter"
	_ "github.com/mattn/go-sqlite3"
	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

// Response defines the JSON contents for response to resource request by HTTP.
type Response struct {
	Success          bool                     `json:"success"`
	Rows             []map[string]interface{} `json:"rows"`
	ErrorType        string                   `json:"errorType"`
	ErrorDescription string                   `json:"errorDescription"`
}

// InputRule is part of InputConfig
type InputRule struct {
	Path        string
	Before      string
	Befores     []string
	Query       string
	Queries     []string
	After       string
	Afters      []string
	Method      string
	Transaction bool
}

// DBConfig is parameters for opening connection to database.
type DBConfig struct {
	Driver   string `yaml:"driver"`
	Filepath string `yaml:"filepath"`
}

// InputConfig is for reading YAML config file.
type InputConfig struct {
	DB    DBConfig    `yaml:"db"`
	Rules []InputRule `yaml:"rules"`
	Port  int         `yaml:"port"`
}

// QuerySet contains all queries to execute for a Rule.
type QuerySet struct {
	Befores     []Query
	Queries     []Query
	Afters      []Query
	Transaction bool
}

func newQuerySet(db *sqlx.DB, rule Rule) (QuerySet, error) {
	befores, err := newQueries(db, rule.Befores)
	if err != nil {
		return QuerySet{}, err
	}

	queries, err := newQueries(db, rule.Queries)
	if err != nil {
		return QuerySet{}, err
	}

	afters, err := newQueries(db, rule.Afters)
	if err != nil {
		return QuerySet{}, err
	}

	return QuerySet{
		Befores:     befores,
		Queries:     queries,
		Afters:      afters,
		Transaction: rule.Transaction,
	}, nil
}

func newQueries(db *sqlx.DB, qss []string) ([]Query, error) {
	queries := []Query{}
	for _, qs := range qss {
		q, err := newQuery(db, qs)
		if err != nil {
			return []Query{}, err
		}
		queries = append(queries, q)
	}
	return queries, nil
}

func newQuery(db *sqlx.DB, qs string) (Query, error) {
	re := regexp.MustCompile(`\{\{(\w+)\}\}`)
	groups := re.FindAllStringSubmatch(qs, -1)

	argKeys := []string{}
	for _, group := range groups {
		argKeys = append(argKeys, group[1])
	}
	replaced := re.ReplaceAllString(qs, "?")
	stmt, err := db.Preparex(replaced)
	if err != nil {
		return Query{}, SQLParseError(errors.Wrap(err, "failed to parse SQL"))
	}
	return Query{
		SQL:     stmt,
		ArgKeys: argKeys,
	}, nil
}

// Rule is part of `Config`, it's set of the routing path and the SQL query.
type Rule struct {
	Befores     []string
	Queries     []string
	Afters      []string
	Path        string
	Method      Method
	Transaction bool
}

// Config contains configs of whole app
type Config struct {
	DB    DBConfig
	Rules []Rule
	Port  int
}

// Config returns one converted to Config struct
func (ic InputConfig) Config() (Config, error) {
	rules, err := normalizeRules(ic.Rules)
	if err != nil {
		return Config{}, err
	}

	port, err := normalizePort(ic.Port)
	if err != nil {
		return Config{}, err
	}

	return Config{
		DB:    ic.DB,
		Rules: rules,
		Port:  port,
	}, nil
}

func normalizePort(orig int) (int, error) {
	if orig == 0 {
		return 80, nil
	}
	return orig, nil
}

func normalizeRules(orig []InputRule) ([]Rule, error) {
	result := []Rule{}
	for _, rule := range orig {
		normalized, err := normalizeRule(rule)
		if err != nil {
			return []Rule{}, err
		}
		result = append(result, normalized)
	}
	return result, nil
}

func normalizeRule(orig InputRule) (Rule, error) {
	befores, err := normalizeBefores(orig.Before, orig.Befores)
	if err != nil {
		return Rule{}, err
	}

	queries, err := normalizeQuery(orig.Query, orig.Queries)
	if err != nil {
		return Rule{}, err
	}

	afters, err := normalizeAfters(orig.After, orig.Afters)
	if err != nil {
		return Rule{}, err
	}

	method, err := newMethod(orig.Method)
	if err != nil {
		return Rule{}, err
	}

	path, err := normalizePath(orig.Path)
	if err != nil {
		return Rule{}, err
	}

	transaction, err := normalizeTransaction(orig.Transaction)
	if err != nil {
		return Rule{}, err
	}

	return Rule{
		Befores:     befores,
		Queries:     queries,
		Afters:      afters,
		Method:      method,
		Path:        path,
		Transaction: transaction,
	}, nil
}

func normalizeTransaction(orig bool) (bool, error) {
	return orig, nil
}

func normalizePath(orig string) (string, error) {
	return orig, nil
}

func normalizeBefores(before string, befores []string) ([]string, error) {
	if before != "" && len(befores) != 0 {
		return []string{}, fmt.Errorf("both of `before` and `befores` can't be defined in a rule")
	} else if before != "" {
		return []string{before}, nil
	} else if len(befores) > 0 {
		return befores, nil
	}
	return []string{}, nil
}

func normalizeAfters(after string, afters []string) ([]string, error) {
	if after != "" && len(afters) != 0 {
		return []string{}, fmt.Errorf("both of `after` and `afters` can't be defined in a rule")
	} else if after != "" {
		return []string{after}, nil
	} else if len(afters) > 0 {
		return afters, nil
	}
	return []string{}, nil
}

func normalizeQuery(query string, queries []string) ([]string, error) {
	if query != "" && len(queries) != 0 {
		return []string{}, fmt.Errorf("both of `query` and `queries` can't be defined in a rule")
	} else if query != "" {
		return []string{query}, nil
	} else if len(queries) > 0 {
		return queries, nil
	}
	return []string{}, fmt.Errorf("at least one SQL query must be given per rule")
}

func newMethod(method string) (Method, error) {
	switch method {
	case "GET":
		return GET, nil
	case "POST":
		return POST, nil
	case "PUT":
		return PUT, nil
	case "PATCH":
		return PATCH, nil
	case "DELETE":
		return DELETE, nil
	case "":
		return GET, nil
	}
	return "", fmt.Errorf("invalid method name")
}

// Method is enum for HTTP methods.
type Method string

const (
	// GET expresses GET method
	GET Method = "GET"

	// POST expresses POST method
	POST = "POST"

	// PUT expresses PUT method
	PUT = "PUT"

	// PATCH expresses PATCH method
	PATCH = "PATCH"

	// DELETE expresses DELETE method
	DELETE = "DELETE"
)

// QueryExecutionError causes in execution SQL query
type QueryExecutionError error

// UnknownArgError causes when given undefined parameters name in path at SOL.
type UnknownArgError error

// SQLParseError causes when failed to parse SQL.
type SQLParseError error

// Query is data set for one SQL query execution
type Query struct {
	SQL     *sqlx.Stmt
	ArgKeys []string
}

// ExecuteWithArgMap executes the Query. `params` is map of param key and value.
func (q Query) ExecuteWithArgMap(params map[string]interface{}) (*sqlx.Rows, error) {
	args := []interface{}{}
	for _, argKey := range q.ArgKeys {
		arg, ok := params[argKey]
		if !ok {
			return nil, UnknownArgError(errors.Errorf("unknown argument name: %s", argKey))
		}
		args = append(args, arg)
	}
	return q.SQL.Queryx(args...)
}

func main() {
	inputConfig := InputConfig{}
	bts, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Println(err)
		return
	}
	yaml.Unmarshal(bts, &inputConfig)
	config, err := inputConfig.Config()
	if err != nil {
		fmt.Printf("invalid configuration: %s\n", err)
		return
	}

	db, err := sqlx.Open(config.DB.Driver, config.DB.Filepath)
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	for _, rule := range config.Rules {
		querySet, err := newQuerySet(db, rule)
		if err != nil {
			fmt.Println(errors.Wrap(err, "failed to compile query"))
			return
		}
		router.Handle(string(rule.Method), rule.Path, createHandler(db, querySet))
	}
	fmt.Println(http.ListenAndServe(fmt.Sprintf(":%d", config.Port), router))
}

func createHandler(db *sqlx.DB, q QuerySet) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		paramMap, err := createParamMap(req, params)
		if err != nil {
			w.Write(createErrorResponseBytes(err))
			return
		}
		rowMaps, err := executeQuerySet(db, q, paramMap)
		if err != nil {
			w.Write(createErrorResponseBytes(err))
			return
		}

		bts, marshalErr := json.Marshal(Response{
			Success: true,
			Rows:    rowMaps,
		})
		if marshalErr != nil {
			panic(marshalErr)
		}
		w.Write(bts)
	}
}

func createErrorResponseBytes(err error) []byte {
	bts, marshalErr := json.Marshal(Response{
		Success:          false,
		ErrorType:        getErrorType(err),
		ErrorDescription: err.Error(),
	})
	if marshalErr != nil {
		panic(marshalErr)
	}
	return bts
}

func executeQuerySet(db *sqlx.DB, querySet QuerySet, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	var op sqlx.Queryer
	if querySet.Transaction {
		var err error
		op, err = db.Beginx()
		if err != nil {
			return nil, err
		}
	} else {
		op = db
	}

	err := executeQueries(op, querySet.Befores, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}

	rowMaps, err := executeQueriesRowMaps(op, querySet.Queries, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}

	err = executeQueries(op, querySet.Afters, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}
	_ = commitPossibly(op)
	return rowMaps, nil
}

func rollbackPossibly(op sqlx.Queryer) bool {
	if tx, ok := op.(*sqlx.Tx); ok {
		tx.Rollback()
		return true
	}
	return false
}

func commitPossibly(op sqlx.Queryer) bool {
	if tx, ok := op.(*sqlx.Tx); ok {
		tx.Commit()
		return true
	}
	return false
}

func getErrorType(err error) string {
	switch err.(type) {
	case QueryExecutionError:
		return "QueryExecutionError"
	}
	return "Unknown"
}

func executeQueries(db sqlx.Queryer, qs []Query, paramMap map[string]interface{}) error {
	_, err := executeQueriesRowMaps(db, qs, paramMap)
	return err
}

func executeQueriesRowMaps(db sqlx.Queryer, qs []Query, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	concated := []map[string]interface{}{}
	for _, q := range qs {
		rowMap, err := executeRowMaps(db, q, paramMap)
		if err != nil {
			return nil, err
		}
		concated = append(concated, rowMap...)
	}
	return concated, nil
}

func executeRowMaps(db sqlx.Queryer, q Query, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	rows, err := q.ExecuteWithArgMap(paramMap)
	if err != nil {
		return nil, QueryExecutionError(errors.Wrap(err, "failed to execute query"))
	}
	res, err := createMapSliceFromRows(rows)
	if err != nil {
		return nil, QueryExecutionError(errors.Wrap(err, "failed to create map slice from results"))
	}
	return res, nil
}

// RequestBodyParseError causes when invalid JSON given as request body.
type RequestBodyParseError error

func createParamMap(req *http.Request, params httprouter.Params) (map[string]interface{}, error) {
	result := make(map[string]interface{})
	for _, param := range params {
		result[param.Key] = param.Value
	}

	reqBody := make(map[string]interface{})
	err := json.NewDecoder(req.Body).Decode(&reqBody)
	defer req.Body.Close()
	if err != io.EOF {
		if err != nil {
			return nil, RequestBodyParseError(errors.Wrap(err, "request body must be only 1 hieralchical key/value pairs"))
		}
		for k, v := range reqBody {
			result[k] = v
		}
	}

	// Note that only string value can be passed by URL Queries.
	q := req.URL.Query()
	for k, v := range q {
		result[k] = v[0]
	}

	return result, nil
}

type mapString []byte

func (bts mapString) MarshalText() ([]byte, error) {
	return bts, nil
}

func createMapSliceFromRows(rows *sqlx.Rows) ([]map[string]interface{}, error) {
	result := []map[string]interface{}{}
	for rows.Next() {
		cols := make(map[string]interface{})
		err := rows.MapScan(cols)
		if err != nil {
			return nil, err
		}

		newCols := make(map[string]interface{})
		for k, v := range cols {
			if str, ok := v.([]byte); ok {
				newCols[k] = mapString(str)
			} else {
				newCols[k] = v
			}
		}
		result = append(result, newCols)
	}
	return result, nil
}
<commit_msg>[Add] charset<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"

	"github.com/jmoiron/sqlx"
	"github.com/julienschmidt/httprouter"
	_ "github.com/mattn/go-sqlite3"
	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

// Response defines the JSON contents for response to resource request by HTTP.
type Response struct {
	Success          bool                     `json:"success"`
	Rows             []map[string]interface{} `json:"rows"`
	ErrorType        string                   `json:"errorType"`
	ErrorDescription string                   `json:"errorDescription"`
}

// InputRule is part of InputConfig
type InputRule struct {
	Path        string
	Before      string
	Befores     []string
	Query       string
	Queries     []string
	After       string
	Afters      []string
	Method      string
	Transaction bool
}

// DBConfig is parameters for opening connection to database.
type DBConfig struct {
	Driver   string `yaml:"driver"`
	Filepath string `yaml:"filepath"`
}

// InputConfig is for reading YAML config file.
type InputConfig struct {
	DB    DBConfig    `yaml:"db"`
	Rules []InputRule `yaml:"rules"`
	Port  int         `yaml:"port"`
}

// QuerySet contains all queries to execute for a Rule.
type QuerySet struct {
	Befores     []Query
	Queries     []Query
	Afters      []Query
	Transaction bool
}

func newQuerySet(db *sqlx.DB, rule Rule) (QuerySet, error) {
	befores, err := newQueries(db, rule.Befores)
	if err != nil {
		return QuerySet{}, err
	}

	queries, err := newQueries(db, rule.Queries)
	if err != nil {
		return QuerySet{}, err
	}

	afters, err := newQueries(db, rule.Afters)
	if err != nil {
		return QuerySet{}, err
	}

	return QuerySet{
		Befores:     befores,
		Queries:     queries,
		Afters:      afters,
		Transaction: rule.Transaction,
	}, nil
}

func newQueries(db *sqlx.DB, qss []string) ([]Query, error) {
	queries := []Query{}
	for _, qs := range qss {
		q, err := newQuery(db, qs)
		if err != nil {
			return []Query{}, err
		}
		queries = append(queries, q)
	}
	return queries, nil
}

func newQuery(db *sqlx.DB, qs string) (Query, error) {
	re := regexp.MustCompile(`\{\{(\w+)\}\}`)
	groups := re.FindAllStringSubmatch(qs, -1)

	argKeys := []string{}
	for _, group := range groups {
		argKeys = append(argKeys, group[1])
	}
	replaced := re.ReplaceAllString(qs, "?")
	stmt, err := db.Preparex(replaced)
	if err != nil {
		return Query{}, SQLParseError(errors.Wrap(err, "failed to parse SQL"))
	}
	return Query{
		SQL:     stmt,
		ArgKeys: argKeys,
	}, nil
}

// Rule is part of `Config`, it's set of the routing path and the SQL query.
type Rule struct {
	Befores     []string
	Queries     []string
	Afters      []string
	Path        string
	Method      Method
	Transaction bool
}

// Config contains configs of whole app
type Config struct {
	DB    DBConfig
	Rules []Rule
	Port  int
}

// Config returns one converted to Config struct
func (ic InputConfig) Config() (Config, error) {
	rules, err := normalizeRules(ic.Rules)
	if err != nil {
		return Config{}, err
	}

	port, err := normalizePort(ic.Port)
	if err != nil {
		return Config{}, err
	}

	return Config{
		DB:    ic.DB,
		Rules: rules,
		Port:  port,
	}, nil
}

func normalizePort(orig int) (int, error) {
	if orig == 0 {
		return 80, nil
	}
	return orig, nil
}

func normalizeRules(orig []InputRule) ([]Rule, error) {
	result := []Rule{}
	for _, rule := range orig {
		normalized, err := normalizeRule(rule)
		if err != nil {
			return []Rule{}, err
		}
		result = append(result, normalized)
	}
	return result, nil
}

func normalizeRule(orig InputRule) (Rule, error) {
	befores, err := normalizeBefores(orig.Before, orig.Befores)
	if err != nil {
		return Rule{}, err
	}

	queries, err := normalizeQuery(orig.Query, orig.Queries)
	if err != nil {
		return Rule{}, err
	}

	afters, err := normalizeAfters(orig.After, orig.Afters)
	if err != nil {
		return Rule{}, err
	}

	method, err := newMethod(orig.Method)
	if err != nil {
		return Rule{}, err
	}

	path, err := normalizePath(orig.Path)
	if err != nil {
		return Rule{}, err
	}

	transaction, err := normalizeTransaction(orig.Transaction)
	if err != nil {
		return Rule{}, err
	}

	return Rule{
		Befores:     befores,
		Queries:     queries,
		Afters:      afters,
		Method:      method,
		Path:        path,
		Transaction: transaction,
	}, nil
}

func normalizeTransaction(orig bool) (bool, error) {
	return orig, nil
}

func normalizePath(orig string) (string, error) {
	return orig, nil
}

func normalizeBefores(before string, befores []string) ([]string, error) {
	if before != "" && len(befores) != 0 {
		return []string{}, fmt.Errorf("both of `before` and `befores` can't be defined in a rule")
	} else if before != "" {
		return []string{before}, nil
	} else if len(befores) > 0 {
		return befores, nil
	}
	return []string{}, nil
}

func normalizeAfters(after string, afters []string) ([]string, error) {
	if after != "" && len(afters) != 0 {
		return []string{}, fmt.Errorf("both of `after` and `afters` can't be defined in a rule")
	} else if after != "" {
		return []string{after}, nil
	} else if len(afters) > 0 {
		return afters, nil
	}
	return []string{}, nil
}

func normalizeQuery(query string, queries []string) ([]string, error) {
	if query != "" && len(queries) != 0 {
		return []string{}, fmt.Errorf("both of `query` and `queries` can't be defined in a rule")
	} else if query != "" {
		return []string{query}, nil
	} else if len(queries) > 0 {
		return queries, nil
	}
	return []string{}, fmt.Errorf("at least one SQL query must be given per rule")
}

func newMethod(method string) (Method, error) {
	switch method {
	case "GET":
		return GET, nil
	case "POST":
		return POST, nil
	case "PUT":
		return PUT, nil
	case "PATCH":
		return PATCH, nil
	case "DELETE":
		return DELETE, nil
	case "":
		return GET, nil
	}
	return "", fmt.Errorf("invalid method name")
}

// Method is enum for HTTP methods.
type Method string

const (
	// GET expresses GET method
	GET Method = "GET"

	// POST expresses POST method
	POST = "POST"

	// PUT expresses PUT method
	PUT = "PUT"

	// PATCH expresses PATCH method
	PATCH = "PATCH"

	// DELETE expresses DELETE method
	DELETE = "DELETE"
)

// QueryExecutionError causes in execution SQL query
type QueryExecutionError error

// UnknownArgError causes when given undefined parameters name in path at SOL.
type UnknownArgError error

// SQLParseError causes when failed to parse SQL.
type SQLParseError error

// Query is data set for one SQL query execution
type Query struct {
	SQL     *sqlx.Stmt
	ArgKeys []string
}

// ExecuteWithArgMap executes the Query. `params` is map of param key and value.
func (q Query) ExecuteWithArgMap(params map[string]interface{}) (*sqlx.Rows, error) {
	args := []interface{}{}
	for _, argKey := range q.ArgKeys {
		arg, ok := params[argKey]
		if !ok {
			return nil, UnknownArgError(errors.Errorf("unknown argument name: %s", argKey))
		}
		args = append(args, arg)
	}
	return q.SQL.Queryx(args...)
}

func main() {
	inputConfig := InputConfig{}
	bts, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Println(err)
		return
	}
	yaml.Unmarshal(bts, &inputConfig)
	config, err := inputConfig.Config()
	if err != nil {
		fmt.Printf("invalid configuration: %s\n", err)
		return
	}

	db, err := sqlx.Open(config.DB.Driver, config.DB.Filepath)
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	for _, rule := range config.Rules {
		querySet, err := newQuerySet(db, rule)
		if err != nil {
			fmt.Println(errors.Wrap(err, "failed to compile query"))
			return
		}
		router.Handle(string(rule.Method), rule.Path, createHandler(db, querySet))
	}
	fmt.Println(http.ListenAndServe(fmt.Sprintf(":%d", config.Port), router))
}

func createHandler(db *sqlx.DB, q QuerySet) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		paramMap, err := createParamMap(req, params)
		if err != nil {
			w.Write(createErrorResponseBytes(err))
			return
		}
		rowMaps, err := executeQuerySet(db, q, paramMap)
		if err != nil {
			w.Write(createErrorResponseBytes(err))
			return
		}

		bts, marshalErr := json.Marshal(Response{
			Success: true,
			Rows:    rowMaps,
		})
		if marshalErr != nil {
			panic(marshalErr)
		}
		w.Write(bts)
	}
}

func createErrorResponseBytes(err error) []byte {
	bts, marshalErr := json.Marshal(Response{
		Success:          false,
		ErrorType:        getErrorType(err),
		ErrorDescription: err.Error(),
	})
	if marshalErr != nil {
		panic(marshalErr)
	}
	return bts
}

func executeQuerySet(db *sqlx.DB, querySet QuerySet, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	var op sqlx.Queryer
	if querySet.Transaction {
		var err error
		op, err = db.Beginx()
		if err != nil {
			return nil, err
		}
	} else {
		op = db
	}

	err := executeQueries(op, querySet.Befores, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}

	rowMaps, err := executeQueriesRowMaps(op, querySet.Queries, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}

	err = executeQueries(op, querySet.Afters, paramMap)
	if err != nil {
		_ = rollbackPossibly(op)
		return nil, err
	}
	_ = commitPossibly(op)
	return rowMaps, nil
}

func rollbackPossibly(op sqlx.Queryer) bool {
	if tx, ok := op.(*sqlx.Tx); ok {
		tx.Rollback()
		return true
	}
	return false
}

func commitPossibly(op sqlx.Queryer) bool {
	if tx, ok := op.(*sqlx.Tx); ok {
		tx.Commit()
		return true
	}
	return false
}

func getErrorType(err error) string {
	switch err.(type) {
	case QueryExecutionError:
		return "QueryExecutionError"
	}
	return "Unknown"
}

func executeQueries(db sqlx.Queryer, qs []Query, paramMap map[string]interface{}) error {
	_, err := executeQueriesRowMaps(db, qs, paramMap)
	return err
}

func executeQueriesRowMaps(db sqlx.Queryer, qs []Query, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	concated := []map[string]interface{}{}
	for _, q := range qs {
		rowMap, err := executeRowMaps(db, q, paramMap)
		if err != nil {
			return nil, err
		}
		concated = append(concated, rowMap...)
	}
	return concated, nil
}

func executeRowMaps(db sqlx.Queryer, q Query, paramMap map[string]interface{}) ([]map[string]interface{}, error) {
	rows, err := q.ExecuteWithArgMap(paramMap)
	if err != nil {
		return nil, QueryExecutionError(errors.Wrap(err, "failed to execute query"))
	}
	res, err := createMapSliceFromRows(rows)
	if err != nil {
		return nil, QueryExecutionError(errors.Wrap(err, "failed to create map slice from results"))
	}
	return res, nil
}

// RequestBodyParseError causes when invalid JSON given as request body.
type RequestBodyParseError error

func createParamMap(req *http.Request, params httprouter.Params) (map[string]interface{}, error) {
	result := make(map[string]interface{})
	for _, param := range params {
		result[param.Key] = param.Value
	}

	reqBody := make(map[string]interface{})
	err := json.NewDecoder(req.Body).Decode(&reqBody)
	defer req.Body.Close()
	if err != io.EOF {
		if err != nil {
			return nil, RequestBodyParseError(errors.Wrap(err, "request body must be only 1 hieralchical key/value pairs"))
		}
		for k, v := range reqBody {
			result[k] = v
		}
	}

	// Note that only string value can be passed by URL Queries.
	q := req.URL.Query()
	for k, v := range q {
		result[k] = v[0]
	}

	return result, nil
}

type mapString []byte

func (bts mapString) MarshalText() ([]byte, error) {
	return bts, nil
}

func createMapSliceFromRows(rows *sqlx.Rows) ([]map[string]interface{}, error) {
	result := []map[string]interface{}{}
	for rows.Next() {
		cols := make(map[string]interface{})
		err := rows.MapScan(cols)
		if err != nil {
			return nil, err
		}

		newCols := make(map[string]interface{})
		for k, v := range cols {
			if str, ok := v.([]byte); ok {
				newCols[k] = mapString(str)
			} else {
				newCols[k] = v
			}
		}
		result = append(result, newCols)
	}
	return result, nil
}
<|endoftext|>


<commit_before>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package main

import (
	"github.com/codegangsta/cli"
	"log"
	"os"
)

func doImport(c *cli.Context) {
	log.Printf("import")

	srcfile := c.Args().First()
	log.Fatal(runImport(srcfile))
}

func doDaemon(c *cli.Context) {
	log.Printf("starting daemon")
	log.Fatal(runDaemon())
}

func main() {
	app := cli.NewApp()
	app.Name = "q3stats"
	app.HideHelp = true
	app.HideVersion = true
	app.Commands = []cli.Command{
		{
			Name:   "import",
			Usage:  "import match logs",
			Action: doImport,
		},
		{
			Name:   "daemon",
			Usage:  "run daemon",
			Action: doDaemon,
		},
		{
			Name: "help",
			Usage: "show command help",
			Action: func(c *cli.Context) {
				cli.ShowCommandHelp(c, "import")
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>main: gofmt<commit_after>// The MIT License (MIT)

// Copyright (c) 2016 Maciej Borzecki

// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:

// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package main

import (
	"github.com/codegangsta/cli"
	"log"
	"os"
)

func doImport(c *cli.Context) {
	log.Printf("import")

	srcfile := c.Args().First()
	log.Fatal(runImport(srcfile))
}

func doDaemon(c *cli.Context) {
	log.Printf("starting daemon")
	log.Fatal(runDaemon())
}

func main() {
	app := cli.NewApp()
	app.Name = "q3stats"
	app.HideHelp = true
	app.HideVersion = true
	app.Commands = []cli.Command{
		{
			Name:   "import",
			Usage:  "import match logs",
			Action: doImport,
		},
		{
			Name:   "daemon",
			Usage:  "run daemon",
			Action: doDaemon,
		},
		{
			Name:  "help",
			Usage: "show command help",
			Action: func(c *cli.Context) {
				cli.ShowCommandHelp(c, "import")
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>