<commit_before>// dbjob.go
package main

import (
	"io"
	"log"

	"code.google.com/p/go-sqlite/go1/sqlite3"
)

const batchSize int = 100

type fileDB struct {
	db *sqlite3.Conn

	insert     *sqlite3.Stmt
	batchCount int

	begin  *sqlite3.Stmt
	commit *sqlite3.Stmt

	get     *sqlite3.Stmt
	getArgs sqlite3.NamedArgs
	//row    sqlite3.RowMap
}

func (fdb *fileDB) Close() {
	if fdb.db != nil {
		if fdb.commit != nil {
			fdb.commit.Exec()
		}

		if fdb.insert != nil {
			fdb.insert.Close()
		}
		if fdb.get != nil {
			fdb.get.Close()
		}
		if fdb.begin != nil {
			fdb.begin.Close()
		}
		if fdb.commit != nil {
			fdb.commit.Close()
		}
	}
}

func newDBConnection(path string) (db *sqlite3.Conn, err error) {
	db, err = sqlite3.Open(path)

	if err == nil {
		err = db.Exec("CREATE TABLE IF NOT EXISTS FILES (PATH varchar not null, SIZE integer not null, MOD_TIME integer not null, MODE integer not null, CHKSUM integer not null, primary key(PATH));")
	}

	if err == nil {
		err = db.Exec("CREATE INDEX IF NOT EXISTS BYSUM on FILES (CHKSUM, SIZE);")
	}

	//db.BusyTimeout(time.Second)

	return
}

func closeDBConnection(db *sqlite3.Conn) {
	db.Exec("COMMIT TRANSACTION;")
	db.Close()
}

func newFileDB(db *sqlite3.Conn) (fdb *fileDB) {
	var err error
	fdb = &fileDB{}

	fdb.insert, err = db.Prepare("insert or replace into FILES (path, size, mod_time, mode, chksum) values ($path, $size, $mod_time, $mode, $chksum);")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.get, err = db.Prepare("select size, mod_time, mode, chksum from FILES where path = $path;")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.begin, err = db.Prepare("BEGIN TRANSACTION;")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.commit, err = db.Prepare("COMMIT TRANSACTION;")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.getArgs = make(sqlite3.NamedArgs)

	err = fdb.begin.Exec()
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	return
}

func (fdb *fileDB) insertOrReplace(f *fileJob) {
	args := make(sqlite3.NamedArgs)
	args["$path"] = f.Fpath
	args["$size"] = f.Info.Size()
	args["$mod_time"] = f.Info.ModTime().UnixNano()
	args["$mode"] = int64(f.Info.Mode())
	args["$chksum"] = int64(f.Chksum)

	defer fdb.insert.Reset()
	err := fdb.insert.Exec(args)
	if err != nil {
		log.Fatal(err)
	}

	if fdb.batchCount += 1; fdb.batchCount >= batchSize {
		err = fdb.commit.Exec()
		if err != nil {
			log.Fatal(err)
		}
		fdb.begin.Exec()
		if err != nil {
			log.Fatal(err)
		}
		fdb.batchCount = 0
	}

	f.Err = NewError(code_NEW_SUM, f, "updating checksum")
	return
}

func (fdb *fileDB) getForPath(f *fileJob) (row sqlite3.RowMap, err error) {
	defer fdb.get.Reset()
	fdb.getArgs["$path"] = f.Fpath
	err = fdb.get.Query(fdb.getArgs)
	switch {
	case err == io.EOF:
		err = nil

	case err != nil:

	default:
		row = make(sqlite3.RowMap)
		err = fdb.get.Scan(row)
	}

	return
}

func CheckInDB(f *fileJob, fdb *fileDB) {
	if f.Err != nil {
		return
	}

	row, err := fdb.getForPath(f)
	switch {
	case err != nil:
		// Problem with db.  Fail.
		log.Fatal(err)

	case row == nil:
		// No entry.  Make one.
		fdb.insertOrReplace(f)

	default:
		// Got a value.  Compare it against calculated values.
		// If file was deliberately changed, replace the row with new cheksum
		switch {
		case !ignoreMTime && row["MOD_TIME"] != f.Info.ModTime().UnixNano():
			fdb.insertOrReplace(f)

		case row["CHKSUM"] != int64(f.Chksum):
			f.Err = NewError(code_BAD_SUM, f, "checksum did not match recorded value")
		}
	}

	return
}

func dbChecker(in chan *fileJob, out chan *fileJob, db *sqlite3.Conn) {
	defer func() { out <- nil }()

	fdb := newFileDB(db)
	defer fdb.Close()

	for f := range in {
		if f == nil {
			return
		}

		if f.Err == nil {
			CheckInDB(f, fdb)
		}

		out <- f
	}
}
<commit_msg>Use builtin begin/commit<commit_after>// dbjob.go
package main

import (
	"io"
	"log"

	"code.google.com/p/go-sqlite/go1/sqlite3"
)

const batchSize int = 100

type fileDB struct {
	db *sqlite3.Conn

	insert     *sqlite3.Stmt
	batchCount int

	get     *sqlite3.Stmt
	getArgs sqlite3.NamedArgs
	//row    sqlite3.RowMap
}

func (fdb *fileDB) Close() {
	if fdb.db != nil {
		fdb.db.Commit()

		if fdb.insert != nil {
			fdb.insert.Close()
		}
		if fdb.get != nil {
			fdb.get.Close()
		}
	}
}

func newDBConnection(path string) (db *sqlite3.Conn, err error) {
	db, err = sqlite3.Open(path)

	if err == nil {
		err = db.Exec("CREATE TABLE IF NOT EXISTS FILES (PATH varchar not null, SIZE integer not null, MOD_TIME integer not null, MODE integer not null, CHKSUM integer not null, primary key(PATH));")
	}

	if err == nil {
		err = db.Exec("CREATE INDEX IF NOT EXISTS BYSUM on FILES (CHKSUM, SIZE);")
	}

	//db.BusyTimeout(time.Second)

	return
}

func closeDBConnection(db *sqlite3.Conn) {
	db.Exec("COMMIT TRANSACTION;")
	db.Close()
}

func newFileDB(db *sqlite3.Conn) (fdb *fileDB) {
	var err error
	fdb = &fileDB{}

	fdb.db = db

	fdb.insert, err = db.Prepare("insert or replace into FILES (path, size, mod_time, mode, chksum) values ($path, $size, $mod_time, $mode, $chksum);")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.get, err = db.Prepare("select size, mod_time, mode, chksum from FILES where path = $path;")
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	fdb.getArgs = make(sqlite3.NamedArgs)

	err = fdb.db.Begin()
	if err != nil {
		fdb.Close()
		log.Fatal(err)
	}

	return
}

func (fdb *fileDB) insertOrReplace(f *fileJob) {
	args := make(sqlite3.NamedArgs)
	args["$path"] = f.Fpath
	args["$size"] = f.Info.Size()
	args["$mod_time"] = f.Info.ModTime().UnixNano()
	args["$mode"] = int64(f.Info.Mode())
	args["$chksum"] = int64(f.Chksum)

	defer fdb.insert.Reset()
	err := fdb.insert.Exec(args)
	if err != nil {
		log.Fatal(err)
	}

	if fdb.batchCount += 1; fdb.batchCount >= batchSize {
		err = fdb.db.Commit()
		if err != nil {
			log.Fatal(err)
		}
		fdb.db.Begin()
		if err != nil {
			log.Fatal(err)
		}
		fdb.batchCount = 0
	}

	f.Err = NewError(code_NEW_SUM, f, "updating checksum")
	return
}

func (fdb *fileDB) getForPath(f *fileJob) (row sqlite3.RowMap, err error) {
	defer fdb.get.Reset()
	fdb.getArgs["$path"] = f.Fpath
	err = fdb.get.Query(fdb.getArgs)
	switch {
	case err == io.EOF:
		err = nil

	case err != nil:

	default:
		row = make(sqlite3.RowMap)
		err = fdb.get.Scan(row)
	}

	return
}

func CheckInDB(f *fileJob, fdb *fileDB) {
	if f.Err != nil {
		return
	}

	row, err := fdb.getForPath(f)
	switch {
	case err != nil:
		// Problem with db.  Fail.
		log.Fatal(err)

	case row == nil:
		// No entry.  Make one.
		fdb.insertOrReplace(f)

	default:
		// Got a value.  Compare it against calculated values.
		// If file was deliberately changed, replace the row with new cheksum
		switch {
		case !ignoreMTime && row["MOD_TIME"] != f.Info.ModTime().UnixNano():
			fdb.insertOrReplace(f)

		case row["CHKSUM"] != int64(f.Chksum):
			f.Err = NewError(code_BAD_SUM, f, "checksum did not match recorded value")
		}
	}

	return
}

func dbChecker(in chan *fileJob, out chan *fileJob, db *sqlite3.Conn) {
	defer func() { out <- nil }()

	fdb := newFileDB(db)
	defer fdb.Close()

	for f := range in {
		if f == nil {
			return
		}

		if f.Err == nil {
			CheckInDB(f, fdb)
		}

		out <- f
	}
}
<|endoftext|>


<commit_before>// Package dburl provides a standardized way of processing database connection
// strings in the form of a URL.
//
// Standard URLs are of the form
// protocol+transport://user:pass@host/dbname?opt1=a&opt2=b
//
// For example, the following are URLs that can be processed using Parse or
// Open:
//
//     postgres://user:pass@localhost/dbname
//     pg://user:pass@localhost/dbname?sslmode=disable
//     mysql://user:pass@localhost/dbname
//     sqlserver://user:pass@remote-host.com/dbname
//     oracle://user:pass@somehost.com/oracledb
//     sqlite:/path/to/file.db
// 	   file:myfile.sqlite3?loc=auto
//
// Protocol aliases:
//
// The following protocol aliases are available, and will be parsed according
// to the rules for their respective driver.
//
//     Database (driver)            | Aliases
//     ------------------------------------------------------------------
//     Microsoft SQL Server (mssql) | ms, sqlserver
//     MySQL (mysql)                | my, mariadb, maria, percona, aurora
//     Oracle (ora)                 | or, oracle, oci8, oci
//     PostgreSQL (postgres)        | pg, postgresql, pgsql
//     SQLite3 (sqlite3)            | sq, sqlite, file
//
package dburl

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"
)

// URL wraps the standard net/url.URL type, adding OriginalScheme, Proto,
// Driver, and DSN strings.
type URL struct {
	url.URL
	OriginalScheme, Proto, Driver, DSN string
}

// String satisfies the stringer interface.
func (u *URL) String() string {
	p := &url.URL{
		Scheme:   u.OriginalScheme,
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return p.String()
}

// Short provides a short description of the user, host, and database.
func (u *URL) Short() string {
	s := u.Driver[:2]
	if s == "po" {
		s = "pg"
	}

	s += ":"

	if u.User != nil {
		if un := u.User.Username(); un != "" {
			s += un + "@"
		}
	}

	if u.Host != "" {
		s += u.Host
	}

	if u.Path != "" && u.Path != "/" {
		s += u.Path
	}

	if u.Opaque != "" {
		s += u.Opaque
	}

	return s
}

// Parse parses a rawurl string and normalizes the scheme.
func Parse(rawurl string) (*URL, error) {
	// parse url
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if u.Scheme == "" {
		return nil, errors.New("invalid database scheme")
	}

	// create url
	v := &URL{URL: *u, OriginalScheme: u.Scheme, Proto: "tcp"}
	v.Scheme = strings.ToLower(v.Scheme)

	// check if +unix or whatever is in the scheme
	if strings.Contains(v.Scheme, "+") {
		p := strings.SplitN(v.Scheme, "+", 2)
		v.Scheme = p[0]
		v.Proto = p[1]
	}

	// check protocol
	if v.Proto != "tcp" && v.Proto != "udp" && v.Proto != "unix" {
		return nil, errors.New("invalid transport protocol")
	}

	// get loader
	loader, ok := loaders[v.Scheme]
	if !ok {
		return nil, errors.New("unknown database type")
	}

	// process
	v.Driver, v.DSN, err = loader(v)
	if err != nil {
		return nil, err
	}

	if v.Driver != "sqlite3" && v.Opaque != "" {
		return Parse(v.OriginalScheme + "://" + v.Opaque)
	}

	return v, nil
}

// Open takes a rawurl like "protocol+transport://user:pass@host/dbname?option1=a&option2=b"
// and creates a standard sql.DB connection.
//
// See Parse for information on formatting URLs to work properly with Open.
func Open(rawurl string) (*sql.DB, error) {
	u, err := Parse(rawurl)
	if err != nil {
		return nil, err
	}

	return sql.Open(u.Driver, u.DSN)
}

// mssqlProcess processes a mssql url and protocol.
func mssqlProcess(u *URL) (string, string, error) {
	var err error

	// build host or domain socket
	host := u.Host
	port := 1433

	// grab dbname
	var dbname string
	if u.Path != "" {
		dbname = u.Path[1:]
	}

	// extract port if present
	pos := strings.Index(host, ":")
	if pos != -1 {
		port, err = strconv.Atoi(host[pos+1:])
		if err != nil {
			return "", "", errors.New("invalid port")
		}
		host = host[:pos]
	}

	// format dsn
	dsn := fmt.Sprintf("server=%s;port=%d", host, port)
	if dbname != "" {
		dsn += ";database=" + dbname
	}

	// add user/pass
	if u.User != nil {
		if user := u.User.Username(); len(user) > 0 {
			dsn += ";user id=" + user
		}
		if pass, ok := u.User.Password(); ok {
			dsn += ";password=" + pass
		}
	}

	// add params
	for k, v := range u.Query() {
		dsn += ";" + k + "=" + v[0]
	}

	return "mssql", dsn, nil
}

// mysqlProcess processes a mssql url and protocol.
func mysqlProcess(u *URL) (string, string, error) {
	// build host or domain socket
	host := u.Host
	dbname := u.Path

	if strings.HasPrefix(dbname, "/") {
		dbname = dbname[1:]
	}

	if u.Proto == "unix" {
		if u.Opaque != "" {
			host = path.Dir(u.Opaque)
			dbname = path.Base(u.Opaque)
		} else {
			host = path.Join(u.Host, path.Dir(u.Path))
			dbname = path.Base(u.Path)
		}
	} else if !strings.Contains(host, ":") {
		// append default port
		host = host + ":3306"
	}

	// create dsn
	dsn := fmt.Sprintf("%s(%s)", u.Proto, host)

	// build user/pass
	if u.User != nil {
		if un := u.User.Username(); len(un) > 0 {
			if up, ok := u.User.Password(); ok {
				un += ":" + up
			}
			dsn = un + "@" + dsn
		}
	}

	// add database name
	dsn += "/" + dbname

	// add params
	params := u.Query().Encode()
	if len(params) > 0 {
		dsn = dsn + "?" + params
	}

	// format
	return "mysql", dsn, nil
}

// oracleProcess processes a mssql url and protocol.
func oracleProcess(u *URL) (string, string, error) {
	if u.User == nil {
		return "", "", errors.New("must provide username and password in oracle dsn")
	}

	// build host or domain socket
	host := u.Host
	dbname := u.Path[1:]

	// build user/pass
	userinfo := ""
	if un := u.User.Username(); len(un) > 0 {
		userinfo = un
		if up, ok := u.User.Password(); ok {
			userinfo = userinfo + "/" + up
		}
	}

	// format
	return "ora", fmt.Sprintf(
		"%s@%s/%s",
		userinfo,
		host,
		dbname,
	), nil
}

// postgresProcess processes a mssql url and protocol.
func postgresProcess(u *URL) (string, string, error) {
	p := &url.URL{
		Scheme:   "postgres",
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return "postgres", p.String(), nil
}

// sqliteProcess processes a mssql url and protocol.
func sqliteProcess(u *URL) (string, string, error) {
	p := u.Opaque
	if u.Path != "" {
		p = u.Path
	}

	if u.Host != "" && u.Host != "localhost" {
		p = path.Join(u.Host, p)
	}

	return "sqlite3", p + u.Query().Encode(), nil
}

var loaders = map[string]func(*URL) (string, string, error){
	// mssql
	"mssql":     mssqlProcess,
	"sqlserver": mssqlProcess,
	"ms":        mssqlProcess,

	// mysql
	"mysql":   mysqlProcess,
	"mariadb": mysqlProcess,
	"maria":   mysqlProcess,
	"percona": mysqlProcess,
	"aurora":  mysqlProcess,
	"my":      mysqlProcess,

	// oracle
	"ora":    oracleProcess,
	"oracle": oracleProcess,
	"oci8":   oracleProcess,
	"oci":    oracleProcess,
	"or":     oracleProcess,

	// postgresql
	"postgres":   postgresProcess,
	"postgresql": postgresProcess,
	"pgsql":      postgresProcess,
	"pg":         postgresProcess,

	// sqlite
	"sqlite3": sqliteProcess,
	"sqlite":  sqliteProcess,
	"file":    sqliteProcess,
	"sq":      sqliteProcess,
}
<commit_msg>Fixing issue with sqlite3 url query parameters<commit_after>// Package dburl provides a standardized way of processing database connection
// strings in the form of a URL.
//
// Standard URLs are of the form
// protocol+transport://user:pass@host/dbname?opt1=a&opt2=b
//
// For example, the following are URLs that can be processed using Parse or
// Open:
//
//     postgres://user:pass@localhost/dbname
//     pg://user:pass@localhost/dbname?sslmode=disable
//     mysql://user:pass@localhost/dbname
//     sqlserver://user:pass@remote-host.com/dbname
//     oracle://user:pass@somehost.com/oracledb
//     sqlite:/path/to/file.db
// 	   file:myfile.sqlite3?loc=auto
//
// Protocol aliases:
//
// The following protocol aliases are available, and will be parsed according
// to the rules for their respective driver.
//
//     Database (driver)            | Aliases
//     ------------------------------------------------------------------
//     Microsoft SQL Server (mssql) | ms, sqlserver
//     MySQL (mysql)                | my, mariadb, maria, percona, aurora
//     Oracle (ora)                 | or, oracle, oci8, oci
//     PostgreSQL (postgres)        | pg, postgresql, pgsql
//     SQLite3 (sqlite3)            | sq, sqlite, file
//
package dburl

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"path"
	"strconv"
	"strings"
)

// URL wraps the standard net/url.URL type, adding OriginalScheme, Proto,
// Driver, and DSN strings.
type URL struct {
	url.URL
	OriginalScheme, Proto, Driver, DSN string
}

// String satisfies the stringer interface.
func (u *URL) String() string {
	p := &url.URL{
		Scheme:   u.OriginalScheme,
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return p.String()
}

// Short provides a short description of the user, host, and database.
func (u *URL) Short() string {
	s := u.Driver[:2]
	if s == "po" {
		s = "pg"
	}

	s += ":"

	if u.User != nil {
		if un := u.User.Username(); un != "" {
			s += un + "@"
		}
	}

	if u.Host != "" {
		s += u.Host
	}

	if u.Path != "" && u.Path != "/" {
		s += u.Path
	}

	if u.Opaque != "" {
		s += u.Opaque
	}

	return s
}

// Parse parses a rawurl string and normalizes the scheme.
func Parse(rawurl string) (*URL, error) {
	// parse url
	u, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if u.Scheme == "" {
		return nil, errors.New("invalid database scheme")
	}

	// create url
	v := &URL{URL: *u, OriginalScheme: u.Scheme, Proto: "tcp"}
	v.Scheme = strings.ToLower(v.Scheme)

	// check if +unix or whatever is in the scheme
	if strings.Contains(v.Scheme, "+") {
		p := strings.SplitN(v.Scheme, "+", 2)
		v.Scheme = p[0]
		v.Proto = p[1]
	}

	// check protocol
	if v.Proto != "tcp" && v.Proto != "udp" && v.Proto != "unix" {
		return nil, errors.New("invalid transport protocol")
	}

	// get loader
	loader, ok := loaders[v.Scheme]
	if !ok {
		return nil, errors.New("unknown database type")
	}

	// process
	v.Driver, v.DSN, err = loader(v)
	if err != nil {
		return nil, err
	}

	if v.Driver != "sqlite3" && v.Opaque != "" {
		return Parse(v.OriginalScheme + "://" + v.Opaque)
	}

	return v, nil
}

// Open takes a rawurl like "protocol+transport://user:pass@host/dbname?option1=a&option2=b"
// and creates a standard sql.DB connection.
//
// See Parse for information on formatting URLs to work properly with Open.
func Open(rawurl string) (*sql.DB, error) {
	u, err := Parse(rawurl)
	if err != nil {
		return nil, err
	}

	return sql.Open(u.Driver, u.DSN)
}

// mssqlProcess processes a mssql url and protocol.
func mssqlProcess(u *URL) (string, string, error) {
	var err error

	// build host or domain socket
	host := u.Host
	port := 1433

	// grab dbname
	var dbname string
	if u.Path != "" {
		dbname = u.Path[1:]
	}

	// extract port if present
	pos := strings.Index(host, ":")
	if pos != -1 {
		port, err = strconv.Atoi(host[pos+1:])
		if err != nil {
			return "", "", errors.New("invalid port")
		}
		host = host[:pos]
	}

	// format dsn
	dsn := fmt.Sprintf("server=%s;port=%d", host, port)
	if dbname != "" {
		dsn += ";database=" + dbname
	}

	// add user/pass
	if u.User != nil {
		if user := u.User.Username(); len(user) > 0 {
			dsn += ";user id=" + user
		}
		if pass, ok := u.User.Password(); ok {
			dsn += ";password=" + pass
		}
	}

	// add params
	for k, v := range u.Query() {
		dsn += ";" + k + "=" + v[0]
	}

	return "mssql", dsn, nil
}

// mysqlProcess processes a mssql url and protocol.
func mysqlProcess(u *URL) (string, string, error) {
	// build host or domain socket
	host := u.Host
	dbname := u.Path

	if strings.HasPrefix(dbname, "/") {
		dbname = dbname[1:]
	}

	if u.Proto == "unix" {
		if u.Opaque != "" {
			host = path.Dir(u.Opaque)
			dbname = path.Base(u.Opaque)
		} else {
			host = path.Join(u.Host, path.Dir(u.Path))
			dbname = path.Base(u.Path)
		}
	} else if !strings.Contains(host, ":") {
		// append default port
		host = host + ":3306"
	}

	// create dsn
	dsn := fmt.Sprintf("%s(%s)", u.Proto, host)

	// build user/pass
	if u.User != nil {
		if un := u.User.Username(); len(un) > 0 {
			if up, ok := u.User.Password(); ok {
				un += ":" + up
			}
			dsn = un + "@" + dsn
		}
	}

	// add database name
	dsn += "/" + dbname

	// add params
	params := u.Query().Encode()
	if len(params) > 0 {
		dsn += "?" + params
	}

	// format
	return "mysql", dsn, nil
}

// oracleProcess processes a mssql url and protocol.
func oracleProcess(u *URL) (string, string, error) {
	if u.User == nil {
		return "", "", errors.New("must provide username and password in oracle dsn")
	}

	// build host or domain socket
	host := u.Host
	dbname := u.Path[1:]

	// build user/pass
	userinfo := ""
	if un := u.User.Username(); len(un) > 0 {
		userinfo = un
		if up, ok := u.User.Password(); ok {
			userinfo = userinfo + "/" + up
		}
	}

	// format
	return "ora", fmt.Sprintf(
		"%s@%s/%s",
		userinfo,
		host,
		dbname,
	), nil
}

// postgresProcess processes a mssql url and protocol.
func postgresProcess(u *URL) (string, string, error) {
	p := &url.URL{
		Scheme:   "postgres",
		Opaque:   u.Opaque,
		User:     u.User,
		Host:     u.Host,
		Path:     u.Path,
		RawPath:  u.RawPath,
		RawQuery: u.RawQuery,
		Fragment: u.Fragment,
	}

	return "postgres", p.String(), nil
}

// sqliteProcess processes a mssql url and protocol.
func sqliteProcess(u *URL) (string, string, error) {
	dsn := u.Opaque
	if u.Path != "" {
		dsn = u.Path
	}

	if u.Host != "" && u.Host != "localhost" {
		dsn = path.Join(u.Host, dsn)
	}

	// add params
	params := u.Query().Encode()
	if len(params) > 0 {
		dsn += "?" + params
	}

	return "sqlite3", dsn, nil
}

var loaders = map[string]func(*URL) (string, string, error){
	// mssql
	"mssql":     mssqlProcess,
	"sqlserver": mssqlProcess,
	"ms":        mssqlProcess,

	// mysql
	"mysql":   mysqlProcess,
	"mariadb": mysqlProcess,
	"maria":   mysqlProcess,
	"percona": mysqlProcess,
	"aurora":  mysqlProcess,
	"my":      mysqlProcess,

	// oracle
	"ora":    oracleProcess,
	"oracle": oracleProcess,
	"oci8":   oracleProcess,
	"oci":    oracleProcess,
	"or":     oracleProcess,

	// postgresql
	"postgres":   postgresProcess,
	"postgresql": postgresProcess,
	"pgsql":      postgresProcess,
	"pg":         postgresProcess,

	// sqlite
	"sqlite3": sqliteProcess,
	"sqlite":  sqliteProcess,
	"file":    sqliteProcess,
	"sq":      sqliteProcess,
}
<|endoftext|>


<commit_before>package easy_middleware

import (
	"net/http"
	"time"
	"fmt"
)

const DELAY_HEADER_KEY = "X-Add-Delay"

// decimal numbers, each with optional fraction and a unit suffix,
// such as "300ms", "-1.5h" or "2h45m".
func Delay() Middleware {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {

			delayHeaderValue := r.Header.Get(DELAY_HEADER_KEY)
			if delayHeaderValue != "" {
				delayDuration, err := time.ParseDuration(delayHeaderValue)
				if err == nil {
					time.Sleep(delayDuration)
				}
			}

			h.ServeHTTP(rw, r)
		})
	}
}

<commit_msg>- Bugfix: Removed unused pkg<commit_after>package easy_middleware

import (
	"net/http"
	"time"
)

const DELAY_HEADER_KEY = "X-Add-Delay"

// decimal numbers, each with optional fraction and a unit suffix,
// such as "300ms", "-1.5h" or "2h45m".
func Delay() Middleware {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {

			delayHeaderValue := r.Header.Get(DELAY_HEADER_KEY)
			if delayHeaderValue != "" {
				delayDuration, err := time.ParseDuration(delayHeaderValue)
				if err == nil {
					time.Sleep(delayDuration)
				}
			}

			h.ServeHTTP(rw, r)
		})
	}
}

<|endoftext|>


<commit_before>package gsnake

import (
    "github.com/howeyc/fsnotify"
    "log"
    "sync"
    "github.com/golang/glog"
    "path/filepath"
)

type Dispatcher struct {
    dir    string
    watcher *fsnotify.Watcher
    status *ProcessStatus
    h *FilesHandler
    textModule TextModule
    pcapModule PcapModule
}

var dispatcher *Dispatcher
func NewDispatcher() (*Dispatcher, error) {
    var err error
    dispatcher, err = newDispatcher(*dir)
    return dispatcher, err
}

func newDispatcher(dir string) (d *Dispatcher, err error) {
    glog.Infof("NewDispatcher")
    d = &Dispatcher{}
    d.watcher, err = fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }

    d.dir = dir
    d.status, err = NewProcessStatus(*statusFile)
    if err != nil || d.status == nil {
        log.Fatal(err)
    }

    d.h, err = NewFilesHandler(dir)
    if err != nil {
        log.Fatal(err)
    }

    d.textModule = &DefaultTextModule{}
    d.pcapModule = &DefaultPcapModule{}
    return d, err
}

func (d *Dispatcher) onCreate(ev *fsnotify.FileEvent) {
    if IsDir(ev.Name) {
        d.watcher.Watch(ev.Name)
        //Ignore this : FIXME if we renamed ev.Name latterly, we should add the new name to the watching list.
    } else {
        if ok, _ := filepath.Match(*filePattern, filepath.Base(ev.Name)); ok {
            d.h.OnFileCreated(ev.Name)
        } else {
            glog.Infof("Create a file <%v> but does not match the file pattern <%v>", ev.Name, *filePattern)
        }
    }
}

func (d *Dispatcher) onDelete(ev *fsnotify.FileEvent) {
    d.status.OnFileDeleted(ev.Name)
}

func (d *Dispatcher) onModify(ev *fsnotify.FileEvent) {
    d.h.OnFileModified(ev.Name)
}

func (d *Dispatcher) watchEvent(wg *sync.WaitGroup) {
    wg.Done()
    for {
        select {
        case ev := <-d.watcher.Event:
            glog.Info("event:", ev, " name=", ev.Name)
            if ev.IsCreate() {
                d.onCreate(ev)
            } else if ev.IsDelete() {
                d.onDelete(ev)
            } else if ev.IsModify() {
                d.onModify(ev)
            } else {
                log.Printf("don't care")
            }
        case err := <-d.watcher.Error:
            log.Println("error:", err)
        }
    }
}

func (d *Dispatcher) Run() {
    glog.Infof("Watching <%v>", d.dir)
    err := d.watcher.Watch(d.dir)
    if err != nil {
        log.Fatal("Watch event of " + d.dir + " FAILED: " + err.Error())
    }

    //start to watch the file event and wait the goroutine started
    var wg sync.WaitGroup
    wg.Add(1)
    go d.watchEvent(&wg)
    wg.Wait()

    //start file handler to run
    d.h.Run()

    d.Close()
}

func (d *Dispatcher) Close() {
    d.watcher.Close()
}

func (d *Dispatcher) RegisterTextModule(m TextModule) {
    d.textModule = m
}

func (d *Dispatcher) RegisterPcapModule(m PcapModule) {
    d.pcapModule = m
}

<commit_msg>use glog instead of log<commit_after>package gsnake

import (
	"github.com/golang/glog"
	"github.com/howeyc/fsnotify"
	"path/filepath"
	"sync"
)

type Dispatcher struct {
	dir        string
	watcher    *fsnotify.Watcher
	status     *ProcessStatus
	h          *FilesHandler
	textModule TextModule
	pcapModule PcapModule
}

var dispatcher *Dispatcher

func NewDispatcher() (*Dispatcher, error) {
	var err error
	dispatcher, err = newDispatcher(*dir)
	return dispatcher, err
}

func newDispatcher(dir string) (d *Dispatcher, err error) {
	glog.Infof("NewDispatcher")
	d = &Dispatcher{}
	d.watcher, err = fsnotify.NewWatcher()
	if err != nil {
		glog.Fatal(err)
	}

	d.dir = dir
	d.status, err = NewProcessStatus(*statusFile)
	if err != nil || d.status == nil {
		glog.Fatal(err)
	}

	d.h, err = NewFilesHandler(dir)
	if err != nil {
		glog.Fatal(err)
	}

	d.textModule = &DefaultTextModule{}
	d.pcapModule = &DefaultPcapModule{}
	return d, err
}

func (d *Dispatcher) onCreate(ev *fsnotify.FileEvent) {
	if IsDir(ev.Name) {
		d.watcher.Watch(ev.Name)
		//Ignore this : FIXME if we renamed ev.Name latterly, we should add the new name to the watching list.
	} else {
		if ok, _ := filepath.Match(*filePattern, filepath.Base(ev.Name)); ok {
			d.h.OnFileCreated(ev.Name)
		} else {
			glog.Infof("Create a file <%v> but does not match the file pattern <%v>", ev.Name, *filePattern)
		}
	}
}

func (d *Dispatcher) onDelete(ev *fsnotify.FileEvent) {
	d.status.OnFileDeleted(ev.Name)
}

func (d *Dispatcher) onModify(ev *fsnotify.FileEvent) {
	d.h.OnFileModified(ev.Name)
}

func (d *Dispatcher) watchEvent(wg *sync.WaitGroup) {
	wg.Done()
	for {
		select {
		case ev := <-d.watcher.Event:
			glog.Info("event:", ev, " name=", ev.Name)
			if ev.IsCreate() {
				d.onCreate(ev)
			} else if ev.IsDelete() {
				d.onDelete(ev)
			} else if ev.IsModify() {
				d.onModify(ev)
			} else {
				glog.Info("don't care this event:", ev)
			}
		case err := <-d.watcher.Error:
			glog.Info("error:", err)
		}
	}
}

func (d *Dispatcher) Run() {
	glog.Infof("Watching <%v>", d.dir)
	err := d.watcher.Watch(d.dir)
	if err != nil {
		glog.Fatal("Watch event of " + d.dir + " FAILED: " + err.Error())
	}

	//start to watch the file event and wait the goroutine started
	var wg sync.WaitGroup
	wg.Add(1)
	go d.watchEvent(&wg)
	wg.Wait()

	//start file handler to run
	d.h.Run()

	d.Close()
}

func (d *Dispatcher) Close() {
	d.watcher.Close()
}

func (d *Dispatcher) RegisterTextModule(m TextModule) {
	d.textModule = m
}

func (d *Dispatcher) RegisterPcapModule(m PcapModule) {
	d.pcapModule = m
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/oschwald/geoip2-golang"
	"html/template"
	"net"
	"net/http"
	"strings"
)

type ErrorMessage struct {
	Error string
}

type ReturnData struct {
	Ip          net.IP
	CountryCode string
	CountryName string
	City        string
	Latitude    float64
	Longitude   float64
}

// Load the database file
var db, liberr = geoip2.Open("geodata/GeoLite2-City.mmdb")

// Load templates.
var htmlTemplates = template.Must(template.ParseFiles("bin/templates/index.html", "bin/templates/error.html"))

func getData(r *http.Request) interface{} {
	vars := mux.Vars(r)
	requestIp := ""
	if val, ok := vars["ip"]; ok {
		// IP is provided in URL.
		requestIp = val
	} else {
		// Take IP from Remote Address.
		address := strings.Split(r.RemoteAddr, ":")
		requestIp = address[0]
	}

	ip := net.ParseIP(requestIp)

	record, err := db.City(ip)
	if err != nil {
		return ErrorMessage{Error: "Unknown IP"}
	}

	return ReturnData{
		Ip:          ip,
		CountryCode: record.Country.IsoCode,
		CountryName: record.Country.Names["en"],
		City:        record.City.Names["en"],
		Latitude:    record.Location.Latitude,
		Longitude:   record.Location.Longitude}
}

func xmlHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/xml")
	retData := getData(r)
	msg, _ := xml.Marshal(retData)
	w.Write(msg)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	retData := getData(r)
	msg, _ := json.Marshal(retData)
	w.Write(msg)
}

func htmlHandler(w http.ResponseWriter, r *http.Request) {
	retData := getData(r)

	switch retData.(type) {
	case ReturnData:
		htmlTemplates.ExecuteTemplate(w, "index.html", retData)
	default:
		htmlTemplates.ExecuteTemplate(w, "error.html", "")
	}
}

func main() {

	if liberr != nil {
		// Country database was not loaded successfully
		fmt.Printf("Error: %s\n", liberr.Error())
		return
	}

	r := mux.NewRouter()
	r.HandleFunc("/{ip}/json", jsonHandler)
	r.HandleFunc("/json", jsonHandler)
	r.HandleFunc("/{ip}/xml", xmlHandler)
	r.HandleFunc("/xml", xmlHandler)
	r.HandleFunc("/{ip}", htmlHandler)
	r.HandleFunc("/", htmlHandler)
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("bin/assets")))

	http.Handle("/", r)
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Improves retrieving IP address when not specified in URL<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"github.com/gorilla/mux"
	"github.com/oschwald/geoip2-golang"
	"html/template"
	"net"
	"net/http"
	"strings"
)

type ErrorMessage struct {
	Error string
}

type ReturnData struct {
	Ip          net.IP
	CountryCode string
	CountryName string
	City        string
	Latitude    float64
	Longitude   float64
}

// Load the database file
var db, liberr = geoip2.Open("geodata/GeoLite2-City.mmdb")

// Load templates.
var htmlTemplates = template.Must(template.ParseFiles("bin/templates/index.html", "bin/templates/error.html"))

func getData(r *http.Request) interface{} {
	vars := mux.Vars(r)
	requestIp := ""
	if val, ok := vars["ip"]; ok {
		// IP is provided in URL.
		requestIp = val
	} else {
		// Take IP from Remote Address.
		address := r.Header.Get("X-Real-IP")
		if address == "" {
			address = r.Header.Get("X-Forwarded-For")
		}
                if address == "" {
			address = r.RemoteAddr
			idx := strings.LastIndex(address, ":")
			address = address[:idx]
		}
		requestIp = address
	}

	ip := net.ParseIP(requestIp)

	record, err := db.City(ip)
	if err != nil {
		return ErrorMessage{Error: "Unknown IP"}
	}

	return ReturnData{
		Ip:          ip,
		CountryCode: record.Country.IsoCode,
		CountryName: record.Country.Names["en"],
		City:        record.City.Names["en"],
		Latitude:    record.Location.Latitude,
		Longitude:   record.Location.Longitude}
}

func xmlHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/xml")
	retData := getData(r)
	msg, _ := xml.Marshal(retData)
	w.Write(msg)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	retData := getData(r)
	msg, _ := json.Marshal(retData)
	w.Write(msg)
}

func htmlHandler(w http.ResponseWriter, r *http.Request) {
	retData := getData(r)

	switch retData.(type) {
	case ReturnData:
		htmlTemplates.ExecuteTemplate(w, "index.html", retData)
	default:
		htmlTemplates.ExecuteTemplate(w, "error.html", "")
	}
}

func main() {

	if liberr != nil {
		// Country database was not loaded successfully
		fmt.Printf("Error: %s\n", liberr.Error())
		return
	}

	r := mux.NewRouter()
	r.HandleFunc("/{ip}/json", jsonHandler)
	r.HandleFunc("/json", jsonHandler)
	r.HandleFunc("/{ip}/xml", xmlHandler)
	r.HandleFunc("/xml", xmlHandler)
	r.HandleFunc("/{ip}", htmlHandler)
	r.HandleFunc("/", htmlHandler)
	r.PathPrefix("/").Handler(http.FileServer(http.Dir("bin/assets")))

	http.Handle("/", r)
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>// api provides common functionality for all the iron.io APIs
package api

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/iron-io/iron_go/config"
)

type URL struct {
	URL      url.URL
	Settings config.Settings
}

var (
	debug bool
)

func dbg(v ...interface{}) {
	if debug {
		fmt.Fprintln(os.Stderr, v...)
	}
}

func init() {
	if os.Getenv("IRON_API_DEBUG") != "" {
		debug = true
		dbg("debugging of api enabled")
	}
}

func Action(cs config.Settings, prefix string, suffix ...string) *URL {
	parts := append([]string{prefix}, suffix...)
	for n, part := range parts {
		parts[n] = url.QueryEscape(part)
	}

	u := &URL{Settings: cs, URL: url.URL{}}
	u.URL.Scheme = cs.Scheme
	u.URL.Host = fmt.Sprintf("%s:%d", url.QueryEscape(cs.Host), cs.Port)
	u.URL.Path = fmt.Sprintf("/%s/projects/%s/%s", cs.ApiVersion, cs.ProjectId, strings.Join(parts, "/"))
	return u
}

func VersionAction(cs config.Settings) *URL {
	u := &URL{Settings: cs, URL: url.URL{Scheme: cs.Scheme}}
	u.URL.Host = fmt.Sprintf("%s:%d", url.QueryEscape(cs.Host), cs.Port)
	u.URL.Path = "/version"
	return u
}

func (u *URL) QueryAdd(key string, format string, value interface{}) *URL {
	query := u.URL.Query()
	query.Add(key, fmt.Sprintf(format, value))
	u.URL.RawQuery = query.Encode()
	return u
}

func (u *URL) Req(method string, in, out interface{}) (err error) {
	var reqBody io.Reader
	if in != nil {
		data, err := json.Marshal(in)
		if err != nil {
			return err
		}
		reqBody = bytes.NewBuffer(data)
	}
	response, err := u.Request(method, reqBody)
	if response != nil {
		defer response.Body.Close()
	}
	if err == nil && out != nil {
		err = json.NewDecoder(response.Body).Decode(out)
		dbg("u:", u, "out:", fmt.Sprintf("%#v\n", out))
	}

	return
}

var MaxRequestRetries = 5

func (u *URL) Request(method string, body io.Reader) (response *http.Response, err error) {
	client := http.Client{}

	var bodyBytes []byte
	if body == nil {
		bodyBytes = []byte{}
	} else {
		bodyBytes, err = ioutil.ReadAll(body)
		if err != nil {
			return nil, err
		}
	}

	request, err := http.NewRequest(method, u.URL.String(), nil)
	if err != nil {
		return nil, err
	}

	request.Header.Set("Authorization", "OAuth "+u.Settings.Token)
	request.Header.Set("Accept", "application/json")
	request.Header.Set("Accept-Encoding", "gzip/deflate")
	request.Header.Set("User-Agent", u.Settings.UserAgent)

	if body != nil {
		request.Header.Set("Content-Type", "application/json")
	}

	dbg("request:", fmt.Sprintf("%#v\n", request))

	for tries := 0; tries <= MaxRequestRetries; tries++ {
		request.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
		response, err = client.Do(request)
		if err != nil {
			if err == io.EOF {
				continue
			}
			return
		}

		if response.StatusCode == http.StatusServiceUnavailable {
			delay := (tries + 1) * 10 // smooth out delays from 0-2
			time.Sleep(time.Duration(delay*delay) * time.Millisecond)
			continue
		}

		break
	}

	// DumpResponse(response)
	if err = ResponseAsError(response); err != nil {
		return
	}

	return
}

func DumpRequest(req *http.Request) {
	out, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%q\n", out)
}

func DumpResponse(response *http.Response) {
	out, err := httputil.DumpResponse(response, true)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%q\n", out)
}

var HTTPErrorDescriptions = map[int]string{
	http.StatusUnauthorized:     "The OAuth token is either not provided or invalid",
	http.StatusNotFound:         "The resource, project, or endpoint being requested doesn't exist.",
	http.StatusMethodNotAllowed: "This endpoint doesn't support that particular verb",
	http.StatusNotAcceptable:    "Required fields are missing",
}

func ResponseAsError(response *http.Response) HTTPResponseError {
	if response.StatusCode == http.StatusOK {
		return nil
	}

	defer response.Body.Close()
	desc, found := HTTPErrorDescriptions[response.StatusCode]
	if found {
		return resErr{response: response, error: response.Status + ": " + desc}
	}

	out := map[string]interface{}{}
	err := json.NewDecoder(response.Body).Decode(&out)
	if err != nil {
		return resErr{response: response, error: fmt.Sprint(response.Status, ": ", err.Error())}
	}
	if msg, ok := out["msg"]; ok {
		return resErr{response: response, error: fmt.Sprint(response.Status, ": ", msg)}
	}

	return resErr{response: response, error: response.Status + ": Unknown API Response"}
}

type HTTPResponseError interface {
	Error() string
	Response() *http.Response
}

type resErr struct {
	error    string
	response *http.Response
}

func (h resErr) Error() string            { return h.error }
func (h resErr) Response() *http.Response { return h.response }
<commit_msg>Added api function to support simple endpoints<commit_after>// api provides common functionality for all the iron.io APIs
package api

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/iron-io/iron_go/config"
)

type URL struct {
	URL      url.URL
	Settings config.Settings
}

var (
	debug bool
)

func dbg(v ...interface{}) {
	if debug {
		fmt.Fprintln(os.Stderr, v...)
	}
}

func init() {
	if os.Getenv("IRON_API_DEBUG") != "" {
		debug = true
		dbg("debugging of api enabled")
	}
}

func Action(cs config.Settings, prefix string, suffix ...string) *URL {
	parts := append([]string{prefix}, suffix...)
	for n, part := range parts {
		parts[n] = url.QueryEscape(part)
	}

	u := &URL{Settings: cs, URL: url.URL{}}
	u.URL.Scheme = cs.Scheme
	u.URL.Host = fmt.Sprintf("%s:%d", url.QueryEscape(cs.Host), cs.Port)
	u.URL.Path = fmt.Sprintf("/%s/projects/%s/%s", cs.ApiVersion, cs.ProjectId, strings.Join(parts, "/"))
	return u
}

func ActionEndpoint(cs config.Settings, endpoint string) *URL {
	u := &URL{Settings: cs, URL: url.URL{}}
	u.URL.Scheme = cs.Scheme
	u.URL.Host = fmt.Sprintf("%s:%d", url.QueryEscape(cs.Host), cs.Port)
	u.URL.Path = fmt.Sprintf("/%s/projects/%s/%s", cs.ApiVersion, cs.ProjectId, endpoint)
	return u
}

func VersionAction(cs config.Settings) *URL {
	u := &URL{Settings: cs, URL: url.URL{Scheme: cs.Scheme}}
	u.URL.Host = fmt.Sprintf("%s:%d", url.QueryEscape(cs.Host), cs.Port)
	u.URL.Path = "/version"
	return u
}

func (u *URL) QueryAdd(key string, format string, value interface{}) *URL {
	query := u.URL.Query()
	query.Add(key, fmt.Sprintf(format, value))
	u.URL.RawQuery = query.Encode()
	return u
}

func (u *URL) Req(method string, in, out interface{}) (err error) {
	var reqBody io.Reader
	if in != nil {
		data, err := json.Marshal(in)
		if err != nil {
			return err
		}
		reqBody = bytes.NewBuffer(data)
	}
	response, err := u.Request(method, reqBody)
	if response != nil {
		defer response.Body.Close()
	}
	if err == nil && out != nil {
		err = json.NewDecoder(response.Body).Decode(out)
		dbg("u:", u, "out:", fmt.Sprintf("%#v\n", out))
	}

	return
}

var MaxRequestRetries = 5

func (u *URL) Request(method string, body io.Reader) (response *http.Response, err error) {
	client := http.Client{}

	var bodyBytes []byte
	if body == nil {
		bodyBytes = []byte{}
	} else {
		bodyBytes, err = ioutil.ReadAll(body)
		if err != nil {
			return nil, err
		}
	}

	request, err := http.NewRequest(method, u.URL.String(), nil)
	if err != nil {
		return nil, err
	}

	request.Header.Set("Authorization", "OAuth "+u.Settings.Token)
	request.Header.Set("Accept", "application/json")
	request.Header.Set("Accept-Encoding", "gzip/deflate")
	request.Header.Set("User-Agent", u.Settings.UserAgent)

	if body != nil {
		request.Header.Set("Content-Type", "application/json")
	}

	dbg("request:", fmt.Sprintf("%#v\n", request))

	for tries := 0; tries <= MaxRequestRetries; tries++ {
		request.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
		response, err = client.Do(request)
		if err != nil {
			if err == io.EOF {
				continue
			}
			return
		}

		if response.StatusCode == http.StatusServiceUnavailable {
			delay := (tries + 1) * 10 // smooth out delays from 0-2
			time.Sleep(time.Duration(delay*delay) * time.Millisecond)
			continue
		}

		break
	}

	// DumpResponse(response)
	if err = ResponseAsError(response); err != nil {
		return
	}

	return
}

func DumpRequest(req *http.Request) {
	out, err := httputil.DumpRequestOut(req, true)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%q\n", out)
}

func DumpResponse(response *http.Response) {
	out, err := httputil.DumpResponse(response, true)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%q\n", out)
}

var HTTPErrorDescriptions = map[int]string{
	http.StatusUnauthorized:     "The OAuth token is either not provided or invalid",
	http.StatusNotFound:         "The resource, project, or endpoint being requested doesn't exist.",
	http.StatusMethodNotAllowed: "This endpoint doesn't support that particular verb",
	http.StatusNotAcceptable:    "Required fields are missing",
}

func ResponseAsError(response *http.Response) HTTPResponseError {
	if response.StatusCode == http.StatusOK {
		return nil
	}

	defer response.Body.Close()
	desc, found := HTTPErrorDescriptions[response.StatusCode]
	if found {
		return resErr{response: response, error: response.Status + ": " + desc}
	}

	out := map[string]interface{}{}
	err := json.NewDecoder(response.Body).Decode(&out)
	if err != nil {
		return resErr{response: response, error: fmt.Sprint(response.Status, ": ", err.Error())}
	}
	if msg, ok := out["msg"]; ok {
		return resErr{response: response, error: fmt.Sprint(response.Status, ": ", msg)}
	}

	return resErr{response: response, error: response.Status + ": Unknown API Response"}
}

type HTTPResponseError interface {
	Error() string
	Response() *http.Response
}

type resErr struct {
	error    string
	response *http.Response
}

func (h resErr) Error() string            { return h.error }
func (h resErr) Response() *http.Response { return h.response }
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2015-2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// Package api is Beehive's RESTful api for introspection and configuration
package api

import (
	"log"
	"net/http"
	"path"
	_ "strconv"

	"github.com/emicklei/go-restful"
	_ "github.com/emicklei/go-restful/swagger"

	"github.com/muesli/beehive/api/context"
	"github.com/muesli/beehive/api/resources/actions"
	"github.com/muesli/beehive/api/resources/bees"
	"github.com/muesli/beehive/api/resources/chains"
	"github.com/muesli/beehive/api/resources/hives"
	"github.com/muesli/smolder"
)

func configFromPathParam(req *restful.Request, resp *restful.Response) {
	rootdir := "./config"

	subpath := req.PathParameter("subpath")
	if len(subpath) == 0 {
		subpath = "index.html"
	}
	actual := path.Join(rootdir, subpath)
	log.Printf("serving %s ... (from %s)\n", actual, req.PathParameter("subpath"))
	http.ServeFile(
		resp.ResponseWriter,
		req.Request,
		actual)
}

func imageFromPathParam(req *restful.Request, resp *restful.Response) {
	rootdir := "./assets/bees"

	subpath := req.PathParameter("subpath")
	actual := path.Join(rootdir, subpath)
	log.Printf("serving %s ... (from %s)\n", actual, req.PathParameter("subpath"))
	http.ServeFile(
		resp.ResponseWriter,
		req.Request,
		actual)
}

// Run sets up the restful API container and an HTTP server go-routine
func Run() {
	// to see what happens in the package, uncomment the following
	//restful.TraceLogger(log.New(os.Stdout, "[restful] ", log.LstdFlags|log.Lshortfile))

	context := &context.APIContext{}

	// Setup web-service
	smolderConfig := smolder.APIConfig{
		BaseURL:    "http://localhost:8181",
		PathPrefix: "v1/",
	}

	wsContainer := smolder.NewSmolderContainer(smolderConfig, nil, nil)
	wsContainer.Router(restful.CurlyRouter{})
	ws := new(restful.WebService)
	ws.Route(ws.GET("/config/").To(configFromPathParam))
	ws.Route(ws.GET("/config/{subpath:*}").To(configFromPathParam))
	ws.Route(ws.GET("/images/{subpath:*}").To(imageFromPathParam))
	wsContainer.Add(ws)

	func(resources ...smolder.APIResource) {
		for _, r := range resources {
			r.Register(wsContainer, smolderConfig, context)
		}
	}(
		&hives.HiveResource{},
		&bees.BeeResource{},
		&chains.ChainResource{},
		&actions.ActionResource{},
	)

	server := &http.Server{Addr: ":8181", Handler: wsContainer}
	go func() {
		log.Fatal(server.ListenAndServe())
	}()
}
<commit_msg>Removed obsolete imports from api.go<commit_after>/*
 *    Copyright (C) 2015-2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

// Package api is Beehive's RESTful api for introspection and configuration
package api

import (
	"log"
	"net/http"
	"path"

	"github.com/emicklei/go-restful"

	"github.com/muesli/beehive/api/context"
	"github.com/muesli/beehive/api/resources/actions"
	"github.com/muesli/beehive/api/resources/bees"
	"github.com/muesli/beehive/api/resources/chains"
	"github.com/muesli/beehive/api/resources/hives"
	"github.com/muesli/smolder"
)

func configFromPathParam(req *restful.Request, resp *restful.Response) {
	rootdir := "./config"

	subpath := req.PathParameter("subpath")
	if len(subpath) == 0 {
		subpath = "index.html"
	}
	actual := path.Join(rootdir, subpath)
	log.Printf("serving %s ... (from %s)\n", actual, req.PathParameter("subpath"))
	http.ServeFile(
		resp.ResponseWriter,
		req.Request,
		actual)
}

func imageFromPathParam(req *restful.Request, resp *restful.Response) {
	rootdir := "./assets/bees"

	subpath := req.PathParameter("subpath")
	actual := path.Join(rootdir, subpath)
	log.Printf("serving %s ... (from %s)\n", actual, req.PathParameter("subpath"))
	http.ServeFile(
		resp.ResponseWriter,
		req.Request,
		actual)
}

// Run sets up the restful API container and an HTTP server go-routine
func Run() {
	// to see what happens in the package, uncomment the following
	//restful.TraceLogger(log.New(os.Stdout, "[restful] ", log.LstdFlags|log.Lshortfile))

	context := &context.APIContext{}

	// Setup web-service
	smolderConfig := smolder.APIConfig{
		BaseURL:    "http://localhost:8181",
		PathPrefix: "v1/",
	}

	wsContainer := smolder.NewSmolderContainer(smolderConfig, nil, nil)
	wsContainer.Router(restful.CurlyRouter{})
	ws := new(restful.WebService)
	ws.Route(ws.GET("/config/").To(configFromPathParam))
	ws.Route(ws.GET("/config/{subpath:*}").To(configFromPathParam))
	ws.Route(ws.GET("/images/{subpath:*}").To(imageFromPathParam))
	wsContainer.Add(ws)

	func(resources ...smolder.APIResource) {
		for _, r := range resources {
			r.Register(wsContainer, smolderConfig, context)
		}
	}(
		&hives.HiveResource{},
		&bees.BeeResource{},
		&chains.ChainResource{},
		&actions.ActionResource{},
	)

	server := &http.Server{Addr: ":8181", Handler: wsContainer}
	go func() {
		log.Fatal(server.ListenAndServe())
	}()
}
<|endoftext|>


<commit_before>// Package api is an API Gateway
package api

import (
	"fmt"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/micro/cli"
	ahandler "github.com/micro/go-api/handler"
	aapi "github.com/micro/go-api/handler/api"
	"github.com/micro/go-api/handler/event"
	ahttp "github.com/micro/go-api/handler/http"
	arpc "github.com/micro/go-api/handler/rpc"
	"github.com/micro/go-api/handler/web"
	"github.com/micro/go-api/resolver"
	"github.com/micro/go-api/resolver/grpc"
	"github.com/micro/go-api/resolver/host"
	rrmicro "github.com/micro/go-api/resolver/micro"
	"github.com/micro/go-api/resolver/path"
	"github.com/micro/go-api/router"
	"github.com/micro/go-api/server"
	httpapi "github.com/micro/go-api/server/http"
	"github.com/micro/go-log"
	"github.com/micro/go-micro"
	"github.com/micro/micro/internal/handler"
	"github.com/micro/micro/internal/helper"
	"github.com/micro/micro/internal/stats"
	"github.com/micro/micro/plugin"
)

var (
	Name         = "go.micro.api"
	Address      = ":8080"
	Handler      = "meta"
	Resolver     = "micro"
	RPCPath      = "/rpc"
	APIPath      = "/"
	ProxyPath    = "/{service:[a-zA-Z0-9]+}"
	Namespace    = "go.micro.api"
	HeaderPrefix = "X-Micro-"
)

func run(ctx *cli.Context, srvOpts ...micro.Option) {
	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("handler")) > 0 {
		Handler = ctx.String("handler")
	}
	if len(ctx.String("namespace")) > 0 {
		Namespace = ctx.String("namespace")
	}
	if len(ctx.String("resolver")) > 0 {
		Resolver = ctx.String("resolver")
	}

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	// Init API
	var opts []server.Option

	if ctx.GlobalBool("enable_acme") {
		hosts := helper.ACMEHosts(ctx)
		opts = append(opts, server.EnableACME(true))
		opts = append(opts, server.ACMEHosts(hosts...))
	} else if ctx.GlobalBool("enable_tls") {
		config, err := helper.TLSConfig(ctx)
		if err != nil {
			fmt.Println(err.Error())
			return
		}

		opts = append(opts, server.EnableTLS(true))
		opts = append(opts, server.TLSConfig(config))
	}

	// create the router
	var h http.Handler
	r := mux.NewRouter()
	h = r

	if ctx.GlobalBool("enable_stats") {
		st := stats.New()
		r.HandleFunc("/stats", st.StatsHandler)
		h = st.ServeHTTP(r)
		st.Start()
		defer st.Stop()
	}

	srvOpts = append(srvOpts, micro.Name(Name))
	if i := time.Duration(ctx.GlobalInt("register_ttl")); i > 0 {
		srvOpts = append(srvOpts, micro.RegisterTTL(i*time.Second))
	}
	if i := time.Duration(ctx.GlobalInt("register_interval")); i > 0 {
		srvOpts = append(srvOpts, micro.RegisterInterval(i*time.Second))
	}

	// initialise service
	service := micro.NewService(srvOpts...)

	// register rpc handler
	log.Logf("Registering RPC Handler at %s", RPCPath)
	r.HandleFunc(RPCPath, handler.RPC)

	// resolver options
	ropts := []resolver.Option{
		resolver.WithNamespace(Namespace),
		resolver.WithHandler(Handler),
	}

	// default resolver
	rr := rrmicro.NewResolver(ropts...)

	switch Resolver {
	case "host":
		rr = host.NewResolver(ropts...)
	case "path":
		rr = path.NewResolver(ropts...)
	case "grpc":
		rr = grpc.NewResolver(ropts...)
	}

	switch Handler {
	case "rpc":
		log.Logf("Registering API RPC Handler at %s", APIPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(arpc.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		rp := arpc.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(rp)
	case "api":
		log.Logf("Registering API Request Handler at %s", APIPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(aapi.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ap := aapi.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(ap)
	case "event":
		log.Logf("Registering API Event Handler at %s", APIPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(event.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ev := event.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(ev)
	case "http", "proxy":
		log.Logf("Registering API HTTP Handler at %s", ProxyPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(ahttp.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ht := ahttp.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(ProxyPath).Handler(ht)
	case "web":
		log.Logf("Registering API Web Handler at %s", APIPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(web.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		w := web.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(w)
	default:
		log.Logf("Registering API Default Handler at %s", APIPath)
		rt := router.NewRouter(
			router.WithNamespace(Namespace),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		r.PathPrefix(APIPath).Handler(handler.Meta(service, rt))
	}

	// reverse wrap handler
	plugins := append(Plugins(), plugin.Plugins()...)
	for i := len(plugins); i > 0; i-- {
		h = plugins[i-1].Handler()(h)
	}

	// create the server
	api := httpapi.NewServer(Address)
	api.Init(opts...)
	api.Handle("/", h)

	// Start API
	if err := api.Start(); err != nil {
		log.Fatal(err)
	}

	// Run server
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}

	// Stop API
	if err := api.Stop(); err != nil {
		log.Fatal(err)
	}
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "api",
		Usage: "Run the api gateway",
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "address",
				Usage:  "Set the api address e.g 0.0.0.0:8080",
				EnvVar: "MICRO_API_ADDRESS",
			},
			cli.StringFlag{
				Name:   "handler",
				Usage:  "Specify the request handler to be used for mapping HTTP requests to services; {api, event, http, rpc}",
				EnvVar: "MICRO_API_HANDLER",
			},
			cli.StringFlag{
				Name:   "namespace",
				Usage:  "Set the namespace used by the API e.g. com.example.api",
				EnvVar: "MICRO_API_NAMESPACE",
			},
			cli.StringFlag{
				Name:   "resolver",
				Usage:  "Set the hostname resolver used by the API {host, path, grpc}",
				EnvVar: "MICRO_API_RESOLVER",
			},
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<commit_msg>update to match go-api<commit_after>// Package api is an API Gateway
package api

import (
	"fmt"
	"net/http"
	"time"

	"github.com/gorilla/mux"
	"github.com/micro/cli"
	ahandler "github.com/micro/go-api/handler"
	aapi "github.com/micro/go-api/handler/api"
	"github.com/micro/go-api/handler/event"
	ahttp "github.com/micro/go-api/handler/http"
	arpc "github.com/micro/go-api/handler/rpc"
	"github.com/micro/go-api/handler/web"
	"github.com/micro/go-api/resolver"
	"github.com/micro/go-api/resolver/grpc"
	"github.com/micro/go-api/resolver/host"
	rrmicro "github.com/micro/go-api/resolver/micro"
	"github.com/micro/go-api/resolver/path"
	"github.com/micro/go-api/router"
	regRouter "github.com/micro/go-api/router/registry"
	"github.com/micro/go-api/server"
	httpapi "github.com/micro/go-api/server/http"
	"github.com/micro/go-log"
	"github.com/micro/go-micro"
	"github.com/micro/micro/internal/handler"
	"github.com/micro/micro/internal/helper"
	"github.com/micro/micro/internal/stats"
	"github.com/micro/micro/plugin"
)

var (
	Name         = "go.micro.api"
	Address      = ":8080"
	Handler      = "meta"
	Resolver     = "micro"
	RPCPath      = "/rpc"
	APIPath      = "/"
	ProxyPath    = "/{service:[a-zA-Z0-9]+}"
	Namespace    = "go.micro.api"
	HeaderPrefix = "X-Micro-"
)

func run(ctx *cli.Context, srvOpts ...micro.Option) {
	if len(ctx.GlobalString("server_name")) > 0 {
		Name = ctx.GlobalString("server_name")
	}
	if len(ctx.String("address")) > 0 {
		Address = ctx.String("address")
	}
	if len(ctx.String("handler")) > 0 {
		Handler = ctx.String("handler")
	}
	if len(ctx.String("namespace")) > 0 {
		Namespace = ctx.String("namespace")
	}
	if len(ctx.String("resolver")) > 0 {
		Resolver = ctx.String("resolver")
	}

	// Init plugins
	for _, p := range Plugins() {
		p.Init(ctx)
	}

	// Init API
	var opts []server.Option

	if ctx.GlobalBool("enable_acme") {
		hosts := helper.ACMEHosts(ctx)
		opts = append(opts, server.EnableACME(true))
		opts = append(opts, server.ACMEHosts(hosts...))
	} else if ctx.GlobalBool("enable_tls") {
		config, err := helper.TLSConfig(ctx)
		if err != nil {
			fmt.Println(err.Error())
			return
		}

		opts = append(opts, server.EnableTLS(true))
		opts = append(opts, server.TLSConfig(config))
	}

	// create the router
	var h http.Handler
	r := mux.NewRouter()
	h = r

	if ctx.GlobalBool("enable_stats") {
		st := stats.New()
		r.HandleFunc("/stats", st.StatsHandler)
		h = st.ServeHTTP(r)
		st.Start()
		defer st.Stop()
	}

	srvOpts = append(srvOpts, micro.Name(Name))
	if i := time.Duration(ctx.GlobalInt("register_ttl")); i > 0 {
		srvOpts = append(srvOpts, micro.RegisterTTL(i*time.Second))
	}
	if i := time.Duration(ctx.GlobalInt("register_interval")); i > 0 {
		srvOpts = append(srvOpts, micro.RegisterInterval(i*time.Second))
	}

	// initialise service
	service := micro.NewService(srvOpts...)

	// register rpc handler
	log.Logf("Registering RPC Handler at %s", RPCPath)
	r.HandleFunc(RPCPath, handler.RPC)

	// resolver options
	ropts := []resolver.Option{
		resolver.WithNamespace(Namespace),
		resolver.WithHandler(Handler),
	}

	// default resolver
	rr := rrmicro.NewResolver(ropts...)

	switch Resolver {
	case "host":
		rr = host.NewResolver(ropts...)
	case "path":
		rr = path.NewResolver(ropts...)
	case "grpc":
		rr = grpc.NewResolver(ropts...)
	}

	switch Handler {
	case "rpc":
		log.Logf("Registering API RPC Handler at %s", APIPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(arpc.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		rp := arpc.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(rp)
	case "api":
		log.Logf("Registering API Request Handler at %s", APIPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(aapi.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ap := aapi.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(ap)
	case "event":
		log.Logf("Registering API Event Handler at %s", APIPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(event.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ev := event.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(ev)
	case "http", "proxy":
		log.Logf("Registering API HTTP Handler at %s", ProxyPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(ahttp.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		ht := ahttp.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(ProxyPath).Handler(ht)
	case "web":
		log.Logf("Registering API Web Handler at %s", APIPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithHandler(web.Handler),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		w := web.NewHandler(
			ahandler.WithNamespace(Namespace),
			ahandler.WithRouter(rt),
			ahandler.WithService(service),
		)
		r.PathPrefix(APIPath).Handler(w)
	default:
		log.Logf("Registering API Default Handler at %s", APIPath)
		rt := regRouter.NewRouter(
			router.WithNamespace(Namespace),
			router.WithResolver(rr),
			router.WithRegistry(service.Options().Registry),
		)
		r.PathPrefix(APIPath).Handler(handler.Meta(service, rt))
	}

	// reverse wrap handler
	plugins := append(Plugins(), plugin.Plugins()...)
	for i := len(plugins); i > 0; i-- {
		h = plugins[i-1].Handler()(h)
	}

	// create the server
	api := httpapi.NewServer(Address)
	api.Init(opts...)
	api.Handle("/", h)

	// Start API
	if err := api.Start(); err != nil {
		log.Fatal(err)
	}

	// Run server
	if err := service.Run(); err != nil {
		log.Fatal(err)
	}

	// Stop API
	if err := api.Stop(); err != nil {
		log.Fatal(err)
	}
}

func Commands(options ...micro.Option) []cli.Command {
	command := cli.Command{
		Name:  "api",
		Usage: "Run the api gateway",
		Action: func(ctx *cli.Context) {
			run(ctx, options...)
		},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:   "address",
				Usage:  "Set the api address e.g 0.0.0.0:8080",
				EnvVar: "MICRO_API_ADDRESS",
			},
			cli.StringFlag{
				Name:   "handler",
				Usage:  "Specify the request handler to be used for mapping HTTP requests to services; {api, event, http, rpc}",
				EnvVar: "MICRO_API_HANDLER",
			},
			cli.StringFlag{
				Name:   "namespace",
				Usage:  "Set the namespace used by the API e.g. com.example.api",
				EnvVar: "MICRO_API_NAMESPACE",
			},
			cli.StringFlag{
				Name:   "resolver",
				Usage:  "Set the hostname resolver used by the API {host, path, grpc}",
				EnvVar: "MICRO_API_RESOLVER",
			},
		},
	}

	for _, p := range Plugins() {
		if cmds := p.Commands(); len(cmds) > 0 {
			command.Subcommands = append(command.Subcommands, cmds...)
		}

		if flags := p.Flags(); len(flags) > 0 {
			command.Flags = append(command.Flags, flags...)
		}
	}

	return []cli.Command{command}
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

// SetupAPI sets up the api for fetching data from the wargaming api
func SetupAPI() error {
	err := CheckAPI()
	if err != nil {
		return err
	}
	flags := other.Flags
	fmt.Println("setting up the api...")
	if len(flags.WGKey) == 0 {
		return errors.New("No wargaming api key defined use `./wotnlclans -help` to get more info")
	}
	fmt.Println("Running api...")
	clanIds := db.GetClanIDs()
	if len(clanIds) == 0 {
		err := SearchForClanIds(flags, true)
		if err != nil {
			other.DevPrint("ERROR: [SearchForClanIds]:", err.Error())
		}
	} else {
		other.DevPrint("Skipping searching for clans")
		GetClanData(clanIds)
	}
	GetIcons()

	RunSchedule()

	return nil
}

// RunSchedule runs GetClanData every view hours
func RunSchedule() {
	go func() {
		count := 0
		for {
			time.Sleep(time.Hour * 4)
			count++
			if count == 12 {
				count = 0
				err := SearchForClanIds(other.Flags, false)
				if err != nil {
					other.DevPrint("ERROR: [SearchForClanIds]:", err.Error())
				}
			} else {
				err := GetClanData()
				if err != nil {
					other.DevPrint("ERROR: [GetClanData]:", err.Error())
				}
			}
		}
	}()
}

// SearchForClanIds searches through all clans for dutch clans and after that saves them in the database
func SearchForClanIds(flags other.FlagsType, isInit bool) error {
	if isInit && flags.SkipStartupIndexing {
		return nil
	}

	clans, err := GetAllClanIds(flags)
	if err != nil {
		return err
	}

	other.DevPrint("Fetched", len(clans), "clan ids")
	clans = FilterOutClans(clans)
	other.DevPrint("Filtered out all dutch clans,", len(clans), "clans")
	// TODO: Remove blacklisted clans and add extra clans to the clans list
	clans = RemovedDuplicates(clans)
	other.DevPrint("Removed all duplicate clans")
	db.SetClanIDs(clans)

	// when this is ran for the first time make sure to get clan list
	GetClanData(clans)

	return nil
}

// GetClanData returns all needed information about clans
// includedClans is not needed but if you have a database
func GetClanData(includedClans ...[]string) error {
	clans := []string{}
	if len(includedClans) > 0 && len(includedClans[0]) > 0 {
		clans = includedClans[0]
	} else {
		clans = db.GetClanIDs()
	}
	toSave := []db.ClanStats{}
	toFetch := SplitToChucks(clans)
	for _, chunk := range toFetch {
		toInclude := map[string]string{"clanID": strings.Join(chunk, "%2C")}

		var info ClanData
		infoOut, err := CallRoute("clanData", toInclude)
		if err != nil {
			continue
		}
		json.Unmarshal([]byte(infoOut), &info)

		var rating ClanRating
		ratingOut, err := CallRoute("clanRating", toInclude)
		if err != nil {
			continue
		}
		json.Unmarshal([]byte(ratingOut), &rating)

		if info.Status != "ok" || rating.Status != "ok" || len(info.Data) != len(rating.Data) {
			continue
		}

		for i := range info.Data {
			cInfo := info.Data[i]
			cRating := rating.Data[i]
			toSave = append(toSave, db.ClanStats{
				Tag:         cInfo.Tag,
				Name:        cInfo.Name,
				Color:       cInfo.Color,
				Members:     cInfo.MembersCount,
				Description: cInfo.DescriptionHTML,
				Motto:       cInfo.Motto,
				ID:          i,
				Emblems: map[string]string{
					"X256.Wowp":   cInfo.Emblems.X256.Wowp,
					"X195.Portal": cInfo.Emblems.X195.Portal,
					"X64.Portal":  cInfo.Emblems.X64.Portal,
					"X64.Wot":     cInfo.Emblems.X64.Wot,
					"X32.Portal":  cInfo.Emblems.X32.Portal,
					"X24.Portal":  cInfo.Emblems.X24.Portal,
				},
				Stats: db.HistoryClanStats{
					Tag:                cInfo.Tag,
					Name:               cInfo.Name,
					ID:                 i,
					Members:            cInfo.MembersCount,
					Battles:            cRating.BattlesCountAvg.Value,
					DailyBattles:       cRating.BattlesCountAvgDaily.Value,
					Efficiency:         cRating.Efficiency.Value,
					FbElo10:            cRating.FbEloRating10.Value,
					FbElo8:             cRating.FbEloRating8.Value,
					FbElo6:             cRating.FbEloRating6.Value,
					FbElo:              cRating.FbEloRating.Value,
					GmElo10:            cRating.GmEloRating10.Value,
					GmElo8:             cRating.GmEloRating8.Value,
					GmElo6:             cRating.GmEloRating6.Value,
					GmElo:              cRating.GmEloRating.Value,
					GlobRating:         cRating.GlobalRatingAvg.Value,
					GlobRatingWeighted: cRating.GlobalRatingWeightedAvg.Value,
					WinRatio:           cRating.WinsRatioAvg.Value,
					V10l:               cRating.V10lAvg.Value,
				},
			})
		}
	}
	db.SetCurrentClansData(toSave)
	return nil
}

// DutchWords is a list of words and small setences that are usualy in dutch clan descriptions
var DutchWords = []string{
	"verplicht", "menselijkheid", "pannenkoeken", "leeftijd", "minimale", "opzoek", "beginnende", "nederlandse", "spelers", "voldoen", "wij zijn", "gezelligheid", "ons op", "Kom erbij", "minimaal", "gemiddelde", "plezier", "samenwerking", "samenwerken", "aangezien", "toegelaten", "goedkeuring", "gebruik", "tijdens",
}

// BannedWords is a list of words that do usualy not exsits in a dutch clan descriptions
var BannedWords = []string{
	"équipe", "jeux", "ambiance", "rigolade",
}

// IsDutch detecst dutch words in setence
func IsDutch(input string) bool {
	if len(input) == 0 {
		return false
	}
	input = strings.ToLower(input)
	returnStatus := false
	for _, word := range DutchWords {
		if strings.Contains(input, word) {
			returnStatus = true
			break
		}
	}
	for _, word := range BannedWords {
		if strings.Contains(input, word) {
			returnStatus = false
			break
		}
	}
	return returnStatus
}

// SplitToChucks splits up a input list in arrays of 100
// This makes it easy to request a lot of things at the same time from the wargaming api
func SplitToChucks(list []string) [][]string {
	toReturn := [][]string{}
	for _, item := range list {
		if len(toReturn) == 0 || len(toReturn[len(toReturn)-1]) == 100 {
			toReturn = append(toReturn, []string{})
		}
		where := len(toReturn) - 1
		toReturn[where] = append(toReturn[where], item)
	}
	return toReturn
}

// FilterOutClans filters out all not dutch clans out of a input list
func FilterOutClans(clanList []string) []string {
	tofetch := SplitToChucks(clanList)
	toReturn := []string{}
	for _, ids := range tofetch {
		rawOut, err := CallRoute("clanDiscription", map[string]string{"clanID": strings.Join(ids, "%2C")}) // %2C = ,
		if err != nil {
			continue
		}
		var out ClanDiscription
		json.Unmarshal([]byte(rawOut), &out)
		if out.Status != "ok" {
			continue
		}
		for clanID, clan := range out.Data {
			if IsDutch(clan.Description) {
				toReturn = append(toReturn, clanID)
				other.DevPrint("found clan:", clan.Tag)
			}
		}
	}
	return toReturn
}

// RemovedDuplicates removes duplicates from an array
func RemovedDuplicates(input []string) (output []string) {
	output = []string{}
	for _, inputItem := range input {
		exsists := false
	checkArr:
		for _, outputItem := range output {
			if inputItem == outputItem {
				exsists = true
				break checkArr
			}
		}
		if !exsists {
			output = append(output, inputItem)
		}
	}
	return output
}

// GetAllClanIds returns all clan ids
func GetAllClanIds(flags other.FlagsType) ([]string, error) {
	ids := []string{}
	page := 0

	for {
		page++
		if page > flags.MaxIndexPages {
			break
		}
		var out TopClans
		outString, err := CallRoute("topClans", map[string]string{
			"pageNum": fmt.Sprintf("%v", page),
		})

		json.Unmarshal([]byte(outString), &out)

		if err != nil {
			return ids, err
		}

		if out.Status != "ok" {
			return ids, errors.New(out.Error.Message)
		}

		if len(out.Data) == 0 {
			break
		}

		for _, clan := range out.Data {
			ids = append(ids, fmt.Sprintf("%v", clan.ClanID))
		}

		if page%10 == 1 {
			other.DevPrint("Fetched", len(ids), "Clans")
		}
	}
	return ids, nil
}
<commit_msg>Added GetIcons to Schedule<commit_after>package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

// SetupAPI sets up the api for fetching data from the wargaming api
func SetupAPI() error {
	err := CheckAPI()
	if err != nil {
		return err
	}
	flags := other.Flags
	fmt.Println("setting up the api...")
	if len(flags.WGKey) == 0 {
		return errors.New("No wargaming api key defined use `./wotnlclans -help` to get more info")
	}
	fmt.Println("Running api...")
	clanIds := db.GetClanIDs()
	if len(clanIds) == 0 {
		err := SearchForClanIds(flags, true)
		if err != nil {
			other.DevPrint("ERROR: [SearchForClanIds]:", err.Error())
		}
	} else {
		other.DevPrint("Skipping searching for clans")
		GetClanData(clanIds)
	}
	GetIcons()

	RunSchedule()

	return nil
}

// RunSchedule runs GetClanData every view hours
func RunSchedule() {
	go func() {
		count := 0
		for {
			time.Sleep(time.Hour * 4)
			count++
			if count == 12 {
				count = 0
				err := SearchForClanIds(other.Flags, false)
				if err != nil {
					other.DevPrint("ERROR: [SearchForClanIds]:", err.Error())
				}
			} else {
				err := GetClanData()
				if err != nil {
					other.DevPrint("ERROR: [GetClanData]:", err.Error())
				}
			}
			GetIcons()
		}
	}()
}

// SearchForClanIds searches through all clans for dutch clans and after that saves them in the database
func SearchForClanIds(flags other.FlagsType, isInit bool) error {
	if isInit && flags.SkipStartupIndexing {
		return nil
	}

	clans, err := GetAllClanIds(flags)
	if err != nil {
		return err
	}

	other.DevPrint("Fetched", len(clans), "clan ids")
	clans = FilterOutClans(clans)
	other.DevPrint("Filtered out all dutch clans,", len(clans), "clans")
	// TODO: Remove blacklisted clans and add extra clans to the clans list
	clans = RemovedDuplicates(clans)
	other.DevPrint("Removed all duplicate clans")
	db.SetClanIDs(clans)

	// when this is ran for the first time make sure to get clan list
	GetClanData(clans)

	return nil
}

// GetClanData returns all needed information about clans
// includedClans is not needed but if you have a database
func GetClanData(includedClans ...[]string) error {
	clans := []string{}
	if len(includedClans) > 0 && len(includedClans[0]) > 0 {
		clans = includedClans[0]
	} else {
		clans = db.GetClanIDs()
	}
	toSave := []db.ClanStats{}
	toFetch := SplitToChucks(clans)
	for _, chunk := range toFetch {
		toInclude := map[string]string{"clanID": strings.Join(chunk, "%2C")}

		var info ClanData
		infoOut, err := CallRoute("clanData", toInclude)
		if err != nil {
			continue
		}
		json.Unmarshal([]byte(infoOut), &info)

		var rating ClanRating
		ratingOut, err := CallRoute("clanRating", toInclude)
		if err != nil {
			continue
		}
		json.Unmarshal([]byte(ratingOut), &rating)

		if info.Status != "ok" || rating.Status != "ok" || len(info.Data) != len(rating.Data) {
			continue
		}

		for i := range info.Data {
			cInfo := info.Data[i]
			cRating := rating.Data[i]
			toSave = append(toSave, db.ClanStats{
				Tag:         cInfo.Tag,
				Name:        cInfo.Name,
				Color:       cInfo.Color,
				Members:     cInfo.MembersCount,
				Description: cInfo.DescriptionHTML,
				Motto:       cInfo.Motto,
				ID:          i,
				Emblems: map[string]string{
					"X256.Wowp":   cInfo.Emblems.X256.Wowp,
					"X195.Portal": cInfo.Emblems.X195.Portal,
					"X64.Portal":  cInfo.Emblems.X64.Portal,
					"X64.Wot":     cInfo.Emblems.X64.Wot,
					"X32.Portal":  cInfo.Emblems.X32.Portal,
					"X24.Portal":  cInfo.Emblems.X24.Portal,
				},
				Stats: db.HistoryClanStats{
					Tag:                cInfo.Tag,
					Name:               cInfo.Name,
					ID:                 i,
					Members:            cInfo.MembersCount,
					Battles:            cRating.BattlesCountAvg.Value,
					DailyBattles:       cRating.BattlesCountAvgDaily.Value,
					Efficiency:         cRating.Efficiency.Value,
					FbElo10:            cRating.FbEloRating10.Value,
					FbElo8:             cRating.FbEloRating8.Value,
					FbElo6:             cRating.FbEloRating6.Value,
					FbElo:              cRating.FbEloRating.Value,
					GmElo10:            cRating.GmEloRating10.Value,
					GmElo8:             cRating.GmEloRating8.Value,
					GmElo6:             cRating.GmEloRating6.Value,
					GmElo:              cRating.GmEloRating.Value,
					GlobRating:         cRating.GlobalRatingAvg.Value,
					GlobRatingWeighted: cRating.GlobalRatingWeightedAvg.Value,
					WinRatio:           cRating.WinsRatioAvg.Value,
					V10l:               cRating.V10lAvg.Value,
				},
			})
		}
	}
	db.SetCurrentClansData(toSave)
	return nil
}

// DutchWords is a list of words and small setences that are usualy in dutch clan descriptions
var DutchWords = []string{
	"verplicht", "menselijkheid", "pannenkoeken", "leeftijd", "minimale", "opzoek", "beginnende", "nederlandse", "spelers", "voldoen", "wij zijn", "gezelligheid", "ons op", "Kom erbij", "minimaal", "gemiddelde", "plezier", "samenwerking", "samenwerken", "aangezien", "toegelaten", "goedkeuring", "gebruik", "tijdens",
}

// BannedWords is a list of words that do usualy not exsits in a dutch clan descriptions
var BannedWords = []string{
	"équipe", "jeux", "ambiance", "rigolade",
}

// IsDutch detecst dutch words in setence
func IsDutch(input string) bool {
	if len(input) == 0 {
		return false
	}
	input = strings.ToLower(input)
	returnStatus := false
	for _, word := range DutchWords {
		if strings.Contains(input, word) {
			returnStatus = true
			break
		}
	}
	for _, word := range BannedWords {
		if strings.Contains(input, word) {
			returnStatus = false
			break
		}
	}
	return returnStatus
}

// SplitToChucks splits up a input list in arrays of 100
// This makes it easy to request a lot of things at the same time from the wargaming api
func SplitToChucks(list []string) [][]string {
	toReturn := [][]string{}
	for _, item := range list {
		if len(toReturn) == 0 || len(toReturn[len(toReturn)-1]) == 100 {
			toReturn = append(toReturn, []string{})
		}
		where := len(toReturn) - 1
		toReturn[where] = append(toReturn[where], item)
	}
	return toReturn
}

// FilterOutClans filters out all not dutch clans out of a input list
func FilterOutClans(clanList []string) []string {
	tofetch := SplitToChucks(clanList)
	toReturn := []string{}
	for _, ids := range tofetch {
		rawOut, err := CallRoute("clanDiscription", map[string]string{"clanID": strings.Join(ids, "%2C")}) // %2C = ,
		if err != nil {
			continue
		}
		var out ClanDiscription
		json.Unmarshal([]byte(rawOut), &out)
		if out.Status != "ok" {
			continue
		}
		for clanID, clan := range out.Data {
			if IsDutch(clan.Description) {
				toReturn = append(toReturn, clanID)
				other.DevPrint("found clan:", clan.Tag)
			}
		}
	}
	return toReturn
}

// RemovedDuplicates removes duplicates from an array
func RemovedDuplicates(input []string) (output []string) {
	output = []string{}
	for _, inputItem := range input {
		exsists := false
	checkArr:
		for _, outputItem := range output {
			if inputItem == outputItem {
				exsists = true
				break checkArr
			}
		}
		if !exsists {
			output = append(output, inputItem)
		}
	}
	return output
}

// GetAllClanIds returns all clan ids
func GetAllClanIds(flags other.FlagsType) ([]string, error) {
	ids := []string{}
	page := 0

	for {
		page++
		if page > flags.MaxIndexPages {
			break
		}
		var out TopClans
		outString, err := CallRoute("topClans", map[string]string{
			"pageNum": fmt.Sprintf("%v", page),
		})

		json.Unmarshal([]byte(outString), &out)

		if err != nil {
			return ids, err
		}

		if out.Status != "ok" {
			return ids, errors.New(out.Error.Message)
		}

		if len(out.Data) == 0 {
			break
		}

		for _, clan := range out.Data {
			ids = append(ids, fmt.Sprintf("%v", clan.ClanID))
		}

		if page%10 == 1 {
			other.DevPrint("Fetched", len(ids), "Clans")
		}
	}
	return ids, nil
}
<|endoftext|>


<commit_before>package td_client

import (
	"io"
	"time"
	"fmt"
	"strconv"
	"net/url"
	"encoding/json"
)

type ListJobsResultElement struct {
	Id string
	Type string
	Database string
	Status string
	Query string
	StartAt time.Time
	EndAt time.Time
	CpuTime float64
	ResultSize int
	ResultUrl string
	Priority int
	RetryLimit int
}

type ListJobsResult []ListJobsResultElement

var listJobsSchema = map[string]interface{} {
	"jobs": []map[string]interface{} {
		{
			"job_id": "",
			"type": Optional{"", "?"},
			"database": "",
			"status": "",
			"query": "",
			"start_at": time.Time{},
			"end_at": time.Time{},
			"cpu_time": Optional{0., 0.},
			"result_size": Optional{0, 0},
			"result": "",
			"priority": 0,
			"retry_limit": 0,
		},
	},
}

var jobStatusSchema = map[string]interface{} {
	"status": "",
	"job_id": "",
	"start_at": Optional{time.Time{}, time.Time{}},
	"created_at": time.Time{},
	"updated_at": time.Time{},
	"end_at": Optional{time.Time{}, time.Time{}},
	"duration": Optional{0., 0.},
	"cpu_time": Optional{0., 0.},
	"result_size": Optional{0, 0},
}

type ShowJobResultDebugElement struct {
	CmdOut string
	StdErr string
}

type ShowJobResult struct {
	Id string
	Type string
	Database string
	Status string
	Query string
	Debug ShowJobResultDebugElement
	Url string
	StartAt time.Time
	EndAt time.Time
	CpuTime float64
	ResultSize int
	ResultUrl string
	Priority int
	RetryLimit int
	HiveResultSchema map[string]interface{}
}

var showJobSchema = map[string]interface{} {
	"type": Optional{"", "?"},
	"database": "",
	"status": "",
	"query": "",
	"start_at": time.Time{},
	"end_at": time.Time{},
	"cpu_time": Optional{"", ""},
	"result_size": Optional{0, 0},
	"result": "",
	"priority": 0,
	"retry_limit": 0,
	"hive_result_schema": map[string]interface{}{},
}

type Query struct {
	Type string
	Query string
	ResultUrl string
	Priority int
	RetryLimit int
}

var submitJobSchema = map[string]interface{} {
	"job": "",
	"job_id": "",
	"database": "",
}

func (client *TDClient) ListJobs() (*ListJobsResult, error) {
	resp, err := client.get("/v3/jobs/list", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, client.buildError(resp, -1, "List jobs failed", nil)
	}
	js, err := client.checkedJson(resp, listJobsSchema)
	if err != nil {
		return nil, err
	}
	jobs := js["jobs"].([]map[string]interface{})
	retval := make(ListJobsResult, len(jobs))
	for i, v := range jobs {
		retval[i] = ListJobsResultElement {
			Id: v["job_id"].(string),
			Type: v["type"].(string),
			Database: v["database"].(string),
			Status: v["status"].(string),
			Query: v["query"].(string),
			StartAt: v["start_at"].(time.Time),
			EndAt: v["end_at"].(time.Time),
			CpuTime: v["cpu_time"].(float64),
			ResultSize: v["result_size"].(int),
			ResultUrl: v["result_url"].(string),
			Priority: v["priority"].(int),
			RetryLimit: v["retry_limit"].(int),
		}
	}
	return &retval, nil
}

func (client *TDClient) ShowJob(jobId string) (*ShowJobResult, error) {
	resp, err := client.get(fmt.Sprintf("/v3/job/show/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, client.buildError(resp, -1, "Show job failed", nil)
	}
	js, err := client.checkedJson(resp, showJobSchema)
	if err != nil {
		return nil, err
	}
	typeStr := js["type"].(string)
	hiveResultSchema := (map[string]interface{})(nil)
	hiveResultSchemaStr := js["hive_result_schema"].(string)
	err = json.Unmarshal([]byte(hiveResultSchemaStr), &hiveResultSchemaStr)
	if err != nil {
		return nil, err
	}
	return &ShowJobResult {
		Id: js["job_id"].(string),
		Type: typeStr,
		Database: js["database"].(string),
		Status: js["status"].(string),
		Query: js["query"].(string),
		Debug: ShowJobResultDebugElement {
			CmdOut: js["debug"].(map[string]string)["cmdout"],
			StdErr: js["debug"].(map[string]string)["stderr"],
		},
		Url: js["url"].(string),
		StartAt: js["start_at"].(time.Time),
		EndAt: js["end_at"].(time.Time),
		CpuTime: js["cpu_time"].(float64),
		ResultSize: js["result_size"].(int),
		ResultUrl: js["result_url"].(string),
		Priority: js["priority"].(int),
		RetryLimit: js["retry_limit"].(int),
		HiveResultSchema: hiveResultSchema,
	}, nil
}

func (client *TDClient) JobStatus(jobId string) (string, error) {
	resp, err := client.get(fmt.Sprintf("/v3/job/status/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Get job status failed", nil)
	}
	js, err := client.checkedJson(resp, jobStatusSchema)
	if err != nil {
		return "", err
	}
	return js["status"].(string), nil
}

func (client *TDClient) JobResult(jobId string, format string, reader func(io.Reader) error) error {
	resp, err := client.get(fmt.Sprintf("/v3/job/result/%s", url.QueryEscape(jobId)), url.Values { "format": { format }})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return client.buildError(resp, -1, "Get job result failed", nil)
	}
	return reader(resp.Body)
}

func (client *TDClient) JobResultEach(jobId string, reader func(interface{}) error) error {
	return client.JobResult(jobId, "msgpack", func(r io.Reader) error {
		dec := client.getMessagePackDecoder(r)
		for {
			v := (interface{})(nil)
			err := dec.Decode(&v)
			if err != nil {
				if err == io.EOF {
					break
				}
				return &APIError {
					Type: GenericError,
					Message: "Invalid MessagePack stream",
					Cause: err,
				}
			}
			err = reader(v)
			if err != nil {
				return &APIError {
					Type: GenericError,
					Message: "Reader returned error status",
					Cause: err,
				}
			}
		}
		return nil
	})
}

func (client *TDClient) KillJob(jobId string) error {
	resp, err := client.get(fmt.Sprintf("/v3/job/kill/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return client.buildError(resp, -1, "Kill job failed", nil)
	}
	return nil
}

func (client *TDClient) SubmitQuery(db string, q Query) (string, error) {
	params := url.Values {}
	params.Set("query", q.Query)
	if q.ResultUrl != "" {
		params.Set("result", q.ResultUrl)
	}
	if q.Priority >= 0 {
		params.Set("priority", strconv.Itoa(q.Priority))
	}
	if q.RetryLimit >= 0 {
		params.Set("retry_limit", strconv.Itoa(q.RetryLimit))
	}
	resp, err := client.post(fmt.Sprintf("/v3/job/issue/%s/%s", url.QueryEscape(q.Type), url.QueryEscape(db)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Query failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}

func (client *TDClient) SubmitExportJob(db string, table string, storageType string, options map[string]string) (string, error) {
	params := dictToValues(options)
	params.Set("storage_type", storageType)
	resp, err := client.post(fmt.Sprintf("/v3/export/run/%s/%s", url.QueryEscape(db), url.QueryEscape(table)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Export failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}

func (client *TDClient) SubmitPartialDeleteJob(db string, table string, to time.Time, from time.Time, options map[string]string) (string, error) {
	params := dictToValues(options)
	if !to.IsZero() {
		params.Set("to", to.UTC().Format(TDAPIDateTime))
	}
	if !from.IsZero() {
		params.Set("from", from.UTC().Format(TDAPIDateTime))
	}
	resp, err := client.post(fmt.Sprintf("/v3/table/partialdelete/%s/%s", url.QueryEscape(db), url.QueryEscape(table)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Partial delete failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}
<commit_msg>Fix job API.<commit_after>package td_client

import (
	"io"
	"time"
	"fmt"
	"strconv"
	"net/url"
)

type ListJobsResultElement struct {
	Id string
	Type string
	Database string
	Status string
	Query string
	StartAt time.Time
	EndAt time.Time
	CpuTime float64
	ResultSize int
	ResultUrl string
	Priority int
	RetryLimit int
}

type ListJobsResult []ListJobsResultElement

var listJobsSchema = map[string]interface{} {
	"jobs": []map[string]interface{} {
		{
			"job_id": "",
			"type": Optional{"", "?"},
			"database": "",
			"status": "",
			"query": "",
			"start_at": time.Time{},
			"end_at": time.Time{},
			"cpu_time": Optional{0., 0.},
			"result_size": Optional{0, 0},
			"result": "",
			"priority": 0,
			"retry_limit": 0,
		},
	},
}

var jobStatusSchema = map[string]interface{} {
	"status": "",
	"job_id": "",
	"start_at": Optional{time.Time{}, time.Time{}},
	"created_at": time.Time{},
	"updated_at": time.Time{},
	"end_at": Optional{time.Time{}, time.Time{}},
	"duration": Optional{0., 0.},
	"cpu_time": Optional{0., 0.},
	"result_size": Optional{0, 0},
}

type ShowJobResultDebugElement struct {
	CmdOut string
	StdErr string
}

type ShowJobResult struct {
	Id string
	Type string
	Database string
	UserName string
	Status string
	Query string
	Debug ShowJobResultDebugElement
	Url string
	Duration int
	CreatedAt time.Time
	UpdatedAt time.Time
	StartAt time.Time
	EndAt time.Time
	CpuTime float64
	ResultSize int
	ResultUrl string
	Priority int
	RetryLimit int
	HiveResultSchema []interface{}
}

var showJobSchema = map[string]interface{} {
	"job_id": "",
	"type": Optional{"", "?"},
	"organization": Optional{"", ""},
	"user_name": "",
	"database": "",
	"status": "",
	"query": "",
	"debug": map[string]interface{} {
		"cmdout": Optional{"", ""},
		"stderr": Optional{"", ""},
	},
	"url": "",
	"duration": Optional{0, 0},
	"created_at": time.Time{},
	"updated_at": time.Time{},
	"start_at": Optional{time.Time{}, time.Time{}},
	"end_at": Optional{time.Time{}, time.Time{}},
	"cpu_time": Optional{0., 0.},
	"result_size": Optional{0, 0},
	"result": "",
	"priority": 0,
	"retry_limit": 0,
	"hive_result_schema": Optional{EmbeddedJSON([]interface{}{}), nil},
}

type Query struct {
	Type string
	Query string
	ResultUrl string
	Priority int
	RetryLimit int
}

var submitJobSchema = map[string]interface{} {
	"job": "",
	"job_id": "",
	"database": "",
}

func (client *TDClient) ListJobs() (*ListJobsResult, error) {
	resp, err := client.get("/v3/jobs/list", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, client.buildError(resp, -1, "List jobs failed", nil)
	}
	js, err := client.checkedJson(resp, listJobsSchema)
	if err != nil {
		return nil, err
	}
	jobs := js["jobs"].([]map[string]interface{})
	retval := make(ListJobsResult, len(jobs))
	for i, v := range jobs {
		retval[i] = ListJobsResultElement {
			Id: v["job_id"].(string),
			Type: v["type"].(string),
			Database: v["database"].(string),
			Status: v["status"].(string),
			Query: v["query"].(string),
			StartAt: v["start_at"].(time.Time),
			EndAt: v["end_at"].(time.Time),
			CpuTime: v["cpu_time"].(float64),
			ResultSize: v["result_size"].(int),
			ResultUrl: v["result"].(string),
			Priority: v["priority"].(int),
			RetryLimit: v["retry_limit"].(int),
		}
	}
	return &retval, nil
}

func (client *TDClient) ShowJob(jobId string) (*ShowJobResult, error) {
	resp, err := client.get(fmt.Sprintf("/v3/job/show/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, client.buildError(resp, -1, "Show job failed", nil)
	}
	js, err := client.checkedJson(resp, showJobSchema)
	if err != nil {
		return nil, err
	}
	typeStr := js["type"].(string)
	hiveResultSchema, _ := js["hive_result_schema"].([]interface{})
	return &ShowJobResult {
		Id: js["job_id"].(string),
		Type: typeStr,
		Database: js["database"].(string),
		UserName: js["user_name"].(string),
		Status: js["status"].(string),
		Query: js["query"].(string),
		Debug: ShowJobResultDebugElement {
			CmdOut: js["debug"].(map[string]interface{})["cmdout"].(string),
			StdErr: js["debug"].(map[string]interface{})["stderr"].(string),
		},
		Url: js["url"].(string),
		CreatedAt: js["created_at"].(time.Time),
		UpdatedAt: js["updated_at"].(time.Time),
		StartAt: js["start_at"].(time.Time),
		EndAt: js["end_at"].(time.Time),
		CpuTime: js["cpu_time"].(float64),
		ResultSize: js["result_size"].(int),
		ResultUrl: js["result"].(string),
		Priority: js["priority"].(int),
		RetryLimit: js["retry_limit"].(int),
		HiveResultSchema: hiveResultSchema,
	}, nil
}

func (client *TDClient) JobStatus(jobId string) (string, error) {
	resp, err := client.get(fmt.Sprintf("/v3/job/status/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Get job status failed", nil)
	}
	js, err := client.checkedJson(resp, jobStatusSchema)
	if err != nil {
		return "", err
	}
	return js["status"].(string), nil
}

func (client *TDClient) JobResult(jobId string, format string, reader func(io.Reader) error) error {
	resp, err := client.get(fmt.Sprintf("/v3/job/result/%s", url.QueryEscape(jobId)), url.Values { "format": { format }})
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return client.buildError(resp, -1, "Get job result failed", nil)
	}
	return reader(resp.Body)
}

func (client *TDClient) JobResultEach(jobId string, reader func(interface{}) error) error {
	return client.JobResult(jobId, "msgpack", func(r io.Reader) error {
		dec := client.getMessagePackDecoder(r)
		for {
			v := (interface{})(nil)
			err := dec.Decode(&v)
			if err != nil {
				if err == io.EOF {
					break
				}
				return &APIError {
					Type: GenericError,
					Message: "Invalid MessagePack stream",
					Cause: err,
				}
			}
			err = reader(v)
			if err != nil {
				return &APIError {
					Type: GenericError,
					Message: "Reader returned error status",
					Cause: err,
				}
			}
		}
		return nil
	})
}

func (client *TDClient) KillJob(jobId string) error {
	resp, err := client.get(fmt.Sprintf("/v3/job/kill/%s", url.QueryEscape(jobId)), nil)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return client.buildError(resp, -1, "Kill job failed", nil)
	}
	return nil
}

func (client *TDClient) SubmitQuery(db string, q Query) (string, error) {
	params := url.Values {}
	params.Set("query", q.Query)
	if q.ResultUrl != "" {
		params.Set("result", q.ResultUrl)
	}
	if q.Priority >= 0 {
		params.Set("priority", strconv.Itoa(q.Priority))
	}
	if q.RetryLimit >= 0 {
		params.Set("retry_limit", strconv.Itoa(q.RetryLimit))
	}
	resp, err := client.post(fmt.Sprintf("/v3/job/issue/%s/%s", url.QueryEscape(q.Type), url.QueryEscape(db)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Query failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}

func (client *TDClient) SubmitExportJob(db string, table string, storageType string, options map[string]string) (string, error) {
	params := dictToValues(options)
	params.Set("storage_type", storageType)
	resp, err := client.post(fmt.Sprintf("/v3/export/run/%s/%s", url.QueryEscape(db), url.QueryEscape(table)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Export failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}

func (client *TDClient) SubmitPartialDeleteJob(db string, table string, to time.Time, from time.Time, options map[string]string) (string, error) {
	params := dictToValues(options)
	if !to.IsZero() {
		params.Set("to", to.UTC().Format(TDAPIDateTime))
	}
	if !from.IsZero() {
		params.Set("from", from.UTC().Format(TDAPIDateTime))
	}
	resp, err := client.post(fmt.Sprintf("/v3/table/partialdelete/%s/%s", url.QueryEscape(db), url.QueryEscape(table)), params)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", client.buildError(resp, -1, "Partial delete failed", nil)
	}
	js, err := client.checkedJson(resp, submitJobSchema)
	if err != nil {
		return "", err
	}
	return js["job_id"].(string), nil
}
<|endoftext|>