<commit_before>package main

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"
)

var (
	// DbErrAlreadyDefined hapens when the given entry already exists,
	// for example a container.
	DbErrAlreadyDefined = fmt.Errorf("The container/snapshot already exists")

	/* NoSuchObjectError is in the case of joins (and probably other) queries,
	 * we don't get back sql.ErrNoRows when no rows are returned, even though we do
	 * on selects without joins. Instead, you can use this error to
	 * propagate up and generate proper 404s to the client when something
	 * isn't found so we don't abuse sql.ErrNoRows any more than we
	 * already do.
	 */
	NoSuchObjectError = fmt.Errorf("No such object")
)

// Profile is here to order Profiles.
type Profile struct {
	name  string
	order int
}

// Profiles will contain a list of all Profiles.
type Profiles []Profile

// CURRENT_SCHEMA contains the current SQLite SQL Schema.
const CURRENT_SCHEMA string = `
CREATE TABLE IF NOT EXISTS certificates (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    fingerprint VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    certificate TEXT NOT NULL,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (key)
);
CREATE TABLE IF NOT EXISTS containers (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    architecture INTEGER NOT NULL,
    type INTEGER NOT NULL,
    ephemeral INTEGER NOT NULL DEFAULT 0,
    stateful INTEGER NOT NULL DEFAULT 0,
    creation_date DATETIME,
    last_use_date DATETIME,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS containers_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, key)
);
CREATE TABLE IF NOT EXISTS containers_devices (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, name)
);
CREATE TABLE IF NOT EXISTS containers_devices_config (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE,
    UNIQUE (container_device_id, key)
);
CREATE TABLE IF NOT EXISTS containers_profiles (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    profile_id INTEGER NOT NULL,
    apply_order INTEGER NOT NULL default 0,
    UNIQUE (container_id, profile_id),
    FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE,
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    cached INTEGER NOT NULL DEFAULT 0,
    fingerprint VARCHAR(255) NOT NULL,
    filename VARCHAR(255) NOT NULL,
    size INTEGER NOT NULL,
    public INTEGER NOT NULL DEFAULT 0,
    auto_update INTEGER NOT NULL DEFAULT 0,
    architecture INTEGER NOT NULL,
    creation_date DATETIME,
    expiry_date DATETIME,
    upload_date DATETIME NOT NULL,
    last_use_date DATETIME,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS images_aliases (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    image_id INTEGER NOT NULL,
    description VARCHAR(255),
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS images_properties (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    type INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images_source (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    server TEXT NOT NULL,
    protocol INTEGER NOT NULL,
    certificate TEXT NOT NULL,
    alias VARCHAR(255) NOT NULL,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS patches (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    applied_at DATETIME NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    description TEXT,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value VARCHAR(255),
    UNIQUE (profile_id, key),
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    UNIQUE (profile_id, name),
    FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (profile_device_id, key),
    FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS schema (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    version INTEGER NOT NULL,
    updated_at DATETIME NOT NULL,
    UNIQUE (version)
);`

// Create the initial (current) schema for a given SQLite DB connection.
func createDb(db *sql.DB) (err error) {
	latestVersion := dbGetSchema(db)

	if latestVersion != 0 {
		return nil
	}

	_, err = db.Exec(CURRENT_SCHEMA)
	if err != nil {
		return err
	}

	// There isn't an entry for schema version, let's put it in.
	insertStmt := `INSERT INTO schema (version, updated_at) values (?, strftime("%s"));`
	_, err = db.Exec(insertStmt, dbGetLatestSchema())
	if err != nil {
		return err
	}

	err = dbProfileCreateDefault(db)
	if err != nil {
		return err
	}

	return dbProfileCreateDocker(db)
}

func dbGetSchema(db *sql.DB) (v int) {
	arg1 := []interface{}{}
	arg2 := []interface{}{&v}
	q := "SELECT max(version) FROM schema"
	err := dbQueryRowScan(db, q, arg1, arg2)
	if err != nil {
		return 0
	}
	return v
}

func dbGetLatestSchema() int {
	return dbUpdates[len(dbUpdates)-1].version
}

// Create a database connection object and return it.
func initializeDbObject(d *Daemon, path string) (err error) {
	var openPath string

	timeout := 5 // TODO - make this command-line configurable?

	// These are used to tune the transaction BEGIN behavior instead of using the
	// similar "locking_mode" pragma (locking for the whole database connection).
	openPath = fmt.Sprintf("%s?_busy_timeout=%d&_txlock=exclusive", path, timeout*1000)

	// Open the database. If the file doesn't exist it is created.
	d.db, err = sql.Open("sqlite3", openPath)
	if err != nil {
		return err
	}

	// Create the DB if it doesn't exist.
	err = createDb(d.db)
	if err != nil {
		return fmt.Errorf("Error creating database: %s", err)
	}

	// Run PRAGMA statements now since they are *per-connection*.
	d.db.Exec("PRAGMA foreign_keys=ON;") // This allows us to use ON DELETE CASCADE

	// Apply any update
	err = dbUpdatesApplyAll(d)
	if err != nil {
		return err
	}

	return nil
}

func isDbLockedError(err error) bool {
	if err == nil {
		return false
	}
	if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
		return true
	}
	if err.Error() == "database is locked" {
		return true
	}
	return false
}

func isNoMatchError(err error) bool {
	if err == nil {
		return false
	}
	if err.Error() == "sql: no rows in result set" {
		return true
	}
	return false
}

func dbBegin(db *sql.DB) (*sql.Tx, error) {
	for i := 0; i < 100; i++ {
		tx, err := db.Begin()
		if err == nil {
			return tx, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbBegin: error %q", err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbBegin: DB still locked")
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func txCommit(tx *sql.Tx) error {
	for i := 0; i < 100; i++ {
		err := tx.Commit()
		if err == nil {
			return nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("Txcommit: error %q", err)
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("Txcommit: db still locked")
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQueryRowScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) error {
	for i := 0; i < 100; i++ {
		err := db.QueryRow(q, args...).Scan(outargs...)
		if err == nil {
			return nil
		}
		if isNoMatchError(err) {
			return err
		}
		if !isDbLockedError(err) {
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryRowScan: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQuery(db *sql.DB, q string, args ...interface{}) (*sql.Rows, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Query(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQuery: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func doDbQueryScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) ([][]interface{}, error) {
	rows, err := db.Query(q, args...)
	if err != nil {
		return [][]interface{}{}, err
	}
	defer rows.Close()
	result := [][]interface{}{}
	for rows.Next() {
		ptrargs := make([]interface{}, len(outargs))
		for i := range outargs {
			switch t := outargs[i].(type) {
			case string:
				str := ""
				ptrargs[i] = &str
			case int:
				integer := 0
				ptrargs[i] = &integer
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		err = rows.Scan(ptrargs...)
		if err != nil {
			return [][]interface{}{}, err
		}
		newargs := make([]interface{}, len(outargs))
		for i := range ptrargs {
			switch t := outargs[i].(type) {
			case string:
				newargs[i] = *ptrargs[i].(*string)
			case int:
				newargs[i] = *ptrargs[i].(*int)
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		result = append(result, newargs)
	}
	err = rows.Err()
	if err != nil {
		return [][]interface{}{}, err
	}
	return result, nil
}

/*
 * . q is the database query
 * . inargs is an array of interfaces containing the query arguments
 * . outfmt is an array of interfaces containing the right types of output
 *   arguments, i.e.
 *      var arg1 string
 *      var arg2 int
 *      outfmt := {}interface{}{arg1, arg2}
 *
 * The result will be an array (one per output row) of arrays (one per output argument)
 * of interfaces, containing pointers to the actual output arguments.
 */
func dbQueryScan(db *sql.DB, q string, inargs []interface{}, outfmt []interface{}) ([][]interface{}, error) {
	for i := 0; i < 100; i++ {
		result, err := doDbQueryScan(db, q, inargs, outfmt)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryscan: query %q inargs %q, DB still locked", q, inargs)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func dbExec(db *sql.DB, q string, args ...interface{}) (sql.Result, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Exec(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbExec: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbExec: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}
<commit_msg>db: Fix int64 handling<commit_after>package main

import (
	"database/sql"
	"fmt"
	"time"

	"github.com/mattn/go-sqlite3"

	"github.com/lxc/lxd/shared"
)

var (
	// DbErrAlreadyDefined hapens when the given entry already exists,
	// for example a container.
	DbErrAlreadyDefined = fmt.Errorf("The container/snapshot already exists")

	/* NoSuchObjectError is in the case of joins (and probably other) queries,
	 * we don't get back sql.ErrNoRows when no rows are returned, even though we do
	 * on selects without joins. Instead, you can use this error to
	 * propagate up and generate proper 404s to the client when something
	 * isn't found so we don't abuse sql.ErrNoRows any more than we
	 * already do.
	 */
	NoSuchObjectError = fmt.Errorf("No such object")
)

// Profile is here to order Profiles.
type Profile struct {
	name  string
	order int
}

// Profiles will contain a list of all Profiles.
type Profiles []Profile

// CURRENT_SCHEMA contains the current SQLite SQL Schema.
const CURRENT_SCHEMA string = `
CREATE TABLE IF NOT EXISTS certificates (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    fingerprint VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    certificate TEXT NOT NULL,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (key)
);
CREATE TABLE IF NOT EXISTS containers (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    architecture INTEGER NOT NULL,
    type INTEGER NOT NULL,
    ephemeral INTEGER NOT NULL DEFAULT 0,
    stateful INTEGER NOT NULL DEFAULT 0,
    creation_date DATETIME,
    last_use_date DATETIME,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS containers_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, key)
);
CREATE TABLE IF NOT EXISTS containers_devices (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    FOREIGN KEY (container_id) REFERENCES containers (id) ON DELETE CASCADE,
    UNIQUE (container_id, name)
);
CREATE TABLE IF NOT EXISTS containers_devices_config (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (container_device_id) REFERENCES containers_devices (id) ON DELETE CASCADE,
    UNIQUE (container_device_id, key)
);
CREATE TABLE IF NOT EXISTS containers_profiles (
    id INTEGER primary key AUTOINCREMENT NOT NULL,
    container_id INTEGER NOT NULL,
    profile_id INTEGER NOT NULL,
    apply_order INTEGER NOT NULL default 0,
    UNIQUE (container_id, profile_id),
    FOREIGN KEY (container_id) REFERENCES containers(id) ON DELETE CASCADE,
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    cached INTEGER NOT NULL DEFAULT 0,
    fingerprint VARCHAR(255) NOT NULL,
    filename VARCHAR(255) NOT NULL,
    size INTEGER NOT NULL,
    public INTEGER NOT NULL DEFAULT 0,
    auto_update INTEGER NOT NULL DEFAULT 0,
    architecture INTEGER NOT NULL,
    creation_date DATETIME,
    expiry_date DATETIME,
    upload_date DATETIME NOT NULL,
    last_use_date DATETIME,
    UNIQUE (fingerprint)
);
CREATE TABLE IF NOT EXISTS images_aliases (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    image_id INTEGER NOT NULL,
    description VARCHAR(255),
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS images_properties (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    type INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS images_source (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    image_id INTEGER NOT NULL,
    server TEXT NOT NULL,
    protocol INTEGER NOT NULL,
    certificate TEXT NOT NULL,
    alias VARCHAR(255) NOT NULL,
    FOREIGN KEY (image_id) REFERENCES images (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS patches (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    applied_at DATETIME NOT NULL,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    name VARCHAR(255) NOT NULL,
    description TEXT,
    UNIQUE (name)
);
CREATE TABLE IF NOT EXISTS profiles_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value VARCHAR(255),
    UNIQUE (profile_id, key),
    FOREIGN KEY (profile_id) REFERENCES profiles(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_id INTEGER NOT NULL,
    name VARCHAR(255) NOT NULL,
    type INTEGER NOT NULL default 0,
    UNIQUE (profile_id, name),
    FOREIGN KEY (profile_id) REFERENCES profiles (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS profiles_devices_config (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    profile_device_id INTEGER NOT NULL,
    key VARCHAR(255) NOT NULL,
    value TEXT,
    UNIQUE (profile_device_id, key),
    FOREIGN KEY (profile_device_id) REFERENCES profiles_devices (id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS schema (
    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    version INTEGER NOT NULL,
    updated_at DATETIME NOT NULL,
    UNIQUE (version)
);`

// Create the initial (current) schema for a given SQLite DB connection.
func createDb(db *sql.DB) (err error) {
	latestVersion := dbGetSchema(db)

	if latestVersion != 0 {
		return nil
	}

	_, err = db.Exec(CURRENT_SCHEMA)
	if err != nil {
		return err
	}

	// There isn't an entry for schema version, let's put it in.
	insertStmt := `INSERT INTO schema (version, updated_at) values (?, strftime("%s"));`
	_, err = db.Exec(insertStmt, dbGetLatestSchema())
	if err != nil {
		return err
	}

	err = dbProfileCreateDefault(db)
	if err != nil {
		return err
	}

	return dbProfileCreateDocker(db)
}

func dbGetSchema(db *sql.DB) (v int) {
	arg1 := []interface{}{}
	arg2 := []interface{}{&v}
	q := "SELECT max(version) FROM schema"
	err := dbQueryRowScan(db, q, arg1, arg2)
	if err != nil {
		return 0
	}
	return v
}

func dbGetLatestSchema() int {
	return dbUpdates[len(dbUpdates)-1].version
}

// Create a database connection object and return it.
func initializeDbObject(d *Daemon, path string) (err error) {
	var openPath string

	timeout := 5 // TODO - make this command-line configurable?

	// These are used to tune the transaction BEGIN behavior instead of using the
	// similar "locking_mode" pragma (locking for the whole database connection).
	openPath = fmt.Sprintf("%s?_busy_timeout=%d&_txlock=exclusive", path, timeout*1000)

	// Open the database. If the file doesn't exist it is created.
	d.db, err = sql.Open("sqlite3", openPath)
	if err != nil {
		return err
	}

	// Create the DB if it doesn't exist.
	err = createDb(d.db)
	if err != nil {
		return fmt.Errorf("Error creating database: %s", err)
	}

	// Run PRAGMA statements now since they are *per-connection*.
	d.db.Exec("PRAGMA foreign_keys=ON;") // This allows us to use ON DELETE CASCADE

	// Apply any update
	err = dbUpdatesApplyAll(d)
	if err != nil {
		return err
	}

	return nil
}

func isDbLockedError(err error) bool {
	if err == nil {
		return false
	}
	if err == sqlite3.ErrLocked || err == sqlite3.ErrBusy {
		return true
	}
	if err.Error() == "database is locked" {
		return true
	}
	return false
}

func isNoMatchError(err error) bool {
	if err == nil {
		return false
	}
	if err.Error() == "sql: no rows in result set" {
		return true
	}
	return false
}

func dbBegin(db *sql.DB) (*sql.Tx, error) {
	for i := 0; i < 100; i++ {
		tx, err := db.Begin()
		if err == nil {
			return tx, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbBegin: error %q", err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbBegin: DB still locked")
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func txCommit(tx *sql.Tx) error {
	for i := 0; i < 100; i++ {
		err := tx.Commit()
		if err == nil {
			return nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("Txcommit: error %q", err)
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("Txcommit: db still locked")
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQueryRowScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) error {
	for i := 0; i < 100; i++ {
		err := db.QueryRow(q, args...).Scan(outargs...)
		if err == nil {
			return nil
		}
		if isNoMatchError(err) {
			return err
		}
		if !isDbLockedError(err) {
			return err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryRowScan: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return fmt.Errorf("DB is locked")
}

func dbQuery(db *sql.DB, q string, args ...interface{}) (*sql.Rows, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Query(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQuery: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func doDbQueryScan(db *sql.DB, q string, args []interface{}, outargs []interface{}) ([][]interface{}, error) {
	rows, err := db.Query(q, args...)
	if err != nil {
		return [][]interface{}{}, err
	}
	defer rows.Close()
	result := [][]interface{}{}
	for rows.Next() {
		ptrargs := make([]interface{}, len(outargs))
		for i := range outargs {
			switch t := outargs[i].(type) {
			case string:
				str := ""
				ptrargs[i] = &str
			case int:
				integer := 0
				ptrargs[i] = &integer
			case int64:
				integer := int64(0)
				ptrargs[i] = &integer
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		err = rows.Scan(ptrargs...)
		if err != nil {
			return [][]interface{}{}, err
		}
		newargs := make([]interface{}, len(outargs))
		for i := range ptrargs {
			switch t := outargs[i].(type) {
			case string:
				newargs[i] = *ptrargs[i].(*string)
			case int:
				newargs[i] = *ptrargs[i].(*int)
			case int64:
				newargs[i] = *ptrargs[i].(*int64)
			default:
				return [][]interface{}{}, fmt.Errorf("Bad interface type: %s", t)
			}
		}
		result = append(result, newargs)
	}
	err = rows.Err()
	if err != nil {
		return [][]interface{}{}, err
	}
	return result, nil
}

/*
 * . q is the database query
 * . inargs is an array of interfaces containing the query arguments
 * . outfmt is an array of interfaces containing the right types of output
 *   arguments, i.e.
 *      var arg1 string
 *      var arg2 int
 *      outfmt := {}interface{}{arg1, arg2}
 *
 * The result will be an array (one per output row) of arrays (one per output argument)
 * of interfaces, containing pointers to the actual output arguments.
 */
func dbQueryScan(db *sql.DB, q string, inargs []interface{}, outfmt []interface{}) ([][]interface{}, error) {
	for i := 0; i < 100; i++ {
		result, err := doDbQueryScan(db, q, inargs, outfmt)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbQuery: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbQueryscan: query %q inargs %q, DB still locked", q, inargs)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}

func dbExec(db *sql.DB, q string, args ...interface{}) (sql.Result, error) {
	for i := 0; i < 100; i++ {
		result, err := db.Exec(q, args...)
		if err == nil {
			return result, nil
		}
		if !isDbLockedError(err) {
			shared.LogDebugf("DbExec: query %q error %q", q, err)
			return nil, err
		}
		time.Sleep(100 * time.Millisecond)
	}

	shared.LogDebugf("DbExec: query %q args %q, DB still locked", q, args)
	shared.PrintStack()
	return nil, fmt.Errorf("DB is locked")
}
<|endoftext|>


<commit_before>package exec

import (
	"os"
	goexec "os/exec"
	"strings"

	"github.com/mlafeldt/chef-runner/log"
)

type RunnerFunc func(args []string) error

func DefaultRunner(args []string) error {
	log.Debugf("exec: %s\n", strings.Join(args, " "))
	cmd := goexec.Command(args[0], args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

var runnerFunc = DefaultRunner

func SetRunnerFunc(f RunnerFunc) {
	runnerFunc = f
}

func RunCommand(args []string) error {
	return runnerFunc(args)
}
<commit_msg>Document exec package<commit_after>// Package exec runs external commands. It's a wrapper around Go's os/exec
// package that allows to stub command execution for testing.
package exec

import (
	"os"
	goexec "os/exec"
	"strings"

	"github.com/mlafeldt/chef-runner/log"
)

// The RunnerFunc type is an adapter to use any function for running commands.
type RunnerFunc func(args []string) error

// DefaultRunner is the default function used to run commands. It calls
// os/exec.Run so that stdout and stderr are written to the terminal.
// DefaultRunner also logs all executed commands.
func DefaultRunner(args []string) error {
	log.Debugf("exec: %s\n", strings.Join(args, " "))
	cmd := goexec.Command(args[0], args[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

var runnerFunc = DefaultRunner

// SetRunnerFunc registers the function f to run all future commands.
func SetRunnerFunc(f RunnerFunc) {
	runnerFunc = f
}

// RunCommand runs the specified command using the currently registered
// RunnerFunc.
func RunCommand(args []string) error {
	return runnerFunc(args)
}
<|endoftext|>


<commit_before>package rebecca

import (
	"reflect"
	"testing"

	"github.com/waterlink/rebecca/driver/fake"
)

func TestExec(t *testing.T) {
	d := fake.NewDriver()
	SetupDriver(d)

	if err := Exec("SOME QUERY $1, $2", 42, "hello"); err != nil {
		t.Fatal()
	}

	actual := d.ReceivedExec()
	expected := fake.ReceivedExec{
		Tx:    nil,
		Query: "SOME QUERY $1, $2",
		Args:  []interface{}{42, "hello"},
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected driver to receive exec %#v, but got %#v", expected, actual)
	}
}

func TestTransactionExec(t *testing.T) {
	d := fake.NewDriver()
	SetupDriver(d)

	tx, err := Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer tx.Rollback()

	if err := tx.Exec("SOME QUERY $1, $2", 42, "hello"); err != nil {
		t.Fatal()
	}

	actual := d.ReceivedExec()
	expected := fake.ReceivedExec{
		Tx:    tx.tx,
		Query: "SOME QUERY $1, $2",
		Args:  []interface{}{42, "hello"},
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected driver to receive exec %#v, but got %#v", expected, actual)
	}
}
<commit_msg>Satisfy linters<commit_after>package rebecca

import (
	"reflect"
	"testing"

	"github.com/waterlink/rebecca/driver/fake"
)

func TestExec(t *testing.T) {
	d := fake.NewDriver()
	SetupDriver(d)

	if err := Exec("SOME QUERY $1, $2", 42, "hello"); err != nil {
		t.Fatal(err)
	}

	actual := d.ReceivedExec()
	expected := fake.ReceivedExec{
		Tx:    nil,
		Query: "SOME QUERY $1, $2",
		Args:  []interface{}{42, "hello"},
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected driver to receive exec %#v, but got %#v", expected, actual)
	}
}

func TestTransactionExec(t *testing.T) {
	d := fake.NewDriver()
	SetupDriver(d)

	tx, err := Begin()
	if err != nil {
		t.Fatal(err)
	}
	defer tx.Rollback()

	if err := tx.Exec("SOME QUERY $1, $2", 42, "hello"); err != nil {
		t.Fatal(err)
	}

	actual := d.ReceivedExec()
	expected := fake.ReceivedExec{
		Tx:    tx.tx,
		Query: "SOME QUERY $1, $2",
		Args:  []interface{}{42, "hello"},
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected driver to receive exec %#v, but got %#v", expected, actual)
	}
}
<|endoftext|>


<commit_before>package sirius

import (
	"time"
)

type Execution struct {
	Ext Extension
	Msg Message
	Cfg ExtensionConfig
}

type ExecutionResult struct {
	Err    error
	Action MessageAction
}

type ExtensionRunner interface {
	Run([]Execution, chan<- ExecutionResult, time.Duration)
}

type AsyncRunner struct{}

func NewExecution(x Extension, m Message, cfg ExtensionConfig) *Execution {
	return &Execution{
		Ext: x,
		Msg: m,
		Cfg: cfg,
	}
}

func NewAsyncRunner() *AsyncRunner {
	return &AsyncRunner{}
}

// Run executes all extensions in exe, and returns all ExecutionResults that
// are received before timeout has elapsed.
func (r *AsyncRunner) Run(exe []Execution, res chan<- ExecutionResult, timeout time.Duration) {
	er := make(chan ExecutionResult, len(exe))

	for _, e := range exe {
		go func(ex Execution, r chan<- ExecutionResult) {
			a, err := ex.Ext.Run(e.Msg, e.Cfg)

			r <- ExecutionResult{
				Err:    err,
				Action: a,
			}
		}(e, er)
	}

Execution:
	for range exe {
		select {
		case <-time.After(timeout):
			break Execution
		case res <- <-er:
		}
	}

	close(res)
}
<commit_msg>Use copied instead of captured Msg and Cfg variables<commit_after>package sirius

import (
	"time"
)

type Execution struct {
	Ext Extension
	Msg Message
	Cfg ExtensionConfig
}

type ExecutionResult struct {
	Err    error
	Action MessageAction
}

type ExtensionRunner interface {
	Run([]Execution, chan<- ExecutionResult, time.Duration)
}

type AsyncRunner struct{}

func NewExecution(x Extension, m Message, cfg ExtensionConfig) *Execution {
	return &Execution{
		Ext: x,
		Msg: m,
		Cfg: cfg,
	}
}

func NewAsyncRunner() *AsyncRunner {
	return &AsyncRunner{}
}

// Run executes all extensions in exe, and returns all ExecutionResults that
// are received before timeout has elapsed.
func (r *AsyncRunner) Run(exe []Execution, res chan<- ExecutionResult, timeout time.Duration) {
	er := make(chan ExecutionResult, len(exe))

	for _, e := range exe {
		go func(ex Execution, r chan<- ExecutionResult) {
			a, err := ex.Ext.Run(ex.Msg, ex.Cfg)

			r <- ExecutionResult{
				Err:    err,
				Action: a,
			}
		}(e, er)
	}

Execution:
	for range exe {
		select {
		case <-time.After(timeout):
			break Execution
		case res <- <-er:
		}
	}

	close(res)
}
<|endoftext|>


<commit_before>package client

import (
	opClient "github.com/coreos-inc/operator-client/pkg/client"
	"github.com/pkg/errors"
	corev1 "k8s.io/api/core/v1"
	v1beta1extensions "k8s.io/api/extensions/v1beta1"
	v1beta1rbac "k8s.io/api/rbac/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type InstallStrategyDeploymentInterface interface {
	CreateRole(role *v1beta1rbac.Role) (*v1beta1rbac.Role, error)
	CreateRoleBinding(roleBinding *v1beta1rbac.RoleBinding) (*v1beta1rbac.RoleBinding, error)
	EnsureServiceAccount(serviceAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error)
	CreateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error)
	CreateOrUpdateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error)
	DeleteDeployment(name string) error
	GetServiceAccountByName(serviceAccountName string) (*corev1.ServiceAccount, error)
	GetOwnedDeployments(owner metav1.ObjectMeta) (*v1beta1extensions.DeploymentList, error)
}

type InstallStrategyDeploymentClientForNamespace struct {
	opClient  opClient.Interface
	Namespace string
}

var _ InstallStrategyDeploymentInterface = &InstallStrategyDeploymentClientForNamespace{}

func NewInstallStrategyDeploymentClient(opClient opClient.Interface, namespace string) InstallStrategyDeploymentInterface {
	return &InstallStrategyDeploymentClientForNamespace{
		opClient:  opClient,
		Namespace: namespace,
	}
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateRole(role *v1beta1rbac.Role) (*v1beta1rbac.Role, error) {
	return c.opClient.KubernetesInterface().RbacV1beta1().Roles(c.Namespace).Create(role)
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateRoleBinding(roleBinding *v1beta1rbac.RoleBinding) (*v1beta1rbac.RoleBinding, error) {
	return c.opClient.KubernetesInterface().RbacV1beta1().RoleBindings(c.Namespace).Create(roleBinding)
}

func (c *InstallStrategyDeploymentClientForNamespace) EnsureServiceAccount(serviceAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error) {
	foundAccount, err := c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Get(serviceAccount.Name, metav1.GetOptions{})
	if err == nil {
		return foundAccount, nil
	}
	if !apierrors.IsNotFound(err) {
		return nil, errors.Wrap(err, "checking for existing serviceacccount failed")
	}

	createdAccount, err := c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Create(serviceAccount)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return nil, errors.Wrap(err, "creating serviceacccount failed")
	}
	if apierrors.IsAlreadyExists(err) {
		return serviceAccount, nil
	}
	return createdAccount, nil
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error) {
	return c.opClient.CreateDeployment(deployment)
}

func (c *InstallStrategyDeploymentClientForNamespace) DeleteDeployment(name string) error {
	return c.opClient.DeleteDeployment(c.Namespace, name, false) // non-cascading delete
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateOrUpdateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error) {
	_, d, err := c.opClient.CreateOrRollingUpdateDeployment(deployment)
	return d, err
}

func (c *InstallStrategyDeploymentClientForNamespace) GetServiceAccountByName(serviceAccountName string) (*corev1.ServiceAccount, error) {
	return c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Get(serviceAccountName, metav1.GetOptions{})
}

func (c *InstallStrategyDeploymentClientForNamespace) GetOwnedDeployments(owner metav1.ObjectMeta) (*v1beta1extensions.DeploymentList, error) {
	return c.opClient.ListDeploymentsWithLabels(c.Namespace, map[string]string{
		"alm-owner-name":      owner.Name,
		"alm-owner-namespace": owner.Namespace,
	})
}
<commit_msg>fix(pkg/client/deployment): use cascading delete for deployments<commit_after>package client

import (
	opClient "github.com/coreos-inc/operator-client/pkg/client"
	"github.com/pkg/errors"
	corev1 "k8s.io/api/core/v1"
	v1beta1extensions "k8s.io/api/extensions/v1beta1"
	v1beta1rbac "k8s.io/api/rbac/v1beta1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type InstallStrategyDeploymentInterface interface {
	CreateRole(role *v1beta1rbac.Role) (*v1beta1rbac.Role, error)
	CreateRoleBinding(roleBinding *v1beta1rbac.RoleBinding) (*v1beta1rbac.RoleBinding, error)
	EnsureServiceAccount(serviceAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error)
	CreateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error)
	CreateOrUpdateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error)
	DeleteDeployment(name string) error
	GetServiceAccountByName(serviceAccountName string) (*corev1.ServiceAccount, error)
	GetOwnedDeployments(owner metav1.ObjectMeta) (*v1beta1extensions.DeploymentList, error)
}

type InstallStrategyDeploymentClientForNamespace struct {
	opClient  opClient.Interface
	Namespace string
}

var _ InstallStrategyDeploymentInterface = &InstallStrategyDeploymentClientForNamespace{}

func NewInstallStrategyDeploymentClient(opClient opClient.Interface, namespace string) InstallStrategyDeploymentInterface {
	return &InstallStrategyDeploymentClientForNamespace{
		opClient:  opClient,
		Namespace: namespace,
	}
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateRole(role *v1beta1rbac.Role) (*v1beta1rbac.Role, error) {
	return c.opClient.KubernetesInterface().RbacV1beta1().Roles(c.Namespace).Create(role)
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateRoleBinding(roleBinding *v1beta1rbac.RoleBinding) (*v1beta1rbac.RoleBinding, error) {
	return c.opClient.KubernetesInterface().RbacV1beta1().RoleBindings(c.Namespace).Create(roleBinding)
}

func (c *InstallStrategyDeploymentClientForNamespace) EnsureServiceAccount(serviceAccount *corev1.ServiceAccount) (*corev1.ServiceAccount, error) {
	foundAccount, err := c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Get(serviceAccount.Name, metav1.GetOptions{})
	if err == nil {
		return foundAccount, nil
	}
	if !apierrors.IsNotFound(err) {
		return nil, errors.Wrap(err, "checking for existing serviceacccount failed")
	}

	createdAccount, err := c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Create(serviceAccount)
	if err != nil && !apierrors.IsAlreadyExists(err) {
		return nil, errors.Wrap(err, "creating serviceacccount failed")
	}
	if apierrors.IsAlreadyExists(err) {
		return serviceAccount, nil
	}
	return createdAccount, nil
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error) {
	return c.opClient.CreateDeployment(deployment)
}

func (c *InstallStrategyDeploymentClientForNamespace) DeleteDeployment(name string) error {
	return c.opClient.DeleteDeployment(c.Namespace, name, true) // cascading delete, e.g. foreground
}

func (c *InstallStrategyDeploymentClientForNamespace) CreateOrUpdateDeployment(deployment *v1beta1extensions.Deployment) (*v1beta1extensions.Deployment, error) {
	_, d, err := c.opClient.CreateOrRollingUpdateDeployment(deployment)
	return d, err
}

func (c *InstallStrategyDeploymentClientForNamespace) GetServiceAccountByName(serviceAccountName string) (*corev1.ServiceAccount, error) {
	return c.opClient.KubernetesInterface().CoreV1().ServiceAccounts(c.Namespace).Get(serviceAccountName, metav1.GetOptions{})
}

func (c *InstallStrategyDeploymentClientForNamespace) GetOwnedDeployments(owner metav1.ObjectMeta) (*v1beta1extensions.DeploymentList, error) {
	return c.opClient.ListDeploymentsWithLabels(c.Namespace, map[string]string{
		"alm-owner-name":      owner.Name,
		"alm-owner-namespace": owner.Namespace,
	})
}
<|endoftext|>


<commit_before>package expr

import (
	"context"

	utilctx "github.com/go-graphite/carbonapi/util/ctx"

	"github.com/ansel1/merry"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	_ "github.com/go-graphite/carbonapi/expr/functions"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type evaluator struct{}

// FetchAndEvalExp fetch data and evalualtes expressions
func (eval evaluator) FetchAndEvalExp(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	config.Config.Limiter.Enter()
	defer config.Config.Limiter.Leave()

	multiFetchRequest := pb.MultiFetchRequest{}
	metricRequestCache := make(map[string]parser.MetricRequest)
	maxDataPoints := utilctx.GetMaxDatapoints(ctx)
	// values related to this particular `target=`
	targetValues := make(map[parser.MetricRequest][]*types.MetricData)

	for _, m := range exp.Metrics() {
		fetchRequest := pb.FetchRequest{
			Name:           m.Metric,
			PathExpression: m.Metric,
			StartTime:      m.From + from,
			StopTime:       m.Until + until,
			MaxDataPoints:  maxDataPoints,
		}
		metricRequest := parser.MetricRequest{
			Metric: fetchRequest.PathExpression,
			From:   fetchRequest.StartTime,
			Until:  fetchRequest.StopTime,
		}

		// avoid multiple requests in a function, E.g divideSeries(a.b, a.b)
		if cachedMetricRequest, ok := metricRequestCache[m.Metric]; ok &&
			cachedMetricRequest.From == metricRequest.From &&
			cachedMetricRequest.Until == metricRequest.Until {
			continue
		}

		// avoid multiple requests in a http request, E.g render?target=a.b&target=a.b
		if _, ok := values[metricRequest]; ok {
			targetValues[metricRequest] = nil
			continue
		}

		// avoid multiple requests from the same target, e.g. target=max(a,asPercent(holtWintersForecast(a),a))
		if _, ok := targetValues[metricRequest]; ok {
			continue
		}

		metricRequestCache[m.Metric] = metricRequest
		targetValues[metricRequest] = nil
		multiFetchRequest.Metrics = append(multiFetchRequest.Metrics, fetchRequest)
	}

	if len(multiFetchRequest.Metrics) > 0 {
		metrics, _, err := config.Config.ZipperInstance.Render(ctx, multiFetchRequest)
		// If we had only partial result, we want to do our best to actually do our job
		if err != nil && merry.HTTPCode(err) >= 400 && exp.Target() != "fallbackSeries" {
			return nil, err
		}
		for _, metric := range metrics {
			metricRequest := metricRequestCache[metric.PathExpression]
			if metric.RequestStartTime != 0 && metric.RequestStopTime != 0 {
				metricRequest.From = metric.RequestStartTime
				metricRequest.Until = metric.RequestStopTime
			}
			data, ok := values[metricRequest]
			if !ok {
				data = make([]*types.MetricData, 0, 1)
			}
			values[metricRequest] = append(data, metric)
		}
	}

	for m := range targetValues {
		targetValues[m] = values[m]
	}

	if config.Config.ZipperInstance.ScaleToCommonStep() {
		targetValues = helper.ScaleValuesToCommonStep(targetValues)
	}

	return eval.Eval(ctx, exp, from, until, targetValues)
}

// Eval evalualtes expressions
func (eval evaluator) Eval(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (results []*types.MetricData, err error) {
	rewritten, targets, err := RewriteExpr(ctx, exp, from, until, values)
	if err != nil {
		return nil, err
	}
	if rewritten {
		for _, target := range targets {
			exp, _, err = parser.ParseExpr(target)
			if err != nil {
				return nil, err
			}
			result, err := eval.FetchAndEvalExp(ctx, exp, from, until, values)
			if err != nil {
				return nil, err
			}
			results = append(results, result...)
		}
		return results, nil
	}
	return EvalExpr(ctx, exp, from, until, values)
}

var _evaluator = evaluator{}

func init() {
	helper.SetEvaluator(_evaluator)
	metadata.SetEvaluator(_evaluator)
}

// FetchAndEvalExp fetch data and evalualtes expressions
func FetchAndEvalExp(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	return _evaluator.FetchAndEvalExp(ctx, e, from, until, values)
}

// Eval is the main expression evaluator
func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		return nil, merry.WithHTTPCode(parser.ErrMissingArgument, 400)
	}

	metadata.FunctionMD.RLock()
	f, ok := metadata.FunctionMD.Functions[e.Target()]
	metadata.FunctionMD.RUnlock()
	if ok {
		v, err := f.Do(ctx, e, from, until, values)
		if err != nil {
			err = merry.WithMessagef(err, "function=%s", e.Target())
		}
		return v, err
	}

	return nil, merry.WithHTTPCode(helper.ErrUnknownFunction(e.Target()), 400)
}

// RewriteExpr expands targets that use applyByNode into a new list of targets.
// eg:
// applyByNode(foo*, 1, "%") -> (true, ["foo1", "foo2"], nil)
// sumSeries(foo) -> (false, nil, nil)
// Assumes that applyByNode only appears as the outermost function.
func RewriteExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (bool, []string, error) {
	if e.IsFunc() {
		metadata.FunctionMD.RLock()
		f, ok := metadata.FunctionMD.RewriteFunctions[e.Target()]
		metadata.FunctionMD.RUnlock()
		if ok {
			return f.Do(ctx, e, from, until, values)
		}
	}
	return false, nil, nil
}
<commit_msg>Improve error messages and set 400 code for parser Errs<commit_after>package expr

import (
	"context"

	utilctx "github.com/go-graphite/carbonapi/util/ctx"

	"github.com/ansel1/merry"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	_ "github.com/go-graphite/carbonapi/expr/functions"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type evaluator struct{}

// FetchAndEvalExp fetch data and evalualtes expressions
func (eval evaluator) FetchAndEvalExp(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	config.Config.Limiter.Enter()
	defer config.Config.Limiter.Leave()

	multiFetchRequest := pb.MultiFetchRequest{}
	metricRequestCache := make(map[string]parser.MetricRequest)
	maxDataPoints := utilctx.GetMaxDatapoints(ctx)
	// values related to this particular `target=`
	targetValues := make(map[parser.MetricRequest][]*types.MetricData)

	for _, m := range exp.Metrics() {
		fetchRequest := pb.FetchRequest{
			Name:           m.Metric,
			PathExpression: m.Metric,
			StartTime:      m.From + from,
			StopTime:       m.Until + until,
			MaxDataPoints:  maxDataPoints,
		}
		metricRequest := parser.MetricRequest{
			Metric: fetchRequest.PathExpression,
			From:   fetchRequest.StartTime,
			Until:  fetchRequest.StopTime,
		}

		// avoid multiple requests in a function, E.g divideSeries(a.b, a.b)
		if cachedMetricRequest, ok := metricRequestCache[m.Metric]; ok &&
			cachedMetricRequest.From == metricRequest.From &&
			cachedMetricRequest.Until == metricRequest.Until {
			continue
		}

		// avoid multiple requests in a http request, E.g render?target=a.b&target=a.b
		if _, ok := values[metricRequest]; ok {
			targetValues[metricRequest] = nil
			continue
		}

		// avoid multiple requests from the same target, e.g. target=max(a,asPercent(holtWintersForecast(a),a))
		if _, ok := targetValues[metricRequest]; ok {
			continue
		}

		metricRequestCache[m.Metric] = metricRequest
		targetValues[metricRequest] = nil
		multiFetchRequest.Metrics = append(multiFetchRequest.Metrics, fetchRequest)
	}

	if len(multiFetchRequest.Metrics) > 0 {
		metrics, _, err := config.Config.ZipperInstance.Render(ctx, multiFetchRequest)
		// If we had only partial result, we want to do our best to actually do our job
		if err != nil && merry.HTTPCode(err) >= 400 && exp.Target() != "fallbackSeries" {
			return nil, err
		}
		for _, metric := range metrics {
			metricRequest := metricRequestCache[metric.PathExpression]
			if metric.RequestStartTime != 0 && metric.RequestStopTime != 0 {
				metricRequest.From = metric.RequestStartTime
				metricRequest.Until = metric.RequestStopTime
			}
			data, ok := values[metricRequest]
			if !ok {
				data = make([]*types.MetricData, 0, 1)
			}
			values[metricRequest] = append(data, metric)
		}
	}

	for m := range targetValues {
		targetValues[m] = values[m]
	}

	if config.Config.ZipperInstance.ScaleToCommonStep() {
		targetValues = helper.ScaleValuesToCommonStep(targetValues)
	}

	return eval.Eval(ctx, exp, from, until, targetValues)
}

// Eval evalualtes expressions
func (eval evaluator) Eval(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (results []*types.MetricData, err error) {
	rewritten, targets, err := RewriteExpr(ctx, exp, from, until, values)
	if err != nil {
		return nil, err
	}
	if rewritten {
		for _, target := range targets {
			exp, _, err = parser.ParseExpr(target)
			if err != nil {
				return nil, err
			}
			result, err := eval.FetchAndEvalExp(ctx, exp, from, until, values)
			if err != nil {
				return nil, err
			}
			results = append(results, result...)
		}
		return results, nil
	}
	return EvalExpr(ctx, exp, from, until, values)
}

var _evaluator = evaluator{}

func init() {
	helper.SetEvaluator(_evaluator)
	metadata.SetEvaluator(_evaluator)
}

// FetchAndEvalExp fetch data and evalualtes expressions
func FetchAndEvalExp(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	return _evaluator.FetchAndEvalExp(ctx, e, from, until, values)
}

// Eval is the main expression evaluator
func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		err := merry.WithMessagef(parser.ErrMissingArgument, "target=%s: %s", e.Target(), parser.ErrMissingArgument)
		return nil, merry.WithHTTPCode(err, 400)
	}

	metadata.FunctionMD.RLock()
	f, ok := metadata.FunctionMD.Functions[e.Target()]
	metadata.FunctionMD.RUnlock()
	if ok {
		v, err := f.Do(ctx, e, from, until, values)
		if err != nil {
			err = merry.WithMessagef(err, "function=%s: %s", e.Target(), err.Error())
			if merry.Is(
				err,
				parser.ErrMissingExpr,
				parser.ErrMissingComma,
				parser.ErrMissingQuote,
				parser.ErrUnexpectedCharacter,
				parser.ErrBadType,
				parser.ErrMissingArgument,
				parser.ErrMissingTimeseries,
				parser.ErrSeriesDoesNotExist,
				parser.ErrUnknownTimeUnits,
			) {
				err = merry.WithHTTPCode(err, 400)
			}
		}
		return v, err
	}

	return nil, merry.WithHTTPCode(helper.ErrUnknownFunction(e.Target()), 400)
}

// RewriteExpr expands targets that use applyByNode into a new list of targets.
// eg:
// applyByNode(foo*, 1, "%") -> (true, ["foo1", "foo2"], nil)
// sumSeries(foo) -> (false, nil, nil)
// Assumes that applyByNode only appears as the outermost function.
func RewriteExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (bool, []string, error) {
	if e.IsFunc() {
		metadata.FunctionMD.RLock()
		f, ok := metadata.FunctionMD.RewriteFunctions[e.Target()]
		metadata.FunctionMD.RUnlock()
		if ok {
			return f.Do(ctx, e, from, until, values)
		}
	}
	return false, nil, nil
}
<|endoftext|>


<commit_before>package rest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"labix.org/v2/mgo"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"sync"
	"testing"
)

// 1
func TestConfig(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_config", nil)

	m.ServeHTTP(res, req)
	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[]}`)
}

// 2
func TestConfigResonseField(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "test", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_config", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"test":[]}`)
}

// 3
func TestInvalidPost(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString("string"))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusBadRequest)
	expect(t, res.Body.String(), `{"error":"invalid character 's' looking for beginning of value"}`)
}

// 4
func TestPost(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString(`{"foo":"bar"}`))

	m.ServeHTTP(res, req)

	resp := res.Body.String()
	expect(t, res.Code, http.StatusCreated)

	var body map[string]string
	json.Unmarshal([]byte(resp), &body)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_create/"+body["_id"], nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)

	resp = res.Body.String()
	expect(t, res.Code, http.StatusOK)

	var body2 map[string]interface{}
	json.Unmarshal([]byte(resp), &body2)

	item, _ := body2["data"].(map[string]string)
	expect(t, body["_id"], item["_id"])
}

// 5
func TestPostIntId(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString(`{"_id":1,"foo":"bar"}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusCreated)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar"}}`)
}

// 6
func TestCollection(t *testing.T) {
	defer dropDb()
	generateData(3, "test_collection")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_collection", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"_id":1,"foo":"bar-3"},{"_id":2,"foo":"bar-2"},{"_id":3,"foo":"bar-1"}]}`)
}

// 7
func TestQuery(t *testing.T) {
	defer dropDb()
	generateData(3, "test_query")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", `/api/v1/test_query?query={"foo":"bar-2"}`, nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"_id":2,"foo":"bar-2"}]}`)
}

// 8
func TestLimitSkipSortSelect(t *testing.T) {
	defer dropDb()
	generateData(20, "test_limit_skip_sort_select")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", `/api/v1/test_limit_skip_sort_select?sort=_id&limit=3&skip=3&select={"_id":0}`, nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"foo":"bar-17"},{"foo":"bar-16"},{"foo":"bar-15"}]}`)
}

// 9
func TestCount(t *testing.T) {
	defer dropDb()
	generateData(3, "test_count")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_count?count=1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":3}`)
}

// 10
func TestNotFound(t *testing.T) {
	defer dropDb()
	generateData(3, "test_not_found")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_not_found/4", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusNotFound)
	expect(t, res.Body.String(), `{"error":"not found"}`)
}

// 11
func TestInvalidPut(t *testing.T) {
	defer dropDb()
	generateData(1, "test_i_put")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_i_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-1"}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("PUT", "/api/v1/test_i_put/1", bytes.NewBufferString(`{"_id":1,"foo":"bar-2","test":1}}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusBadRequest)
	expect(t, res.Body.String(), `{"error":"invalid character '}' after top-level value"}`)

}

// 12
func TestPut(t *testing.T) {
	defer dropDb()
	generateData(1, "test_put")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-1"}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("PUT", "/api/v1/test_put/1", bytes.NewBufferString(`{"_id":1,"foo":"bar-2","test":1}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"updated":1}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-2","test":1}}`)

}

// 13
func TestDelete(t *testing.T) {
	defer dropDb()
	generateData(1, "test_delete")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("DELETE", "/api/v1/test_delete/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"removed":1}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_delete/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusNotFound)
	expect(t, res.Body.String(), `{"error":"not found"}`)

}

// 14
func TestApplyHandler(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}), func() (int, string) {
		return http.StatusForbidden, "test forbidden"
	})

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_apply", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusForbidden)
	expect(t, res.Body.String(), `test forbidden`)
}

// 15
func TestAutoIncrement(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", true}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_autoinc", bytes.NewBufferString(`{"foo":"bar"}`))
	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusCreated)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar"}}`)
}

// helpers
var db *mgo.Database
var wg sync.WaitGroup
var once sync.Once

func generateData(n int, name string) {
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	for i := range make([]int, n) {
		res := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/api/v1/"+name,
			bytes.NewBufferString(
				fmt.Sprintf(`{"_id":%d,"foo":"bar-%d"}`, i+1, n-i)))
		m.ServeHTTP(res, req)
	}
}

func dropDb() {
	wg.Done()
	go func() {
		once.Do(func() {
			wg.Wait()
			db.DropDatabase()
		})
	}()
}

func getDb() *mgo.Database {
	if db == nil {
		wg.Add(15) // test cases counter
		var mongoUri string
		if os.Getenv("WERCKER_MONGODB_HOST") > 0 {
			mongoUri = os.Getenv("WERCKER_MONGODB_HOST") + os.Getenv("WERCKER_MONGODB_PORT")
		} else {
			mongoUri = "localhost"
		}
		session, _ := mgo.Dial(mongoUri)
		db = session.DB("_rest_test")
		db.DropDatabase()
	}
	return db
}

func expect(t *testing.T, a interface{}, b interface{}) {
	if a != b {
		t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
	}
}

// API prefix for routing it right.
<commit_msg>test env hotfix<commit_after>package rest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"labix.org/v2/mgo"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"sync"
	"testing"
)

// 1
func TestConfig(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_config", nil)

	m.ServeHTTP(res, req)
	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[]}`)
}

// 2
func TestConfigResonseField(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "test", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_config", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"test":[]}`)
}

// 3
func TestInvalidPost(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString("string"))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusBadRequest)
	expect(t, res.Body.String(), `{"error":"invalid character 's' looking for beginning of value"}`)
}

// 4
func TestPost(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString(`{"foo":"bar"}`))

	m.ServeHTTP(res, req)

	resp := res.Body.String()
	expect(t, res.Code, http.StatusCreated)

	var body map[string]string
	json.Unmarshal([]byte(resp), &body)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_create/"+body["_id"], nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)

	resp = res.Body.String()
	expect(t, res.Code, http.StatusOK)

	var body2 map[string]interface{}
	json.Unmarshal([]byte(resp), &body2)

	item, _ := body2["data"].(map[string]string)
	expect(t, body["_id"], item["_id"])
}

// 5
func TestPostIntId(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_create", bytes.NewBufferString(`{"_id":1,"foo":"bar"}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusCreated)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar"}}`)
}

// 6
func TestCollection(t *testing.T) {
	defer dropDb()
	generateData(3, "test_collection")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_collection", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"_id":1,"foo":"bar-3"},{"_id":2,"foo":"bar-2"},{"_id":3,"foo":"bar-1"}]}`)
}

// 7
func TestQuery(t *testing.T) {
	defer dropDb()
	generateData(3, "test_query")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", `/api/v1/test_query?query={"foo":"bar-2"}`, nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"_id":2,"foo":"bar-2"}]}`)
}

// 8
func TestLimitSkipSortSelect(t *testing.T) {
	defer dropDb()
	generateData(20, "test_limit_skip_sort_select")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", `/api/v1/test_limit_skip_sort_select?sort=_id&limit=3&skip=3&select={"_id":0}`, nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":[{"foo":"bar-17"},{"foo":"bar-16"},{"foo":"bar-15"}]}`)
}

// 9
func TestCount(t *testing.T) {
	defer dropDb()
	generateData(3, "test_count")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_count?count=1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":3}`)
}

// 10
func TestNotFound(t *testing.T) {
	defer dropDb()
	generateData(3, "test_not_found")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_not_found/4", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusNotFound)
	expect(t, res.Body.String(), `{"error":"not found"}`)
}

// 11
func TestInvalidPut(t *testing.T) {
	defer dropDb()
	generateData(1, "test_i_put")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_i_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-1"}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("PUT", "/api/v1/test_i_put/1", bytes.NewBufferString(`{"_id":1,"foo":"bar-2","test":1}}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusBadRequest)
	expect(t, res.Body.String(), `{"error":"invalid character '}' after top-level value"}`)

}

// 12
func TestPut(t *testing.T) {
	defer dropDb()
	generateData(1, "test_put")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-1"}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("PUT", "/api/v1/test_put/1", bytes.NewBufferString(`{"_id":1,"foo":"bar-2","test":1}`))

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"updated":1}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_put/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar-2","test":1}}`)

}

// 13
func TestDelete(t *testing.T) {
	defer dropDb()
	generateData(1, "test_delete")
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("DELETE", "/api/v1/test_delete/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusOK)
	expect(t, res.Body.String(), `{"data":{"removed":1}}`)

	res = httptest.NewRecorder()
	req, _ = http.NewRequest("GET", "/api/v1/test_delete/1", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusNotFound)
	expect(t, res.Body.String(), `{"error":"not found"}`)

}

// 14
func TestApplyHandler(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}), func() (int, string) {
		return http.StatusForbidden, "test forbidden"
	})

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("GET", "/api/v1/test_apply", nil)

	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusForbidden)
	expect(t, res.Body.String(), `test forbidden`)
}

// 15
func TestAutoIncrement(t *testing.T) {
	defer dropDb()
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", true}))

	res := httptest.NewRecorder()
	req, _ := http.NewRequest("POST", "/api/v1/test_autoinc", bytes.NewBufferString(`{"foo":"bar"}`))
	m.ServeHTTP(res, req)

	expect(t, res.Code, http.StatusCreated)
	expect(t, res.Body.String(), `{"data":{"_id":1,"foo":"bar"}}`)
}

// helpers
var db *mgo.Database
var wg sync.WaitGroup
var once sync.Once

func generateData(n int, name string) {
	m := martini.Classic()
	m.Group("/api/v1", Rest(Config{getDb(), "data", false}))

	for i := range make([]int, n) {
		res := httptest.NewRecorder()
		req, _ := http.NewRequest("POST", "/api/v1/"+name,
			bytes.NewBufferString(
				fmt.Sprintf(`{"_id":%d,"foo":"bar-%d"}`, i+1, n-i)))
		m.ServeHTTP(res, req)
	}
}

func dropDb() {
	wg.Done()
	go func() {
		once.Do(func() {
			wg.Wait()
			db.DropDatabase()
		})
	}()
}

func getDb() *mgo.Database {
	if db == nil {
		wg.Add(15) // test cases counter
		var mongoUri string
		if len(os.Getenv("WERCKER_MONGODB_HOST")) > 0 {
			mongoUri = os.Getenv("WERCKER_MONGODB_HOST") + os.Getenv("WERCKER_MONGODB_PORT")
		} else {
			mongoUri = "localhost"
		}
		session, _ := mgo.Dial(mongoUri)
		db = session.DB("_rest_test")
		db.DropDatabase()
	}
	return db
}

func expect(t *testing.T, a interface{}, b interface{}) {
	if a != b {
		t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
	}
}

// API prefix for routing it right.
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/codegangsta/cli"
	"github.com/hashicorp/serf/command/agent"
	"github.com/hashicorp/serf/serf"
)

func StartMaster(c *cli.Context) {
	rpcAddress := c.GlobalString("rpc")
	rpcAuthKey := c.GlobalString("rpc-auth")
	masterAddress := c.String("master")
	listenAddress := c.String("listen")

	fields := strings.Split(listenAddress, ":")
	if len(fields) != 2 {
		log.Fatalf("listen requires host:port! %s is not valid", listenAddress)
	}
	bindAddr := fields[0]
	bindPort, err := strconv.Atoi(fields[1])
	if err != nil {
		log.Fatalf("Unable to parse %s into port", fields[1])
	}
	serfConfig := serf.DefaultConfig()
	serfConfig.MemberlistConfig.BindAddr = bindAddr
	serfConfig.MemberlistConfig.BindPort = bindPort
	if c.IsSet("name") {
		serfConfig.NodeName = c.String("name")
	}
	serfConfig.Tags = map[string]string{"master": "true"}

	log.Printf("Starting master on %s", listenAddress)
	log.Printf("Starting master RPC listener on %s", rpcAddress)
	//TODO pull this crap into the Master type
	logOutput := os.Stderr
	logWriter := agent.NewLogWriter(123)
	a, err := agent.Create(agent.DefaultConfig(), serfConfig, logWriter)
	if err != nil {
		log.Fatalf("Unable to create agent: %s", err)
	}

	//register event handlers
	meh := MasterEventHandler{}
	a.RegisterEventHandler(&meh)

	if err := a.Start(); err != nil {
		log.Fatalf("Unable to start agent: %s", err)
	}

	log.Printf("Joining cluster by way of %s", masterAddress)
	n, err := a.Join([]string{masterAddress}, true)
	if n > 0 {
		log.Printf("Cluster joined; %d nodes participating", n)
	}
	if err != nil {
		log.Fatalf("unable to join cluster: %s", err)
	}

	members := a.Serf().Members()
	log.Printf("%d nodes currently in cluster:", len(members))
	for _, m := range members {
		log.Printf("  %s %s:%d %v %s", m.Name, m.Addr, m.Port, m.Tags, m.Status)
	}

	rpcListener, err := net.Listen("tcp", rpcAddress)
	if err != nil {
		log.Fatalf("Error starting RPC listener: %s", err)
	}
	//TODO should we listen for shutdown signals and close the agent properly?
	agent.NewAgentIPC(a, rpcAuthKey, rpcListener, logOutput, logWriter)
	select {}

}

type MasterEventHandler struct {
	//Agent   *agent.Agent //TODO is this necessary? how do we rebroadcast a query that comes in over RPC?
	Queries []*serf.Query
	sync.Mutex
}

func (m *MasterEventHandler) HandleEvent(e serf.Event) {
	switch evt := e.(type) {
	case *serf.Query:
		//query := e.(*serf.Query)
		log.Printf("%s: payload %q", evt.EventType(), evt.Payload)
		m.Lock()
		defer m.Unlock()
		m.Queries = append(m.Queries, evt)
		//TODO broadcast this query? it came in over RPC...
		log.Print("TODO broadcast this query!?????")
		err := evt.Respond([]byte("fuck"))
		if err != nil {
			log.Printf("Unable to respond to query: %s", err)
		}
		//(name string, payload []byte, params *QueryParam) (*QueryResponse, error)
		//TODO what query params? (filternodes, filtertags, requestack, timeout)
		/* dont do this; it causes infinite queries to be handled by the master
		_, err := m.Agent.Serf().Query(query.Name, query.Payload, nil)
		if err != nil {
			log.Printf("[ERROR] unable to rebroadcast query: %s", err)
			return
		}
		*/
	case serf.UserEvent:
		log.Printf("%s: %s with payload %q (coalescable: %t)", evt.EventType(), evt.Name, evt.Payload, evt.Coalesce)
	case serf.MemberEvent:
		log.Printf("%s: %v", evt.EventType(), evt.Members)
	default:
		log.Printf("[EVENT] %s", evt.EventType())
	}
}
<commit_msg>UDP is fucky on OSX, this works fine<commit_after>package main

import (
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/codegangsta/cli"
	"github.com/hashicorp/serf/command/agent"
	"github.com/hashicorp/serf/serf"
)

func StartMaster(c *cli.Context) {
	rpcAddress := c.GlobalString("rpc")
	rpcAuthKey := c.GlobalString("rpc-auth")
	masterAddress := c.String("master")
	listenAddress := c.String("listen")

	fields := strings.Split(listenAddress, ":")
	if len(fields) != 2 {
		log.Fatalf("listen requires host:port! %s is not valid", listenAddress)
	}
	bindAddr := fields[0]
	bindPort, err := strconv.Atoi(fields[1])
	if err != nil {
		log.Fatalf("Unable to parse %s into port", fields[1])
	}
	serfConfig := serf.DefaultConfig()
	serfConfig.MemberlistConfig.BindAddr = bindAddr
	serfConfig.MemberlistConfig.BindPort = bindPort
	if c.IsSet("name") {
		serfConfig.NodeName = c.String("name")
	}
	serfConfig.Tags = map[string]string{"master": "true"}

	log.Printf("Starting master on %s", listenAddress)
	log.Printf("Starting master RPC listener on %s", rpcAddress)
	//TODO pull this crap into the Master type
	logOutput := os.Stderr
	logWriter := agent.NewLogWriter(123)
	a, err := agent.Create(agent.DefaultConfig(), serfConfig, logWriter)
	if err != nil {
		log.Fatalf("Unable to create agent: %s", err)
	}

	//register event handlers
	meh := MasterEventHandler{}
	a.RegisterEventHandler(&meh)

	if err := a.Start(); err != nil {
		log.Fatalf("Unable to start agent: %s", err)
	}

	log.Printf("Joining cluster by way of %s", masterAddress)
	n, err := a.Join([]string{masterAddress}, true)
	if n > 0 {
		log.Printf("Cluster joined; %d nodes participating", n)
	}
	if err != nil {
		log.Fatalf("unable to join cluster: %s", err)
	}

	members := a.Serf().Members()
	log.Printf("%d nodes currently in cluster:", len(members))
	for _, m := range members {
		log.Printf("  %s %s:%d %v %s", m.Name, m.Addr, m.Port, m.Tags, m.Status)
	}

	rpcListener, err := net.Listen("tcp", rpcAddress)
	if err != nil {
		log.Fatalf("Error starting RPC listener: %s", err)
	}
	//TODO should we listen for shutdown signals and close the agent properly?
	agent.NewAgentIPC(a, rpcAuthKey, rpcListener, logOutput, logWriter)
	select {}

}

type MasterEventHandler struct {
	//Agent   *agent.Agent //TODO is this necessary? how do we rebroadcast a query that comes in over RPC?
	Queries []*serf.Query
	sync.Mutex
}

func (m *MasterEventHandler) HandleEvent(e serf.Event) {
	switch evt := e.(type) {
	case *serf.Query:
		//query := e.(*serf.Query)
		log.Printf("%s: payload %q", evt.EventType(), evt.Payload)
		m.Lock()
		defer m.Unlock()
		m.Queries = append(m.Queries, evt)
		// we dont respond to queries :)
		/*
			err := evt.Respond([]byte("fuck"))
			if err != nil {
				log.Printf("Unable to respond to query: %s", err)
			}
		*/
	case serf.UserEvent:
		log.Printf("%s: %s with payload %q (coalescable: %t)", evt.EventType(), evt.Name, evt.Payload, evt.Coalesce)
	case serf.MemberEvent:
		log.Printf("%s: %v", evt.EventType(), evt.Members)
	default:
		log.Printf("[EVENT] %s", evt.EventType())
	}
}
<|endoftext|>


<commit_before>/*
multitemplate is a package for executing templates from multiple template languages.
This package is the revel connector for multitemplate. Below is the details on how to use
multitemplate in your revel application.

app.conf
  // multitemplate must be added to your module list in your app.conf
  module.template=github.com/acsellers/multitemplate/revel
  module.jobs=github.com/revel/revel/modules/jobs

app/controllers/init.go
  // You can set DefaultLayouts for different content types, which are
  // found in your normal template paths
  multitemplate.DefaultLayout[multitemplate.HTML] = "layouts/app.html"

  // You must initialize the library in revel's OnAppStart function.
	revel.OnAppStart(multitemplate.Init)

app/init.go
  // If you want to use the auto-refresh functionality in DevMode, you need to add the
  // multitemplate.ReloadFilter to the revel.Filters list.
  func init() {
    revel.Filters = []revel.Filter{
      revel.PanicFilter,             // Recover from panics and display an error page instead.
      revel.RouterFilter,            // Use the routing table to select the right Action
      multitemplate.ReloadFilter,    // Filter for the multitemplate auto-refresh of templates
      revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
      ...

app/controllers/base_controller.go

  // Import the controllers area from multitemplate's revel connector.
  // Note that the package name that folder is multitemplate, so it is easier to know
  // where the Controller is from.
  import "github.com/acsellers/multitemplate/revel/app/controllers"

  // Simply replace any *revel.Controllers with multitemplate.Controllers
  type BaseController struct {
    multitemplate.Controller
    Txn *database.Transaction
  }

And that's it!
*/
package multitemplaterevel
<commit_msg>Add integration guide for revel to godoc<commit_after>/*
multitemplate is a package for executing templates from multiple template languages.
This package is the revel connector for multitemplate. Below is the details on how to
setup multitemplate so you can use it in your revel application.

app.conf
  // multitemplate must be added to your module list in your app.conf
  module.template=github.com/acsellers/multitemplate/revel
  module.jobs=github.com/revel/revel/modules/jobs

app/controllers/init.go

  func init() {
          // You can set DefaultLayouts for different content types, which are
          // found in your normal template paths
          multitemplate.DefaultLayout[multitemplate.HTML] = "layouts/app.html"

          // You must initialize the library in revel's OnAppStart function.
          revel.OnAppStart(multitemplate.Init)
  }

app/init.go

  // import the multitemplate/revel/app/controllers package and any sub-languages you want
  // to load
  import (
          _ "github.com/acsellers/multitemplate/bham"
          "github.com/acsellers/multitemplate/revel/app/controllers"
          "github.com/revel/revel"
  )

  // If you want to use the auto-refresh functionality in DevMode, you need to add the
  // multitemplate.ReloadFilter to the revel.Filters list.
  func init() {
          revel.Filters = []revel.Filter{
                  revel.PanicFilter,             // Recover from panics and display an error page instead.
                  revel.RouterFilter,            // Use the routing table to select the right Action
                  multitemplate.ReloadFilter,    // Filter for the multitemplate auto-refresh of templates
                  revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
                  ...

app/controllers/base_controller.go

  // Import the controllers area from multitemplate's revel connector.
  // Note that the package name that folder is multitemplate, so it is easier to know
  // where the Controller is from.
  import "github.com/acsellers/multitemplate/revel/app/controllers"

  // Simply replace any *revel.Controllers with multitemplate.Controllers
  type BaseController struct {
          multitemplate.Controller
          Txn *database.Transaction
  }

And that's it!
*/
package multitemplaterevel
<|endoftext|>


<commit_before>package revfluent

import (
    "fmt"
    "github.com/fluent/fluent-logger-golang/fluent"
    "github.com/revel/revel"
)

var (
    Logger *fluent.Fluent // Global logger
)

// Reads configuration and connects to fluentd
func Init() {
    timeout, err := time.ParseDuration(revel.Config.StringDefault("revfluent.timeout", fluent.defaultTimeout.String()))
    if err != nil {
        revel.ERROR.Panic(err)
    }

    port := revel.Config.IntDefault("revfluent.port", fluent.defaultPort)
    host := revel.Config.StringDefault("revfluent.host", fluent.defaultHost)
    network := revel.Config.StringDefault("revfluent.network", fluent.defaultNetwork)
    socketPath := revel.Config.StringDefault("revfluent.socketPath", fluent.defaultSocketPath)
    bufferLimit := revel.Config.IntDefault("revfluent.bufferLimit", fluent.defaultBufferLimit)
    retryWait := revel.Config.IntDefault("revfluent.retryWait", fluent.defaultRetryWait)
    maxRetry := revel.Config.IntDefault("revfluent.maxRetry", fluent.defaultMaxRetry)
    tagPrefix := revel.Config.StringDefault("revfluent.tagPrefix", revel.Config.String("app.name"))

    config := fluent.Config{
        FluentPort:       port,
        FluentHost:       host,
        FluentNetwork:    network,
        FluentSocketPath: socketPath,
        Timeout:          timeout,
        BufferLimit:      bufferLimit,
        RetryWait:        retryWait,
        MaxRetry:         maxRetry,
        TagPrefix:        tagPrefix,
    }

    revel.INFO.Printf("Connecting Fluentd: %s:%i", host, port)

    Logger, err := fluent.New(config)
    if err != nil {
        revel.ERROR.Panic("Failed to connect Fluentd: %s", err)
    }
}
<commit_msg>Removed default values<commit_after>package revfluent

import (
    "github.com/fluent/fluent-logger-golang/fluent"
    "github.com/revel/revel"
    "time"
)

var (
    Logger *fluent.Fluent // Global logger
)

// Reads configuration and connects to fluentd
func Init() {
    timeString, _ := revel.Config.String("revfluent.timeout")
    timeout, err := time.ParseDuration(timeString)
    if err != nil {
        revel.ERROR.Panic(err)
    }

    port, _ := revel.Config.Int("revfluent.port")
    host, _ := revel.Config.String("revfluent.host")
    network, _ := revel.Config.String("revfluent.network")
    socketPath, _ := revel.Config.String("revfluent.socketPath")
    bufferLimit, _ := revel.Config.Int("revfluent.bufferLimit")
    retryWait, _ := revel.Config.Int("revfluent.retryWait")
    maxRetry, _ := revel.Config.Int("revfluent.maxRetry")

    appName, _ := revel.Config.String("app.name")
    tagPrefix := revel.Config.StringDefault("revfluent.tagPrefix", appName)

    config := fluent.Config{
        FluentPort:       port,
        FluentHost:       host,
        FluentNetwork:    network,
        FluentSocketPath: socketPath,
        Timeout:          timeout,
        BufferLimit:      bufferLimit,
        RetryWait:        retryWait,
        MaxRetry:         maxRetry,
        TagPrefix:        tagPrefix,
    }

    revel.INFO.Printf("Connecting Fluentd: %s:%i", host, port)

    Logger, err := fluent.New(config)
    if err != nil {
        revel.ERROR.Panic("Failed to connect Fluentd: %s", err)
    }
}
<|endoftext|>


<commit_before>package sirius

type Extension interface {
	Run(Message) (error, MessageAction)
}

type CfgExtension interface {
	RunWithConfig(Message, ExtensionConfig)
}

type cfg_int int
type cfg_bool bool
type cfg_float float64
type cfg_list []string

type EID string

type ExtensionLoader interface {
	Load(EID) Extension
}

type ExtensionConfig map[string]interface{}

type InvalidConfig struct {
	Key string
	msg string
}

func (ic InvalidConfig) Error() string {
	return ic.msg
}

func (cfg ExtensionConfig) read(key string, def interface{}) interface{} {
	if val, ok := cfg[key]; ok {
		return val
	}

	return def
}

func (cfg ExtensionConfig) integer(key string, def int) int {
	if val, ok := cfg[key]; ok {
		switch b := val.(type) {
		case int:
			return b
		}
	}

	return def
}

func (cfg ExtensionConfig) boolean(key string, def bool) bool {
	if val, ok := cfg[key]; ok {
		switch b := val.(type) {
		case bool:
			return b
		case int:
			// Require explicit 0 or 1
			if b == 0 {
				return false
			} else if b == 1 {
				return true
			}
		}
	}

	return def
}

func (cfg ExtensionConfig) float(key string, def float64) float64 {
	if val, ok := cfg[key]; ok {
		switch f := val.(type) {
		case float32:
			return float64(f)
		case float64:
			return f
		}
	}

	return def
}

func (cfg ExtensionConfig) list(key string, def []string) []string {
	if val, ok := cfg[key]; ok {
		switch l := val.(type) {
		case []string:
			return l
		}

	}

	return def
}
<commit_msg>Export ExtensionConfig retrieval functions<commit_after>package sirius

type Extension interface {
	Run(Message) (error, MessageAction)
}

type CfgExtension interface {
	RunWithConfig(Message, ExtensionConfig)
}

type cfg_int int
type cfg_bool bool
type cfg_float float64
type cfg_list []string

type EID string

type ExtensionLoader interface {
	Load(EID) Extension
}

type ExtensionConfig map[string]interface{}

type InvalidConfig struct {
	Key string
	msg string
}

func (ic InvalidConfig) Error() string {
	return ic.msg
}

func (cfg ExtensionConfig) Read(key string, def interface{}) interface{} {
	if val, ok := cfg[key]; ok {
		return val
	}

	return def
}

func (cfg ExtensionConfig) Integer(key string, def int) int {
	if val, ok := cfg[key]; ok {
		switch b := val.(type) {
		case int:
			return b
		}
	}

	return def
}

func (cfg ExtensionConfig) Boolean(key string, def bool) bool {
	if val, ok := cfg[key]; ok {
		switch b := val.(type) {
		case bool:
			return b
		case int:
			// Require explicit 0 or 1
			if b == 0 {
				return false
			} else if b == 1 {
				return true
			}
		}
	}

	return def
}

func (cfg ExtensionConfig) Float(key string, def float64) float64 {
	if val, ok := cfg[key]; ok {
		switch f := val.(type) {
		case float32:
			return float64(f)
		case float64:
			return f
		}
	}

	return def
}

func (cfg ExtensionConfig) List(key string, def []string) []string {
	if val, ok := cfg[key]; ok {
		switch l := val.(type) {
		case []string:
			return l
		}

	}

	return def
}
<|endoftext|>


<commit_before>// Copyright 2016 Qubit Digital Ltd.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logspray is a collection of tools for streaming and indexing
// large volumes of dynamic logs.

package indexer

import (
	"context"
	"fmt"
	"io"
	"path/filepath"
	"sync"
	"time"

	"github.com/QubitProducts/logspray/proto/logspray"
	"github.com/QubitProducts/logspray/ql"
	"github.com/oklog/ulid"
)

// Shard represents a logging session.
type Shard struct {
	id         string
	shardStart time.Time
	dataDir    string
	batchSize  uint

	filesLock sync.Mutex
	files     map[string]*ShardFile
	pbfiles   map[string]*ShardFile

	writeRaw bool
	writePB  bool

	cacheLock  sync.Mutex
	labelCache map[string]map[string]struct{}
}

// ShardFile tracks our offset into a protobuf serialisation.
type ShardFile struct {
	writer io.WriteCloser
	fn     string
	pb     bool
	id     string

	sync.RWMutex
	headersSent bool
	labels      map[string]string
	offset      int64
}

func newShard(startTime time.Time, baseDir, id string, writeRaw, writePB bool) (*Shard, error) {
	dataDir := filepath.Join(baseDir, fmt.Sprintf("%d", startTime.Unix()))
	return &Shard{
		dataDir:    dataDir,
		id:         id,
		shardStart: startTime,
		writePB:    writePB,
		writeRaw:   writeRaw,

		files:   map[string]*ShardFile{},
		pbfiles: map[string]*ShardFile{},

		labelCache: map[string]map[string]struct{}{},
	}, nil
}

func (s *Shard) writeMessage(ctx context.Context, m *logspray.Message, shardKey string, labels map[string]string) error {
	// THere's a horrid mess of locking here that should be tidied
	// up
	s.filesLock.Lock()

	var ok bool
	var rawf *ShardFile
	if s.writeRaw {
		rawf, ok = s.files[m.StreamID]
		if !ok {
			uid := ulid.MustParse(m.StreamID)
			dir := filepath.Join(s.dataDir, shardKey, s.id)
			rawfn := filepath.Join(dir, uid.String()+".log")

			rawf = &ShardFile{
				fn:     rawfn,
				offset: 0,
				pb:     false,
			}
			s.files[m.StreamID] = rawf
		}
	}

	var pbf *ShardFile
	if s.writePB {
		pbf, ok = s.pbfiles[m.StreamID]
		if !ok {
			uid := ulid.MustParse(m.StreamID)
			dir := filepath.Join(s.dataDir, shardKey, s.id)
			pbfn := filepath.Join(dir, uid.String()+".pb.log")

			pbf = &ShardFile{
				fn:     pbfn,
				pb:     true,
				id:     m.StreamID,
				labels: labels,
			}
			s.pbfiles[m.StreamID] = pbf
		}
	}

	/* update labels cache */
	if !ok {
		s.cacheLock.Lock()
		for k, v := range labels {
			_, ok := s.labelCache[k]
			if !ok {
				s.labelCache[k] = map[string]struct{}{}
			}
			s.labelCache[k][v] = struct{}{}
		}
		s.cacheLock.Unlock()
	}

	s.cacheLock.Lock()
	for k, v := range m.Labels {
		_, ok := s.labelCache[k]
		if !ok {
			s.labelCache[k] = map[string]struct{}{}
		}
		s.labelCache[k][v] = struct{}{}
	}
	s.cacheLock.Unlock()

	s.filesLock.Unlock()

	var err error
	if s.writePB {
		err = pbf.writeMessageToFile(ctx, m)
		if err != nil {
			return err
		}
	}

	if s.writeRaw {
		err = rawf.writeMessageToFile(ctx, m)
	}

	return err
}

func (s *Shard) close() {
	s.cacheLock.Lock()
	defer s.cacheLock.Unlock()

	s.labelCache = nil
	for fs := range s.files {
		s.files[fs].writer.Close()
	}
	for fs := range s.pbfiles {
		s.pbfiles[fs].writer.Close()
	}
}

// Labels retursn the list of labels in the label cache
func (s *Shard) Labels() []string {
	res := []string{}
	if s == nil {
		return res
	}
	s.cacheLock.Lock()
	for k := range s.labelCache {
		res = append(res, k)
	}
	s.cacheLock.Unlock()
	return res
}

// LabelValues returns all the known values for a given label.
func (s *Shard) LabelValues(name string) []string {
	res := []string{}
	if s == nil {
		return res
	}

	s.cacheLock.Lock()
	for v := range s.labelCache[name] {
		res = append(res, v)
	}
	s.cacheLock.Unlock()
	return res
}

func (s *Shard) findFiles(from, to time.Time) []*ShardFile {
	var fs []*ShardFile
	if s == nil {
		return fs
	}

	s.filesLock.Lock()

	for _, f := range s.pbfiles {
		fs = append(fs, f)
	}

	s.filesLock.Unlock()

	return fs
}

// Search this shard for queries between the provided time frames
// for message matched by the provided match function.
func (s *Shard) Search(ctx context.Context, msgFunc logspray.MessageFunc, matcher ql.MatchFunc, from, to time.Time, count, offset uint64, reverse bool) error {
	if s == nil {
		return nil
	}

	fs := s.findFiles(from, to)
	for _, f := range fs {
		if err := f.Search(ctx, msgFunc, matcher, from, to, count, offset, reverse); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>indexer: don't crash on non-ulid id<commit_after>// Copyright 2016 Qubit Digital Ltd.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logspray is a collection of tools for streaming and indexing
// large volumes of dynamic logs.

package indexer

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"path/filepath"
	"sync"
	"time"

	"github.com/QubitProducts/logspray/proto/logspray"
	"github.com/QubitProducts/logspray/ql"
	"github.com/oklog/ulid"
)

// Shard represents a logging session.
type Shard struct {
	id         string
	shardStart time.Time
	dataDir    string
	batchSize  uint

	filesLock sync.Mutex
	files     map[string]*ShardFile
	pbfiles   map[string]*ShardFile

	writeRaw bool
	writePB  bool

	cacheLock  sync.Mutex
	labelCache map[string]map[string]struct{}
}

// ShardFile tracks our offset into a protobuf serialisation.
type ShardFile struct {
	writer io.WriteCloser
	fn     string
	pb     bool
	id     string

	sync.RWMutex
	headersSent bool
	labels      map[string]string
	offset      int64
}

func newShard(startTime time.Time, baseDir, id string, writeRaw, writePB bool) (*Shard, error) {
	dataDir := filepath.Join(baseDir, fmt.Sprintf("%d", startTime.Unix()))
	return &Shard{
		dataDir:    dataDir,
		id:         id,
		shardStart: startTime,
		writePB:    writePB,
		writeRaw:   writeRaw,

		files:   map[string]*ShardFile{},
		pbfiles: map[string]*ShardFile{},

		labelCache: map[string]map[string]struct{}{},
	}, nil
}

func (s *Shard) writeMessage(ctx context.Context, m *logspray.Message, shardKey string, labels map[string]string) error {
	// THere's a horrid mess of locking here that should be tidied
	// up
	s.filesLock.Lock()

	var ok bool
	var rawf *ShardFile
	if s.writeRaw {
		rawf, ok = s.files[m.StreamID]
		if !ok {
			uid, err := ulid.Parse(m.StreamID)
			if err != nil {
				uid = base64.StdEncoding.EncodeToString(m.StreamID)
			}

			dir := filepath.Join(s.dataDir, shardKey, s.id)
			rawfn := filepath.Join(dir, uid.String()+".log")

			rawf = &ShardFile{
				fn:     rawfn,
				offset: 0,
				pb:     false,
			}
			s.files[m.StreamID] = rawf
		}
	}

	var pbf *ShardFile
	if s.writePB {
		pbf, ok = s.pbfiles[m.StreamID]
		if !ok {
			uid, err := ulid.Parse(m.StreamID)
			if err != nil {
				uid = base64.StdEncoding.EncodeToString(m.StreamID)
			}
			dir := filepath.Join(s.dataDir, shardKey, s.id)
			pbfn := filepath.Join(dir, uid.String()+".pb.log")

			pbf = &ShardFile{
				fn:     pbfn,
				pb:     true,
				id:     m.StreamID,
				labels: labels,
			}
			s.pbfiles[m.StreamID] = pbf
		}
	}

	/* update labels cache */
	if !ok {
		s.cacheLock.Lock()
		for k, v := range labels {
			_, ok := s.labelCache[k]
			if !ok {
				s.labelCache[k] = map[string]struct{}{}
			}
			s.labelCache[k][v] = struct{}{}
		}
		s.cacheLock.Unlock()
	}

	s.cacheLock.Lock()
	for k, v := range m.Labels {
		_, ok := s.labelCache[k]
		if !ok {
			s.labelCache[k] = map[string]struct{}{}
		}
		s.labelCache[k][v] = struct{}{}
	}
	s.cacheLock.Unlock()

	s.filesLock.Unlock()

	var err error
	if s.writePB {
		err = pbf.writeMessageToFile(ctx, m)
		if err != nil {
			return err
		}
	}

	if s.writeRaw {
		err = rawf.writeMessageToFile(ctx, m)
	}

	return err
}

func (s *Shard) close() {
	s.cacheLock.Lock()
	defer s.cacheLock.Unlock()

	s.labelCache = nil
	for fs := range s.files {
		s.files[fs].writer.Close()
	}
	for fs := range s.pbfiles {
		s.pbfiles[fs].writer.Close()
	}
}

// Labels retursn the list of labels in the label cache
func (s *Shard) Labels() []string {
	res := []string{}
	if s == nil {
		return res
	}
	s.cacheLock.Lock()
	for k := range s.labelCache {
		res = append(res, k)
	}
	s.cacheLock.Unlock()
	return res
}

// LabelValues returns all the known values for a given label.
func (s *Shard) LabelValues(name string) []string {
	res := []string{}
	if s == nil {
		return res
	}

	s.cacheLock.Lock()
	for v := range s.labelCache[name] {
		res = append(res, v)
	}
	s.cacheLock.Unlock()
	return res
}

func (s *Shard) findFiles(from, to time.Time) []*ShardFile {
	var fs []*ShardFile
	if s == nil {
		return fs
	}

	s.filesLock.Lock()

	for _, f := range s.pbfiles {
		fs = append(fs, f)
	}

	s.filesLock.Unlock()

	return fs
}

// Search this shard for queries between the provided time frames
// for message matched by the provided match function.
func (s *Shard) Search(ctx context.Context, msgFunc logspray.MessageFunc, matcher ql.MatchFunc, from, to time.Time, count, offset uint64, reverse bool) error {
	if s == nil {
		return nil
	}

	fs := s.findFiles(from, to)
	for _, f := range fs {
		if err := f.Search(ctx, msgFunc, matcher, from, to, count, offset, reverse); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>