<commit_before>package db

import (
	"database/sql"

	"github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

// SQLDB is a sql database repository implementing the DB interface.
type SQLDB struct {
	sqlx *sqlx.DB
}

// Ensure SQLDB implements DB.
var _ DB = (*SQLDB)(nil)

// NewSQLDB returns an SQLDB.
func NewSQLDB(sqlDB *sql.DB, driverName string) (*SQLDB, error) {
	db := &SQLDB{
		sqlx: sqlx.NewDb(sqlDB, driverName),
	}
	if err := db.sqlx.Ping(); err != nil {
		return nil, err
	}
	return db, nil
}

// AddGHInstallation implements the DB interface.
func (db *SQLDB) AddGHInstallation(installationID, accountID, senderID int) error {
	// INSERT IGNORE so any duplicates are ignored
	_, err := db.sqlx.Exec("INSERT IGNORE INTO gh_installations (installation_id, account_id, sender_id) VALUES (?, ?, ?)",
		installationID, accountID, senderID,
	)
	return err
}

// RemoveGHInstallation implements the DB interface.
func (db *SQLDB) RemoveGHInstallation(installationID int) error {
	_, err := db.sqlx.Exec("DELETE FROM gh_installations WHERE installation_id = ?", installationID)
	return err
}

// GetGHInstallation implements the DB interface.
func (db *SQLDB) GetGHInstallation(installationID int) (*GHInstallation, error) {
	var row struct {
		ID             int            `db:"id"`
		InstallationID int            `db:"installation_id"`
		AccountID      int            `db:"account_id"`
		SenderID       int            `db:"sender_id"`
		EnabledAt      mysql.NullTime `db:"enabled_at"`
	}
	err := db.sqlx.Get(&row, "SELECT id, installation_id, account_id, sender_id, enabled_at FROM gh_installations WHERE installation_id = ?", installationID)
	switch {
	case err == sql.ErrNoRows:
		return nil, nil
	case err != nil:
		return nil, err
	}
	ghi := &GHInstallation{
		ID:             row.ID,
		InstallationID: row.InstallationID,
		AccountID:      row.AccountID,
		SenderID:       row.SenderID,
	}
	if row.EnabledAt.Valid {
		ghi.enabledAt = row.EnabledAt.Time
	}
	return ghi, nil
}

// ListTools implements the DB interface.
func (db *SQLDB) ListTools() ([]Tool, error) {
	var tools []Tool
	err := db.sqlx.Select(&tools, "SELECT id, name, path, args, `regexp` FROM tools")
	return tools, err
}

// StartAnalysis implements the DB interface.
func (db *SQLDB) StartAnalysis(ghInstallationID, repositoryID int) (*Analysis, error) {
	analysis := NewAnalysis()
	result, err := db.sqlx.Exec("INSERT INTO analysis (gh_installation_id, repository_id) VALUES (?, ?)", ghInstallationID, repositoryID)
	if err != nil {
		return nil, err
	}
	analysisID, err := result.LastInsertId()
	analysis.ID = int(analysisID)
	return analysis, err
}

// FinishAnalysis implements the DB interface.
func (db *SQLDB) FinishAnalysis(analysisID int, status AnalysisStatus, analysis *Analysis) error {
	if analysis == nil {
		_, err := db.sqlx.Exec("UPDATE analysis SET status = ? WHERE id = ?", string(status), analysisID)
		return err
	}
	_, err := db.sqlx.Exec("UPDATE analysis SET status = ?, clone_duration = ?, deps_duration = ?, total_duration = ? WHERE id = ?",
		string(status), analysis.CloneDuration, analysis.DepsDuration, analysis.TotalDuration, analysisID,
	)
	if err != nil {
		return err
	}
	for toolID, tool := range analysis.Tools {
		toolResult, err := db.sqlx.Exec("INSERT INTO analysis_tool (analysis_id, tool_id, duration) VALUES (?, ?, ?)", analysisID, toolID, tool.Duration)
		if err != nil {
			return err
		}

		toolAnalysisID, err := toolResult.LastInsertId()
		if err != nil {
			return err
		}

		for _, issue := range tool.Issues {
			_, err := db.sqlx.Exec("INSERT INTO issues (analysis_tool_id, path, line, hunk_pos, issue) VALUES(?, ?, ?, ?, ?)",
				toolAnalysisID, issue.Path, issue.Line, issue.HunkPos, issue.Issue,
			)
			if err != nil {
				return err
			}
		}

	}
	return nil
}

// GetAnalysis implements the DB interface.
func (db *SQLDB) GetAnalysis(analysisID int) (*Analysis, error) {
	analysis := NewAnalysis()

	err := db.sqlx.Get(analysis, `
SELECT id, gh_installation_id, repository_id, status, clone_duration, deps_duration, total_duration, created_at
FROM analysis WHERE id = ?`, analysisID)
	if err != nil {
		return nil, err
	}
	if analysis == nil {
		return nil, nil
	}

	var toolIssues []struct {
		ToolID   int            `db:"tool_id"`
		Name     string         `db:"name"`
		URL      string         `db:"url"`
		Duration Duration       `db:"duration"`
		Path     sql.NullString `db:"path"`
		Line     sql.NullInt64  `db:"line"`
		HunkPos  sql.NullInt64  `db:"hunk_pos"`
		Issue    sql.NullString `db:"issue"`
	}

	// get all the tools and issues if they have them
	err = db.sqlx.Select(&toolIssues, `
   SELECT at.tool_id, at.duration, i.path, i.line, i.hunk_pos, i.issue,
		  t.name, t.url
     FROM analysis_tool at
	 JOIN tools t ON (at.tool_id = t.id)
LEFT JOIN issues i ON (i.analysis_tool_id = at.id)
    WHERE at.analysis_id = ?`,
		analysisID,
	)
	if err != nil {
		return nil, err
	}

	for _, issue := range toolIssues {
		toolID := ToolID(issue.ToolID)
		if _, ok := analysis.Tools[toolID]; !ok {
			analysis.Tools[toolID] = AnalysisTool{
				Tool:     &Tool{ID: toolID, Name: issue.Name, URL: issue.URL},
				ToolID:   toolID,
				Duration: issue.Duration,
			}
		}

		if issue.Issue.Valid {
			at := analysis.Tools[toolID]
			at.Issues = append(at.Issues, Issue{
				Path:    issue.Path.String,
				Line:    int(issue.Line.Int64),
				HunkPos: int(issue.HunkPos.Int64),
				Issue:   issue.Issue.String,
			})
			analysis.Tools[toolID] = at
		}
	}

	return analysis, nil
}
<commit_msg>Bug fix prod vs dev sql differences<commit_after>package db

import (
	"database/sql"

	"github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

// SQLDB is a sql database repository implementing the DB interface.
type SQLDB struct {
	sqlx *sqlx.DB
}

// Ensure SQLDB implements DB.
var _ DB = (*SQLDB)(nil)

// NewSQLDB returns an SQLDB.
func NewSQLDB(sqlDB *sql.DB, driverName string) (*SQLDB, error) {
	db := &SQLDB{
		sqlx: sqlx.NewDb(sqlDB, driverName),
	}
	if err := db.sqlx.Ping(); err != nil {
		return nil, err
	}
	return db, nil
}

// AddGHInstallation implements the DB interface.
func (db *SQLDB) AddGHInstallation(installationID, accountID, senderID int) error {
	// INSERT IGNORE so any duplicates are ignored
	_, err := db.sqlx.Exec("INSERT IGNORE INTO gh_installations (installation_id, account_id, sender_id) VALUES (?, ?, ?)",
		installationID, accountID, senderID,
	)
	return err
}

// RemoveGHInstallation implements the DB interface.
func (db *SQLDB) RemoveGHInstallation(installationID int) error {
	_, err := db.sqlx.Exec("DELETE FROM gh_installations WHERE installation_id = ?", installationID)
	return err
}

// GetGHInstallation implements the DB interface.
func (db *SQLDB) GetGHInstallation(installationID int) (*GHInstallation, error) {
	var row struct {
		ID             int            `db:"id"`
		InstallationID int            `db:"installation_id"`
		AccountID      int            `db:"account_id"`
		SenderID       int            `db:"sender_id"`
		EnabledAt      mysql.NullTime `db:"enabled_at"`
	}
	err := db.sqlx.Get(&row, "SELECT id, installation_id, account_id, sender_id, enabled_at FROM gh_installations WHERE installation_id = ?", installationID)
	switch {
	case err == sql.ErrNoRows:
		return nil, nil
	case err != nil:
		return nil, err
	}
	ghi := &GHInstallation{
		ID:             row.ID,
		InstallationID: row.InstallationID,
		AccountID:      row.AccountID,
		SenderID:       row.SenderID,
	}
	if row.EnabledAt.Valid {
		ghi.enabledAt = row.EnabledAt.Time
	}
	return ghi, nil
}

// ListTools implements the DB interface.
func (db *SQLDB) ListTools() ([]Tool, error) {
	var tools []Tool
	err := db.sqlx.Select(&tools, "SELECT id, name, path, args, `regexp` FROM tools")
	return tools, err
}

// StartAnalysis implements the DB interface.
func (db *SQLDB) StartAnalysis(ghInstallationID, repositoryID int) (*Analysis, error) {
	analysis := NewAnalysis()
	result, err := db.sqlx.Exec("INSERT INTO analysis (gh_installation_id, repository_id) VALUES (?, ?)", ghInstallationID, repositoryID)
	if err != nil {
		return nil, err
	}
	analysisID, err := result.LastInsertId()
	analysis.ID = int(analysisID)
	return analysis, err
}

// FinishAnalysis implements the DB interface.
func (db *SQLDB) FinishAnalysis(analysisID int, status AnalysisStatus, analysis *Analysis) error {
	if analysis == nil {
		_, err := db.sqlx.Exec("UPDATE analysis SET status = ? WHERE id = ?", string(status), analysisID)
		return err
	}
	_, err := db.sqlx.Exec("UPDATE analysis SET status = ?, clone_duration = SEC_TO_TIME(?), deps_duration = SEC_TO_TIME(?), total_duration = SEC_TO_TIME(?) WHERE id = ?",
		string(status), analysis.CloneDuration, analysis.DepsDuration, analysis.TotalDuration, analysisID,
	)
	if err != nil {
		return err
	}
	for toolID, tool := range analysis.Tools {
		toolResult, err := db.sqlx.Exec("INSERT INTO analysis_tool (analysis_id, tool_id, SEC_TO_TIME(duration)) VALUES (?, ?, ?)", analysisID, toolID, tool.Duration)
		if err != nil {
			return err
		}

		toolAnalysisID, err := toolResult.LastInsertId()
		if err != nil {
			return err
		}

		for _, issue := range tool.Issues {
			_, err := db.sqlx.Exec("INSERT INTO issues (analysis_tool_id, path, line, hunk_pos, issue) VALUES(?, ?, ?, ?, ?)",
				toolAnalysisID, issue.Path, issue.Line, issue.HunkPos, issue.Issue,
			)
			if err != nil {
				return err
			}
		}

	}
	return nil
}

// GetAnalysis implements the DB interface.
func (db *SQLDB) GetAnalysis(analysisID int) (*Analysis, error) {
	analysis := NewAnalysis()

	err := db.sqlx.Get(analysis, `
SELECT id, gh_installation_id, repository_id, status, clone_duration, deps_duration, total_duration, created_at
FROM analysis WHERE id = ?`, analysisID)
	if err != nil {
		return nil, err
	}
	if analysis == nil {
		return nil, nil
	}

	var toolIssues []struct {
		ToolID   int            `db:"tool_id"`
		Name     string         `db:"name"`
		URL      string         `db:"url"`
		Duration Duration       `db:"duration"`
		Path     sql.NullString `db:"path"`
		Line     sql.NullInt64  `db:"line"`
		HunkPos  sql.NullInt64  `db:"hunk_pos"`
		Issue    sql.NullString `db:"issue"`
	}

	// get all the tools and issues if they have them
	err = db.sqlx.Select(&toolIssues, `
   SELECT at.tool_id, at.duration, i.path, i.line, i.hunk_pos, i.issue,
		  t.name, t.url
     FROM analysis_tool at
	 JOIN tools t ON (at.tool_id = t.id)
LEFT JOIN issues i ON (i.analysis_tool_id = at.id)
    WHERE at.analysis_id = ?`,
		analysisID,
	)
	if err != nil {
		return nil, err
	}

	for _, issue := range toolIssues {
		toolID := ToolID(issue.ToolID)
		if _, ok := analysis.Tools[toolID]; !ok {
			analysis.Tools[toolID] = AnalysisTool{
				Tool:     &Tool{ID: toolID, Name: issue.Name, URL: issue.URL},
				ToolID:   toolID,
				Duration: issue.Duration,
			}
		}

		if issue.Issue.Valid {
			at := analysis.Tools[toolID]
			at.Issues = append(at.Issues, Issue{
				Path:    issue.Path.String,
				Line:    int(issue.Line.Int64),
				HunkPos: int(issue.HunkPos.Int64),
				Issue:   issue.Issue.String,
			})
			analysis.Tools[toolID] = at
		}
	}

	return analysis, nil
}
<|endoftext|>


<commit_before>package gumble

import (
	"crypto/tls"
	"net"
	"time"

	"github.com/layeh/gumble/gumble/MumbleProto"
)

// Config holds the configuration data used by Client.
type Config struct {
	// User name used when authenticating with the server.
	Username string
	// Password used when authenticating with the server. A password is not
	// usually required to connect to a server.
	Password string
	// Server address, including port (e.g. localhost:64738).
	Address string
	Tokens  AccessTokens

	// AudioInterval is the interval at which audio packets are sent. Valid
	// values are 10ms, 20ms, 40ms, and 60ms.
	AudioInterval time.Duration

	// AudioDataBytes is the number of bytes that an audio frame can use.
	AudioDataBytes int

	TLSConfig tls.Config
	// If non-nil, this function will be called after the connection to the
	// server has been made. If it returns nil, the connection will stay alive,
	// otherwise, it will be closed and Client.Connect will return the returned
	// error.
	TLSVerify func(state *tls.ConnectionState) error
	Dialer    net.Dialer
}

// NewConfig returns a new Config struct with default values set.
func NewConfig() *Config {
	return &Config{
		AudioInterval:  AudioDefaultInterval,
		AudioDataBytes: AudioDefaultDataBytes,
	}
}

// GetAudioFrameSize returns the appropriate audio frame size, based off of the
// audio interval.
func (c *Config) GetAudioFrameSize() int {
	return int(c.AudioInterval/AudioDefaultInterval) * AudioDefaultFrameSize
}

// AccessTokens are additional passwords that can be provided to the server to
// gain access to restricted channels.
type AccessTokens []string

func (at AccessTokens) writeMessage(client *Client) error {
	packet := MumbleProto.Authenticate{
		Tokens: at,
	}
	proto := protoMessage{&packet}
	return proto.writeMessage(client)
}
<commit_msg>add timeout to NewConfig()'s *Config<commit_after>package gumble

import (
	"crypto/tls"
	"net"
	"time"

	"github.com/layeh/gumble/gumble/MumbleProto"
)

// Config holds the configuration data used by Client.
type Config struct {
	// User name used when authenticating with the server.
	Username string
	// Password used when authenticating with the server. A password is not
	// usually required to connect to a server.
	Password string
	// Server address, including port (e.g. localhost:64738).
	Address string
	Tokens  AccessTokens

	// AudioInterval is the interval at which audio packets are sent. Valid
	// values are 10ms, 20ms, 40ms, and 60ms.
	AudioInterval time.Duration

	// AudioDataBytes is the number of bytes that an audio frame can use.
	AudioDataBytes int

	TLSConfig tls.Config
	// If non-nil, this function will be called after the connection to the
	// server has been made. If it returns nil, the connection will stay alive,
	// otherwise, it will be closed and Client.Connect will return the returned
	// error.
	TLSVerify func(state *tls.ConnectionState) error
	Dialer    net.Dialer
}

// NewConfig returns a new Config struct with default values set.
func NewConfig() *Config {
	return &Config{
		AudioInterval:  AudioDefaultInterval,
		AudioDataBytes: AudioDefaultDataBytes,
		Dialer: net.Dialer{
			Timeout: time.Second * 20,
		},
	}
}

// GetAudioFrameSize returns the appropriate audio frame size, based off of the
// audio interval.
func (c *Config) GetAudioFrameSize() int {
	return int(c.AudioInterval/AudioDefaultInterval) * AudioDefaultFrameSize
}

// AccessTokens are additional passwords that can be provided to the server to
// gain access to restricted channels.
type AccessTokens []string

func (at AccessTokens) writeMessage(client *Client) error {
	packet := MumbleProto.Authenticate{
		Tokens: at,
	}
	proto := protoMessage{&packet}
	return proto.writeMessage(client)
}
<|endoftext|>


<commit_before>package canal

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/siddontang/go-mysql/schema"
)

const (
	UpdateAction = "update"
	InsertAction = "insert"
	DeleteAction = "delete"
)

type RowsEvent struct {
	Table  *schema.Table
	Action string
	// changed row list
	// binlog has three update event version, v0, v1 and v2.
	// for v1 and v2, the rows number must be even.
	// Two rows for one event, format is [before update row, after update row]
	// for update v0, only one row for a event, and we don't support this version.
	Rows [][]interface{}
}

func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent {
	e := new(RowsEvent)

	e.Table = table
	e.Action = action
	e.Rows = rows

	return e
}

// Get primary keys in one row for a table, a table may use multi fields as the PK
func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) {
	indexes := table.PKColumns
	if len(indexes) == 0 {
		return nil, errors.Errorf("table %s has no PK", table)
	} else if len(table.Columns) != len(row) {
		return nil, errors.Errorf("table %s has %d columns, but row data %v len is %d", table,
			len(table.Columns), row, len(row))
	}

	values := make([]interface{}, 0, len(indexes))

	for _, index := range indexes {
		values = append(values, row[index])
	}

	return values, nil
}

// String implements fmt.Stringer interface.
func (r *RowsEvent) String() string {
	return fmt.Sprintf("%s %s %v", r.Action, r.Table, r.Rows)
}
<commit_msg>add function of get term field's value by field name (#124)<commit_after>package canal

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/siddontang/go-mysql/schema"
)

const (
	UpdateAction = "update"
	InsertAction = "insert"
	DeleteAction = "delete"
)

type RowsEvent struct {
	Table  *schema.Table
	Action string
	// changed row list
	// binlog has three update event version, v0, v1 and v2.
	// for v1 and v2, the rows number must be even.
	// Two rows for one event, format is [before update row, after update row]
	// for update v0, only one row for a event, and we don't support this version.
	Rows [][]interface{}
}

func newRowsEvent(table *schema.Table, action string, rows [][]interface{}) *RowsEvent {
	e := new(RowsEvent)

	e.Table = table
	e.Action = action
	e.Rows = rows

	return e
}

// Get primary keys in one row for a table, a table may use multi fields as the PK
func GetPKValues(table *schema.Table, row []interface{}) ([]interface{}, error) {
	indexes := table.PKColumns
	if len(indexes) == 0 {
		return nil, errors.Errorf("table %s has no PK", table)
	} else if len(table.Columns) != len(row) {
		return nil, errors.Errorf("table %s has %d columns, but row data %v len is %d", table,
			len(table.Columns), row, len(row))
	}

	values := make([]interface{}, 0, len(indexes))

	for _, index := range indexes {
		values = append(values, row[index])
	}

	return values, nil
}

// Get term column's value
func GetColumnValue(table *schema.Table, column string, row []interface{}) (interface{}, error) {
	index := table.FindColumn(column)
	if index == -1 {
		return nil, errors.Errorf("table %s has no column name %s", table, column)
	}

	return row[index], nil
}

// String implements fmt.Stringer interface.
func (r *RowsEvent) String() string {
	return fmt.Sprintf("%s %s %v", r.Action, r.Table, r.Rows)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"

	"github.com/arschles/gocons/log"
	"gopkg.in/yaml.v2"
)

const defaultConsFileName = "gocons.yaml"

func getConsfile() (*Consfile, error) {
	b, err := ioutil.ReadFile(defaultConsFileName)
	if err != nil {
		return nil, err
	}
	f := &Consfile{}
	if err := yaml.Unmarshal(b, f); err != nil {
		return nil, err
	}
	return f, nil
}

func getConsfileOrDie() *Consfile {
	consfile, err := getConsfile()
	if err != nil {
		log.Die("error getting consfile [%s]", err)
		return nil
	}
	return consfile
}

type Consfile struct {
	Version int      `yaml:"version"`
	Plugins []string `yaml:"repos"`
	Targets []Target `yaml:"targets"`
}

type Target struct {
	Name        string   `yaml:"name"`
	Description string   `yaml:"description"`
	Depends     string   `yaml:"depends"`
	Commands    []string `yaml:"commands"`
}
<commit_msg>adding global env vars<commit_after>package main

import (
	"fmt"
	"io/ioutil"

	"github.com/arschles/gocons/log"
	"gopkg.in/yaml.v2"
)

const defaultConsFileName = "gocons.yaml"

func getConsfile() (*Consfile, error) {
	b, err := ioutil.ReadFile(defaultConsFileName)
	if err != nil {
		return nil, err
	}
	f := &Consfile{}
	if err := yaml.Unmarshal(b, f); err != nil {
		return nil, err
	}
	return f, nil
}

func getConsfileOrDie() *Consfile {
	consfile, err := getConsfile()
	if err != nil {
		log.Die("error getting consfile [%s]", err)
		return nil
	}
	return consfile
}

type Consfile struct {
	Version int      `yaml:"version"`
	Envs    []Env    `yaml:"environment_vars"`
	Plugins []string `yaml:"repos"`
	Targets []Target `yaml:"targets"`
}

type Env struct {
	Name string `yaml:"name"`
	Val  string `yaml:"val"`
}

func (e Env) String() string {
	return fmt.Sprintf("%s=%s", e.Name, e.Val)
}

type Envs []Env

func (e Envs) Strings() []string {
	strs := make([]string, len(e))
	for i, env := range e {
		strs[i] = env.String()
	}
	return strs
}

type Target struct {
	Name        string   `yaml:"name"`
	Description string   `yaml:"description"`
	Depends     string   `yaml:"depends"`
	Commands    []string `yaml:"commands"`
}
<|endoftext|>


<commit_before>// Package trigram is a dumb trigram index
package trigram

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// a special (and invalid) trigram that holds all the document IDs
const tAllDocIDs T = 0xFFFFFFFF

// Extract returns a list of trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var allDocIDs []DocID

	var trigrams []T

	for id, d := range docs {
		ts := Extract(d, trigrams)
		docid := DocID(id)
		allDocIDs = append(allDocIDs, docid)
		for _, t := range ts {
			idx[t] = append(idx[t], docid)
		}
		trigrams = trigrams[:0]
	}

	idx[tAllDocIDs] = allDocIDs

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) DocID {

	id := DocID(len(idx[tAllDocIDs]))

	ts := Extract(s, nil)
	for _, t := range ts {
		idx[t] = append(idx[t], id)
	}

	idx[tAllDocIDs] = append(idx[tAllDocIDs], id)

	return id
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	if len(ts) == 0 {
		return idx[tAllDocIDs]
	}

	midx := 0
	mtri := ts[midx]

	for i, t := range ts {
		if len(idx[t]) < len(idx[mtri]) {
			midx = i
			mtri = t
		}
	}

	ts[0], ts[midx] = ts[midx], ts[0]

	return idx.Filter(idx[mtri], ts[1:])
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts []T) []DocID {
	for _, t := range ts {
		docs = intersect(docs, idx[t])
	}

	return docs
}

func intersect(a, b []DocID) []DocID {

	// TODO(dgryski): reduce allocations by reusing A

	var aidx, bidx int

	var result []DocID

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx >= len(a) || bidx >= len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx >= len(a) {
				break scan
			}
		}

		for bidx < len(b) && a[aidx] > b[bidx] {
			bidx++
			if bidx >= len(b) {
				break scan
			}
		}
	}

	return result
}
<commit_msg>Sort all the trigrams before intersection<commit_after>// Package trigram is a dumb trigram index
package trigram

import (
	"sort"
)

// T is a trigram
type T uint32

func (t T) String() string {
	b := [3]byte{byte(t >> 16), byte(t >> 8), byte(t)}
	return string(b[:])
}

// DocID is a document ID
type DocID uint32

// Index is a trigram index
type Index map[T][]DocID

// a special (and invalid) trigram that holds all the document IDs
const tAllDocIDs T = 0xFFFFFFFF

// Extract returns a list of trigrams in s
func Extract(s string, trigrams []T) []T {

	for i := 0; i <= len(s)-3; i++ {
		t := T(uint32(s[i])<<16 | uint32(s[i+1])<<8 | uint32(s[i+2]))
		trigrams = appendIfUnique(trigrams, t)
	}

	return trigrams
}

func appendIfUnique(t []T, n T) []T {
	for _, v := range t {
		if v == n {
			return t
		}
	}

	return append(t, n)
}

// NewIndex returns an index for the strings in docs
func NewIndex(docs []string) Index {

	idx := make(Index)

	var allDocIDs []DocID

	var trigrams []T

	for id, d := range docs {
		ts := Extract(d, trigrams)
		docid := DocID(id)
		allDocIDs = append(allDocIDs, docid)
		for _, t := range ts {
			idx[t] = append(idx[t], docid)
		}
		trigrams = trigrams[:0]
	}

	idx[tAllDocIDs] = allDocIDs

	return idx
}

// Add adds a new string to the search index
func (idx Index) Add(s string) DocID {

	id := DocID(len(idx[tAllDocIDs]))

	ts := Extract(s, nil)
	for _, t := range ts {
		idx[t] = append(idx[t], id)
	}

	idx[tAllDocIDs] = append(idx[tAllDocIDs], id)

	return id
}

// for sorting
type docList []DocID

func (d docList) Len() int           { return len(d) }
func (d docList) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
func (d docList) Less(i, j int) bool { return d[i] < d[j] }

// Sort ensures all the document IDs are in order.
func (idx Index) Sort() {
	for _, v := range idx {
		dl := docList(v)
		if !sort.IsSorted(dl) {
			sort.Sort(dl)
		}
	}
}

// Query returns a list of document IDs that match the trigrams in the query s
func (idx Index) Query(s string) []DocID {
	ts := Extract(s, nil)
	return idx.QueryTrigrams(ts)
}

type tfList struct {
	tri  []T
	freq []int
}

func (tf tfList) Len() int { return len(tf.tri) }
func (tf tfList) Swap(i, j int) {
	tf.tri[i], tf.tri[j] = tf.tri[j], tf.tri[i]
	tf.freq[i], tf.freq[j] = tf.freq[j], tf.freq[i]
}
func (tf tfList) Less(i, j int) bool { return tf.freq[i] < tf.freq[j] }

// QueryTrigrams returns a list of document IDs that match the trigram set ts
func (idx Index) QueryTrigrams(ts []T) []DocID {

	if len(ts) == 0 {
		return idx[tAllDocIDs]
	}

	var freq []int

	for _, t := range ts {
		ln := len(idx[t])
		if ln == 0 {
			return nil
		}
		freq = append(freq, ln)
	}

	sort.Sort(tfList{ts, freq})

	ids := idx.Filter(idx[ts[0]], ts[1:])

	return ids
}

// Filter removes documents that don't contain the specified trigrams
func (idx Index) Filter(docs []DocID, ts []T) []DocID {
	for _, t := range ts {
		docs = intersect(docs, idx[t])
	}

	return docs
}

func intersect(a, b []DocID) []DocID {

	// TODO(dgryski): reduce allocations by reusing A

	var aidx, bidx int

	var result []DocID

scan:
	for aidx < len(a) && bidx < len(b) {
		if a[aidx] == b[bidx] {
			result = append(result, a[aidx])
			aidx++
			bidx++
			if aidx >= len(a) || bidx >= len(b) {
				break scan
			}
		}

		for a[aidx] < b[bidx] {
			aidx++
			if aidx >= len(a) {
				break scan
			}
		}

		for bidx < len(b) && a[aidx] > b[bidx] {
			bidx++
			if bidx >= len(b) {
				break scan
			}
		}
	}

	return result
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package appstatus can attach/reterieve an application-specific response
// status to/from an error. It designed to prevent accidental exposure of
// internal statuses to RPC clients, for example Spanner's statuses.
package appstatus
<commit_msg>Expand appstatus godoc<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package appstatus can attach/reterieve an application-specific response
// status to/from an error. It designed to prevent accidental exposure of
// internal statuses to RPC clients, for example Spanner's statuses.
//
// Attaching a status
//
// Use ToError, Error and Errorf to create new status-annotated errors.
// Use Attach and Attachf to annotate existing errors with a status.
//
//   if req.PageSize < 0  {
//      return appstatus.Errorf(codes.InvalidArgument, "page size cannot be negative")
//   }
//   if err := checkState(); err != nil {
//     return appstatus.Attachf(err, codes.PreconditionFailed, "invalid state")
//   }
//
// This may be done deep in the function call hierarchy.
//
// Do not use appstatus in code where you don't explicitly intend to return a
// specific status. This is unnecessary because any unrecognized error is
// treated as internal and because it is explicitly prohibited to attach a
// status to an error chain multiple times. When a status is attached, the
// package supports its propagation all the way to the requester
// unless there is code that explicitly throws it away.
//
// Returning a status
//
// Use GRPCifyAndLog right before returning the error from a gRPC method
// handler. Usually it is done in a Postlude of a service decorator, see
// ../cmd/svcdec.
//
//   func NewMyServer() pb.MyServer {
//     return &pb.DecoratedMyServer{
//       Service:  &actualImpl{},
//       Postlude: func(ctx context.Context, methodName string, rsp proto.Message, err error) error {
//         return appstatus.GRPCifyAndLog(ctx, err)
//       },
//     }
//   }
//
// It recognizes only appstatus-annotated errors and treats any other error as
// internal. This behavior is important to avoid accidentally returning errors
// from Spanner or other client library that also uses grpc/status package to
// communicate status code from *other* services. For example, if a there is a
// typo in Spanner SQL statement, spanner package may return a status-annotated
// errors with code NotFound. In this case, our RPC must respond with internal
// error and not NotFound.
package appstatus
<|endoftext|>


<commit_before>package app

import (
	"RPGit/api"
	"RPGit/app/services"
	"RPGit/crons"

	"github.com/revel/revel"
	"github.com/revel/revel/modules/jobs/app/jobs"
)

func init() {
	// Filters is the default set of global filters.
	revel.Filters = []revel.Filter{
		api.PanicFilter,               // Recover from panics and display an error page instead.
		revel.RouterFilter,            // Use the routing table to select the right Action
		revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
		revel.ParamsFilter,            // Parse parameters into Controller.Params.
		HeaderFilter,                  // Add some security based headers
		revel.InterceptorFilter,       // Run interceptors around the action.
		revel.CompressFilter,          // Compress the result.
		revel.ActionInvoker,           // Invoke the action.
	}

	// Register custom template helpers
	services.RegisterHelpers()

	// register startup functions with OnAppStart
	revel.OnAppStart(func() {
		// Init database
		services.InitDatabase()

		// Defines CRONS
		jobs.Schedule("cron.import", crons.Import{})
		jobs.Now(crons.WarmCache{})
		jobs.Now(crons.FullImport{})
	})
}

// TODO turn this into revel.HeaderFilter
// should probably also have a filter for CSRF
// not sure if it can go in the same filter or not
var HeaderFilter = func(c *revel.Controller, fc []revel.Filter) {
	// Add some common security headers
	c.Response.Out.Header().Add("X-Frame-Options", "SAMEORIGIN")
	c.Response.Out.Header().Add("X-XSS-Protection", "1; mode=block")
	c.Response.Out.Header().Add("X-Content-Type-Options", "nosniff")
	c.Response.Out.Header().Add("Cache-Control", "s-maxage=100000, private")

	fc[0](c, fc[1:]) // Execute the next filter stage.
}
<commit_msg>Revel panic filter is back<commit_after>package app

import (
	"RPGit/app/services"
	"RPGit/crons"

	"github.com/revel/revel"
	"github.com/revel/revel/modules/jobs/app/jobs"
)

func init() {
	// Filters is the default set of global filters.
	revel.Filters = []revel.Filter{
		revel.PanicFilter,             // Recover from panics and display an error page instead.
		revel.RouterFilter,            // Use the routing table to select the right Action
		revel.FilterConfiguringFilter, // A hook for adding or removing per-Action filters.
		revel.ParamsFilter,            // Parse parameters into Controller.Params.
		HeaderFilter,                  // Add some security based headers
		revel.InterceptorFilter,       // Run interceptors around the action.
		revel.CompressFilter,          // Compress the result.
		revel.ActionInvoker,           // Invoke the action.
	}

	// Register custom template helpers
	services.RegisterHelpers()

	// register startup functions with OnAppStart
	revel.OnAppStart(func() {
		// Init database
		services.InitDatabase()

		// Defines CRONS
		jobs.Schedule("cron.import", crons.Import{})
		jobs.Now(crons.WarmCache{})
		jobs.Now(crons.FullImport{})
	})
}

// TODO turn this into revel.HeaderFilter
// should probably also have a filter for CSRF
// not sure if it can go in the same filter or not
var HeaderFilter = func(c *revel.Controller, fc []revel.Filter) {
	// Add some common security headers
	c.Response.Out.Header().Add("X-Frame-Options", "SAMEORIGIN")
	c.Response.Out.Header().Add("X-XSS-Protection", "1; mode=block")
	c.Response.Out.Header().Add("X-Content-Type-Options", "nosniff")
	c.Response.Out.Header().Add("Cache-Control", "s-maxage=100000, private")

	fc[0](c, fc[1:]) // Execute the next filter stage.
}
<|endoftext|>


<commit_before>// Package oplogc provides an easy to use client interface for the oplog service.
//
// See https://github.com/dailymotion/oplog for more information on oplog.
//
// In case of a connection failure recovery the ack mechanism allows you to handle operations in parallel
// without loosing track of which operation has been handled.
//
// See cmd/oplog-tail for another usage example.
package oplogc

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

// Options is the subscription options
type Options struct {
	// Path of the state file where to persiste the current oplog position.
	// If empty string, the state is not stored.
	StateFile string
	// AllowReplication activates replication if the state file is not found.
	// When false, a consumer with no state file will only get future operations.
	AllowReplication bool
	// Password to access password protected oplog
	Password string
	// Proxy to be used to access oplog
	Proxy string
	// Filters to apply on the oplog output
	Filter Filter
}

// Filter contains arguments to filter the oplog output
type Filter struct {
	// A list of types to filter on
	Types []string
	// A list of parent type/id to filter on
	Parents []string
}

// Consumer holds all the information required to connect to an oplog server
type Consumer struct {
	// URL of the oplog
	url string
	// options for the consumer's subscription
	options Options
	// lastID is the current most advanced acked event id
	lastID string
	// saved is true when current lastID is persisted
	saved bool
	// processing is true when a process loop is in progress
	processing bool
	// mu is a mutex used to coordinate access to lastID and saved properties
	mu *sync.RWMutex
	// http is the client used to connect to the oplog
	http http.Client
	// body points to the current streamed response body
	body io.ReadCloser
	// ife holds all event ids sent to the consumer but no yet acked
	ife *inFlightEvents
	// ack is a channel to ack the operations
	ack chan Operation
	// stop is a channel used to stop the process loop
	stop chan struct{}
}

// ErrAccessDenied is returned by Subscribe when the oplog requires a password
// different from the one provided in options.
var ErrAccessDenied = errors.New("invalid credentials")

// ErrResumeFailed is returned when the requested last id was not found by the
// oplog server. This may happen when the last id is very old or size of the
// oplog capped collection is too small for the load.
//
// When this error happen, the consumer may choose to either ignore the lost events
// or force a full replication.
var ErrResumeFailed = errors.New("resume failed")

// ErrorWritingState is returned when the last processed id can't be written to
// the state file.
var ErrWritingState = errors.New("writing state file failed")

// Subscribe creates a Consumer to connect to the given URL.
func Subscribe(url string, options Options) *Consumer {
	qs := ""
	if len(options.Filter.Parents) > 0 {
		parents := strings.Join(options.Filter.Parents, ",")
		if parents != "" {
			qs += "?parents="
			qs += parents
		}
	}
	if len(options.Filter.Types) > 0 {
		types := strings.Join(options.Filter.Types, ",")
		if types != "" {
			if qs == "" {
				qs += "?"
			} else {
				qs += "&"
			}
			qs += "types="
			qs += types
		}
	}

	var proxyFunc func(*http.Request) (*neturl.URL, error) = nil

	if len(options.Proxy) > 0 {
		url_proxy, _ := neturl.ParseRequestURI(options.Proxy)
		proxyFunc = http.ProxyURL(url_proxy)
	}

	c := &Consumer{
		url:     strings.Join([]string{url, qs}, ""),
		options: options,
		ife:     newInFlightEvents(),
		mu:      &sync.RWMutex{},
		ack:     make(chan Operation),
		http:    http.Client{Transport: &http.Transport{Proxy: proxyFunc}},
	}

	return c
}

// Start reads the oplog output and send operations back thru the returned ops channel.
// The caller must then call the Done() method on operation when it has been handled.
// Failing to call Done() the operations would prevent any resume in case of connection
// failure or restart of the process.
//
// Any errors are return on the errs channel. In all cases, the Start() method will
// try to reconnect and/or ignore the error. It is the callers responsability to stop
// the process loop by calling the Stop() method.
//
// When the loop has ended, a message is sent thru the done channel.
func (c *Consumer) Start() (ops chan Operation, errs chan error, done chan bool) {
	ops = make(chan Operation)
	errs = make(chan error)
	done = make(chan bool)

	// Ensure we never have more than one process loop running
	if c.processing {
		panic("Can't run two process loops in parallel")
	}
	c.processing = true

	c.mu.Lock()
	c.stop = make(chan struct{})
	stop := c.stop
	c.mu.Unlock()

	// Recover the last event id saved from a previous excution
	lastID, err := c.loadLastEventID()
	if err != nil {
		errs <- err
		return
	}
	c.lastID = lastID

	wg := sync.WaitGroup{}

	// SSE stream reading
	stopReadStream := make(chan struct{}, 1)
	wg.Add(1)
	go c.readStream(ops, errs, stopReadStream, &wg)

	// Periodic (non blocking) saving of the last id when needed
	stopStateSaving := make(chan struct{}, 1)
	if c.options.StateFile != "" {
		wg.Add(1)
		go c.periodicStateSaving(errs, stopStateSaving, &wg)
	}

	go func() {
		for {
			select {
			case <-stop:
				// If a stop is requested, we ensure all go routines are stopped
				close(stopReadStream)
				close(stopStateSaving)
				if c.body != nil {
					// Closing the body will ensure readStream isn't blocked in IO wait
					c.body.Close()
				}
				wg.Wait()
				c.processing = false
				done <- true
				return
			case op := <-c.ack:
				if op.Event == "reset" {
					c.ife.Unlock()
				}
				if idx := c.ife.pull(op.ID); idx == 0 {
					c.SetLastID(op.ID)
				}
			}
		}
	}()

	return
}

// Stop instructs the Start() loop to stop
func (c *Consumer) Stop() {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.stop != nil {
		close(c.stop)
		c.stop = nil
	}
}

// readStream maintains a connection to the oplog stream and read sent events as they are coming
func (c *Consumer) readStream(ops chan<- Operation, errs chan<- error, stop <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.connect()
	d := newDecoder(c.body)
	op := Operation{}
	op.ack = c.ack
	backoff := time.Second
	for {
		err := d.next(&op)
		select {
		case <-stop:
			return
		default:
			// proceed
		}
		if err != nil {
			errs <- err
			for {
				time.Sleep(backoff)
				if backoff < 30*time.Second {
					backoff *= 2
				}
				if err = c.connect(); err == nil {
					d = newDecoder(c.body)
					break
				}
				errs <- err
			}
			continue
		}

		c.ife.push(op.ID)
		if op.Event == "reset" {
			// We must not process any further operation until the "reset" operation
			// is not acke
			c.ife.Lock()
		}
		select {
		case <-stop:
			return
		default:
			ops <- op
		}

		// reset backoff on success
		backoff = time.Second
	}
}

// periodicStateSaving saves the lastID into a file every seconds if it has been updated
func (c *Consumer) periodicStateSaving(errs chan<- error, stop <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case <-stop:
			return
		case <-time.After(time.Second):
			c.mu.RLock()
			saved := c.saved
			lastID := c.lastID
			c.mu.RUnlock()
			if saved {
				continue
			}
			if err := c.saveLastEventID(lastID); err != nil {
				errs <- ErrWritingState
			}
			c.mu.Lock()
			c.saved = lastID == c.lastID
			c.mu.Unlock()
		}
	}
}

// LastID returns the most advanced acked event id
func (c *Consumer) LastID() string {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.lastID
}

// SetLastID sets the last id to the given value and informs the save go routine
func (c *Consumer) SetLastID(id string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.lastID = id
	c.saved = false
}

// connect tries to connect to the oplog event stream
func (c *Consumer) connect() (err error) {
	if c.body != nil {
		c.body.Close()
	}
	// Usable dummy body in case of connection error
	c.body = ioutil.NopCloser(bytes.NewBuffer([]byte{}))

	req, err := http.NewRequest("GET", c.url, nil)
	if err != nil {
		return
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	lastID := c.LastID()
	if len(lastID) > 0 {
		req.Header.Set("Last-Event-ID", lastID)
	}
	if c.options.Password != "" {
		req.SetBasicAuth("", c.options.Password)
	}
	res, err := c.http.Do(req)
	if err != nil {
		return
	}
	if res.StatusCode == 403 || res.StatusCode == 401 {
		err = ErrAccessDenied
		return
	}
	if res.StatusCode != 200 {
		message, _ := ioutil.ReadAll(res.Body)
		err = fmt.Errorf("HTTP error %d: %s", res.StatusCode, string(message))
		return
	}
	c.body = res.Body
	return
}

// loadLastEventID tries to read the last event id from the state file.
//
// If the StateFile option was not set, the id will always be an empty string
// as for tailing only future events.
//
// If the StateFile option is set but no file exists, the last event id is
// initialized to "0" in order to request a full replication if AllowReplication
// option is set to true or to an empty string otherwise (start at present).
func (c *Consumer) loadLastEventID() (id string, err error) {
	if c.options.StateFile == "" {
		return "", nil
	}
	_, err = os.Stat(c.options.StateFile)
	if os.IsNotExist(err) {
		if c.options.AllowReplication {
			// full replication
			id = "0"
		} else {
			// start at NOW()
			id = ""
		}
		err = nil
	} else if err == nil {
		var content []byte
		content, err = ioutil.ReadFile(c.options.StateFile)
		if err != nil {
			return
		}
		if match, _ := regexp.Match("^(?:[0-9]{0,13}|[0-9a-f]{24})$", content); !match {
			err = errors.New("state file contains invalid data")
		}
		id = string(content)
	}
	return
}

// saveLastEventID persiste the last event id into a file
func (c *Consumer) saveLastEventID(id string) error {
	return ioutil.WriteFile(c.options.StateFile, []byte(id), 0644)
}
<commit_msg>Mytho handle errors<commit_after>// Package oplogc provides an easy to use client interface for the oplog service.
//
// See https://github.com/dailymotion/oplog for more information on oplog.
//
// In case of a connection failure recovery the ack mechanism allows you to handle operations in parallel
// without loosing track of which operation has been handled.
//
// See cmd/oplog-tail for another usage example.
package oplogc

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	neturl "net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

// Options is the subscription options
type Options struct {
	// Path of the state file where to persiste the current oplog position.
	// If empty string, the state is not stored.
	StateFile string
	// AllowReplication activates replication if the state file is not found.
	// When false, a consumer with no state file will only get future operations.
	AllowReplication bool
	// Password to access password protected oplog
	Password string
	// Proxy to be used to access oplog
	Proxy string
	// Filters to apply on the oplog output
	Filter Filter
}

// Filter contains arguments to filter the oplog output
type Filter struct {
	// A list of types to filter on
	Types []string
	// A list of parent type/id to filter on
	Parents []string
}

// Consumer holds all the information required to connect to an oplog server
type Consumer struct {
	// URL of the oplog
	url string
	// options for the consumer's subscription
	options Options
	// lastID is the current most advanced acked event id
	lastID string
	// saved is true when current lastID is persisted
	saved bool
	// processing is true when a process loop is in progress
	processing bool
	// mu is a mutex used to coordinate access to lastID and saved properties
	mu *sync.RWMutex
	// http is the client used to connect to the oplog
	http http.Client
	// body points to the current streamed response body
	body io.ReadCloser
	// ife holds all event ids sent to the consumer but no yet acked
	ife *inFlightEvents
	// ack is a channel to ack the operations
	ack chan Operation
	// stop is a channel used to stop the process loop
	stop chan struct{}
}

// ErrAccessDenied is returned by Subscribe when the oplog requires a password
// different from the one provided in options.
var ErrAccessDenied = errors.New("invalid credentials")

// ErrResumeFailed is returned when the requested last id was not found by the
// oplog server. This may happen when the last id is very old or size of the
// oplog capped collection is too small for the load.
//
// When this error happen, the consumer may choose to either ignore the lost events
// or force a full replication.
var ErrResumeFailed = errors.New("resume failed")

// ErrorWritingState is returned when the last processed id can't be written to
// the state file.
var ErrWritingState = errors.New("writing state file failed")

// Subscribe creates a Consumer to connect to the given URL.
func Subscribe(url string, options Options) *Consumer {
	qs := ""
	if len(options.Filter.Parents) > 0 {
		parents := strings.Join(options.Filter.Parents, ",")
		if parents != "" {
			qs += "?parents="
			qs += parents
		}
	}
	if len(options.Filter.Types) > 0 {
		types := strings.Join(options.Filter.Types, ",")
		if types != "" {
			if qs == "" {
				qs += "?"
			} else {
				qs += "&"
			}
			qs += "types="
			qs += types
		}
	}

	var proxyFunc func(*http.Request) (*neturl.URL, error) = nil
	if len(options.Proxy) > 0 {
		urlProxy, err := neturl.Parse(options.Proxy)
		if err != nil {
			panic(err)
		}
		proxyFunc = http.ProxyURL(urlProxy)
	}

	c := &Consumer{
		url:     strings.Join([]string{url, qs}, ""),
		options: options,
		ife:     newInFlightEvents(),
		mu:      &sync.RWMutex{},
		ack:     make(chan Operation),
		http:    http.Client{Transport: &http.Transport{Proxy: proxyFunc}},
	}

	return c
}

// Start reads the oplog output and send operations back thru the returned ops channel.
// The caller must then call the Done() method on operation when it has been handled.
// Failing to call Done() the operations would prevent any resume in case of connection
// failure or restart of the process.
//
// Any errors are return on the errs channel. In all cases, the Start() method will
// try to reconnect and/or ignore the error. It is the callers responsability to stop
// the process loop by calling the Stop() method.
//
// When the loop has ended, a message is sent thru the done channel.
func (c *Consumer) Start() (ops chan Operation, errs chan error, done chan bool) {
	ops = make(chan Operation)
	errs = make(chan error)
	done = make(chan bool)

	// Ensure we never have more than one process loop running
	if c.processing {
		panic("Can't run two process loops in parallel")
	}
	c.processing = true

	c.mu.Lock()
	c.stop = make(chan struct{})
	stop := c.stop
	c.mu.Unlock()

	// Recover the last event id saved from a previous excution
	lastID, err := c.loadLastEventID()
	if err != nil {
		errs <- err
		return
	}
	c.lastID = lastID

	wg := sync.WaitGroup{}

	// SSE stream reading
	stopReadStream := make(chan struct{}, 1)
	wg.Add(1)
	go c.readStream(ops, errs, stopReadStream, &wg)

	// Periodic (non blocking) saving of the last id when needed
	stopStateSaving := make(chan struct{}, 1)
	if c.options.StateFile != "" {
		wg.Add(1)
		go c.periodicStateSaving(errs, stopStateSaving, &wg)
	}

	go func() {
		for {
			select {
			case <-stop:
				// If a stop is requested, we ensure all go routines are stopped
				close(stopReadStream)
				close(stopStateSaving)
				if c.body != nil {
					// Closing the body will ensure readStream isn't blocked in IO wait
					c.body.Close()
				}
				wg.Wait()
				c.processing = false
				done <- true
				return
			case op := <-c.ack:
				if op.Event == "reset" {
					c.ife.Unlock()
				}
				if idx := c.ife.pull(op.ID); idx == 0 {
					c.SetLastID(op.ID)
				}
			}
		}
	}()

	return
}

// Stop instructs the Start() loop to stop
func (c *Consumer) Stop() {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.stop != nil {
		close(c.stop)
		c.stop = nil
	}
}

// readStream maintains a connection to the oplog stream and read sent events as they are coming
func (c *Consumer) readStream(ops chan<- Operation, errs chan<- error, stop <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	c.connect()
	d := newDecoder(c.body)
	op := Operation{}
	op.ack = c.ack
	backoff := time.Second
	for {
		err := d.next(&op)
		select {
		case <-stop:
			return
		default:
			// proceed
		}
		if err != nil {
			errs <- err
			for {
				time.Sleep(backoff)
				if backoff < 30*time.Second {
					backoff *= 2
				}
				if err = c.connect(); err == nil {
					d = newDecoder(c.body)
					break
				}
				errs <- err
			}
			continue
		}

		c.ife.push(op.ID)
		if op.Event == "reset" {
			// We must not process any further operation until the "reset" operation
			// is not acke
			c.ife.Lock()
		}
		select {
		case <-stop:
			return
		default:
			ops <- op
		}

		// reset backoff on success
		backoff = time.Second
	}
}

// periodicStateSaving saves the lastID into a file every seconds if it has been updated
func (c *Consumer) periodicStateSaving(errs chan<- error, stop <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case <-stop:
			return
		case <-time.After(time.Second):
			c.mu.RLock()
			saved := c.saved
			lastID := c.lastID
			c.mu.RUnlock()
			if saved {
				continue
			}
			if err := c.saveLastEventID(lastID); err != nil {
				errs <- ErrWritingState
			}
			c.mu.Lock()
			c.saved = lastID == c.lastID
			c.mu.Unlock()
		}
	}
}

// LastID returns the most advanced acked event id
func (c *Consumer) LastID() string {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.lastID
}

// SetLastID sets the last id to the given value and informs the save go routine
func (c *Consumer) SetLastID(id string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.lastID = id
	c.saved = false
}

// connect tries to connect to the oplog event stream
func (c *Consumer) connect() (err error) {
	if c.body != nil {
		c.body.Close()
	}
	// Usable dummy body in case of connection error
	c.body = ioutil.NopCloser(bytes.NewBuffer([]byte{}))

	req, err := http.NewRequest("GET", c.url, nil)
	if err != nil {
		return
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	lastID := c.LastID()
	if len(lastID) > 0 {
		req.Header.Set("Last-Event-ID", lastID)
	}
	if c.options.Password != "" {
		req.SetBasicAuth("", c.options.Password)
	}
	res, err := c.http.Do(req)
	if err != nil {
		return
	}
	if res.StatusCode == 403 || res.StatusCode == 401 {
		err = ErrAccessDenied
		return
	}
	if res.StatusCode != 200 {
		message, _ := ioutil.ReadAll(res.Body)
		err = fmt.Errorf("HTTP error %d: %s", res.StatusCode, string(message))
		return
	}
	c.body = res.Body
	return
}

// loadLastEventID tries to read the last event id from the state file.
//
// If the StateFile option was not set, the id will always be an empty string
// as for tailing only future events.
//
// If the StateFile option is set but no file exists, the last event id is
// initialized to "0" in order to request a full replication if AllowReplication
// option is set to true or to an empty string otherwise (start at present).
func (c *Consumer) loadLastEventID() (id string, err error) {
	if c.options.StateFile == "" {
		return "", nil
	}
	_, err = os.Stat(c.options.StateFile)
	if os.IsNotExist(err) {
		if c.options.AllowReplication {
			// full replication
			id = "0"
		} else {
			// start at NOW()
			id = ""
		}
		err = nil
	} else if err == nil {
		var content []byte
		content, err = ioutil.ReadFile(c.options.StateFile)
		if err != nil {
			return
		}
		if match, _ := regexp.Match("^(?:[0-9]{0,13}|[0-9a-f]{24})$", content); !match {
			err = errors.New("state file contains invalid data")
		}
		id = string(content)
	}
	return
}

// saveLastEventID persiste the last event id into a file
func (c *Consumer) saveLastEventID(id string) error {
	return ioutil.WriteFile(c.options.StateFile, []byte(id), 0644)
}
<|endoftext|>


<commit_before>// The main binary is the KnowYourCity server.
package main

import (
	"encoding/json"
	"log"
	"flag"
	"net/http"
	"flag"

	"github.com/Code4SierraLeone/KnowYourCity/base"

	"gopkg.in/mgo.v2"
	"github.com/bitly/go-nsq"
	"github.com/fatih/color"
	"github.com/go-errors/errors"
	"github.com/gorilla/mux"
	"github.com/gorilla/securecookie"
)

var hashKey = securecookie.GenerateRandomKey(32)
var blockKey = securecookie.GenerateRandomKey(32)
var sc = securecookie.New(hashKey, blockKey)

// Configures variables
var (
	serveraddr string
)

// main
func main() {
	serveraddr = *flag.String("srv", ":9004", "-srv=addr; set server listening address.Default :9004")
	flag.Parse()

	dbSession, err := dialDB()
	if err != nil {
		log.Fatal(err)
	}

	router := mux.NewRouter()
	setAccountRoutes(router, dbSession)

	// TODO: Switch to TLS in prod(samson)
	http.ListenAndServe(serveraddr, router)
}

// dialDB
func dialDB() (*mgo.Session, error) {
	return mgo.Dial("localhost")
}

// setAccountRoutes
// REST good practics: trailing slash denotes a directory, while the lack of it denotes a file/resource
func setAccountRoutes(router *mux.Router, dbSession *mgo.Session) {
	router.HandleFunc("/account/register", withCORS(withClient(DbSession(db, SecretCode(Register))))).Methods("POST")
	router.HandleFunc("/account/login", withCORS(withClient(DbSession(db, Login)))).Methods("POST")
	router.HandleFunc("/account/password/forgot", withCORS(withClient(DbSession(db, ForgotPass)))).Methods("POST")
	router.HandleFunc("/page/account/password/reset", withCORS(withClient(DbSession(db, ResetPassPage)))).Methods("POST")
	router.HandleFunc("/account/password/reset", withCORS(withClient(DbSession(db, ResetPass)))).Methods("POST")
	router.HandleFunc("/account/2factor", withCORS(withClient(DbSession(db, L2Factor)))).Methods("POST")
	router.HandleFunc("/account/logout", withCORS(withClient(DbSession(db, Logout)))).Methods("POST")

	router.HandleFunc("/keys/code/get", withCORS(withClient(DbSession(dbSession, GetSecretCode))))
	router.HandleFunc("/keys/code/set", withCORS(withClient(DbSession(dbSession, SetSecretCode))))
}

// ReportFatal
func ReportFatal(rw http.ResponseWriter, r *http.Request, err error) {
	base.RespondError(rw, r, err)
}

// LogError logs errors to file and stderr
func LogError(err error) {
	color.Set(color.BgRed)
	base.Error.Println(errors.Wrap(err, 1).ErrorStack())
	color.Unset()
}

// LogWarning logs warnings to stdout
func LogWarning(info interface{}) {
	color.Set(color.BgYellow)
	base.Warning.Println(info)
	color.Unset()
}

// LogDebug logs debug info to stdout
func LogDebug(info interface{}) {
	color.Set(color.BgCyan)
	base.Debug.Println(info)
	color.Unset()
}

// LogInfo logs info to
func LogInfo(info interface{}) {
	color.Set(color.BgGreen)
	base.Info.Println(info)
	color.Unset()
}

/*
*/

<commit_msg>Removes duplicate imports<commit_after>// The main binary is the KnowYourCity server.
package main

import (
	"encoding/json"
	"log"
	"flag"
	"net/http"

	"github.com/Code4SierraLeone/KnowYourCity/base"

	"gopkg.in/mgo.v2"
	"github.com/bitly/go-nsq"
	"github.com/fatih/color"
	"github.com/go-errors/errors"
	"github.com/gorilla/mux"
	"github.com/gorilla/securecookie"
)

var hashKey = securecookie.GenerateRandomKey(32)
var blockKey = securecookie.GenerateRandomKey(32)
var sc = securecookie.New(hashKey, blockKey)

// Configures variables
var (
	serveraddr string
)

// main
func main() {
	serveraddr = *flag.String("srv", ":9004", "-srv=addr; set server listening address.Default :9004")
	flag.Parse()

	dbSession, err := dialDB()
	if err != nil {
		log.Fatal(err)
	}

	router := mux.NewRouter()
	setAccountRoutes(router, dbSession)

	// TODO: Switch to TLS in prod(samson)
	http.ListenAndServe(serveraddr, router)
}

// dialDB
func dialDB() (*mgo.Session, error) {
	return mgo.Dial("localhost")
}

// setAccountRoutes
// REST good practics: trailing slash denotes a directory, while the lack of it denotes a file/resource
func setAccountRoutes(router *mux.Router, dbSession *mgo.Session) {
	router.HandleFunc("/account/register", withCORS(withClient(DbSession(db, SecretCode(Register))))).Methods("POST")
	router.HandleFunc("/account/login", withCORS(withClient(DbSession(db, Login)))).Methods("POST")
	router.HandleFunc("/account/password/forgot", withCORS(withClient(DbSession(db, ForgotPass)))).Methods("POST")
	router.HandleFunc("/page/account/password/reset", withCORS(withClient(DbSession(db, ResetPassPage)))).Methods("POST")
	router.HandleFunc("/account/password/reset", withCORS(withClient(DbSession(db, ResetPass)))).Methods("POST")
	router.HandleFunc("/account/2factor", withCORS(withClient(DbSession(db, L2Factor)))).Methods("POST")
	router.HandleFunc("/account/logout", withCORS(withClient(DbSession(db, Logout)))).Methods("POST")

	router.HandleFunc("/keys/code/get", withCORS(withClient(DbSession(dbSession, GetSecretCode))))
	router.HandleFunc("/keys/code/set", withCORS(withClient(DbSession(dbSession, SetSecretCode))))
}

// ReportFatal
func ReportFatal(rw http.ResponseWriter, r *http.Request, err error) {
	base.RespondError(rw, r, err)
}

// LogError logs errors to file and stderr
func LogError(err error) {
	color.Set(color.BgRed)
	base.Error.Println(errors.Wrap(err, 1).ErrorStack())
	color.Unset()
}

// LogWarning logs warnings to stdout
func LogWarning(info interface{}) {
	color.Set(color.BgYellow)
	base.Warning.Println(info)
	color.Unset()
}

// LogDebug logs debug info to stdout
func LogDebug(info interface{}) {
	color.Set(color.BgCyan)
	base.Debug.Println(info)
	color.Unset()
}

// LogInfo logs info to
func LogInfo(info interface{}) {
	color.Set(color.BgGreen)
	base.Info.Println(info)
	color.Unset()
}

/*
*/

<|endoftext|>


<commit_before>package vcs

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/iancmcc/jig/config"
)

var (
	// Git is the singleton driver
	Git VCS = &gitVCS{}

	absolute = regexp.MustCompile(`(remote: )?([\w\s]+):\s+()(\d+)()(.*)`)
	relative = regexp.MustCompile(`(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)`)
)

// GitVCS is a git driver
type gitVCS struct {
}

func parseProgress(repo string, r io.Reader) (<-chan Progress, <-chan bool) {
	out := make(chan Progress)
	scanner := bufio.NewScanner(r)
	scanner.Split(split)
	done := make(chan bool)
	go func() {
		seen := map[string]struct{}{}
		for scanner.Scan() {
			var (
				begin bool
				end   bool
				match []string
			)
			text := strings.TrimSpace(scanner.Text())
			if match = relative.FindStringSubmatch(text); match == nil {
				match = absolute.FindStringSubmatch(text)
			}
			if len(match) == 0 {
				continue
			}
			if strings.HasSuffix(text, "done.") {
				end = true
			}
			op := strings.TrimSpace(match[2])
			if strings.HasPrefix(op, "reused") {
				continue
			}
			cur, _ := strconv.Atoi(match[4])
			max, _ := strconv.Atoi(match[5])
			if _, ok := seen[op]; !ok {
				seen[op] = struct{}{}
				begin = true
			}
			prog := Progress{
				repo,
				begin,
				end,
				op,
				cur,
				max,
			}
			out <- prog
		}
		close(out)
		close(done)
	}()
	return out, done
}

func (g *gitVCS) run(repo, wd string, progress bool, cmd string, args ...string) <-chan Progress {
	if progress {
		args = append([]string{cmd, "--progress"}, args...)
	} else {
		args = append([]string{cmd}, args...)

	}
	command := exec.Command("git", args...)
	command.Dir = wd
	progout, _ := command.StderrPipe()
	command.Start()
	result, done := parseProgress(repo, progout)
	go func() {
		<-done
		command.Wait()
	}()
	return result
}

func prepareDir(dir string) error {
	return os.MkdirAll(filepath.Dir(dir), os.ModeDir|0775)
}

// Clone satisfies the VCS interface
func (g *gitVCS) Clone(r *config.Repo, dir string) (<-chan Progress, error) {
	if err := prepareDir(dir); err != nil {
		return nil, err
	}
	out := make(chan Progress)
	go func() {
		defer close(out)
		for p := range g.run(r.Repo, ".", true, "clone", r.Repo, dir) {
			out <- p
		}
		for p := range g.run(r.Repo, dir, true, "fetch", "--all") {
			out <- p
		}
		g.run(r.Repo, dir, false, "branch", "--track", "develop", "origin/develop")
		g.run(r.Repo, dir, false, "branch", "--track", "master", "origin/master")
		g.run(r.Repo, dir, false, "flow", "init", "-d")
	}()
	return out, nil
}

// Pull satisfies the VCS interface
func (g *gitVCS) Pull(r *config.Repo, dir string) (<-chan Progress, error) {
	return g.run(r.Repo, dir, true, "pull"), nil
}

// Checkout satisfies the VCS interface
func (g *gitVCS) Checkout(r *config.Repo, dir string) (<-chan Progress, error) {
	return g.run(r.Repo, dir, true, "checkout", r.Ref), nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '
'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil

}
<commit_msg>Don't use progress on git checkout, as it is not supported on all git versions<commit_after>package vcs

import (
	"bufio"
	"bytes"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/iancmcc/jig/config"
)

var (
	// Git is the singleton driver
	Git VCS = &gitVCS{}

	absolute = regexp.MustCompile(`(remote: )?([\w\s]+):\s+()(\d+)()(.*)`)
	relative = regexp.MustCompile(`(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)`)
)

// GitVCS is a git driver
type gitVCS struct {
}

func parseProgress(repo string, r io.Reader) (<-chan Progress, <-chan bool) {
	out := make(chan Progress)
	scanner := bufio.NewScanner(r)
	scanner.Split(split)
	done := make(chan bool)
	go func() {
		seen := map[string]struct{}{}
		for scanner.Scan() {
			var (
				begin bool
				end   bool
				match []string
			)
			text := strings.TrimSpace(scanner.Text())
			if match = relative.FindStringSubmatch(text); match == nil {
				match = absolute.FindStringSubmatch(text)
			}
			if len(match) == 0 {
				continue
			}
			if strings.HasSuffix(text, "done.") {
				end = true
			}
			op := strings.TrimSpace(match[2])
			if strings.HasPrefix(op, "reused") {
				continue
			}
			cur, _ := strconv.Atoi(match[4])
			max, _ := strconv.Atoi(match[5])
			if _, ok := seen[op]; !ok {
				seen[op] = struct{}{}
				begin = true
			}
			prog := Progress{
				repo,
				begin,
				end,
				op,
				cur,
				max,
			}
			out <- prog
		}
		close(out)
		close(done)
	}()
	return out, done
}

func (g *gitVCS) run(repo, wd string, progress bool, cmd string, args ...string) <-chan Progress {
	if progress {
		args = append([]string{cmd, "--progress"}, args...)
	} else {
		args = append([]string{cmd}, args...)

	}
	command := exec.Command("git", args...)
	command.Dir = wd
	progout, _ := command.StderrPipe()
	command.Start()
	result, done := parseProgress(repo, progout)
	go func() {
		<-done
		command.Wait()
	}()
	return result
}

func prepareDir(dir string) error {
	return os.MkdirAll(filepath.Dir(dir), os.ModeDir|0775)
}

// Clone satisfies the VCS interface
func (g *gitVCS) Clone(r *config.Repo, dir string) (<-chan Progress, error) {
	if err := prepareDir(dir); err != nil {
		return nil, err
	}
	out := make(chan Progress)
	go func() {
		defer close(out)
		for p := range g.run(r.Repo, ".", true, "clone", r.Repo, dir) {
			out <- p
		}
		for p := range g.run(r.Repo, dir, true, "fetch", "--all") {
			out <- p
		}
		g.run(r.Repo, dir, false, "branch", "--track", "develop", "origin/develop")
		g.run(r.Repo, dir, false, "branch", "--track", "master", "origin/master")
		g.run(r.Repo, dir, false, "flow", "init", "-d")
	}()
	return out, nil
}

// Pull satisfies the VCS interface
func (g *gitVCS) Pull(r *config.Repo, dir string) (<-chan Progress, error) {
	return g.run(r.Repo, dir, true, "pull"), nil
}

// Checkout satisfies the VCS interface
func (g *gitVCS) Checkout(r *config.Repo, dir string) (<-chan Progress, error) {
	return g.run(r.Repo, dir, false, "checkout", r.Ref), nil
}

// dropCR drops a terminal \r from the data.
func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

func split(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '
'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil

}
<|endoftext|>


<commit_before>package platform

import (
	"os"
	"strconv"
	"strings"
)

type EnvFlag struct {
	Name    string
	AltName string
}

func (f EnvFlag) GetValue(defaultValue string) string {
	if v, found := os.LookupEnv(f.Name); found {
		return v
	}
	if len(f.AltName) > 0 {
		if v, found := os.LookupEnv(f.AltName); found {
			return v
		}
	}

	return defaultValue
}

func (f EnvFlag) GetValueAsInt(defaultValue int) int {
	const PlaceHolder = "xxxxxx"
	s := f.GetValue(PlaceHolder)
	if s == PlaceHolder {
		return defaultValue
	}
	v, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		return defaultValue
	}
	return int(v)
}

func NormalizeEnvName(name string) string {
	return strings.Replace(strings.ToUpper(strings.TrimSpace(name)), ".", "_", -1)
}
<commit_msg>get asset location<commit_after>package platform

import (
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

type EnvFlag struct {
	Name    string
	AltName string
}

func (f EnvFlag) GetValue(defaultValue string) string {
	if v, found := os.LookupEnv(f.Name); found {
		return v
	}
	if len(f.AltName) > 0 {
		if v, found := os.LookupEnv(f.AltName); found {
			return v
		}
	}

	return defaultValue
}

func (f EnvFlag) GetValueAsInt(defaultValue int) int {
	const PlaceHolder = "xxxxxx"
	s := f.GetValue(PlaceHolder)
	if s == PlaceHolder {
		return defaultValue
	}
	v, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		return defaultValue
	}
	return int(v)
}

func NormalizeEnvName(name string) string {
	return strings.Replace(strings.ToUpper(strings.TrimSpace(name)), ".", "_", -1)
}

var assetPath = "/"

func init() {
	defAssetLocation, err := os.Executable()
	if err == nil {
		defAssetLocation = filepath.Dir(defAssetLocation)
		assetPath = (EnvFlag{
			Name: "v2ray.location.asset",
		}).GetValue(defAssetLocation)
	}
}

func GetAssetLocation(file string) string {
	return filepath.Join(assetPath, file)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"time"

	"github.com/garyburd/redigo/redis"
)

const (
	//Max number or ms for processing message
	MAX_RAND_PROCESS_TIME = 1000

	MODE_UNKNOWN = iota
	MODE_GENERATOR
	MODE_CONSUMER
)

type (
	Message string

	//Message with error
	BadMessage struct {
		msg Message
		err string
	}

	ProcessedMessage struct {
		duration time.Duration
		msg      Message
		worker   int
	}

	Consumer struct {
		pool          *redis.Pool
		queue         string
		errQueue      string
		in            chan Message
		out           chan ProcessedMessage
		bad           chan BadMessage
		maxGoroutines int
		stop          chan struct{}
		isActive      bool
	}
)

//String representation of BadMessage
func (b *BadMessage) String() string {
	return fmt.Sprintf("m:%q e:%q", b.msg, b.err)
}

//Create new Consumer struct
func NewConsumer(p *redis.Pool, q string, eq string, mg int) *Consumer {
	return &Consumer{
		pool:          p,
		queue:         q,
		errQueue:      eq,
		in:            make(chan Message),
		out:           make(chan ProcessedMessage),
		bad:           make(chan BadMessage),
		maxGoroutines: mg,
		stop:          make(chan struct{}),
	}
}

//Redis message listner
func (c *Consumer) Process(in chan Message, out chan ProcessedMessage) {
	c.in = in
	c.out = out

	for i := 1; i <= c.maxGoroutines; i++ {
		go c.RunWorker(i)
	}
	log.Printf("%d workers started.\n", c.maxGoroutines)

	c.isActive = true

	pc := c.pool.Get()
	defer pc.Close()
	defer func(c *Consumer) {
		log.Printf("%d workers stopped.\n", c.maxGoroutines)
	}(c)
	defer c.Close()

	for {
		select {
		case <-c.stop:
			//dirty hack, should wait before closing c.out
			//while switching from consumer to generator mode
			time.Sleep(time.Second)
			c.stop <- struct{}{}
			return
		default:
			msg, err := pc.Do("BRPOP", c.queue, 0)
			if err != nil {
				log.Println("messages: unable to get from redis:", err)
				return
			}

			v, err := redis.Values(msg, err)
			LogIf(err)

			s, err := redis.String(v[1], err)
			LogIf(err)

			c.in <- Message(s)
		}
	}
}

//Runs message consuming
func (c *Consumer) Start() {
	in := make(chan Message)
	out := make(chan ProcessedMessage)
	bad := make(chan BadMessage)

	if !c.Ping() {
		close(out)
	}

	go c.ProcessErrors(bad)
	go c.Process(in, out)

	go func(cc *Consumer) {
		for {
			select {
			case <-cc.out:
			}
		}
	}(c)
}

//Stops all workers
func (c *Consumer) Stop() {
	if !c.IsActive() {
		return
	}
	c.stop <- struct{}{}
	<-c.stop
}

//Makes primary work for random milliseconds.
func (c *Consumer) RunWorker(wid int) {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	processTime := time.Duration(r.Intn(MAX_RAND_PROCESS_TIME))

	for m := range c.in {
		if prob() {
			c.bad <- BadMessage{msg: m, err: fmt.Sprintf("Error code %d", processTime)}
		} else {
			time.Sleep(time.Millisecond * processTime)
			c.out <- ProcessedMessage{duration: processTime, msg: m, worker: wid}
		}
	}
}

//Closes working channels
func (c *Consumer) Close() {
	if !c.IsActive() {
		return
	}
	close(c.bad)
	close(c.in)
	close(c.out)
	c.isActive = false
}

//Get bad messages from chan and call PushError
func (c *Consumer) ProcessErrors(bad chan BadMessage) {
	pc := c.pool.Get()
	defer pc.Close()

	c.bad = bad

	for b := range c.bad {
		_, err := pc.Do("LPUSH", c.errQueue, b.String())
		if err != nil {
			log.Println("errors: unable to push to redis:", err)
		}
	}
}

//Returns consumer state
func (c *Consumer) IsActive() bool {
	return c.isActive
}

//Pings Redis
func (c *Consumer) Ping() bool {
	pc := c.pool.Get()
	defer pc.Close()

	p, err := pc.Do("PING")
	if err != nil {
		return false
	}
	return p == "PONG"
}
<commit_msg>Fixed restoring consumer.<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"time"

	"github.com/garyburd/redigo/redis"
)

const (
	//Max number or ms for processing message
	MAX_RAND_PROCESS_TIME = 1000

	MODE_UNKNOWN = iota
	MODE_GENERATOR
	MODE_CONSUMER
)

type (
	Message string

	//Message with error
	BadMessage struct {
		msg Message
		err string
	}

	ProcessedMessage struct {
		duration time.Duration
		msg      Message
		worker   int
	}

	Consumer struct {
		pool          *redis.Pool
		queue         string
		errQueue      string
		in            chan Message
		out           chan ProcessedMessage
		bad           chan BadMessage
		maxGoroutines int
		stop          chan struct{}
		isActive      bool
	}
)

//String representation of BadMessage
func (b *BadMessage) String() string {
	return fmt.Sprintf("m:%q e:%q", b.msg, b.err)
}

//Create new Consumer struct
func NewConsumer(p *redis.Pool, q string, eq string, mg int) *Consumer {
	return &Consumer{
		pool:          p,
		queue:         q,
		errQueue:      eq,
		in:            make(chan Message),
		out:           make(chan ProcessedMessage),
		bad:           make(chan BadMessage),
		maxGoroutines: mg,
		stop:          make(chan struct{}),
	}
}

//Redis message listner
func (c *Consumer) Process(in chan Message, out chan ProcessedMessage) {
	c.in = in
	c.out = out

	for i := 1; i <= c.maxGoroutines; i++ {
		go c.RunWorker(i)
	}
	log.Printf("%d workers started.\n", c.maxGoroutines)

	c.isActive = true

	pc := c.pool.Get()
	defer pc.Close()
	defer func(c *Consumer) {
		log.Printf("%d workers stopped.\n", c.maxGoroutines)
	}(c)
	defer c.Close()

	for {
		select {
		case <-c.stop:
			//dirty hack, should wait before closing c.out
			//while switching from consumer to generator mode
			time.Sleep(time.Second)
			c.stop <- struct{}{}
			return
		default:
			msg, err := pc.Do("BRPOP", c.queue, 0)
			if err != nil {
				log.Println("messages: unable to get from redis:", err)
				return
			}

			v, err := redis.Values(msg, err)
			LogIf(err)

			s, err := redis.String(v[1], err)
			LogIf(err)

			c.in <- Message(s)
		}
	}
}

//Runs message consuming
func (c *Consumer) Start() {
	in := make(chan Message)
	out := make(chan ProcessedMessage)
	bad := make(chan BadMessage)

	if !c.Ping() {
		close(out)
		c.out = out
		return
	}

	go c.ProcessErrors(bad)
	go c.Process(in, out)

	go func(cc *Consumer) {
		for {
			select {
			case <-cc.out:
			}
		}
	}(c)
}

//Stops all workers
func (c *Consumer) Stop() {
	if !c.IsActive() {
		return
	}
	c.stop <- struct{}{}
	<-c.stop
}

//Makes primary work for random milliseconds.
func (c *Consumer) RunWorker(wid int) {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	processTime := time.Duration(r.Intn(MAX_RAND_PROCESS_TIME))

	for m := range c.in {
		if prob() {
			c.bad <- BadMessage{msg: m, err: fmt.Sprintf("Error code %d", processTime)}
		} else {
			time.Sleep(time.Millisecond * processTime)
			c.out <- ProcessedMessage{duration: processTime, msg: m, worker: wid}
		}
	}
}

//Closes working channels
func (c *Consumer) Close() {
	if !c.IsActive() {
		return
	}
	close(c.bad)
	close(c.in)
	close(c.out)
	c.isActive = false
}

//Get bad messages from chan and call PushError
func (c *Consumer) ProcessErrors(bad chan BadMessage) {
	pc := c.pool.Get()
	defer pc.Close()

	c.bad = bad

	for b := range c.bad {
		_, err := pc.Do("LPUSH", c.errQueue, b.String())
		if err != nil {
			log.Println("errors: unable to push to redis:", err)
		}
	}
}

//Returns consumer state
func (c *Consumer) IsActive() bool {
	return c.isActive
}

//Pings Redis
func (c *Consumer) Ping() bool {
	pc := c.pool.Get()
	defer pc.Close()

	p, err := pc.Do("PING")
	if err != nil {
		return false
	}
	return p == "PONG"
}
<|endoftext|>


<commit_before>package resource

import (
	"code.google.com/p/go-uuid/uuid"
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/jmmcatee/cracklord/common"
	"sync"
)

// TODO: Add function for adding tools and assign a UUID

const (
	ERROR_AUTH    = "Call to resource did not have the proper authentication token."
	ERROR_NO_TOOL = "Tool specified does not exit."
)

type Queue struct {
	stack map[string]common.Tasker
	tools []common.Tooler
	sync.RWMutex
	hardware map[string]bool
}

func NewResourceQueue() Queue {
	return Queue{
		stack:    map[string]common.Tasker{},
		tools:    []common.Tooler{},
		hardware: map[string]bool{},
	}
}

func (q *Queue) AddTool(tooler common.Tooler) {
	// Add the hardware used by the tool
	q.hardware[tooler.Requirements()] = true

	tooler.SetUUID(uuid.New())
	q.tools = append(q.tools, tooler)
	log.WithFields(log.Fields{
		"toolid":  tooler.UUID(),
		"name":    tooler.Name(),
		"version": tooler.Version(),
	}).Debug("Tool added")
}

// Task RPC functions
func (q *Queue) Ping(ping int, pong *int) error {
	q.Lock()
	defer q.Unlock()

	pong = &ping

	return nil
}

func (q *Queue) ResourceHardware(rpc common.RPCCall, hw *map[string]bool) error {
	q.RLock()
	defer q.RUnlock()

	*hw = q.hardware

	return nil
}

func (q *Queue) AddTask(rpc common.RPCCall, rj *common.Job) error {
	log.WithFields(log.Fields{
		"name": rpc.Job.Name,
		"uuid": rpc.Job.UUID,
	}).Info("Job added")

	log.WithFields(log.Fields{
		"uuid":       rpc.Job.UUID,
		"parameters": rpc.Job.Parameters,
	})

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.AddTask: %v", err)
		}
	}()

	// variable to hold the tasker
	var tasker common.Tasker
	var err error
	// loop through common.Toolers for matching tool
	q.Lock()
	defer q.Unlock()
	for i, _ := range q.tools {
		if q.tools[i].UUID() == rpc.Job.ToolUUID {
			tasker, err = q.tools[i].NewTask(rpc.Job)
			if err != nil {
				return err
			}
		}
	}

	// Check if no tool was found and return error
	if tasker == nil {
		log.Warn("An error occured, we could not find the tool requested")
		return errors.New(ERROR_NO_TOOL)
	}
	log.WithFields(log.Fields{
		"task": rpc.Job.UUID,
	}).Debug("Tasker created")

	// Looks good so lets add to the stack
	if q.stack == nil {
		q.stack = make(map[string]common.Tasker)
	}

	q.stack[rpc.Job.UUID] = tasker

	// Everything should be paused by the control queue so start this job
	err = q.stack[rpc.Job.UUID].Run()
	if err != nil {
		log.Debug("Error starting task on resource")
		return errors.New("Error starting task on the resource: " + err.Error())
	}

	// Grab the status and return that job to the control queue
	*rj = q.stack[rpc.Job.UUID].Status()

	return nil
}

func (q *Queue) TaskStatus(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", j.UUID).Debug("Attempting to gather task status")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskStatus: %v", err)
		}
	}()

	// Grab the task specified by the UUID and return its status
	q.Lock()
	defer q.Unlock()
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok != false {
		log.WithField("task", j.UUID).Error("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	*j = q.stack[rpc.Job.UUID].Status()

	return nil
}

func (q *Queue) TaskPause(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", j.UUID).Debug("Attempting to pause task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskPause: %v", err)
		}
	}()

	// Grab the task specified by the UUID
	q.Lock()
	defer q.Unlock()
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok {
		log.WithField("task", j.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Pause the task
	err := q.stack[rpc.Job.UUID].Pause()
	if err != nil {
		// return the error but quit the job with status Failed
		// This is a definied behavior that we will not for all tools
		q.stack[rpc.Job.UUID].Quit()
		return err
	}

	*j = q.stack[rpc.Job.UUID].Status()

	log.WithField("task", j.UUID).Debug("Task paused successfully")

	return nil
}

func (q *Queue) TaskRun(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", rpc.Job.UUID).Debug("Attempting to run task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskRun: %v", err)
		}
	}()

	// Grab the task specified by the UUID
	q.Lock()
	defer q.Unlock()
	log.WithField("Stack", q.stack).Debug("Stack")
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok == false {
		log.WithField("task", rpc.Job.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Start or resume the task
	err := q.stack[rpc.Job.UUID].Run()
	if err != nil {
		return err
	}

	*j = q.stack[rpc.Job.UUID].Status()

	log.WithField("task", j.UUID).Debug("Task ran successfully")

	return nil

}

func (q *Queue) TaskQuit(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", j.UUID).Debug("Attempting to quit task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskQuit: %v", err)
		}
	}()

	// Grab a lock and set the unlock on return
	q.Lock()
	defer q.Unlock()

	// Grab the task specified by the UUID
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok != false {
		log.WithField("task", j.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Quit the task and return the final result
	*j = q.stack[rpc.Job.UUID].Quit()

	// Remove quit job from stack
	delete(q.stack, rpc.Job.UUID)

	log.WithField("task", j.UUID).Debug("Task ran successfully")

	return nil
}

// Queue Tasks

func (q *Queue) ResourceTools(rpc common.RPCCall, tools *[]common.Tool) error {
	log.Debug("Gathering all tools")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.ResourceTools: %v", err)
		}
	}()

	q.RLock()
	defer q.RUnlock()

	var ts []common.Tool

	for i, _ := range q.tools {
		var tool common.Tool
		tool.Name = q.tools[i].Name()
		tool.Type = q.tools[i].Type()
		tool.Version = q.tools[i].Version()
		tool.UUID = q.tools[i].UUID()
		tool.Parameters = q.tools[i].Parameters()
		tool.Requirements = q.tools[i].Requirements()

		log.WithFields(log.Fields{
			"UUID": tool.UUID,
			"name": tool.Name,
			"type": tool.Type,
			"ver":  tool.Version,
		}).Debug("Tool added")

		ts = append(ts, tool)
	}

	*tools = ts

	return nil
}

func (q *Queue) AllTaskStatus(rpc common.RPCCall, j *[]common.Job) error {
	log.Debug("Gathering all Task Status")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.AllTaskStatus: %v", err)
		}
	}()

	log.Debug("Gathering status on all jobs")

	// Loop through any tasks in the stack and update their status while
	// grabing the Job object output
	var jobs []common.Job

	q.Lock()

	for i, _ := range q.stack {
		jobs = append(jobs, q.stack[i].Status())
	}

	*j = jobs

	q.Unlock()

	return nil
}
<commit_msg>Fixed long running blank UUID error bug<commit_after>package resource

import (
	"code.google.com/p/go-uuid/uuid"
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/jmmcatee/cracklord/common"
	"sync"
)

// TODO: Add function for adding tools and assign a UUID

const (
	ERROR_AUTH    = "Call to resource did not have the proper authentication token."
	ERROR_NO_TOOL = "Tool specified does not exit."
)

type Queue struct {
	stack map[string]common.Tasker
	tools []common.Tooler
	sync.RWMutex
	hardware map[string]bool
}

func NewResourceQueue() Queue {
	return Queue{
		stack:    map[string]common.Tasker{},
		tools:    []common.Tooler{},
		hardware: map[string]bool{},
	}
}

func (q *Queue) AddTool(tooler common.Tooler) {
	// Add the hardware used by the tool
	q.hardware[tooler.Requirements()] = true

	tooler.SetUUID(uuid.New())
	q.tools = append(q.tools, tooler)
	log.WithFields(log.Fields{
		"toolid":  tooler.UUID(),
		"name":    tooler.Name(),
		"version": tooler.Version(),
	}).Debug("Tool added")
}

// Task RPC functions
func (q *Queue) Ping(ping int, pong *int) error {
	q.Lock()
	defer q.Unlock()

	pong = &ping

	return nil
}

func (q *Queue) ResourceHardware(rpc common.RPCCall, hw *map[string]bool) error {
	q.RLock()
	defer q.RUnlock()

	*hw = q.hardware

	return nil
}

func (q *Queue) AddTask(rpc common.RPCCall, rj *common.Job) error {
	log.WithFields(log.Fields{
		"name": rpc.Job.Name,
		"uuid": rpc.Job.UUID,
	}).Info("Job added")

	log.WithFields(log.Fields{
		"uuid":       rpc.Job.UUID,
		"parameters": rpc.Job.Parameters,
	})

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.AddTask: %v", err)
		}
	}()

	// variable to hold the tasker
	var tasker common.Tasker
	var err error
	// loop through common.Toolers for matching tool
	q.Lock()
	defer q.Unlock()
	for i, _ := range q.tools {
		if q.tools[i].UUID() == rpc.Job.ToolUUID {
			tasker, err = q.tools[i].NewTask(rpc.Job)
			if err != nil {
				return err
			}
		}
	}

	// Check if no tool was found and return error
	if tasker == nil {
		log.Warn("An error occured, we could not find the tool requested")
		return errors.New(ERROR_NO_TOOL)
	}
	log.WithFields(log.Fields{
		"task": rpc.Job.UUID,
	}).Debug("Tasker created")

	// Looks good so lets add to the stack
	if q.stack == nil {
		q.stack = make(map[string]common.Tasker)
	}

	q.stack[rpc.Job.UUID] = tasker

	// Everything should be paused by the control queue so start this job
	err = q.stack[rpc.Job.UUID].Run()
	if err != nil {
		log.Debug("Error starting task on resource")
		return errors.New("Error starting task on the resource: " + err.Error())
	}

	// Grab the status and return that job to the control queue
	*rj = q.stack[rpc.Job.UUID].Status()

	return nil
}

func (q *Queue) TaskStatus(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", rpc.Job.UUID).Debug("Attempting to gather task status")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskStatus: %v", err)
		}
	}()

	// Grab the task specified by the UUID and return its status
	q.Lock()
	defer q.Unlock()
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if !ok {
		log.WithField("task", rpc.Job.UUID).Error("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	*j = q.stack[rpc.Job.UUID].Status()

	return nil
}

func (q *Queue) TaskPause(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", j.UUID).Debug("Attempting to pause task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskPause: %v", err)
		}
	}()

	// Grab the task specified by the UUID
	q.Lock()
	defer q.Unlock()
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok {
		log.WithField("task", j.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Pause the task
	err := q.stack[rpc.Job.UUID].Pause()
	if err != nil {
		// return the error but quit the job with status Failed
		// This is a definied behavior that we will not for all tools
		q.stack[rpc.Job.UUID].Quit()
		return err
	}

	*j = q.stack[rpc.Job.UUID].Status()

	log.WithField("task", j.UUID).Debug("Task paused successfully")

	return nil
}

func (q *Queue) TaskRun(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", rpc.Job.UUID).Debug("Attempting to run task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskRun: %v", err)
		}
	}()

	// Grab the task specified by the UUID
	q.Lock()
	defer q.Unlock()
	log.WithField("Stack", q.stack).Debug("Stack")
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok == false {
		log.WithField("task", rpc.Job.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Start or resume the task
	err := q.stack[rpc.Job.UUID].Run()
	if err != nil {
		return err
	}

	*j = q.stack[rpc.Job.UUID].Status()

	log.WithField("task", j.UUID).Debug("Task ran successfully")

	return nil

}

func (q *Queue) TaskQuit(rpc common.RPCCall, j *common.Job) error {
	log.WithField("task", j.UUID).Debug("Attempting to quit task")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.TaskQuit: %v", err)
		}
	}()

	// Grab a lock and set the unlock on return
	q.Lock()
	defer q.Unlock()

	// Grab the task specified by the UUID
	_, ok := q.stack[rpc.Job.UUID]

	// Check for a bad UUID
	if ok != false {
		log.WithField("task", j.UUID).Debug("Task with UUID provided does not exist.")
		return errors.New("Task with UUID provided does not exist.")
	}

	// Quit the task and return the final result
	*j = q.stack[rpc.Job.UUID].Quit()

	// Remove quit job from stack
	delete(q.stack, rpc.Job.UUID)

	log.WithField("task", j.UUID).Debug("Task ran successfully")

	return nil
}

// Queue Tasks

func (q *Queue) ResourceTools(rpc common.RPCCall, tools *[]common.Tool) error {
	log.Debug("Gathering all tools")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.ResourceTools: %v", err)
		}
	}()

	q.RLock()
	defer q.RUnlock()

	var ts []common.Tool

	for i, _ := range q.tools {
		var tool common.Tool
		tool.Name = q.tools[i].Name()
		tool.Type = q.tools[i].Type()
		tool.Version = q.tools[i].Version()
		tool.UUID = q.tools[i].UUID()
		tool.Parameters = q.tools[i].Parameters()
		tool.Requirements = q.tools[i].Requirements()

		log.WithFields(log.Fields{
			"UUID": tool.UUID,
			"name": tool.Name,
			"type": tool.Type,
			"ver":  tool.Version,
		}).Debug("Tool added")

		ts = append(ts, tool)
	}

	*tools = ts

	return nil
}

func (q *Queue) AllTaskStatus(rpc common.RPCCall, j *[]common.Job) error {
	log.Debug("Gathering all Task Status")

	// Add a defered catch for panic from within the tools
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("Recovered from Panic in Resource.AllTaskStatus: %v", err)
		}
	}()

	log.Debug("Gathering status on all jobs")

	// Loop through any tasks in the stack and update their status while
	// grabing the Job object output
	var jobs []common.Job

	q.Lock()

	for i, _ := range q.stack {
		jobs = append(jobs, q.stack[i].Status())
	}

	*j = jobs

	q.Unlock()

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vec

import "math"

// Vectorize returns a function g(xs) that applies f to each x in xs.
//
// f may be evaluated in parallel and in any order.
func Vectorize(f func(float64) float64) func(xs []float64) []float64 {
	return func(xs []float64) []float64 {
		return Map(f, xs)
	}
}

// Map returns f(x) for each x in xs.
//
// f may be evaluated in parallel and in any order.
func Map(f func(float64) float64, xs []float64) []float64 {
	// TODO(austin) Parallelize
	res := make([]float64, len(xs))
	for i, x := range xs {
		res[i] = f(x)
	}
	return res
}

// Linspace returns num values spaced evenly between lo and hi,
// inclusive. If num is 1, this returns an array consisting of lo.
func Linspace(lo, hi float64, num int) []float64 {
	res := make([]float64, num)
	if num == 1 {
		res[0] = lo
		return res
	}
	for i := 0; i < num; i++ {
		res[i] = lo + float64(i)*(hi-lo)/float64(num-1)
	}
	return res
}

// Logspace returns num values spaced evenly on a logarithmic scale
// between base**lo and base**hi, inclusive.
func Logspace(lo, hi float64, num int, base float64) []float64 {
	res := Linspace(lo, hi, num)
	for i, x := range res {
		res[i] = math.Pow(base, x)
	}
	return res
}

// Sum returns the sum of xs.
func Sum(xs []float64) float64 {
	sum := 0.0
	for _, x := range xs {
		sum += x
	}
	return sum
}
<commit_msg>vec: vector concatenation utility<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package vec

import "math"

// Vectorize returns a function g(xs) that applies f to each x in xs.
//
// f may be evaluated in parallel and in any order.
func Vectorize(f func(float64) float64) func(xs []float64) []float64 {
	return func(xs []float64) []float64 {
		return Map(f, xs)
	}
}

// Map returns f(x) for each x in xs.
//
// f may be evaluated in parallel and in any order.
func Map(f func(float64) float64, xs []float64) []float64 {
	// TODO(austin) Parallelize
	res := make([]float64, len(xs))
	for i, x := range xs {
		res[i] = f(x)
	}
	return res
}

// Linspace returns num values spaced evenly between lo and hi,
// inclusive. If num is 1, this returns an array consisting of lo.
func Linspace(lo, hi float64, num int) []float64 {
	res := make([]float64, num)
	if num == 1 {
		res[0] = lo
		return res
	}
	for i := 0; i < num; i++ {
		res[i] = lo + float64(i)*(hi-lo)/float64(num-1)
	}
	return res
}

// Logspace returns num values spaced evenly on a logarithmic scale
// between base**lo and base**hi, inclusive.
func Logspace(lo, hi float64, num int, base float64) []float64 {
	res := Linspace(lo, hi, num)
	for i, x := range res {
		res[i] = math.Pow(base, x)
	}
	return res
}

// Sum returns the sum of xs.
func Sum(xs []float64) float64 {
	sum := 0.0
	for _, x := range xs {
		sum += x
	}
	return sum
}

// Concat returns the concatenation of its arguments. It does not
// modify its inputs.
func Concat(xss ...[]float64) []float64 {
	total := 0
	for _, xs := range xss {
		total += len(xs)
	}
	out := make([]float64, total)
	pos := 0
	for _, xs := range xss {
		pos += copy(out[pos:], xs)
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright 2014 Oleku Konko All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// This module is a Terminal  API for the Go Programming Language.
// The protocols were written in pure Go and works on windows and unix systems

package ts

import (
	"fmt"
	"testing"
)

func ExampleGetSize() {
	size, _ := GetSize()
	fmt.Println(size.Col())  // Get Width
	fmt.Println(size.Row())  // Get Height
	fmt.Println(size.PosX()) // Get X position
	fmt.Println(size.PosY()) // Get Y position
}

func TestSize(t *testing.T) {
	size, err := GetSize()

	if err != nil {
		t.
		t.Fatal(err)
	}
	if size.Col() == 0 || size.Row() == 0 {
		t.Fatalf("Screen Size Failed")
	}
}
<commit_msg>Minor Typo<commit_after>// Copyright 2014 Oleku Konko All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// This module is a Terminal  API for the Go Programming Language.
// The protocols were written in pure Go and works on windows and unix systems

package ts

import (
	"fmt"
	"testing"
)

func ExampleGetSize() {
	size, _ := GetSize()
	fmt.Println(size.Col())  // Get Width
	fmt.Println(size.Row())  // Get Height
	fmt.Println(size.PosX()) // Get X position
	fmt.Println(size.PosY()) // Get Y position
}

func TestSize(t *testing.T) {
	size, err := GetSize()

	if err != nil {
		t.Fatal(err)
	}
	if size.Col() == 0 || size.Row() == 0 {
		t.Fatalf("Screen Size Failed")
	}
}
<|endoftext|>