<commit_before>package sqlitecache

import (
	"database/sql"
	"fmt"
	"github.com/aktau/gofinance/fquery"
	"github.com/aktau/gofinance/util"
	"github.com/coopernurse/gorp"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

/* a version of the fquery.HistEntry struct with a symbol appended,
 * so we can query it easier. */
type dbHistEntry struct {
	Symbol   string
	Date     time.Time
	Open     float64
	Close    float64
	AdjClose float64
	High     float64
	Low      float64
	Volume   int64
}

type dbHistMeta struct {
	Symbol string
	Amount int64

	MinDate int64
	MaxDate int64

	/* processed versions of MinDate/MaxDate */
	rMinDate time.Time
	rMaxDate time.Time
}

var (
	VERBOSITY = 0
)

type SqliteCache struct {
	fquery.Source

	gorp        *gorp.DbMap
	quoteExpiry time.Duration
}

func New(path string, src fquery.Source) (*SqliteCache, error) {
	db, err := sql.Open("sqlite3", path)
	if err != nil {
		return nil, err
	}

	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
	if VERBOSITY >= 2 {
		dbmap.TraceOn("", log.New(os.Stdout, "dbmap: ", log.Lmicroseconds))
	}

	c := &SqliteCache{src, dbmap, 30 * time.Second}

	c.gorp.AddTableWithName(fquery.Quote{}, "quotes").SetKeys(false, "Symbol")
	c.gorp.AddTableWithName(dbHistEntry{}, "histquotes").SetKeys(false, "Symbol", "Date")

	err = c.gorp.CreateTablesIfNotExists()
	if err != nil {
		c.Close()
		return nil, err
	}

	/* support date range queries over all symbols */
	_, err = c.gorp.Exec(`CREATE INDEX IF NOT EXISTS hq_date_idx ON histquotes (Date)`)
	if err != nil {
		c.Close()
		return nil, err
	}

	return c, nil
}

func (c *SqliteCache) SetQuoteExpiry(dur time.Duration) {
	c.quoteExpiry = dur
}

func (c *SqliteCache) HasQuote(symbol string) bool {
	return false
}

func (c *SqliteCache) HasHist(symbol string, start *time.Time, end *time.Time) bool {
	return false
}

func (c *SqliteCache) Close() error {
	return c.gorp.Db.Close()
}

/* TODO: escape your strings, symbols could be user input */
func (c *SqliteCache) Quote(symbols []string) ([]fquery.Quote, error) {
	/* fetch all the quotes we have */
	quotedSymbols := quoteSymbols(symbols)

	var results []fquery.Quote
	cutoff := time.Now().Add(-c.quoteExpiry)
	_, err := c.gorp.Select(&results,
		fmt.Sprintf(
			"SELECT * FROM quotes WHERE Symbol IN (%v) AND Updated >= datetime(%v, 'unixepoch')",
			strings.Join(quotedSymbols, ","), cutoff.Unix()))
	if err != nil {
		/* if an error occured, just patch through to the source */
		vprintln("sqlitecache: error while fetching quotes, ", err, ", will use underlying source")
		return c.Source.Quote(symbols)
	}

	/* in case no error occured, check which ones were not in the cache,
	 * they need to be added to the list of quotes to fetch from the src */
	quoteMap := fquery.QuotesToMap(results)
	toFetch := make([]string, 0, len(symbols))
	for _, symbol := range symbols {
		if _, ok := quoteMap[symbol]; !ok {
			toFetch = append(toFetch, symbol)
			vprintln(symbol, "was NOT fetched from cache!")
		} else {
			vprintln(symbol, "was fetched from cache!")
		}
	}

	/* fetch all missing items, store in cache and add to the results we
	 * already got from the cache */
	fetched, err := c.Source.Quote(toFetch)
	if err != nil {
		vprintf("sqlitecache: error while fetching either of %v: %v\n", toFetch, err)
		return results, nil
	}

	err = c.mergeQuotes(fetched...)
	if err != nil {
		vprintf("sqlitecache: error, could not merge quotes of %v into cache, %v\n", toFetch, err)
	}

	results = append(results, fetched...)

	return results, nil
}

/* I consider this to be an extremely dirty function, it should be split up
 * and possibly (hopefully) simplified*/
func (c *SqliteCache) Hist(symbols []string) (map[string]fquery.Hist, error) {
	/* fetch all the historical results we have in the cache */
	symbolSet := sliceToSet(symbols)

	/* find out how many entries we have per symbol (we do this so we can
	 * process the cached symbols and fetch the missing ones in parallel */
	sqliteCastToUnix := func(selector string) string {
		return `strftime('%s',` + selector + `)`
	}
	var meta []*dbHistMeta
	_, err := c.gorp.Select(&meta,
		fmt.Sprintf(
			`SELECT Symbol, count(Symbol) AS Amount,
			        %v AS MinDate,
					%v AS MaxDate
			 FROM histquotes
			 WHERE Symbol IN (%v)
			 GROUP BY Symbol`,
			sqliteCastToUnix("min(Date)"),
			sqliteCastToUnix("max(Date)"),
			strings.Join(quoteSymbols(symbols), ",")))
	if err != nil {
		/* if an error occured, just patch through to the source */
		vprintln("sqlitecache: error while fetching history metadata, ", err, ", will use underlying source")
		return c.Source.Hist(symbols)
	}

	/* convert the timestamps from UNIX timestamps to time.Time */
	for _, m := range meta {
		m.rMinDate = time.Unix(m.MinDate, 0)
		m.rMaxDate = time.Unix(m.MaxDate, 0)
	}

	/* convert the meta list to a map, for easier searching */
	metaMap := make(map[string]*dbHistMeta, len(meta))
	for _, m := range meta {
		metaMap[m.Symbol] = m
	}

	yesterday := Yesterday()
	missing := make([]string, 0, len(symbols))
	for _, s := range symbols {
		m, found := metaMap[s]
		if !found {
			vprintln(s, "was NOT fetched from cache!")
			missing = append(missing, s)
		} else {
			if m.rMaxDate.Before(yesterday) {
				vprintln(s, "was NOT fetched from cache! (historical prices "+
					"not recent enough), last available date was",
					m.rMaxDate.Format("02/01/2006"))
				missing = append(missing, s)
				delete(symbolSet, s)
			} else {
				vprintln(s, "was fetched from cache!")
			}
		}
	}

	hist := make(map[string]fquery.Hist, len(symbols))
	var (
		histmutex sync.Mutex
		wg        sync.WaitGroup
	)

	if len(symbolSet) > 0 {
		adjSymbols := setToSlice(symbolSet)
		vprintln("fetching", adjSymbols, "from the SQLite db")

		/* now query the actual entries */
		var results []dbHistEntry
		_, err = c.gorp.Select(&results,
			fmt.Sprintf(
				`SELECT * FROM histquotes WHERE Symbol IN (%v) ORDER BY Symbol, Date`,
				strings.Join(quoteSymbols(adjSymbols), ",")))
		if err != nil {
			/* if an error occured, just patch through to the source */
			vprintln("sqlitecache: error while fetching historical quotes, ", err, ", will use underlying source")
			return c.Source.Hist(symbols)
		}

		/* process the dbHistEntries into fquery.Hist structures */
		wg.Add(1)
		go func() {
			defer wg.Done()

			var (
				cursymb string
				curhist fquery.Hist
			)
			for _, e := range results {
				/* detect transitions between symbols */
				if cursymb != e.Symbol {
					fmt.Println("CHANGING SYMBOL:", cursymb, e.Symbol)
					/* if this is not the first iteration, finish up the hist
					 * entry we've been working on */
					if len(cursymb) != 0 {
						histmutex.Lock()
						hist[cursymb] = curhist
						histmutex.Unlock()
					}

					/* move to the following symbol */
					info := metaMap[e.Symbol]
					cursymb = e.Symbol
					curhist = fquery.Hist{
						Symbol:  e.Symbol,
						From:    info.rMinDate,
						To:      info.rMaxDate,
						Entries: make([]fquery.HistEntry, 0, info.Amount),
					}
				}

				curhist.Entries = append(curhist.Entries, normalizeHistEntry(&e))
			}

			/* add the last one too */
			histmutex.Lock()
			hist[cursymb] = curhist
			histmutex.Unlock()
		}()
	}

	/* fetch the missing entries from the original source */
	if len(missing) > 0 {
		wg.Add(1)
		go func() {
			defer wg.Done()

			/* fetch the missing symbols */
			missingMap, err := c.Source.Hist(missing)
			if err != nil {
				vprintln("sqlitecache: error occured while fetching missing", missing, "hist. quotes,", err)
			} else {
				missinglist := make([]fquery.Hist, 0, len(missingMap))
				for symbol, history := range missingMap {
					histmutex.Lock()
					hist[symbol] = history
					histmutex.Unlock()

					missinglist = append(missinglist, history)
				}

				/* add the previously missing items to the cache, so we don't have to
				 * request them again */
				c.mergeHistory(missinglist...)
			}
		}()
	}

	wg.Wait()

	return hist, nil
}

/*
func (c *SqliteCache) HistLimit(symbol []string, start time.Time, end time.Time) (map[string]fquery.Hist, error) {
	return nil, fmt.Errorf("not supported")
}

func (c *SqliteCache) DividendHist(symbol []string) (map[string]fquery.DividendHist, error) {
	return nil, fmt.Errorf("not supported")
}

func (c *SqliteCache) DividendHistLimit(symbol []string, start time.Time, end time.Time) (map[string]fquery.DividendHist, error) {
	return nil, fmt.Errorf("not supported")
}
*/

func (c *SqliteCache) String() string {
	return "SQLite cache, backed by: " + c.String()
}

func (c *SqliteCache) mergeQuotes(quotes ...fquery.Quote) error {
	if len(quotes) == 0 {
		return nil
	}

	tx, err := c.gorp.Begin()
	if err != nil {
		return err
	}

	for _, quote := range quotes {
		vprintln("merging quote: ", quote.Symbol)
		count, err := tx.Update(&quote)
		if err == nil && count == 1 {
			continue
		}
		vprintln("sqlitecache: error while UPDATE'ing symbol", quote.Symbol,
			"err:", err, ", count:", count)

		/* update didn't work, so try insert */
		err = tx.Insert(&quote)
		if err != nil {
			vprintln("sqlitecache: error while INSERTing", err)
		}
	}

	return tx.Commit()
}

func (c *SqliteCache) mergeHistory(hists ...fquery.Hist) error {
	if len(hists) == 0 {
		return nil
	}

	tx, err := c.gorp.Begin()
	if err != nil {
		return err
	}

	for _, hist := range hists {
		/* delete values we're going to override anyway */
		vprintln("deleting potentially already stored history for symbol",
			hist.Symbol, "from", hist.From, "to", hist.To)
		delquery := `DELETE
				     FROM  histquotes
					 WHERE Symbol = '%v'
					 AND   Date BETWEEN datetime(%v, 'unixepoch')
							    AND     datetime(%v, 'unixepoch')`
		delquery = fmt.Sprintf(delquery, hist.Symbol, hist.From.Unix(), hist.To.Unix())
		_, err := tx.Exec(delquery)
		if err != nil {
			vprintln("sqlitecache: mergeHistory: error while deleting, query:",
				delquery, "error:", err)
		}

		vprintln("merging history:", hist.Symbol)
		for _, entry := range hist.Entries {
			/* insert */
			err := tx.Insert(newDbHistEntry(hist.Symbol, &entry))
			if err != nil {
				vprintln("sqlitecache: mergeHistory: error while INSERTing", err)
			}
		}
	}

	return tx.Commit()
}

func vprintln(a ...interface{}) (int, error) {
	if VERBOSITY > 0 {
		return fmt.Println(a...)
	}

	return 0, nil
}

func vprintf(format string, a ...interface{}) (int, error) {
	if VERBOSITY > 0 {
		return fmt.Printf(format, a...)
	}

	return 0, nil
}

func quoteSymbols(symbols []string) []string {
	return util.MapStr(func(s string) string {
		return `'` + s + `'`
	}, symbols)
}

func normalizeHistEntry(e *dbHistEntry) fquery.HistEntry {
	return fquery.HistEntry{
		Date:     util.YearMonthDay(e.Date),
		Open:     e.Open,
		Close:    e.Close,
		AdjClose: e.AdjClose,
		Low:      e.Low,
		High:     e.High,
		Volume:   e.Volume,
	}
}

func newDbHistEntry(symbol string, e *fquery.HistEntry) *dbHistEntry {
	return &dbHistEntry{
		Symbol:   symbol,
		Date:     time.Time(e.Date),
		Open:     e.Open,
		Close:    e.Close,
		AdjClose: e.AdjClose,
		Low:      e.Low,
		High:     e.High,
		Volume:   e.Volume,
	}
}

func Yesterday() time.Time {
	const day = 24 * time.Hour
	return time.Now().Add(-1 * day)
}

func sliceToSet(xs []string) map[string]bool {
	m := make(map[string]bool, len(xs))
	for _, x := range xs {
		m[x] = true
	}
	return m
}

func setToSlice(m map[string]bool) []string {
	xs := make([]string, len(m))
	idx := 0
	for str := range m {
		xs[idx] = str
		idx++
	}
	return xs
}
<commit_msg>sqlitecache: tune some pragmas<commit_after>package sqlitecache

import (
	"database/sql"
	"fmt"
	"github.com/aktau/gofinance/fquery"
	"github.com/aktau/gofinance/util"
	"github.com/coopernurse/gorp"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"strings"
	"sync"
	"time"
)

/* a version of the fquery.HistEntry struct with a symbol appended,
 * so we can query it easier. */
type dbHistEntry struct {
	Symbol   string
	Date     time.Time
	Open     float64
	Close    float64
	AdjClose float64
	High     float64
	Low      float64
	Volume   int64
}

type dbHistMeta struct {
	Symbol string
	Amount int64

	MinDate int64
	MaxDate int64

	/* processed versions of MinDate/MaxDate */
	rMinDate time.Time
	rMaxDate time.Time
}

var (
	VERBOSITY = 0
)

type SqliteCache struct {
	fquery.Source

	gorp        *gorp.DbMap
	quoteExpiry time.Duration
}

func New(path string, src fquery.Source) (*SqliteCache, error) {
	db, err := sql.Open("sqlite3", path)
	if err != nil {
		return nil, err
	}

	db.Exec("PRAGMA synchronous=NORMAL")
	db.Exec("PRAGMA journal_mode=WAL")
	db.Exec("PRAGMA temp_store=MEMORY")
	db.Exec("PRAGMA cache_size=4096")

	dbmap := &gorp.DbMap{Db: db, Dialect: gorp.SqliteDialect{}}
	if VERBOSITY >= 2 {
		dbmap.TraceOn("", log.New(os.Stdout, "dbmap: ", log.Lmicroseconds))
	}

	c := &SqliteCache{src, dbmap, 30 * time.Second}

	c.gorp.AddTableWithName(fquery.Quote{}, "quotes").SetKeys(false, "Symbol")
	c.gorp.AddTableWithName(dbHistEntry{}, "histquotes").SetKeys(false, "Symbol", "Date")

	err = c.gorp.CreateTablesIfNotExists()
	if err != nil {
		c.Close()
		return nil, err
	}

	/* support date range queries over all symbols */
	_, err = c.gorp.Exec(`CREATE INDEX IF NOT EXISTS hq_date_idx ON histquotes (Date)`)
	if err != nil {
		c.Close()
		return nil, err
	}

	return c, nil
}

func (c *SqliteCache) SetQuoteExpiry(dur time.Duration) {
	c.quoteExpiry = dur
}

func (c *SqliteCache) HasQuote(symbol string) bool {
	return false
}

func (c *SqliteCache) HasHist(symbol string, start *time.Time, end *time.Time) bool {
	return false
}

func (c *SqliteCache) Close() error {
	return c.gorp.Db.Close()
}

/* TODO: escape your strings, symbols could be user input */
func (c *SqliteCache) Quote(symbols []string) ([]fquery.Quote, error) {
	/* fetch all the quotes we have */
	quotedSymbols := quoteSymbols(symbols)

	var results []fquery.Quote
	cutoff := time.Now().Add(-c.quoteExpiry)
	_, err := c.gorp.Select(&results,
		fmt.Sprintf(
			"SELECT * FROM quotes WHERE Symbol IN (%v) AND Updated >= datetime(%v, 'unixepoch')",
			strings.Join(quotedSymbols, ","), cutoff.Unix()))
	if err != nil {
		/* if an error occured, just patch through to the source */
		vprintln("sqlitecache: error while fetching quotes, ", err, ", will use underlying source")
		return c.Source.Quote(symbols)
	}

	/* in case no error occured, check which ones were not in the cache,
	 * they need to be added to the list of quotes to fetch from the src */
	quoteMap := fquery.QuotesToMap(results)
	toFetch := make([]string, 0, len(symbols))
	for _, symbol := range symbols {
		if _, ok := quoteMap[symbol]; !ok {
			toFetch = append(toFetch, symbol)
			vprintln(symbol, "was NOT fetched from cache!")
		} else {
			vprintln(symbol, "was fetched from cache!")
		}
	}

	/* fetch all missing items, store in cache and add to the results we
	 * already got from the cache */
	fetched, err := c.Source.Quote(toFetch)
	if err != nil {
		vprintf("sqlitecache: error while fetching either of %v: %v\n", toFetch, err)
		return results, nil
	}

	err = c.mergeQuotes(fetched...)
	if err != nil {
		vprintf("sqlitecache: error, could not merge quotes of %v into cache, %v\n", toFetch, err)
	}

	results = append(results, fetched...)

	return results, nil
}

/* I consider this to be an extremely dirty function, it should be split up
 * and possibly (hopefully) simplified*/
func (c *SqliteCache) Hist(symbols []string) (map[string]fquery.Hist, error) {
	/* fetch all the historical results we have in the cache */
	symbolSet := sliceToSet(symbols)

	/* find out how many entries we have per symbol (we do this so we can
	 * process the cached symbols and fetch the missing ones in parallel */
	sqliteCastToUnix := func(selector string) string {
		return `strftime('%s',` + selector + `)`
	}
	var meta []*dbHistMeta
	_, err := c.gorp.Select(&meta,
		fmt.Sprintf(
			`SELECT Symbol, count(Symbol) AS Amount,
			        %v AS MinDate,
					%v AS MaxDate
			 FROM histquotes
			 WHERE Symbol IN (%v)
			 GROUP BY Symbol`,
			sqliteCastToUnix("min(Date)"),
			sqliteCastToUnix("max(Date)"),
			strings.Join(quoteSymbols(symbols), ",")))
	if err != nil {
		/* if an error occured, just patch through to the source */
		vprintln("sqlitecache: error while fetching history metadata, ", err, ", will use underlying source")
		return c.Source.Hist(symbols)
	}

	/* convert the timestamps from UNIX timestamps to time.Time */
	for _, m := range meta {
		m.rMinDate = time.Unix(m.MinDate, 0)
		m.rMaxDate = time.Unix(m.MaxDate, 0)
	}

	/* convert the meta list to a map, for easier searching */
	metaMap := make(map[string]*dbHistMeta, len(meta))
	for _, m := range meta {
		metaMap[m.Symbol] = m
	}

	yesterday := Yesterday()
	missing := make([]string, 0, len(symbols))
	for _, s := range symbols {
		m, found := metaMap[s]
		if !found {
			vprintln(s, "was NOT fetched from cache!")
			missing = append(missing, s)
		} else {
			if m.rMaxDate.Before(yesterday) {
				vprintln(s, "was NOT fetched from cache! (historical prices "+
					"not recent enough), last available date was",
					m.rMaxDate.Format("02/01/2006"))
				missing = append(missing, s)
				delete(symbolSet, s)
			} else {
				vprintln(s, "was fetched from cache!")
			}
		}
	}

	hist := make(map[string]fquery.Hist, len(symbols))
	var (
		histmutex sync.Mutex
		wg        sync.WaitGroup
	)

	if len(symbolSet) > 0 {
		adjSymbols := setToSlice(symbolSet)
		vprintln("fetching", adjSymbols, "from the SQLite db")

		/* now query the actual entries */
		var results []dbHistEntry
		_, err = c.gorp.Select(&results,
			fmt.Sprintf(
				`SELECT * FROM histquotes WHERE Symbol IN (%v) ORDER BY Symbol, Date`,
				strings.Join(quoteSymbols(adjSymbols), ",")))
		if err != nil {
			/* if an error occured, just patch through to the source */
			vprintln("sqlitecache: error while fetching historical quotes, ", err, ", will use underlying source")
			return c.Source.Hist(symbols)
		}

		/* process the dbHistEntries into fquery.Hist structures */
		wg.Add(1)
		go func() {
			defer wg.Done()

			var (
				cursymb string
				curhist fquery.Hist
			)
			for _, e := range results {
				/* detect transitions between symbols */
				if cursymb != e.Symbol {
					fmt.Println("CHANGING SYMBOL:", cursymb, e.Symbol)
					/* if this is not the first iteration, finish up the hist
					 * entry we've been working on */
					if len(cursymb) != 0 {
						histmutex.Lock()
						hist[cursymb] = curhist
						histmutex.Unlock()
					}

					/* move to the following symbol */
					info := metaMap[e.Symbol]
					cursymb = e.Symbol
					curhist = fquery.Hist{
						Symbol:  e.Symbol,
						From:    info.rMinDate,
						To:      info.rMaxDate,
						Entries: make([]fquery.HistEntry, 0, info.Amount),
					}
				}

				curhist.Entries = append(curhist.Entries, normalizeHistEntry(&e))
			}

			/* add the last one too */
			histmutex.Lock()
			hist[cursymb] = curhist
			histmutex.Unlock()
		}()
	}

	/* fetch the missing entries from the original source */
	if len(missing) > 0 {
		wg.Add(1)
		go func() {
			defer wg.Done()

			/* fetch the missing symbols */
			missingMap, err := c.Source.Hist(missing)
			if err != nil {
				vprintln("sqlitecache: error occured while fetching missing", missing, "hist. quotes,", err)
			} else {
				missinglist := make([]fquery.Hist, 0, len(missingMap))
				for symbol, history := range missingMap {
					histmutex.Lock()
					hist[symbol] = history
					histmutex.Unlock()

					missinglist = append(missinglist, history)
				}

				/* add the previously missing items to the cache, so we don't have to
				 * request them again */
				c.mergeHistory(missinglist...)
			}
		}()
	}

	wg.Wait()

	return hist, nil
}

/*
func (c *SqliteCache) HistLimit(symbol []string, start time.Time, end time.Time) (map[string]fquery.Hist, error) {
	return nil, fmt.Errorf("not supported")
}

func (c *SqliteCache) DividendHist(symbol []string) (map[string]fquery.DividendHist, error) {
	return nil, fmt.Errorf("not supported")
}

func (c *SqliteCache) DividendHistLimit(symbol []string, start time.Time, end time.Time) (map[string]fquery.DividendHist, error) {
	return nil, fmt.Errorf("not supported")
}
*/

func (c *SqliteCache) String() string {
	return "SQLite cache, backed by: " + c.String()
}

func (c *SqliteCache) mergeQuotes(quotes ...fquery.Quote) error {
	if len(quotes) == 0 {
		return nil
	}

	tx, err := c.gorp.Begin()
	if err != nil {
		return err
	}

	for _, quote := range quotes {
		vprintln("merging quote: ", quote.Symbol)
		count, err := tx.Update(&quote)
		if err == nil && count == 1 {
			continue
		}
		vprintln("sqlitecache: error while UPDATE'ing symbol", quote.Symbol,
			"err:", err, ", count:", count)

		/* update didn't work, so try insert */
		err = tx.Insert(&quote)
		if err != nil {
			vprintln("sqlitecache: error while INSERTing", err)
		}
	}

	return tx.Commit()
}

func (c *SqliteCache) mergeHistory(hists ...fquery.Hist) error {
	if len(hists) == 0 {
		return nil
	}

	tx, err := c.gorp.Begin()
	if err != nil {
		return err
	}

	for _, hist := range hists {
		/* delete values we're going to override anyway */
		vprintln("deleting potentially already stored history for symbol",
			hist.Symbol, "from", hist.From, "to", hist.To)
		delquery := `DELETE
				     FROM  histquotes
					 WHERE Symbol = '%v'
					 AND   Date BETWEEN datetime(%v, 'unixepoch')
							    AND     datetime(%v, 'unixepoch')`
		delquery = fmt.Sprintf(delquery, hist.Symbol, hist.From.Unix(), hist.To.Unix())
		_, err := tx.Exec(delquery)
		if err != nil {
			vprintln("sqlitecache: mergeHistory: error while deleting, query:",
				delquery, "error:", err)
		}

		vprintln("merging history:", hist.Symbol)
		for _, entry := range hist.Entries {
			/* insert */
			err := tx.Insert(newDbHistEntry(hist.Symbol, &entry))
			if err != nil {
				vprintln("sqlitecache: mergeHistory: error while INSERTing", err)
			}
		}
	}

	return tx.Commit()
}

func vprintln(a ...interface{}) (int, error) {
	if VERBOSITY > 0 {
		return fmt.Println(a...)
	}

	return 0, nil
}

func vprintf(format string, a ...interface{}) (int, error) {
	if VERBOSITY > 0 {
		return fmt.Printf(format, a...)
	}

	return 0, nil
}

func quoteSymbols(symbols []string) []string {
	return util.MapStr(func(s string) string {
		return `'` + s + `'`
	}, symbols)
}

func normalizeHistEntry(e *dbHistEntry) fquery.HistEntry {
	return fquery.HistEntry{
		Date:     util.YearMonthDay(e.Date),
		Open:     e.Open,
		Close:    e.Close,
		AdjClose: e.AdjClose,
		Low:      e.Low,
		High:     e.High,
		Volume:   e.Volume,
	}
}

func newDbHistEntry(symbol string, e *fquery.HistEntry) *dbHistEntry {
	return &dbHistEntry{
		Symbol:   symbol,
		Date:     time.Time(e.Date),
		Open:     e.Open,
		Close:    e.Close,
		AdjClose: e.AdjClose,
		Low:      e.Low,
		High:     e.High,
		Volume:   e.Volume,
	}
}

func Yesterday() time.Time {
	const day = 24 * time.Hour
	return time.Now().Add(-1 * day)
}

func sliceToSet(xs []string) map[string]bool {
	m := make(map[string]bool, len(xs))
	for _, x := range xs {
		m[x] = true
	}
	return m
}

func setToSlice(m map[string]bool) []string {
	xs := make([]string, len(m))
	idx := 0
	for str := range m {
		xs[idx] = str
		idx++
	}
	return xs
}
<|endoftext|>


<commit_before>package proxy

import (
	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/common/utils/clair"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/common/utils/notary"
	//	"github.com/vmware/harbor/src/ui/api"
	"github.com/vmware/harbor/src/ui/config"
	"github.com/vmware/harbor/src/ui/projectmanager"
	"github.com/vmware/harbor/src/ui/projectmanager/pms"

	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"strings"
)

type contextKey string

const (
	manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
	imageInfoCtxKey    = contextKey("ImageInfo")
	//TODO: temp solution, remove after vmware/harbor#2242 is resolved.
	tokenUsername = "admin"
)

// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder

// NotaryEndpoint , exported for testing.
var NotaryEndpoint = config.InternalNotaryEndpoint()

// EnvChecker is the instance of envPolicyChecker
var EnvChecker = envPolicyChecker{}

// MatchPullManifest checks if the request looks like a request to pull manifest.  If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
	//TODO: add user agent check.
	if req.Method != http.MethodGet {
		return false, "", ""
	}
	re := regexp.MustCompile(manifestURLPattern)
	s := re.FindStringSubmatch(req.URL.Path)
	if len(s) == 3 {
		s[1] = strings.TrimSuffix(s[1], "/")
		return true, s[1], s[2]
	}
	return false, "", ""
}

// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type policyChecker interface {
	// contentTrustEnabled returns whether a project has enabled content trust.
	contentTrustEnabled(name string) bool
	// vulnerablePolicy  returns whether a project has enabled vulnerable, and the project's severity.
	vulnerablePolicy(name string) (bool, models.Severity)
}

//For testing
type envPolicyChecker struct{}

func (ec envPolicyChecker) contentTrustEnabled(name string) bool {
	return os.Getenv("PROJECT_CONTENT_TRUST") == "1"
}
func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
	return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY"))
}

type pmsPolicyChecker struct {
	pm projectmanager.ProjectManager
}

func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
	project, err := pc.pm.Get(name)
	if err != nil {
		log.Errorf("Unexpected error when getting the project, error: %v", err)
		return true
	}
	return project.EnableContentTrust
}
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
	project, err := pc.pm.Get(name)
	if err != nil {
		log.Errorf("Unexpected error when getting the project, error: %v", err)
		return true, models.SevUnknown
	}
	return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity)
}

// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
func newPMSPolicyChecker(pm projectmanager.ProjectManager) policyChecker {
	return &pmsPolicyChecker{
		pm: pm,
	}
}

// TODO: Get project manager with PM factory.
func getPolicyChecker() policyChecker {
	if config.WithAdmiral() {
		return newPMSPolicyChecker(pms.NewProjectManager(config.AdmiralEndpoint(), ""))
	}
	return EnvChecker
}

type imageInfo struct {
	repository  string
	tag         string
	projectName string
	digest      string
}

type urlHandler struct {
	next http.Handler
}

//TODO: wrap a ResponseWriter to get the status code?

func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("in url handler, path: %s", req.URL.Path)
	req.URL.Path = strings.TrimPrefix(req.URL.Path, RegistryProxyPrefix)
	flag, repository, tag := MatchPullManifest(req)
	if flag {
		components := strings.SplitN(repository, "/", 2)
		if len(components) < 2 {
			http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest)
			return
		}
		rec = httptest.NewRecorder()
		uh.next.ServeHTTP(rec, req)
		if rec.Result().StatusCode != http.StatusOK {
			copyResp(rec, rw)
			return
		}
		digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
		img := imageInfo{
			repository:  repository,
			tag:         tag,
			projectName: components[0],
			digest:      digest,
		}
		log.Debugf("image info of the request: %#v", img)

		ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
		req = req.WithContext(ctx)

	}
	uh.next.ServeHTTP(rw, req)
}

type contentTrustHandler struct {
	next http.Handler
}

func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw == nil || !config.WithNotary() {
		cth.next.ServeHTTP(rw, req)
		return
	}
	img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
	if !getPolicyChecker().contentTrustEnabled(img.projectName) {
		cth.next.ServeHTTP(rw, req)
		return
	}
	match, err := matchNotaryDigest(img)
	if err != nil {
		http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError)
		return
	}
	if !match {
		log.Debugf("digest mismatch, failing the response.")
		http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed)
		return
	}
	cth.next.ServeHTTP(rw, req)
}

type vulnerableHandler struct {
	next http.Handler
}

func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw == nil || !config.WithClair() {
		vh.next.ServeHTTP(rw, req)
		return
	}
	img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
	projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
	if !projectVulnerableEnabled {
		vh.next.ServeHTTP(rw, req)
		return
	}
	overview, err := dao.GetImgScanOverview(img.digest)
	if err != nil {
		log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err)
		http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed)
		return
	}
	if overview == nil {
		log.Debugf("cannot get the image scan overview info, failing the response.")
		http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed)
		return
	}
	imageSev := overview.Sev
	if imageSev > int(projectVulnerableSeverity) {
		log.Debugf("the image severity is higher then project setting, failing the response.")
		http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed)
		return
	}
	vh.next.ServeHTTP(rw, req)
}

type funnelHandler struct {
	next http.Handler
}

func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw != nil {
		log.Debugf("Return the original response as no the interceptor takes action.")
		copyResp(rec, rw)
		return
	}
	fu.next.ServeHTTP(rw, req)
}

func matchNotaryDigest(img imageInfo) (bool, error) {
	targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
	if err != nil {
		return false, err
	}
	for _, t := range targets {
		if t.Tag == img.tag {
			log.Debugf("found tag: %s in notary, try to match digest.")
			d, err := notary.DigestFromTarget(t)
			if err != nil {
				return false, err
			}
			return img.digest == d, nil
		}
	}
	log.Debugf("image: %#v, not found in notary", img)
	return false, nil
}

func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
	for k, v := range rec.Header() {
		rw.Header()[k] = v
	}
	rw.WriteHeader(rec.Result().StatusCode)
	rw.Write(rec.Body.Bytes())
}
<commit_msg>to get travis status<commit_after>package proxy

import (
	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/common/utils/clair"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/common/utils/notary"
	//	"github.com/vmware/harbor/src/ui/api"
	"github.com/vmware/harbor/src/ui/config"
	"github.com/vmware/harbor/src/ui/projectmanager"
	"github.com/vmware/harbor/src/ui/projectmanager/pms"

	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"strings"
)

type contextKey string

const (
	manifestURLPattern = `^/v2/((?:[a-z0-9]+(?:[._-][a-z0-9]+)*/)+)manifests/([\w][\w.:-]{0,127})`
	imageInfoCtxKey    = contextKey("ImageInfo")
	//TODO: temp solution, remove after vmware/harbor#2242 is resolved.
	tokenUsername = "admin"
)

// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder

// NotaryEndpoint , exported for testing.
var NotaryEndpoint = config.InternalNotaryEndpoint()

// EnvChecker is the instance of envPolicyChecker
var EnvChecker = envPolicyChecker{}

// MatchPullManifest checks if the request looks like a request to pull manifest.  If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
func MatchPullManifest(req *http.Request) (bool, string, string) {
	//TODO: add user agent check.
	if req.Method != http.MethodGet {
		return false, "", ""
	}
	re := regexp.MustCompile(manifestURLPattern)
	s := re.FindStringSubmatch(req.URL.Path)
	if len(s) == 3 {
		s[1] = strings.TrimSuffix(s[1], "/")
		return true, s[1], s[2]
	}
	return false, "", ""
}

// policyChecker checks the policy of a project by project name, to determine if it's needed to check the image's status under this project.
type policyChecker interface {
	// contentTrustEnabled returns whether a project has enabled content trust.
	contentTrustEnabled(name string) bool
	// vulnerablePolicy  returns whether a project has enabled vulnerable, and the project's severity.
	vulnerablePolicy(name string) (bool, models.Severity)
}

//For testing
type envPolicyChecker struct{}

func (ec envPolicyChecker) contentTrustEnabled(name string) bool {
	return os.Getenv("PROJECT_CONTENT_TRUST") == "1"
}
func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
	return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY"))
}

type pmsPolicyChecker struct {
	pm projectmanager.ProjectManager
}

func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
	project, err := pc.pm.Get(name)
	if err != nil {
		log.Errorf("Unexpected error when getting the project, error: %v", err)
		return true
	}
	return project.EnableContentTrust
}
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
	project, err := pc.pm.Get(name)
	if err != nil {
		log.Errorf("Unexpected error when getting the project, error: %v", err)
		return true, models.SevUnknown
	}
	return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity)
}

// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
func newPMSPolicyChecker(pm projectmanager.ProjectManager) policyChecker {
	return &pmsPolicyChecker{
		pm: pm,
	}
}

// TODO: Get project manager with PM factory.
func getPolicyChecker() policyChecker {
	if config.WithAdmiral() {
		return newPMSPolicyChecker(pms.NewProjectManager(config.AdmiralEndpoint(), ""))
	}
	return EnvChecker
}

type imageInfo struct {
	repository  string
	tag         string
	projectName string
	digest      string
}

type urlHandler struct {
	next http.Handler
}

//TODO: wrap a ResponseWriter to get the status code?

func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	log.Debugf("in url handler, path: %s", req.URL.Path)
	req.URL.Path = strings.TrimPrefix(req.URL.Path, RegistryProxyPrefix)
	flag, repository, tag := MatchPullManifest(req)
	if flag {
		components := strings.SplitN(repository, "/", 2)
		if len(components) < 2 {
			http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest)
			return
		}
		rec = httptest.NewRecorder()
		uh.next.ServeHTTP(rec, req)
		if rec.Result().StatusCode != http.StatusOK {
			copyResp(rec, rw)
			return
		}
		digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
		img := imageInfo{
			repository:  repository,
			tag:         tag,
			projectName: components[0],
			digest:      digest,
		}
		log.Debugf("image info of the request: %#v", img)
		ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
		req = req.WithContext(ctx)
	}
	uh.next.ServeHTTP(rw, req)
}

type contentTrustHandler struct {
	next http.Handler
}

func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw == nil || !config.WithNotary() {
		cth.next.ServeHTTP(rw, req)
		return
	}
	img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
	if !getPolicyChecker().contentTrustEnabled(img.projectName) {
		cth.next.ServeHTTP(rw, req)
		return
	}
	match, err := matchNotaryDigest(img)
	if err != nil {
		http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError)
		return
	}
	if !match {
		log.Debugf("digest mismatch, failing the response.")
		http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed)
		return
	}
	cth.next.ServeHTTP(rw, req)
}

type vulnerableHandler struct {
	next http.Handler
}

func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw == nil || !config.WithClair() {
		vh.next.ServeHTTP(rw, req)
		return
	}
	img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
	projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
	if !projectVulnerableEnabled {
		vh.next.ServeHTTP(rw, req)
		return
	}
	overview, err := dao.GetImgScanOverview(img.digest)
	if err != nil {
		log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err)
		http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed)
		return
	}
	if overview == nil {
		log.Debugf("cannot get the image scan overview info, failing the response.")
		http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed)
		return
	}
	imageSev := overview.Sev
	if imageSev > int(projectVulnerableSeverity) {
		log.Debugf("the image severity is higher then project setting, failing the response.")
		http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed)
		return
	}
	vh.next.ServeHTTP(rw, req)
}

type funnelHandler struct {
	next http.Handler
}

func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	imgRaw := req.Context().Value(imageInfoCtxKey)
	if imgRaw != nil {
		log.Debugf("Return the original response as no the interceptor takes action.")
		copyResp(rec, rw)
		return
	}
	fu.next.ServeHTTP(rw, req)
}

func matchNotaryDigest(img imageInfo) (bool, error) {
	targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
	if err != nil {
		return false, err
	}
	for _, t := range targets {
		if t.Tag == img.tag {
			log.Debugf("found tag: %s in notary, try to match digest.")
			d, err := notary.DigestFromTarget(t)
			if err != nil {
				return false, err
			}
			return img.digest == d, nil
		}
	}
	log.Debugf("image: %#v, not found in notary", img)
	return false, nil
}

func copyResp(rec *httptest.ResponseRecorder, rw http.ResponseWriter) {
	for k, v := range rec.Header() {
		rw.Header()[k] = v
	}
	rw.WriteHeader(rec.Result().StatusCode)
	rw.Write(rec.Body.Bytes())
}
<|endoftext|>


<commit_before>// Copyright 2017 The Bazel Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package starlarktest defines utilities for testing Starlark programs.
//
// Clients can call LoadAssertModule to load a module that defines
// several functions useful for testing.  See assert.star for its
// definition.
//
// The assert.error function, which reports errors to the current Go
// testing.T, requires that clients call SetTest(thread, t) before use.
package starlarktest // import "go.starlark.net/starlarktest"

import (
	"fmt"
	"go/build"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"go.starlark.net/starlark"
	"go.starlark.net/starlarkstruct"
)

const localKey = "Reporter"

// A Reporter is a value to which errors may be reported.
// It is satisfied by *testing.T.
type Reporter interface {
	Error(args ...interface{})
}

// SetReporter associates an error reporter (such as a testing.T in
// a Go test) with the Starlark thread so that Starlark programs may
// report errors to it.
func SetReporter(thread *starlark.Thread, r Reporter) {
	thread.SetLocal(localKey, r)
}

// GetReporter returns the Starlark thread's error reporter.
// It must be preceded by a call to SetReporter.
func GetReporter(thread *starlark.Thread) Reporter {
	r, ok := thread.Local(localKey).(Reporter)
	if !ok {
		panic("internal error: starlarktest.SetReporter was not called")
	}
	return r
}

var (
	once      sync.Once
	assert    starlark.StringDict
	assertErr error
)

// LoadAssertModule loads the assert module.
// It is concurrency-safe and idempotent.
func LoadAssertModule() (starlark.StringDict, error) {
	once.Do(func() {
		predeclared := starlark.StringDict{
			"error":   starlark.NewBuiltin("error", error_),
			"catch":   starlark.NewBuiltin("catch", catch),
			"matches": starlark.NewBuiltin("matches", matches),
			"module":  starlark.NewBuiltin("module", starlarkstruct.MakeModule),
			"_freeze": starlark.NewBuiltin("freeze", freeze),
		}
		filename := DataFile("starlarktest", "assert.star")
		thread := new(starlark.Thread)
		assert, assertErr = starlark.ExecFile(thread, filename, nil, predeclared)
	})
	return assert, assertErr
}

// catch(f) evaluates f() and returns its evaluation error message
// if it failed or None if it succeeded.
func catch(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	var fn starlark.Callable
	if err := starlark.UnpackArgs("catch", args, kwargs, "fn", &fn); err != nil {
		return nil, err
	}
	if _, err := starlark.Call(thread, fn, nil, nil); err != nil {
		return starlark.String(err.Error()), nil
	}
	return starlark.None, nil
}

// matches(pattern, str) reports whether string str matches the regular expression pattern.
func matches(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	var pattern, str string
	if err := starlark.UnpackArgs("matches", args, kwargs, "pattern", &pattern, "str", &str); err != nil {
		return nil, err
	}
	ok, err := regexp.MatchString(pattern, str)
	if err != nil {
		return nil, fmt.Errorf("matches: %s", err)
	}
	return starlark.Bool(ok), nil
}

// error(x) reports an error to the Go test framework.
func error_(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("error: got %d arguments, want 1", len(args))
	}
	buf := new(strings.Builder)
	stk := thread.CallStack()
	stk.Pop()
	fmt.Fprintf(buf, "%sError: ", stk)
	if s, ok := starlark.AsString(args[0]); ok {
		buf.WriteString(s)
	} else {
		buf.WriteString(args[0].String())
	}
	GetReporter(thread).Error(buf.String())
	return starlark.None, nil
}

// freeze(x) freezes its operand.
func freeze(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	if len(kwargs) > 0 {
		return nil, fmt.Errorf("freeze does not accept keyword arguments")
	}
	if len(args) != 1 {
		return nil, fmt.Errorf("freeze got %d arguments, wants 1", len(args))
	}
	args[0].Freeze()
	return args[0], nil
}

// DataFile returns the effective filename of the specified
// test data resource.  The function abstracts differences between
// 'go build', under which a test runs in its package directory,
// and Blaze, under which a test runs in the root of the tree.
var DataFile = func(pkgdir, filename string) string {
	return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename)
}
<commit_msg>Adding support for Bazel test to use DataFile (#251)<commit_after>// Copyright 2017 The Bazel Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package starlarktest defines utilities for testing Starlark programs.
//
// Clients can call LoadAssertModule to load a module that defines
// several functions useful for testing.  See assert.star for its
// definition.
//
// The assert.error function, which reports errors to the current Go
// testing.T, requires that clients call SetTest(thread, t) before use.
package starlarktest // import "go.starlark.net/starlarktest"

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"go.starlark.net/starlark"
	"go.starlark.net/starlarkstruct"
)

const localKey = "Reporter"

// A Reporter is a value to which errors may be reported.
// It is satisfied by *testing.T.
type Reporter interface {
	Error(args ...interface{})
}

// SetReporter associates an error reporter (such as a testing.T in
// a Go test) with the Starlark thread so that Starlark programs may
// report errors to it.
func SetReporter(thread *starlark.Thread, r Reporter) {
	thread.SetLocal(localKey, r)
}

// GetReporter returns the Starlark thread's error reporter.
// It must be preceded by a call to SetReporter.
func GetReporter(thread *starlark.Thread) Reporter {
	r, ok := thread.Local(localKey).(Reporter)
	if !ok {
		panic("internal error: starlarktest.SetReporter was not called")
	}
	return r
}

var (
	once      sync.Once
	assert    starlark.StringDict
	assertErr error
)

// LoadAssertModule loads the assert module.
// It is concurrency-safe and idempotent.
func LoadAssertModule() (starlark.StringDict, error) {
	once.Do(func() {
		predeclared := starlark.StringDict{
			"error":   starlark.NewBuiltin("error", error_),
			"catch":   starlark.NewBuiltin("catch", catch),
			"matches": starlark.NewBuiltin("matches", matches),
			"module":  starlark.NewBuiltin("module", starlarkstruct.MakeModule),
			"_freeze": starlark.NewBuiltin("freeze", freeze),
		}
		filename := DataFile("starlarktest", "assert.star")
		thread := new(starlark.Thread)
		assert, assertErr = starlark.ExecFile(thread, filename, nil, predeclared)
	})
	return assert, assertErr
}

// catch(f) evaluates f() and returns its evaluation error message
// if it failed or None if it succeeded.
func catch(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	var fn starlark.Callable
	if err := starlark.UnpackArgs("catch", args, kwargs, "fn", &fn); err != nil {
		return nil, err
	}
	if _, err := starlark.Call(thread, fn, nil, nil); err != nil {
		return starlark.String(err.Error()), nil
	}
	return starlark.None, nil
}

// matches(pattern, str) reports whether string str matches the regular expression pattern.
func matches(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	var pattern, str string
	if err := starlark.UnpackArgs("matches", args, kwargs, "pattern", &pattern, "str", &str); err != nil {
		return nil, err
	}
	ok, err := regexp.MatchString(pattern, str)
	if err != nil {
		return nil, fmt.Errorf("matches: %s", err)
	}
	return starlark.Bool(ok), nil
}

// error(x) reports an error to the Go test framework.
func error_(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	if len(args) != 1 {
		return nil, fmt.Errorf("error: got %d arguments, want 1", len(args))
	}
	buf := new(strings.Builder)
	stk := thread.CallStack()
	stk.Pop()
	fmt.Fprintf(buf, "%sError: ", stk)
	if s, ok := starlark.AsString(args[0]); ok {
		buf.WriteString(s)
	} else {
		buf.WriteString(args[0].String())
	}
	GetReporter(thread).Error(buf.String())
	return starlark.None, nil
}

// freeze(x) freezes its operand.
func freeze(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
	if len(kwargs) > 0 {
		return nil, fmt.Errorf("freeze does not accept keyword arguments")
	}
	if len(args) != 1 {
		return nil, fmt.Errorf("freeze got %d arguments, wants 1", len(args))
	}
	args[0].Freeze()
	return args[0], nil
}

// DataFile returns the effective filename of the specified
// test data resource.  The function abstracts differences between
// 'go build', under which a test runs in its package directory,
// and Blaze, under which a test runs in the root of the tree.
var DataFile = func(pkgdir, filename string) string {
	// Check if we're being run by Bazel and change directories if so.
	// TEST_SRCDIR and TEST_WORKSPACE are set by the Bazel test runner, so that makes a decent check
	testSrcdir := os.Getenv("TEST_SRCDIR")
	testWorkspace := os.Getenv("TEST_WORKSPACE")
	if testSrcdir != "" && testWorkspace != "" {
		return filepath.Join(testSrcdir, "net_starlark_go", pkgdir, filename)
	}

	return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename)
}
<|endoftext|>


<commit_before>package config

var DefaultConfig = &Config{
	Apibase:  "https://mackerel.io",
	Root:     "/var/lib/mackerel-agent",
	Pidfile:  "/var/run/mackerel-agent.pid",
	Conffile: "/etc/mackerel-agent/mackerel-agent.conf",
	Roles:    []string{},
	Verbose:  false,
	Connection: ConnectionConfig{
		Post_Metrics_Dequeue_Delay_Seconds: 30,     // Check the metric values queue for each half minutes
		Post_Metrics_Retry_Delay_Seconds:   5 * 60, // Wait 5 minutes before retrying metric value posts
		Post_Metrics_Retry_Max:             30,     // Retry up to 30 times (5min * 30 = 2.5hrs)
		Post_Metrics_Buffer_Size:           6 * 60, // Keep metric values of 6 hours span in the queue
	},
}
<commit_msg>double retry limit<commit_after>package config

var DefaultConfig = &Config{
	Apibase:  "https://mackerel.io",
	Root:     "/var/lib/mackerel-agent",
	Pidfile:  "/var/run/mackerel-agent.pid",
	Conffile: "/etc/mackerel-agent/mackerel-agent.conf",
	Roles:    []string{},
	Verbose:  false,
	Connection: ConnectionConfig{
		Post_Metrics_Dequeue_Delay_Seconds: 30,     // Check the metric values queue for each half minutes
		Post_Metrics_Retry_Delay_Seconds:   5 * 60, // Wait 5 minutes before retrying metric value posts
		Post_Metrics_Retry_Max:             60,     // Retry up to 60 times (5min * 60 = 5hrs)
		Post_Metrics_Buffer_Size:           6 * 60, // Keep metric values of 6 hours span in the queue
	},
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"regexp"
	"runtime"

	"github.com/gorilla/websocket"
	sm "github.com/ifn/go-statemachine"
)

type DeskMsg struct {
	Desk [][]string `json:"desk"`
}

type PlayerMsg struct {
	Cmd  sm.EventType `json:"command"`
	Card string       `json:"card"`
}

//

var Order map[string]int = map[string]int{
	"6": 6, "7": 7, "8": 8, "9": 9, "10": 10,
	"J": 11,
	"Q": 12,
	"K": 13,
	"A": 14,
}

func higher(c0, c1, t string) int {
	// c0 and c1 have the same suit
	if c0[0] == c1[0] {
		if Order[c0[1:]] > Order[c1[1:]] {
			return 1
		}
		if Order[c0[1:]] < Order[c1[1:]] {
			return -1
		}
		return 0
	}
	// c0 is trump, c1 is not
	if c0[:1] == t {
		return 1
	}
	// c1 is trump, c0 is not
	if c1[:1] == t {
		return -1
	}
	// suits are different, both are not trump
	return -2
}

var CardRE *regexp.Regexp = regexp.MustCompile(`[SCHD]([6-9JQKA]|10)`)

func isValid(c string) bool {
	return CardRE.MatchString(c)
}

//

const (
	cmdStart sm.EventType = iota
	cmdMove

	cmdCount
)

const (
	stateCollection sm.State = iota

	stateAttack
	stateDefense

	stateCount
)

type roundResult bool

const (
	Beat roundResult = true
)

func stateToString(s sm.State) string {
	return [...]string{
		stateCollection: "COLLECTION",
		stateDefense:    "DEFENSE",
		stateAttack:     "ATTACK",
	}[s]
}

func cmdToString(t sm.EventType) string {
	return [...]string{
		cmdStart: "START",
		cmdMove:  "MOVE",
	}[t]
}

//

type cmdArgs struct {
	conn *playerConn
	card string
}

//

type gameState struct {
	// 1. fields that don't change during a game

	sm  *sm.StateMachine
	hub *hub

	// trump suit
	trump string

	// 2. fields that don't change during a round

	// attacker that started a round
	aconnStart *playerConn
	// defender
	dconn *playerConn

	// 3. fields that change during a round

	// attacker
	aconn *playerConn
	// card that should be beaten
	cardToBeat string
}

func (self *gameState) nextPlayer(c *playerConn) *playerConn {
	return self.hub.conns.(*mapRing).Next(c).(*playerConn)
}

func (self *gameState) finishRound(res roundResult) {
	switch res {
	case Beat:
		self.aconn = self.dconn
	case !Beat:
		self.aconn = self.nextPlayer(self.dconn)
	}
	self.dconn = self.nextPlayer(self.aconn)
}

func (self *gameState) dealCards() {
	for _ = range self.hub.conns.Enumerate() {
	}
}

//

func logOutOfTurn(pconn *playerConn) {
	log.Printf("out of turn: %v", pconn.conn.RemoteAddr())
}

func logWontBeat(c1, c2, t string) {
	log.Printf("%v won't bit %v, trump is ", c1, c2, t)
}

//

func NewGameState() *gameState {
	gst := new(gameState)

	gst.sm = sm.New(stateCollection, uint(stateCount), uint(cmdCount))

	gst.sm.On(cmdStart,
		[]sm.State{stateCollection},
		gst.handleStartInCollection,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack},
		gst.handleMoveInAttack,
	)
	gst.sm.On(cmdMove,
		[]sm.State{stateDefense},
		gst.handleMoveInDefense,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack, stateDefense},
		gst.showDesk,
	)

	gst.hub = NewHub()

	return gst
}

// event handlers
// event handlers are actually transition functions.
// in case error event handler should neither change the gameState,
// nor return the state value different from passed to it as an argument.

func (self *gameState) showDesk(s sm.State, e *sm.Event) sm.State {
	desk, err := json.Marshal(DeskMsg{})
	if err != nil {
		log.Println(err)
		return s
	}

	self.hub.bcastChan <- desk

	return s
}

func (self *gameState) handleStartInCollection(s sm.State, e *sm.Event) sm.State {
	self.dealCards()

	return stateAttack
}

func (self *gameState) handleMoveInAttack(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.aconn {
		logOutOfTurn(conn)
		return s
	}

	// attacker sent the card
	if card != "" {
		self.cardToBeat = card
		return stateDefense
	}

	// attacker sent no card

	aconn := self.nextPlayer(self.aconn)
	if aconn == self.dconn {
		aconn = self.nextPlayer(aconn)
	}

	// check if all attackers have been polled
	if aconn == self.aconnStart {
		self.finishRound(Beat)
		return stateAttack
	}

	self.aconn = aconn
	return stateAttack
}

func (self *gameState) handleMoveInDefense(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.dconn {
		logOutOfTurn(conn)
		return s
	}

	// defender takes the cards
	if card == "" {
		self.finishRound(!Beat)
		return stateAttack
	}

	// check that the sent card is capable to beat
	if higher(card, self.cardToBeat, self.trump) != 1 {
		logWontBeat(card, self.cardToBeat, self.trump)
		return s
	}

	return stateAttack
}

//

type playerConn struct {
	gst *gameState

	conn      *websocket.Conn
	hubToConn chan []byte
}

func (self *playerConn) write() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	for {
		select {
		case m, ok := <-self.hubToConn:
			if !ok {
				return
			}
			//TODO: text or binary?
			err := self.conn.WriteMessage(websocket.TextMessage, m)
			if err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (self *playerConn) read() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	var m PlayerMsg

	for {
		err := self.conn.ReadJSON(&m)
		if err != nil {
			log.Println(err)
			return
		}

		var event *sm.Event
		switch m.Cmd {
		case cmdStart:
			event = &sm.Event{cmdStart, nil}
		case cmdMove:
			event = &sm.Event{cmdMove, cmdArgs{self, m.Card}}
		default:
			log.Printf("unknown command: %v", m.Cmd)
			continue
		}

		err = self.gst.sm.Emit(event)
		if err != nil {
			log.Println(err)
			return //TODO: replace by continue?
		}
	}
}

//

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

func playerHandler(gst *gameState) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Println(err)
			return
		}

		p := &playerConn{gst, conn, make(chan []byte)}

		gst.hub.regChan <- p
		defer func() {
			gst.hub.unregChan <- p
		}()

		go p.write()
		p.read()
	}
}

//

func startDurakSrv() error {
	gst := NewGameState()

	http.HandleFunc("/", playerHandler(gst))

	return http.ListenAndServe(":"+os.Getenv("PORT"), nil)
}

func main() {
	err := startDurakSrv()
	if err != nil {
		log.Fatal(err)
	}
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}
<commit_msg>ignoring ErrIllegalEvent<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"os"
	"regexp"
	"runtime"

	"github.com/gorilla/websocket"
	sm "github.com/ifn/go-statemachine"
)

type DeskMsg struct {
	Desk [][]string `json:"desk"`
}

type PlayerMsg struct {
	Cmd  sm.EventType `json:"command"`
	Card string       `json:"card"`
}

//

var Order map[string]int = map[string]int{
	"6": 6, "7": 7, "8": 8, "9": 9, "10": 10,
	"J": 11,
	"Q": 12,
	"K": 13,
	"A": 14,
}

func higher(c0, c1, t string) int {
	// c0 and c1 have the same suit
	if c0[0] == c1[0] {
		if Order[c0[1:]] > Order[c1[1:]] {
			return 1
		}
		if Order[c0[1:]] < Order[c1[1:]] {
			return -1
		}
		return 0
	}
	// c0 is trump, c1 is not
	if c0[:1] == t {
		return 1
	}
	// c1 is trump, c0 is not
	if c1[:1] == t {
		return -1
	}
	// suits are different, both are not trump
	return -2
}

var CardRE *regexp.Regexp = regexp.MustCompile(`[SCHD]([6-9JQKA]|10)`)

func isValid(c string) bool {
	return CardRE.MatchString(c)
}

//

const (
	cmdStart sm.EventType = iota
	cmdMove

	cmdCount
)

const (
	stateCollection sm.State = iota

	stateAttack
	stateDefense

	stateCount
)

type roundResult bool

const (
	Beat roundResult = true
)

func stateToString(s sm.State) string {
	return [...]string{
		stateCollection: "COLLECTION",
		stateDefense:    "DEFENSE",
		stateAttack:     "ATTACK",
	}[s]
}

func cmdToString(t sm.EventType) string {
	return [...]string{
		cmdStart: "START",
		cmdMove:  "MOVE",
	}[t]
}

//

type cmdArgs struct {
	conn *playerConn
	card string
}

//

type gameState struct {
	// 1. fields that don't change during a game

	sm  *sm.StateMachine
	hub *hub

	// trump suit
	trump string

	// 2. fields that don't change during a round

	// attacker that started a round
	aconnStart *playerConn
	// defender
	dconn *playerConn

	// 3. fields that change during a round

	// attacker
	aconn *playerConn
	// card that should be beaten
	cardToBeat string
}

func (self *gameState) nextPlayer(c *playerConn) *playerConn {
	return self.hub.conns.(*mapRing).Next(c).(*playerConn)
}

func (self *gameState) finishRound(res roundResult) {
	switch res {
	case Beat:
		self.aconn = self.dconn
	case !Beat:
		self.aconn = self.nextPlayer(self.dconn)
	}
	self.dconn = self.nextPlayer(self.aconn)
}

func (self *gameState) dealCards() {
	for _ = range self.hub.conns.Enumerate() {
	}
}

//

func logOutOfTurn(pconn *playerConn) {
	log.Printf("out of turn: %v", pconn.conn.RemoteAddr())
}

func logWontBeat(c1, c2, t string) {
	log.Printf("%v won't bit %v, trump is ", c1, c2, t)
}

//

func NewGameState() *gameState {
	gst := new(gameState)

	gst.sm = sm.New(stateCollection, uint(stateCount), uint(cmdCount))

	gst.sm.On(cmdStart,
		[]sm.State{stateCollection},
		gst.handleStartInCollection,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack},
		gst.handleMoveInAttack,
	)
	gst.sm.On(cmdMove,
		[]sm.State{stateDefense},
		gst.handleMoveInDefense,
	)

	gst.sm.On(cmdMove,
		[]sm.State{stateAttack, stateDefense},
		gst.showDesk,
	)

	gst.hub = NewHub()

	return gst
}

// event handlers
// event handlers are actually transition functions.
// in case error event handler should neither change the gameState,
// nor return the state value different from passed to it as an argument.

func (self *gameState) showDesk(s sm.State, e *sm.Event) sm.State {
	desk, err := json.Marshal(DeskMsg{})
	if err != nil {
		log.Println(err)
		return s
	}

	self.hub.bcastChan <- desk

	return s
}

func (self *gameState) handleStartInCollection(s sm.State, e *sm.Event) sm.State {
	self.dealCards()

	return stateAttack
}

func (self *gameState) handleMoveInAttack(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.aconn {
		logOutOfTurn(conn)
		return s
	}

	// attacker sent the card
	if card != "" {
		self.cardToBeat = card
		return stateDefense
	}

	// attacker sent no card

	aconn := self.nextPlayer(self.aconn)
	if aconn == self.dconn {
		aconn = self.nextPlayer(aconn)
	}

	// check if all attackers have been polled
	if aconn == self.aconnStart {
		self.finishRound(Beat)
		return stateAttack
	}

	self.aconn = aconn
	return stateAttack
}

func (self *gameState) handleMoveInDefense(s sm.State, e *sm.Event) sm.State {
	conn := e.Data.(cmdArgs).conn
	card := e.Data.(cmdArgs).card

	// check that it's conn's turn to move
	if conn != self.dconn {
		logOutOfTurn(conn)
		return s
	}

	// defender takes the cards
	if card == "" {
		self.finishRound(!Beat)
		return stateAttack
	}

	// check that the sent card is capable to beat
	if higher(card, self.cardToBeat, self.trump) != 1 {
		logWontBeat(card, self.cardToBeat, self.trump)
		return s
	}

	return stateAttack
}

//

type playerConn struct {
	gst *gameState

	conn      *websocket.Conn
	hubToConn chan []byte
}

func (self *playerConn) write() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	for {
		select {
		case m, ok := <-self.hubToConn:
			if !ok {
				return
			}
			//TODO: text or binary?
			err := self.conn.WriteMessage(websocket.TextMessage, m)
			if err != nil {
				log.Println(err)
				return
			}
		}
	}
}

func (self *playerConn) read() {
	defer func() {
		err := self.conn.Close()
		if err != nil {
			log.Println(err)
		}
	}()

	var m PlayerMsg

	for {
		err := self.conn.ReadJSON(&m)
		if err != nil {
			log.Println(err)
			return
		}

		var event *sm.Event
		switch m.Cmd {
		case cmdStart:
			event = &sm.Event{cmdStart, nil}
		case cmdMove:
			event = &sm.Event{cmdMove, cmdArgs{self, m.Card}}
		default:
			log.Printf("unknown command: %v", m.Cmd)
			continue
		}

		err = self.gst.sm.Emit(event)
		if err != nil {
			log.Println(err)
		}
	}
}

//

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin:     func(r *http.Request) bool { return true },
}

func playerHandler(gst *gameState) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		conn, err := upgrader.Upgrade(w, r, nil)
		if err != nil {
			log.Println(err)
			return
		}

		p := &playerConn{gst, conn, make(chan []byte)}

		gst.hub.regChan <- p
		defer func() {
			gst.hub.unregChan <- p
		}()

		go p.write()
		p.read()
	}
}

//

func startDurakSrv() error {
	gst := NewGameState()

	http.HandleFunc("/", playerHandler(gst))

	return http.ListenAndServe(":"+os.Getenv("PORT"), nil)
}

func main() {
	err := startDurakSrv()
	if err != nil {
		log.Fatal(err)
	}
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
}
<|endoftext|>


<commit_before>// Interpreter is a scheme source code interpreter.
// It owns a role of API for executing scheme program.
// Interpreter embeds Parser and delegates syntactic analysis to it.

package scheme

import (
	"fmt"
	"strings"
	"text/scanner"
)

type Interpreter struct {
	ObjectBase
	*Parser
	topLevel Binding
}

func NewInterpreter(source string) *Interpreter {
	return &Interpreter{Parser: NewParser(source), topLevel: BuiltinProcedures()}
}

// Load new source code with current environment
func (i *Interpreter) ReloadSourceCode(source string) {
	i.Parser = NewParser(source)
}

func (i *Interpreter) PrintResult(dumpAST bool) {
	for _, result := range i.EvalSource(dumpAST) {
		fmt.Println(result)
	}
}

func (i *Interpreter) EvalSource(dumpAST bool) (results []string) {
	defer func() {
		if err := recover(); err != nil {
			results = append(results, fmt.Sprintf("*** ERROR: %s", err))
		}
	}()

	for i.Peek() != scanner.EOF {
		expression := i.Parser.Parse(i)
		if dumpAST {
			fmt.Printf("\n*** AST ***\n")
			i.DumpAST(expression, 0)
			fmt.Printf("\n*** Result ***\n")
		}

		if expression == nil {
			return
		}
		results = append(results, expression.Eval().String())
	}
	return
}

func (i *Interpreter) bind(identifier string, object Object) {
	i.topLevel[identifier] = object
}

func (i *Interpreter) binding() Binding {
	return i.topLevel
}

func (i *Interpreter) scopedBinding() Binding {
	return i.topLevel
}

func (i *Interpreter) DumpAST(object Object, indentLevel int) {
	if object == nil {
		return
	}
	switch object.(type) {
	case *Application:
		i.printWithIndent("Application", indentLevel)
		i.DumpAST(object.(*Application).procedure, indentLevel+1)
		i.DumpAST(object.(*Application).arguments, indentLevel+1)
	case *Pair:
		pair := object.(*Pair)
		if pair.Car == nil && pair.Cdr == nil {
			i.printWithIndent("()", indentLevel)
			return
		}
		i.printWithIndent("Pair", indentLevel)
		i.DumpAST(pair.Car, indentLevel+1)
		i.DumpAST(pair.Cdr, indentLevel+1)
	case *String:
		i.printWithIndent(fmt.Sprintf("String(%s)", object), indentLevel)
	case *Number:
		i.printWithIndent(fmt.Sprintf("Number(%s)", object), indentLevel)
	case *Boolean:
		i.printWithIndent(fmt.Sprintf("Boolean(%s)", object), indentLevel)
	case *Variable:
		i.printWithIndent(fmt.Sprintf("Variable(%s)", object.(*Variable).identifier), indentLevel)
	case *Definition:
		i.printWithIndent("Definition", indentLevel)
		i.DumpAST(object.(*Definition).variable, indentLevel+1)
		i.DumpAST(object.(*Definition).value, indentLevel+1)
	case *Procedure:
		i.printWithIndent("Procedure", indentLevel)
		i.DumpAST(object.(*Procedure).arguments, indentLevel+1)
		i.DumpAST(object.(*Procedure).body, indentLevel+1)
	case *Set:
		i.printWithIndent("Set", indentLevel)
		i.DumpAST(object.(*Set).variable, indentLevel+1)
		i.DumpAST(object.(*Set).value, indentLevel+1)
	case *If:
		i.printWithIndent("If", indentLevel)
		i.DumpAST(object.(*If).condition, indentLevel+1)
		i.DumpAST(object.(*If).trueBody, indentLevel+1)
		i.DumpAST(object.(*If).falseBody, indentLevel+1)
	case *Cond:
	case *And:
	case *Or:
	case *Begin:
	case *Do:
	case *Iterator:
	}
}

func (i *Interpreter) printWithIndent(text string, indentLevel int) {
	fmt.Printf("%s%s\n", strings.Repeat(" ", indentLevel), text)
}
<commit_msg>Accept new syntax form AST dumping<commit_after>// Interpreter is a scheme source code interpreter.
// It owns a role of API for executing scheme program.
// Interpreter embeds Parser and delegates syntactic analysis to it.

package scheme

import (
	"fmt"
	"strings"
	"text/scanner"
)

type Interpreter struct {
	ObjectBase
	*Parser
	topLevel Binding
}

func NewInterpreter(source string) *Interpreter {
	return &Interpreter{Parser: NewParser(source), topLevel: BuiltinProcedures()}
}

// Load new source code with current environment
func (i *Interpreter) ReloadSourceCode(source string) {
	i.Parser = NewParser(source)
}

func (i *Interpreter) PrintResult(dumpAST bool) {
	results := i.EvalSource(dumpAST)
	if dumpAST {
		fmt.Printf("\n*** Result ***\n")
	}
	for _, result := range results {
		fmt.Println(result)
	}
}

func (i *Interpreter) EvalSource(dumpAST bool) (results []string) {
	defer func() {
		if err := recover(); err != nil {
			results = append(results, fmt.Sprintf("*** ERROR: %s", err))
		}
	}()

	for i.Peek() != scanner.EOF {
		expression := i.Parser.Parse(i)
		if dumpAST {
			fmt.Printf("\n*** AST ***\n")
			i.DumpAST(expression, 0)
		}

		if expression == nil {
			return
		}
		results = append(results, expression.Eval().String())
	}
	return
}

func (i *Interpreter) bind(identifier string, object Object) {
	i.topLevel[identifier] = object
}

func (i *Interpreter) binding() Binding {
	return i.topLevel
}

func (i *Interpreter) scopedBinding() Binding {
	return i.topLevel
}

func (i *Interpreter) DumpAST(object Object, indentLevel int) {
	if object == nil {
		return
	}
	switch object.(type) {
	case *Application:
		i.printWithIndent("Application", indentLevel)
		i.DumpAST(object.(*Application).procedure, indentLevel+1)
		i.DumpAST(object.(*Application).arguments, indentLevel+1)
	case *Pair:
		pair := object.(*Pair)
		if pair.Car == nil && pair.Cdr == nil {
			i.printWithIndent("()", indentLevel)
			return
		}
		i.printWithIndent("Pair", indentLevel)
		i.DumpAST(pair.Car, indentLevel+1)
		i.DumpAST(pair.Cdr, indentLevel+1)
	case *String:
		i.printWithIndent(fmt.Sprintf("String(%s)", object), indentLevel)
	case *Number:
		i.printWithIndent(fmt.Sprintf("Number(%s)", object), indentLevel)
	case *Boolean:
		i.printWithIndent(fmt.Sprintf("Boolean(%s)", object), indentLevel)
	case *Variable:
		i.printWithIndent(fmt.Sprintf("Variable(%s)", object.(*Variable).identifier), indentLevel)
	case *Definition:
		i.printWithIndent("Definition", indentLevel)
		i.DumpAST(object.(*Definition).variable, indentLevel+1)
		i.DumpAST(object.(*Definition).value, indentLevel+1)
	case *Procedure:
		i.printWithIndent("Procedure", indentLevel)
		i.DumpAST(object.(*Procedure).arguments, indentLevel+1)
		i.DumpAST(object.(*Procedure).body, indentLevel+1)
	case *Set:
		i.printWithIndent("Set", indentLevel)
		i.DumpAST(object.(*Set).variable, indentLevel+1)
		i.DumpAST(object.(*Set).value, indentLevel+1)
	case *If:
		i.printWithIndent("If", indentLevel)
		i.DumpAST(object.(*If).condition, indentLevel+1)
		i.DumpAST(object.(*If).trueBody, indentLevel+1)
		i.DumpAST(object.(*If).falseBody, indentLevel+1)
	case *Cond:
		i.printWithIndent("Cond", indentLevel)
		for _, element := range object.(*Cond).cases {
			i.DumpAST(element, indentLevel+1)
		}
		if object.(*Cond).elseBody != nil {
			i.DumpAST(object.(*Cond).elseBody, indentLevel+1)
		}
	case *And:
		i.printWithIndent("And", indentLevel)
		i.DumpAST(object.(*And).body, indentLevel+1)
	case *Or:
		i.printWithIndent("Or", indentLevel)
		i.DumpAST(object.(*Or).body, indentLevel+1)
	case *Begin:
		i.printWithIndent("Begin", indentLevel)
		i.DumpAST(object.(*Begin).body, indentLevel+1)
	case *Do:
		i.printWithIndent("Do", indentLevel)
		for _, iterator := range object.(*Do).iterators {
			i.DumpAST(iterator, indentLevel+1)
		}
		i.DumpAST(object.(*Do).testBody, indentLevel+1)
		i.DumpAST(object.(*Do).continueBody, indentLevel+1)
	case *Iterator:
		i.printWithIndent("Iterator", indentLevel)
		i.DumpAST(object.(*Iterator).variable, indentLevel+1)
		i.DumpAST(object.(*Iterator).value, indentLevel+1)
		i.DumpAST(object.(*Iterator).update, indentLevel+1)
	}
}

func (i *Interpreter) printWithIndent(text string, indentLevel int) {
	fmt.Printf("%s%s\n", strings.Repeat(" ", indentLevel), text)
}
<|endoftext|>


<commit_before>package config

import (
"time"
"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
"github.com/coreos/etcd/client"
"sync"
"fmt"
)

const (
    activeNodeKey     = "/activeNode"
    dataKey           = "/data"
    heartbeatTimeout  = 3
    heartbeatInterval = 1
)

type EtcdConfig struct {
    nodeId    string
    isActive  bool
    endpoints []string
    prefix    string
    kapi      client.KeysAPI
    client    client.Client
    cache     map[string]string
    lock      sync.RWMutex
}

func NewEtcdConfig(nodeId string, endpoints []string, prefix string) (ret *EtcdConfig, err error) {
    ret = new(EtcdConfig)
    ret.endpoints = endpoints
    ret.prefix = prefix
    cfg := client.Config{
        Endpoints:               ret.endpoints,
        Transport:               client.DefaultTransport,
        // set timeout per request to fail fast when the target endpoint is unavailable
        HeaderTimeoutPerRequest: time.Second,
    }
    ret.isActive = false
    ret.client, err = client.New(cfg)
    if err != nil {
        return
    }
    ret.kapi = client.NewKeysAPI(ret.client)
    go ret.keepalive()
    go ret.watchForUpdate()
    return
}

func (self *EtcdConfig) GetMacByIp(ip string) string {
    self.lock.RLock()
    defer self.lock.RUnlock()
    if !self.isActive {
        return ""
    }
    mac, ok := self.cache[ip]
    if !ok {
        return ""
    }
    return mac
}

func (self *EtcdConfig) keepalive() {
    grabOption:= client.SetOptions{
      PrevExist: client.PrevNoExist,
      TTL: time.Second * heartbeatTimeout,
    }
    heartbeatOption:= client.SetOptions{
      TTL: time.Second * heartbeatTimeout,
    }
    for {
      var opt *client.SetOptions
      if self.IsActive() {
        opt = &heartbeatOption
      } else {
        opt = &grabOption
      }
      _, err := self.kapi.Set(context.Background(), self.prefix+activeNodeKey, self.nodeId, opt)
      self.setIsActive(err == nil)
      switch err := err.(type) {
      case *client.Error:
      case nil:
      default:
        //TODO: unknown error!
        fmt.Println(err)
      }
      time.Sleep(heartbeatInterval)
    }
 }

func (self *EtcdConfig) setIsActive(b bool) {
    self.lock.Lock()
    self.isActive = b
    self.lock.Unlock()
}

func (self *EtcdConfig) IsActive() (b bool) {
    self.lock.RLock()
    b = self.isActive
    self.lock.RUnlock()
    return
}

func (self *EtcdConfig) watchForUpdate() {
  //fetch
  // watch data, update cache
}

<commit_msg>fix hearbeat interval<commit_after>package config

import (
"time"
"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
"github.com/coreos/etcd/client"
"sync"
"fmt"
)

const (
    activeNodeKey     = "/activeNode"
    dataKey           = "/data"
    heartbeatTimeout  = 3
    heartbeatInterval = 1
)

type EtcdConfig struct {
    nodeId    string
    isActive  bool
    endpoints []string
    prefix    string
    kapi      client.KeysAPI
    client    client.Client
    cache     map[string]string
    lock      sync.RWMutex
}

func NewEtcdConfig(nodeId string, endpoints []string, prefix string) (ret *EtcdConfig, err error) {
    ret = new(EtcdConfig)
    ret.endpoints = endpoints
    ret.prefix = prefix
    cfg := client.Config{
        Endpoints:               ret.endpoints,
        Transport:               client.DefaultTransport,
        // set timeout per request to fail fast when the target endpoint is unavailable
        HeaderTimeoutPerRequest: time.Second,
    }
    ret.isActive = false
    ret.client, err = client.New(cfg)
    if err != nil {
        return
    }
    ret.kapi = client.NewKeysAPI(ret.client)
    go ret.keepalive()
    go ret.watchForUpdate()
    return
}

func (self *EtcdConfig) GetMacByIp(ip string) string {
    self.lock.RLock()
    defer self.lock.RUnlock()
    if !self.isActive {
        return ""
    }
    mac, ok := self.cache[ip]
    if !ok {
        return ""
    }
    return mac
}

func (self *EtcdConfig) keepalive() {
    grabOption:= client.SetOptions{
      PrevExist: client.PrevNoExist,
      TTL: time.Second * heartbeatTimeout,
    }
    heartbeatOption:= client.SetOptions{
      TTL: time.Second * heartbeatTimeout,
    }
    for {
      var opt *client.SetOptions
      if self.IsActive() {
        opt = &heartbeatOption
      } else {
        opt = &grabOption
      }
      _, err := self.kapi.Set(context.Background(), self.prefix+activeNodeKey, self.nodeId, opt)
      self.setIsActive(err == nil)
      switch err := err.(type) {
      case *client.Error:
      case nil:
      default:
        //TODO: unknown error!
        fmt.Println(err)
      }
      time.Sleep(heartbeatInterval * time.Second)
    }
 }

func (self *EtcdConfig) setIsActive(b bool) {
    self.lock.Lock()
    self.isActive = b
    self.lock.Unlock()
}

func (self *EtcdConfig) IsActive() (b bool) {
    self.lock.RLock()
    b = self.isActive
    self.lock.RUnlock()
    return
}

func (self *EtcdConfig) watchForUpdate() {
  //fetch
  resp, err := self.kapi.Get(context.Background(), self.prefix+dataKey, nil)
  if err != nil {
    // exit
    return
  }
  fmt.Println(resp)
  // watch data, update cache
}

<|endoftext|>


<commit_before>package workflowhelpers

import (
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/commandstarter"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers/internal"
)

type remoteResource interface {
	Create()
	Destroy()
	ShouldRemain() bool
}

type testSuiteConfig interface {
	internal.AdminUserConfig
	internal.SpaceAndOrgConfig
	internal.UserConfig
	internal.AdminClientConfig
	internal.ClientConfig

	GetApiEndpoint() string
	GetSkipSSLValidation() bool

	GetNamePrefix() string
	GetScaledTimeout(time.Duration) time.Duration
}

type ReproducibleTestSuiteSetup struct {
	shortTimeout time.Duration
	longTimeout  time.Duration

	organizationName string
	spaceName        string

	TestUser  remoteResource
	TestSpace internal.Space

	regularUserContext UserContext
	adminUserContext   UserContext

	SkipSSLValidation bool

	SkipUserCreation bool

	originalCfHomeDir string
	currentCfHomeDir  string
}

const RUNAWAY_QUOTA_MEM_LIMIT = "99999G"

func NewTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, "10G")

	return NewTestContextSuiteSetup(config, testSpace, config.GetUseExistingUser())
}

func NewSmokeTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, "10G")

	return NewTestContextSuiteSetup(config, testSpace, true)
}

func NewRunawayAppTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, RUNAWAY_QUOTA_MEM_LIMIT)

	return NewTestContextSuiteSetup(config, testSpace, config.GetUseExistingUser())
}

func NewTestContextSuiteSetup(config testSuiteConfig, testSpace internal.Space, skipUserCreation bool) *ReproducibleTestSuiteSetup {
	var testUser *internal.TestUser
	useTestClient := false
	if config.GetExistingClient() != "" && config.GetExistingClientSecret() != "" {
		testUser = internal.NewTestClient(config, commandstarter.NewCommandStarter())
		skipUserCreation = true
		useTestClient = true
	} else {
		testUser = internal.NewTestUser(config, commandstarter.NewCommandStarter())
	}

	var adminUser *internal.TestUser
	useAdminClient := false
	if config.GetAdminClient() != "" && config.GetAdminClientSecret() != "" {
		adminUser = internal.NewAdminClient(config, commandstarter.NewCommandStarter())
		useAdminClient = true
	} else {
		adminUser = internal.NewAdminUser(config, commandstarter.NewCommandStarter())
	}

	shortTimeout := config.GetScaledTimeout(1 * time.Minute)
	regularUserContext := NewUserContext(config.GetApiEndpoint(), testUser, testSpace, config.GetSkipSSLValidation(), shortTimeout)
	adminUserContext := NewUserContext(config.GetApiEndpoint(), adminUser, nil, config.GetSkipSSLValidation(), shortTimeout)
	regularUserContext.UseClientCredentials = useTestClient
	adminUserContext.UseClientCredentials = useAdminClient

	return NewBaseTestSuiteSetup(config, testSpace, testUser, regularUserContext, adminUserContext, skipUserCreation)
}

func NewBaseTestSuiteSetup(config testSuiteConfig, testSpace internal.Space, testUser remoteResource, regularUserContext, adminUserContext UserContext, skipUserCreation bool) *ReproducibleTestSuiteSetup {
	shortTimeout := config.GetScaledTimeout(1 * time.Minute)

	return &ReproducibleTestSuiteSetup{
		shortTimeout: shortTimeout,
		longTimeout:  config.GetScaledTimeout(5 * time.Minute),

		regularUserContext: regularUserContext,
		adminUserContext:   adminUserContext,

		SkipUserCreation: skipUserCreation,
		TestSpace:        testSpace,
		TestUser:         testUser,
	}
}

func (testSetup ReproducibleTestSuiteSetup) GetOrganizationName() string {
	return testSetup.TestSpace.OrganizationName()
}

func (testSetup ReproducibleTestSuiteSetup) ShortTimeout() time.Duration {
	return testSetup.shortTimeout
}

func (testSetup ReproducibleTestSuiteSetup) LongTimeout() time.Duration {
	return testSetup.longTimeout
}

func (testSetup *ReproducibleTestSuiteSetup) Setup() {
	AsUser(testSetup.AdminUserContext(), testSetup.shortTimeout, func() {
		testSetup.TestSpace.Create()
		if !testSetup.SkipUserCreation {
			testSetup.TestUser.Create()
		}
		testSetup.regularUserContext.AddUserToSpace()
	})
	testSetup.originalCfHomeDir, testSetup.currentCfHomeDir = testSetup.regularUserContext.SetCfHomeDir()
	testSetup.regularUserContext.Login()
	testSetup.regularUserContext.TargetSpace()
}

func (testSetup *ReproducibleTestSuiteSetup) Teardown() {
	testSetup.regularUserContext.Logout()
	testSetup.regularUserContext.UnsetCfHomeDir(testSetup.originalCfHomeDir, testSetup.currentCfHomeDir)

	AsUser(testSetup.AdminUserContext(), testSetup.shortTimeout, func() {
		if !testSetup.TestUser.ShouldRemain() {
			if !testSetup.SkipUserCreation {
				testSetup.TestUser.Destroy()
			}
		}

		testSetup.TestSpace.Destroy()
	})
}

func (testSetup *ReproducibleTestSuiteSetup) AdminUserContext() UserContext {
	return testSetup.adminUserContext
}

func (testSetup *ReproducibleTestSuiteSetup) RegularUserContext() UserContext {
	return testSetup.regularUserContext
}
<commit_msg>Skip adding user to space when using client credentials [#161763601]<commit_after>package workflowhelpers

import (
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/commandstarter"
	"github.com/cloudfoundry-incubator/cf-test-helpers/workflowhelpers/internal"
)

type remoteResource interface {
	Create()
	Destroy()
	ShouldRemain() bool
}

type testSuiteConfig interface {
	internal.AdminUserConfig
	internal.SpaceAndOrgConfig
	internal.UserConfig
	internal.AdminClientConfig
	internal.ClientConfig

	GetApiEndpoint() string
	GetSkipSSLValidation() bool

	GetNamePrefix() string
	GetScaledTimeout(time.Duration) time.Duration
}

type ReproducibleTestSuiteSetup struct {
	shortTimeout time.Duration
	longTimeout  time.Duration

	organizationName string
	spaceName        string

	TestUser  remoteResource
	TestSpace internal.Space

	regularUserContext UserContext
	adminUserContext   UserContext

	SkipSSLValidation bool

	SkipUserCreation bool

	originalCfHomeDir string
	currentCfHomeDir  string
}

const RUNAWAY_QUOTA_MEM_LIMIT = "99999G"

func NewTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, "10G")

	return NewTestContextSuiteSetup(config, testSpace, config.GetUseExistingUser())
}

func NewSmokeTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, "10G")

	return NewTestContextSuiteSetup(config, testSpace, true)
}

func NewRunawayAppTestSuiteSetup(config testSuiteConfig) *ReproducibleTestSuiteSetup {
	testSpace := internal.NewRegularTestSpace(config, RUNAWAY_QUOTA_MEM_LIMIT)

	return NewTestContextSuiteSetup(config, testSpace, config.GetUseExistingUser())
}

func NewTestContextSuiteSetup(config testSuiteConfig, testSpace internal.Space, skipUserCreation bool) *ReproducibleTestSuiteSetup {
	var testUser *internal.TestUser
	useTestClient := false
	if config.GetExistingClient() != "" && config.GetExistingClientSecret() != "" {
		testUser = internal.NewTestClient(config, commandstarter.NewCommandStarter())
		skipUserCreation = true
		useTestClient = true
	} else {
		testUser = internal.NewTestUser(config, commandstarter.NewCommandStarter())
	}

	var adminUser *internal.TestUser
	useAdminClient := false
	if config.GetAdminClient() != "" && config.GetAdminClientSecret() != "" {
		adminUser = internal.NewAdminClient(config, commandstarter.NewCommandStarter())
		useAdminClient = true
	} else {
		adminUser = internal.NewAdminUser(config, commandstarter.NewCommandStarter())
	}

	shortTimeout := config.GetScaledTimeout(1 * time.Minute)
	regularUserContext := NewUserContext(config.GetApiEndpoint(), testUser, testSpace, config.GetSkipSSLValidation(), shortTimeout)
	adminUserContext := NewUserContext(config.GetApiEndpoint(), adminUser, nil, config.GetSkipSSLValidation(), shortTimeout)
	regularUserContext.UseClientCredentials = useTestClient
	adminUserContext.UseClientCredentials = useAdminClient

	return NewBaseTestSuiteSetup(config, testSpace, testUser, regularUserContext, adminUserContext, skipUserCreation)
}

func NewBaseTestSuiteSetup(config testSuiteConfig, testSpace internal.Space, testUser remoteResource, regularUserContext, adminUserContext UserContext, skipUserCreation bool) *ReproducibleTestSuiteSetup {
	shortTimeout := config.GetScaledTimeout(1 * time.Minute)

	return &ReproducibleTestSuiteSetup{
		shortTimeout: shortTimeout,
		longTimeout:  config.GetScaledTimeout(5 * time.Minute),

		regularUserContext: regularUserContext,
		adminUserContext:   adminUserContext,

		SkipUserCreation: skipUserCreation,
		TestSpace:        testSpace,
		TestUser:         testUser,
	}
}

func (testSetup ReproducibleTestSuiteSetup) GetOrganizationName() string {
	return testSetup.TestSpace.OrganizationName()
}

func (testSetup ReproducibleTestSuiteSetup) ShortTimeout() time.Duration {
	return testSetup.shortTimeout
}

func (testSetup ReproducibleTestSuiteSetup) LongTimeout() time.Duration {
	return testSetup.longTimeout
}

func (testSetup *ReproducibleTestSuiteSetup) Setup() {
	AsUser(testSetup.AdminUserContext(), testSetup.shortTimeout, func() {
		testSetup.TestSpace.Create()
		if !testSetup.SkipUserCreation {
			testSetup.TestUser.Create()
		}
		if !testSetup.RegularUserContext().UseClientCredentials {
			testSetup.regularUserContext.AddUserToSpace()
		}
	})
	testSetup.originalCfHomeDir, testSetup.currentCfHomeDir = testSetup.regularUserContext.SetCfHomeDir()
	testSetup.regularUserContext.Login()
	testSetup.regularUserContext.TargetSpace()
}

func (testSetup *ReproducibleTestSuiteSetup) Teardown() {
	testSetup.regularUserContext.Logout()
	testSetup.regularUserContext.UnsetCfHomeDir(testSetup.originalCfHomeDir, testSetup.currentCfHomeDir)

	AsUser(testSetup.AdminUserContext(), testSetup.shortTimeout, func() {
		if !testSetup.TestUser.ShouldRemain() {
			if !testSetup.SkipUserCreation {
				testSetup.TestUser.Destroy()
			}
		}

		testSetup.TestSpace.Destroy()
	})
}

func (testSetup *ReproducibleTestSuiteSetup) AdminUserContext() UserContext {
	return testSetup.adminUserContext
}

func (testSetup *ReproducibleTestSuiteSetup) RegularUserContext() UserContext {
	return testSetup.regularUserContext
}
<|endoftext|>