<commit_before>package cassandra

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gocql/gocql"
	"github.com/uluyol/fabbench/db"
	"github.com/uluyol/fabbench/recorders"
)

type conf struct {
	ClientRetries        *int   `json:"clientRetries",omitempty`
	NumRetries           *int   `json:"numRetries,omitempty"`
	NumConns             *int   `json:"numConns,omitempty"`
	ReadConsistency      string `json:"readConsistency"`
	WriteConsistency     string `json:"writeConsistency"`
	Keyspace             string `json:"keyspace"`
	Table                string `json:"table"`
	ReplicationFactor    *int   `json:"replicationFactor,omitempty"`
	KeyCaching           string `json:"keyCaching"`
	CompactionStrategy   string `json:"compactionStrategy"`
	LeveledSSTableSizeMB *int   `json:"leveledSSTableSizeMB"`
	Timeout              string `json:"timeout"`

	TraceData *string `json:"traceData",omitempty`
	TraceRate *int32  `json:"traceRate",omitempty`
}

func newInt(v int) *int { return &v }

func (c *conf) fillDefaults() {
	intFields := []struct {
		val **int
		def int
	}{
		{&c.ClientRetries, 10},
		{&c.NumRetries, 4},
		{&c.ReplicationFactor, 3},
		{&c.NumConns, 4},
		{&c.LeveledSSTableSizeMB, 160},
	}
	for _, f := range intFields {
		if *f.val == nil {
			*f.val = newInt(f.def)
		}
	}

	strFields := []struct {
		val *string
		def string
	}{
		{&c.ReadConsistency, "ONE"},
		{&c.WriteConsistency, "ONE"},
		{&c.Keyspace, "fabbench"},
		{&c.Table, "udata"},
		{&c.KeyCaching, "ALL"},
		{&c.CompactionStrategy, "LeveledCompactionStrategy"},
		{&c.Timeout, "5s"},
	}
	for _, f := range strFields {
		if *f.val == "" {
			*f.val = f.def
		}
	}
}

func parseConsistency(s string) (c gocql.Consistency, err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
				return
			}
			panic(e)
		}
	}()
	return gocql.ParseConsistency(s), nil
}

type client struct {
	cluster          *gocql.ClusterConfig
	readConsistency  gocql.Consistency
	writeConsistency gocql.Consistency
	conf             *conf

	session struct {
		once sync.Once
		s    *gocql.Session
		err  error
	}

	rat     *recorders.AsyncTrace
	wat     *recorders.AsyncTrace
	rtracer gocql.Tracer
	wtracer gocql.Tracer

	getQPool sync.Pool
	putQPool sync.Pool

	opCount int32
}

func (c *client) getSession() (*gocql.Session, error) {
	c.session.once.Do(func() {
		c.session.s, c.session.err = c.cluster.CreateSession()
		c.rtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.rat.C))
		c.wtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.wat.C))
	})

	return c.session.s, c.session.err
}

func (c *client) traceQuery(q *gocql.Query, read bool) *gocql.Query {
	if c.rtracer != nil && c.wtracer != nil {
		tracer := c.rtracer
		if !read {
			tracer = c.wtracer
		}
		if atomic.AddInt32(&c.opCount, 1)%*c.conf.TraceRate == 0 {
			return q.Trace(tracer)
		}
	}
	return q
}

func (c *client) getQuery(s *gocql.Session) *gocql.Query {
	t := c.getQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("SELECT vval FROM " + c.conf.Table + " WHERE vkey = ?")
	q.Consistency(c.readConsistency)
	return q
}

func (c *client) cacheGetQuery(q *gocql.Query) { c.getQPool.Put(q) }

func (c *client) putQuery(s *gocql.Session) *gocql.Query {
	t := c.putQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("INSERT INTO " + c.conf.Table + " (vkey, vval) VALUES (?, ?)")
	q.Consistency(c.writeConsistency)
	return q
}

func (c *client) cachePutQuery(q *gocql.Query) { c.putQPool.Put(q) }

func newClient(hosts []string, cfg *conf) (db.DB, error) {
	timeout, err := time.ParseDuration(cfg.Timeout)
	if err != nil {
		return nil, fmt.Errorf("error parsing timeout as time.Duration: %v")
	}

	readConsistency, err := parseConsistency(cfg.ReadConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid read consistency: %v")
	}
	writeConsistency, err := parseConsistency(cfg.WriteConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid write consistency: %v")
	}

	cluster := gocql.NewCluster(hosts...)
	cluster.ProtoVersion = 4
	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cfg.NumRetries}
	cluster.Timeout = timeout
	cluster.SocketKeepalive = 30 * time.Second
	cluster.NumConns = *cfg.NumConns

	// Test that we can create a session.
	// We don't know if the Keyspace exists yet,
	// so we can't actually create the session used for gets/puts now.
	// We'll create it when the first request starts executing.
	//
	// For now, just make sure we can connect so we can give sane errors.
	s, err := cluster.CreateSession()
	if err != nil {
		return nil, fmt.Errorf("unable to connect to db: %v", err)
	}
	s.Close()

	cluster.Keyspace = cfg.Keyspace

	var rat, wat *recorders.AsyncTrace
	if cfg.TraceRate != nil && cfg.TraceData != nil {
		rf, err := os.Create(*cfg.TraceData + "-ro.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open ro trace file: %v", err)
		}
		wf, err := os.Create(*cfg.TraceData + "-wo.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open wo trace file: %v", err)
		}
		rat = recorders.NewAsyncTrace(rf)
		wat = recorders.NewAsyncTrace(wf)
		go rat.Consume()
		go wat.Consume()
	}

	c := &client{
		cluster:          cluster,
		readConsistency:  readConsistency,
		writeConsistency: writeConsistency,
		conf:             cfg,
		rat:              rat,
		wat:              wat,
	}
	return c, nil
}

func (c *client) Init(ctx context.Context) error {
	ks := c.cluster.Keyspace
	c.cluster.Keyspace = ""

	defer func() {
		c.cluster.Keyspace = ks
	}()

	session, err := c.cluster.CreateSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db to make table: %v", err)
	}
	defer session.Close()

	q := session.Query(fmt.Sprintf(
		"CREATE KEYSPACE %s WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': %d}",
		ks, *c.conf.ReplicationFactor))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create keyspace %s: %v", ks, err)
	}
	q = session.Query(fmt.Sprintf(
		"CREATE TABLE %s.%s (vkey varchar primary key, vval varchar) WITH compaction = {'class': '%s', 'sstable_size_in_mb': %d} AND caching = {'keys': '%s'}",
		ks, c.conf.Table, c.conf.CompactionStrategy, *c.conf.LeveledSSTableSizeMB, c.conf.KeyCaching))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create table %s: %v", c.conf.Table, err)
	}
	return nil
}

func exponentialRetry(maxTries int, f func() error) error {
	err := f()
	if err == nil {
		return nil
	}
	sleep := 5 * time.Millisecond
	maxSleep := time.Second
	for retry := 0; retry < maxTries; retry++ {
		time.Sleep(sleep)
		err = f()
		if err == nil {
			return nil
		}
		sleep *= 5
		if sleep > maxSleep {
			sleep = maxSleep
		}
	}
	return err
}

func (c *client) Get(ctx context.Context, key string) (string, error) {
	s, err := c.getSession()
	if err != nil {
		return "", fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.getQuery(s)
	var v string
	err = exponentialRetry(*c.conf.ClientRetries, func() error {
		return c.traceQuery(q.Bind(key).WithContext(ctx), true).Scan(&v)
	})
	c.cacheGetQuery(q)
	return v, err
}

func (c *client) Put(ctx context.Context, key, val string) error {
	s, err := c.getSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.putQuery(s)
	err = exponentialRetry(*c.conf.ClientRetries, func() error {
		return c.traceQuery(q.Bind(key, val).WithContext(ctx), false).Exec()
	})
	c.cachePutQuery(q)
	return err
}

func (c *client) Close() error {
	c.rat.Close()
	c.wat.Close()
	return nil
}

func makeDB(hosts []string, data []byte) (db.DB, error) {
	var cfg conf
	if err := json.Unmarshal(data, &cfg); err != nil {
		return nil, fmt.Errorf("invalid cassandra config: %v", err)
	}
	cfg.fillDefaults()
	return newClient(hosts, &cfg)
}

func init() {
	db.Register("cassandra", makeDB)
}
<commit_msg>db/cassandra: check that tracing in on before closing<commit_after>package cassandra

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gocql/gocql"
	"github.com/uluyol/fabbench/db"
	"github.com/uluyol/fabbench/recorders"
)

type conf struct {
	ClientRetries        *int   `json:"clientRetries",omitempty`
	NumRetries           *int   `json:"numRetries,omitempty"`
	NumConns             *int   `json:"numConns,omitempty"`
	ReadConsistency      string `json:"readConsistency"`
	WriteConsistency     string `json:"writeConsistency"`
	Keyspace             string `json:"keyspace"`
	Table                string `json:"table"`
	ReplicationFactor    *int   `json:"replicationFactor,omitempty"`
	KeyCaching           string `json:"keyCaching"`
	CompactionStrategy   string `json:"compactionStrategy"`
	LeveledSSTableSizeMB *int   `json:"leveledSSTableSizeMB"`
	Timeout              string `json:"timeout"`

	TraceData *string `json:"traceData",omitempty`
	TraceRate *int32  `json:"traceRate",omitempty`
}

func newInt(v int) *int { return &v }

func (c *conf) fillDefaults() {
	intFields := []struct {
		val **int
		def int
	}{
		{&c.ClientRetries, 10},
		{&c.NumRetries, 4},
		{&c.ReplicationFactor, 3},
		{&c.NumConns, 4},
		{&c.LeveledSSTableSizeMB, 160},
	}
	for _, f := range intFields {
		if *f.val == nil {
			*f.val = newInt(f.def)
		}
	}

	strFields := []struct {
		val *string
		def string
	}{
		{&c.ReadConsistency, "ONE"},
		{&c.WriteConsistency, "ONE"},
		{&c.Keyspace, "fabbench"},
		{&c.Table, "udata"},
		{&c.KeyCaching, "ALL"},
		{&c.CompactionStrategy, "LeveledCompactionStrategy"},
		{&c.Timeout, "5s"},
	}
	for _, f := range strFields {
		if *f.val == "" {
			*f.val = f.def
		}
	}
}

func parseConsistency(s string) (c gocql.Consistency, err error) {
	defer func() {
		if e := recover(); e != nil {
			if er, ok := e.(error); ok {
				err = er
				return
			}
			panic(e)
		}
	}()
	return gocql.ParseConsistency(s), nil
}

type client struct {
	cluster          *gocql.ClusterConfig
	readConsistency  gocql.Consistency
	writeConsistency gocql.Consistency
	conf             *conf

	session struct {
		once sync.Once
		s    *gocql.Session
		err  error
	}

	rat     *recorders.AsyncTrace
	wat     *recorders.AsyncTrace
	rtracer gocql.Tracer
	wtracer gocql.Tracer

	getQPool sync.Pool
	putQPool sync.Pool

	opCount int32
}

func (c *client) getSession() (*gocql.Session, error) {
	c.session.once.Do(func() {
		c.session.s, c.session.err = c.cluster.CreateSession()
		c.rtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.rat.C))
		c.wtracer = gocql.NewTraceWriter(c.session.s, recorders.NewTraceConsumer(c.wat.C))
	})

	return c.session.s, c.session.err
}

func (c *client) traceQuery(q *gocql.Query, read bool) *gocql.Query {
	if c.rtracer != nil && c.wtracer != nil {
		tracer := c.rtracer
		if !read {
			tracer = c.wtracer
		}
		if atomic.AddInt32(&c.opCount, 1)%*c.conf.TraceRate == 0 {
			return q.Trace(tracer)
		}
	}
	return q
}

func (c *client) getQuery(s *gocql.Session) *gocql.Query {
	t := c.getQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("SELECT vval FROM " + c.conf.Table + " WHERE vkey = ?")
	q.Consistency(c.readConsistency)
	return q
}

func (c *client) cacheGetQuery(q *gocql.Query) { c.getQPool.Put(q) }

func (c *client) putQuery(s *gocql.Session) *gocql.Query {
	t := c.putQPool.Get()
	if t != nil {
		if q, ok := t.(*gocql.Query); ok {
			return q
		}
	}
	q := s.Query("INSERT INTO " + c.conf.Table + " (vkey, vval) VALUES (?, ?)")
	q.Consistency(c.writeConsistency)
	return q
}

func (c *client) cachePutQuery(q *gocql.Query) { c.putQPool.Put(q) }

func newClient(hosts []string, cfg *conf) (db.DB, error) {
	timeout, err := time.ParseDuration(cfg.Timeout)
	if err != nil {
		return nil, fmt.Errorf("error parsing timeout as time.Duration: %v")
	}

	readConsistency, err := parseConsistency(cfg.ReadConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid read consistency: %v")
	}
	writeConsistency, err := parseConsistency(cfg.WriteConsistency)
	if err != nil {
		return nil, fmt.Errorf("invalid write consistency: %v")
	}

	cluster := gocql.NewCluster(hosts...)
	cluster.ProtoVersion = 4
	cluster.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: *cfg.NumRetries}
	cluster.Timeout = timeout
	cluster.SocketKeepalive = 30 * time.Second
	cluster.NumConns = *cfg.NumConns

	// Test that we can create a session.
	// We don't know if the Keyspace exists yet,
	// so we can't actually create the session used for gets/puts now.
	// We'll create it when the first request starts executing.
	//
	// For now, just make sure we can connect so we can give sane errors.
	s, err := cluster.CreateSession()
	if err != nil {
		return nil, fmt.Errorf("unable to connect to db: %v", err)
	}
	s.Close()

	cluster.Keyspace = cfg.Keyspace

	var rat, wat *recorders.AsyncTrace
	if cfg.TraceRate != nil && cfg.TraceData != nil {
		rf, err := os.Create(*cfg.TraceData + "-ro.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open ro trace file: %v", err)
		}
		wf, err := os.Create(*cfg.TraceData + "-wo.gz")
		if err != nil {
			return nil, fmt.Errorf("unable to open wo trace file: %v", err)
		}
		rat = recorders.NewAsyncTrace(rf)
		wat = recorders.NewAsyncTrace(wf)
		go rat.Consume()
		go wat.Consume()
	}

	c := &client{
		cluster:          cluster,
		readConsistency:  readConsistency,
		writeConsistency: writeConsistency,
		conf:             cfg,
		rat:              rat,
		wat:              wat,
	}
	return c, nil
}

func (c *client) Init(ctx context.Context) error {
	ks := c.cluster.Keyspace
	c.cluster.Keyspace = ""

	defer func() {
		c.cluster.Keyspace = ks
	}()

	session, err := c.cluster.CreateSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db to make table: %v", err)
	}
	defer session.Close()

	q := session.Query(fmt.Sprintf(
		"CREATE KEYSPACE %s WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': %d}",
		ks, *c.conf.ReplicationFactor))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create keyspace %s: %v", ks, err)
	}
	q = session.Query(fmt.Sprintf(
		"CREATE TABLE %s.%s (vkey varchar primary key, vval varchar) WITH compaction = {'class': '%s', 'sstable_size_in_mb': %d} AND caching = {'keys': '%s'}",
		ks, c.conf.Table, c.conf.CompactionStrategy, *c.conf.LeveledSSTableSizeMB, c.conf.KeyCaching))
	if err := q.WithContext(ctx).Exec(); err != nil {
		return fmt.Errorf("unable to create table %s: %v", c.conf.Table, err)
	}
	return nil
}

func exponentialRetry(maxTries int, f func() error) error {
	err := f()
	if err == nil {
		return nil
	}
	sleep := 5 * time.Millisecond
	maxSleep := time.Second
	for retry := 0; retry < maxTries; retry++ {
		time.Sleep(sleep)
		err = f()
		if err == nil {
			return nil
		}
		sleep *= 5
		if sleep > maxSleep {
			sleep = maxSleep
		}
	}
	return err
}

func (c *client) Get(ctx context.Context, key string) (string, error) {
	s, err := c.getSession()
	if err != nil {
		return "", fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.getQuery(s)
	var v string
	err = exponentialRetry(*c.conf.ClientRetries, func() error {
		return c.traceQuery(q.Bind(key).WithContext(ctx), true).Scan(&v)
	})
	c.cacheGetQuery(q)
	return v, err
}

func (c *client) Put(ctx context.Context, key, val string) error {
	s, err := c.getSession()
	if err != nil {
		return fmt.Errorf("unable to connect to db: %v", err)
	}
	q := c.putQuery(s)
	err = exponentialRetry(*c.conf.ClientRetries, func() error {
		return c.traceQuery(q.Bind(key, val).WithContext(ctx), false).Exec()
	})
	c.cachePutQuery(q)
	return err
}

func (c *client) Close() error {
	if c.rat != nil {
		c.rat.Close()
	}
	if c.wat != nil {
		c.wat.Close()
	}
	return nil
}

func makeDB(hosts []string, data []byte) (db.DB, error) {
	var cfg conf
	if err := json.Unmarshal(data, &cfg); err != nil {
		return nil, fmt.Errorf("invalid cassandra config: %v", err)
	}
	cfg.fillDefaults()
	return newClient(hosts, &cfg)
}

func init() {
	db.Register("cassandra", makeDB)
}
<|endoftext|>


<commit_before>package dbsearch

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"github.com/iostrovok/go-dbsearch/dbsearch/sqler"
	"github.com/iostrovok/go-iutils/iutils"
	_ "github.com/lib/pq"
	"log"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"time"
)

var m sync.Mutex

//
type OneRow struct {
	Name    string
	DBName  string
	Type    string
	IsArray bool
}

type AllRows struct {
	DBList map[string]*OneRow
	List   map[string]*OneRow
	Done   bool
}

type Searcher struct {
	db  *sql.DB
	log bool
}

func (s *Searcher) SetDebug(is_debug ...bool) {

	if len(is_debug) > 0 {
		s.log = is_debug[0]
	} else {
		s.log = true
	}
}

func (s *Searcher) Ping() error {

	if s.db == nil {
		return fmt.Errorf("can't connect to DB")
	}

	if err := s.db.Ping(); err != nil {
		return err
	}

	return nil
}

func DBI(poolSize int, dsn string, stop_error ...bool) (*Searcher, error) {

	s := new(Searcher)

	db, _ := sql.Open("postgres", dsn)

	if err := db.Ping(); err != nil {
		if len(stop_error) > 0 && stop_error[0] {
			log.Fatalf("DB Error: %s\n", err)
		}
		return nil, errors.New(fmt.Sprintf("DB Error: %s\n", err))
	}

	s.db = db
	s.db.SetMaxOpenConns(poolSize)

	return s, nil
}

func (s *Searcher) GetCount(sqlLine string, values []interface{}) (int, error) {
	var count int
	err := s.db.QueryRow(sqlLine, values...).Scan(&count)
	return count, err
}

func (s *Searcher) GetOne(mType *AllRows, sqlLine string, values ...[]interface{}) (map[string]interface{}, error) {

	sqlLine += " LIMIT 1 OFFSET 0 "

	value := []interface{}{}
	if len(values) > 0 {
		value = values[0]
	}

	list, err := s.Get(mType, sqlLine, value)
	empty := map[string]interface{}{}

	if err != nil {
		return empty, err
	}

	if len(list) > 0 {
		return list[0], nil
	}

	return empty, nil
}

func (s *Searcher) Get(mType *AllRows, sqlLine string, values []interface{}) ([]map[string]interface{}, error) {

	Out := make([]map[string]interface{}, 0)

	if s.log {
		log.Printf("dbsearch.Get: %s\n", sqlLine)
		log.Printf("%v\n", values)
	}

	rows, err := s.db.Query(sqlLine, values...)
	if err != nil {
		return Out, err
	}
	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return Out, err
	}

	rawResult := make([]interface{}, 0)
	for i := 0; i < len(cols); i++ {
		t, find := mType.DBList[cols[i]]
		if !find {
			log.Fatalf("dbsearch.Get not found column: %s!", cols[i])
		}

		switch t.Type {
		case "datetime", "date":
			datetime := new(*time.Time)
			rawResult = append(rawResult, datetime)
		case "int", "numeric":
			rawResult = append(rawResult, new(int))
		case "bigint":
			rawResult = append(rawResult, new(int64))
		default:
			rawResult = append(rawResult, make([]byte, 0))
		}
	}

	dest := make([]interface{}, len(cols)) // A temporary interface{} slice
	for i, _ := range rawResult {
		dest[i] = &rawResult[i] // Put pointers to each string in the interface slice
	}

	for rows.Next() {
		if err := rows.Scan(dest...); err != nil {
			log.Fatal(err)
		}

		result := map[string]interface{}{}
		for i, raw := range rawResult {
			// cols[i] - Column name
			if s.log {
				log.Printf("parseArray. %s: %s\n", cols[i], raw)
			}

			result[cols[i]] = convertType(cols[i], mType, raw)
		}

		Out = append(Out, result)
	}

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}

	return Out, nil
}

func convertType(Name string, mType *AllRows, raw_in interface{}) interface{} {
	t, find := mType.DBList[Name]
	if !find {
		log.Fatal("Not found!")
	}

	if raw_in == nil {
		if t.IsArray {
			switch t.Type {
			case "text", "date", "datetime":
				return []string{}
			case "bigint", "int64", "int":
				return []int{}
			}
			return []interface{}{}
		} else {
			return nil
		}
	}

	switch t.Type {
	case "text":
		raw := raw_in.([]byte)
		if t.IsArray {
			return parseArray(string(raw))
		} else {
			return string(raw)
		}
	case "json", "jsonb":
		line := iutils.AnyToString(raw_in)
		if line == "" {
			line = "{}"
		}
		raw := []byte(line)
		var res map[string]interface{}
		err := json.Unmarshal(raw, &res)
		if err != nil {
			log.Fatal("error:", err)
		}
		return res
	case "bigint", "int64", "int":
		if t.IsArray {
			return parseIntArray(raw_in)
		} else {
			return iutils.AnyToInt(raw_in)
		}
	case "date", "datetime":
		return raw_in
	}

	return nil
}

func (mT *AllRows) PreInit(p interface{}) {
	if !mT.Done {
		m.Lock()
		mT.iPrepare(p)
		spew.Dump(mT)
		m.Unlock()
	}
}

func (aRows *AllRows) iPrepare(s interface{}) {
	st := reflect.TypeOf(s)

	aRows.Done = true
	aRows.List = make(map[string]*OneRow, 0)
	aRows.DBList = make(map[string]*OneRow, 0)
	Count := 0
	for true {
		field := st.Field(Count)

		if field.Name == "" {
			break
		}

		Count++

		dbname := field.Tag.Get("db")
		oRow := OneRow{
			Name:    field.Name,
			DBName:  dbname,
			Type:    field.Tag.Get("type"),
			IsArray: false,
		}
		if field.Tag.Get("is_array") == "yes" {
			oRow.IsArray = true
		}
		aRows.List[field.Name] = &oRow
		aRows.DBList[dbname] = &oRow
	}
}

func Prepare(s interface{}) *AllRows {
	aRows := AllRows{}
	aRows.iPrepare(s)
	return &aRows
}

func (self *Searcher) GetRowsCount(table string) (int, error) {
	return self.GetCount(fmt.Sprintf("SELECT count(*) FROM %s", table), make([]interface{}, 0))
}

/*
*************************** ARRAY PARSER START ******************************
 */

// construct a regexp to extract values:
var (
	unquotedRe  = regexp.MustCompile(`([^",\\{}\s]|NULL)+,`)
	_arrayValue = fmt.Sprintf("\"(%s)+\",", `[^"\\]|\\"|\\\\`)
	quotedRe    = regexp.MustCompile(_arrayValue)

	noNumbers      = regexp.MustCompile(`[^0-9]+`)
	noNumbersStart = regexp.MustCompile(`^[^0-9]+`)
	noNumbersEnd   = regexp.MustCompile(`[^0-9]+$`)
)

func parseIntArray(s interface{}) []int {
	str := strings.TrimSpace(iutils.AnyToString(s))
	str = noNumbersStart.ReplaceAllString(str, "")
	str = noNumbersEnd.ReplaceAllString(str, "")
	return iutils.AnyToIntArray(noNumbers.Split(str, -1))
}

func parseArray(line string) []string {

	out := []string{}
	if line == "{}" {
		return out
	}

	if len(line)-1 != strings.LastIndex(line, "}") || strings.Index(line, "{") != 0 {
		return out
	}

	/* Removes lead & last {} and adds "," to end of string */
	line = line[0:]
	line = line[:len(line)-1] + ","

	for len(line) > 0 {
		s := ""
		if strings.Index(line, `"`) != 0 {
			s = unquotedRe.FindString(line)
			line = line[strings.Index(line, ",")+1:]
			s = strings.TrimSuffix(s, ",")

			/* counvert NULL to empty string6 however we need nil string */
			if s == "NULL" {
				s = ""
			}
		} else {
			s = quotedRe.FindString(line)
			line = line[len(s):]
			s = strings.TrimPrefix(s, "\"")
			s = strings.TrimSuffix(s, "\",")
			s = strings.Join(strings.Split(s, "\\\\"), "\\")
			s = strings.Join(strings.Split(s, "\\\""), "\"")
		}
		out = append(out, s)
	}

	return out
}

/*
*************************** ARRAY PARSER FINISH **************************
 */

func (s *Searcher) Do(sql string, values ...interface{}) {
	_, err := s.db.Exec(sql, values...)
	if err != nil {
		log.Fatal(err)
	}
}

func (s *Searcher) Insert(table string, data map[string]interface{}) {
	sql, values := sqler.InsertLine(table, data)
	s.DoCommit(sql, values)
}

func (s *Searcher) Delete(table string, data_where map[string]interface{}) {
	sql, values := sqler.DeleteLine(table, data_where)
	s.DoCommit(sql, values)
}

func (s *Searcher) Update(table string, data_where map[string]interface{}, data_update map[string]interface{}) {
	sql, values := sqler.UpdateLine(table, data_update, data_where)
	s.DoCommit(sql, values)
}

func (s *Searcher) DoCommit(sql string, values []interface{}) {

	if s.log {
		log.Printf("DoCommit: %s\n", sql)
	}

	txn, err := s.db.Begin()
	if err != nil {
		log.Fatal(err)
	}

	_, err = txn.Exec(sql, values...)
	if err != nil {
		log.Fatal(err)
	}

	err = txn.Commit()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Bug fix. Removes working with string through slice[...len(str)...].<commit_after>package dbsearch

import (
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"github.com/iostrovok/go-dbsearch/dbsearch/sqler"
	"github.com/iostrovok/go-iutils/iutils"
	_ "github.com/lib/pq"
	"log"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"time"
)

var m sync.Mutex

//
type OneRow struct {
	Name    string
	DBName  string
	Type    string
	IsArray bool
}

type AllRows struct {
	DBList map[string]*OneRow
	List   map[string]*OneRow
	Done   bool
}

type Searcher struct {
	db  *sql.DB
	log bool
}

func (s *Searcher) SetDebug(is_debug ...bool) {

	if len(is_debug) > 0 {
		s.log = is_debug[0]
	} else {
		s.log = true
	}
}

func (s *Searcher) Ping() error {

	if s.db == nil {
		return fmt.Errorf("can't connect to DB")
	}

	if err := s.db.Ping(); err != nil {
		return err
	}

	return nil
}

func DBI(poolSize int, dsn string, stop_error ...bool) (*Searcher, error) {

	s := new(Searcher)

	db, _ := sql.Open("postgres", dsn)

	if err := db.Ping(); err != nil {
		if len(stop_error) > 0 && stop_error[0] {
			log.Fatalf("DB Error: %s\n", err)
		}
		return nil, errors.New(fmt.Sprintf("DB Error: %s\n", err))
	}

	s.db = db
	s.db.SetMaxOpenConns(poolSize)

	return s, nil
}

func (s *Searcher) GetCount(sqlLine string, values []interface{}) (int, error) {
	var count int
	err := s.db.QueryRow(sqlLine, values...).Scan(&count)
	return count, err
}

func (s *Searcher) GetOne(mType *AllRows, sqlLine string, values ...[]interface{}) (map[string]interface{}, error) {

	sqlLine += " LIMIT 1 OFFSET 0 "

	value := []interface{}{}
	if len(values) > 0 {
		value = values[0]
	}

	list, err := s.Get(mType, sqlLine, value)
	empty := map[string]interface{}{}

	if err != nil {
		return empty, err
	}

	if len(list) > 0 {
		return list[0], nil
	}

	return empty, nil
}

func (s *Searcher) Get(mType *AllRows, sqlLine string, values []interface{}) ([]map[string]interface{}, error) {

	Out := make([]map[string]interface{}, 0)

	if s.log {
		log.Printf("dbsearch.Get: %s\n", sqlLine)
		log.Printf("%v\n", values)
	}

	rows, err := s.db.Query(sqlLine, values...)
	if err != nil {
		return Out, err
	}
	defer rows.Close()

	cols, err := rows.Columns()
	if err != nil {
		return Out, err
	}

	rawResult := make([]interface{}, 0)
	for i := 0; i < len(cols); i++ {
		t, find := mType.DBList[cols[i]]
		if !find {
			log.Fatalf("dbsearch.Get not found column: %s!", cols[i])
		}

		switch t.Type {
		case "datetime", "date":
			datetime := new(*time.Time)
			rawResult = append(rawResult, datetime)
		case "int", "numeric":
			rawResult = append(rawResult, new(int))
		case "bigint":
			rawResult = append(rawResult, new(int64))
		default:
			rawResult = append(rawResult, make([]byte, 0))
		}
	}

	dest := make([]interface{}, len(cols)) // A temporary interface{} slice
	for i, _ := range rawResult {
		dest[i] = &rawResult[i] // Put pointers to each string in the interface slice
	}

	for rows.Next() {
		if err := rows.Scan(dest...); err != nil {
			log.Fatal(err)
		}

		result := map[string]interface{}{}
		for i, raw := range rawResult {
			// cols[i] - Column name
			if s.log {
				log.Printf("parseArray. %s: %s\n", cols[i], raw)
			}

			result[cols[i]] = convertType(cols[i], mType, raw)
		}

		Out = append(Out, result)
	}

	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}

	return Out, nil
}

func convertType(Name string, mType *AllRows, raw_in interface{}) interface{} {
	t, find := mType.DBList[Name]
	if !find {
		log.Fatal("Not found!")
	}

	if raw_in == nil {
		if t.IsArray {
			switch t.Type {
			case "text", "date", "datetime":
				return []string{}
			case "bigint", "int64", "int":
				return []int{}
			}
			return []interface{}{}
		} else {
			return nil
		}
	}

	switch t.Type {
	case "text":
		raw := raw_in.([]byte)
		if t.IsArray {
			return parseArray(string(raw))
		} else {
			return string(raw)
		}
	case "json", "jsonb":
		line := iutils.AnyToString(raw_in)
		if line == "" {
			line = "{}"
		}
		raw := []byte(line)
		var res map[string]interface{}
		err := json.Unmarshal(raw, &res)
		if err != nil {
			log.Fatal("error:", err)
		}
		return res
	case "bigint", "int64", "int":
		if t.IsArray {
			return parseIntArray(raw_in)
		} else {
			return iutils.AnyToInt(raw_in)
		}
	case "date", "datetime":
		return raw_in
	}

	return nil
}

func (mT *AllRows) PreInit(p interface{}) {
	if !mT.Done {
		m.Lock()
		mT.iPrepare(p)
		spew.Dump(mT)
		m.Unlock()
	}
}

func (aRows *AllRows) iPrepare(s interface{}) {
	st := reflect.TypeOf(s)

	aRows.Done = true
	aRows.List = make(map[string]*OneRow, 0)
	aRows.DBList = make(map[string]*OneRow, 0)
	Count := 0
	for true {
		field := st.Field(Count)

		if field.Name == "" {
			break
		}

		Count++

		dbname := field.Tag.Get("db")
		oRow := OneRow{
			Name:    field.Name,
			DBName:  dbname,
			Type:    field.Tag.Get("type"),
			IsArray: false,
		}
		if field.Tag.Get("is_array") == "yes" {
			oRow.IsArray = true
		}
		aRows.List[field.Name] = &oRow
		aRows.DBList[dbname] = &oRow
	}
}

func Prepare(s interface{}) *AllRows {
	aRows := AllRows{}
	aRows.iPrepare(s)
	return &aRows
}

func (self *Searcher) GetRowsCount(table string) (int, error) {
	return self.GetCount(fmt.Sprintf("SELECT count(*) FROM %s", table), make([]interface{}, 0))
}

/*
*************************** ARRAY PARSER START ******************************
 */

// construct a regexp to extract values:
var (
	unquotedRe  = regexp.MustCompile(`([^",\\{}\s]|NULL)+,`)
	_arrayValue = fmt.Sprintf("\"(%s)+\",", `[^"\\]|\\"|\\\\`)
	quotedRe    = regexp.MustCompile(_arrayValue)

	noNumbers      = regexp.MustCompile(`[^0-9]+`)
	noNumbersStart = regexp.MustCompile(`^[^0-9]+`)
	noNumbersEnd   = regexp.MustCompile(`[^0-9]+$`)
)

func parseIntArray(s interface{}) []int {
	str := strings.TrimSpace(iutils.AnyToString(s))
	str = noNumbersStart.ReplaceAllString(str, "")
	str = noNumbersEnd.ReplaceAllString(str, "")
	return iutils.AnyToIntArray(noNumbers.Split(str, -1))
}

func parseArray(line string) []string {

	out := []string{}
	if line == "{}" {
		return out
	}

	if len(line)-1 != strings.LastIndex(line, "}") || strings.Index(line, "{") != 0 {
		return out
	}

	/* Removes lead & last {} and adds "," to end of string */
	line = strings.TrimPrefix(line, "{")
	line = strings.TrimSuffix(line, "}") + ","

	for len(line) > 0 {
		s := ""
		if strings.Index(line, `"`) != 0 {
			s = unquotedRe.FindString(line)
			line = line[strings.Index(line, ",")+1:]
			s = strings.TrimSuffix(s, ",")

			/* counvert NULL to empty string6 however we need nil string */
			if s == "NULL" {
				s = ""
			}
		} else {
			s = quotedRe.FindString(line)
			line = line[len(s):]
			s = strings.TrimPrefix(s, "\"")
			s = strings.TrimSuffix(s, "\",")
			s = strings.Join(strings.Split(s, "\\\\"), "\\")
			s = strings.Join(strings.Split(s, "\\\""), "\"")
		}
		out = append(out, s)
	}

	return out
}

/*
*************************** ARRAY PARSER FINISH **************************
 */

func (s *Searcher) Do(sql string, values ...interface{}) {
	_, err := s.db.Exec(sql, values...)
	if err != nil {
		log.Fatal(err)
	}
}

func (s *Searcher) Insert(table string, data map[string]interface{}) {
	sql, values := sqler.InsertLine(table, data)
	s.DoCommit(sql, values)
}

func (s *Searcher) Delete(table string, data_where map[string]interface{}) {
	sql, values := sqler.DeleteLine(table, data_where)
	s.DoCommit(sql, values)
}

func (s *Searcher) Update(table string, data_where map[string]interface{}, data_update map[string]interface{}) {
	sql, values := sqler.UpdateLine(table, data_update, data_where)
	s.DoCommit(sql, values)
}

func (s *Searcher) DoCommit(sql string, values []interface{}) {

	if s.log {
		log.Printf("DoCommit: %s\n", sql)
	}

	txn, err := s.db.Begin()
	if err != nil {
		log.Fatal(err)
	}

	_, err = txn.Exec(sql, values...)
	if err != nil {
		log.Fatal(err)
	}

	err = txn.Commit()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package decomposition

import (
	"errors"

	"github.com/ready-steady/linear/decomposition"
)

// CovPCA performs principal component analysis on an m-by-m covariance matrix
// Σ. The principal components U and their variances Λ are returned in the
// descending order of the variances.
//
// By definition, the variances should be nonnegative. Due to finite-precision
// arithmetics, however, some close-to-zero variances might turn out to be
// negative. If the absolute value of a negative variance is smaller than the
// tolerance ε, the function nullifies that variance and proceeds without any
// errors; otherwise, an error is returned.
func CovPCA(Σ []float64, m uint, ε float64) (U []float64, Λ []float64, err error) {
	U = make([]float64, m*m)
	Λ = make([]float64, m)

	if err = decomposition.SymmetricEigen(Σ, U, Λ, m); err != nil {
		return nil, nil, err
	}

	for i := uint(0); i < m; i++ {
		if Λ[i] < 0 {
			if -Λ[i] < ε {
				Λ[i] = 0
			} else {
				return nil, nil, errors.New("the matrix should be positive semidefinite")
			}
		}
	}

	// Λ is in the ascending order. Reverse!
	for i, j := uint(0), m-1; i < j; i, j = i+1, j-1 {
		Λ[i], Λ[j] = Λ[j], Λ[i]
		for k := uint(0); k < m; k++ {
			U[i*m+k], U[j*m+k] = U[j*m+k], U[i*m+k]
		}
	}

	return
}
<commit_msg>decomposition: make a cosmetic adjustment<commit_after>package decomposition

import (
	"errors"

	"github.com/ready-steady/linear/decomposition"
)

// CovPCA performs principal component analysis on an m-by-m covariance matrix
// Σ. The principal components U and their variances Λ are returned in the
// descending order of the variances.
//
// By definition, the variances should be nonnegative. Due to finite-precision
// arithmetics, however, some close-to-zero variances might turn out to be
// negative. If the absolute value of a negative variance is smaller than the
// tolerance ε, the function nullifies that variance and proceeds without any
// errors; otherwise, an error is returned.
func CovPCA(Σ []float64, m uint, ε float64) (U []float64, Λ []float64, err error) {
	U = make([]float64, m*m)
	Λ = make([]float64, m)

	if err = decomposition.SymmetricEigen(Σ, U, Λ, m); err != nil {
		return nil, nil, err
	}

	for i := uint(0); i < m; i++ {
		if Λ[i] < 0.0 {
			if -Λ[i] < ε {
				Λ[i] = 0.0
			} else {
				return nil, nil, errors.New("the matrix should be positive semidefinite")
			}
		}
	}

	// Λ is in the ascending order. Reverse!
	for i, j := uint(0), m-1; i < j; i, j = i+1, j-1 {
		Λ[i], Λ[j] = Λ[j], Λ[i]
		for k := uint(0); k < m; k++ {
			U[i*m+k], U[j*m+k] = U[j*m+k], U[i*m+k]
		}
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2017 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package topdown

import (
	"encoding/json"
	"fmt"
	"math/big"
	"strconv"
	"time"

	"github.com/open-policy-agent/opa/ast"
	"github.com/open-policy-agent/opa/topdown/builtins"
)

type nowKeyID string

var nowKey = nowKeyID("time.now_ns")

func builtinTimeNowNanos(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error {

	exist, ok := bctx.Cache.Get(nowKey)
	var now *ast.Term

	if !ok {
		curr := time.Now()
		now = ast.NewTerm(ast.Number(int64ToJSONNumber(curr.UnixNano())))
		bctx.Cache.Put(nowKey, now)
	} else {
		now = exist.(*ast.Term)
	}

	return iter(now)
}

func builtinTimeParseNanos(a, b ast.Value) (ast.Value, error) {

	format, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}

	value, err := builtins.StringOperand(b, 2)
	if err != nil {
		return nil, err
	}

	result, err := time.Parse(string(format), string(value))
	if err != nil {
		return nil, err
	}

	return ast.Number(int64ToJSONNumber(result.UnixNano())), nil
}

func builtinTimeParseRFC3339Nanos(a ast.Value) (ast.Value, error) {

	value, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}

	result, err := time.Parse(time.RFC3339, string(value))
	if err != nil {
		return nil, err
	}

	return ast.Number(int64ToJSONNumber(result.UnixNano())), nil
}
func builtinParseDurationNanos(a ast.Value) (ast.Value, error) {

	duration, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}
	value, err := time.ParseDuration(string(duration))
	if err != nil {
		return nil, err
	}
	return ast.Number(int64ToJSONNumber(int64(value))), nil
}

func builtinDate(a ast.Value) (ast.Value, error) {

	value, err := builtins.NumberOperand(a, 1)
	if err != nil {
		return nil, err
	}

	f := builtins.NumberToFloat(value)
	i64, acc := f.Int64()
	if acc != big.Exact {
		return nil, fmt.Errorf("timestamp too big")
	}

	t := time.Unix(0, i64).UTC()
	year, month, day := t.Date()
	result := ast.Array{ast.IntNumberTerm(year), ast.IntNumberTerm(int(month)), ast.IntNumberTerm(day)}
	return result, nil
}

func builtinClock(a ast.Value) (ast.Value, error) {

	value, err := builtins.NumberOperand(a, 1)
	if err != nil {
		return nil, err
	}

	f := builtins.NumberToFloat(value)
	i64, acc := f.Int64()
	if acc != big.Exact {
		return nil, fmt.Errorf("timestamp too big")
	}

	t := time.Unix(0, i64).UTC()
	hour, minute, second := t.Clock()
	result := ast.Array{ast.IntNumberTerm(hour), ast.IntNumberTerm(minute), ast.IntNumberTerm(second)}
	return result, nil
}

func builtinWeekday(a ast.Value) (ast.Value, error) {

	value, err := builtins.NumberOperand(a, 1)
	if err != nil {
		return nil, err
	}

	f := builtins.NumberToFloat(value)
	i64, acc := f.Int64()
	if acc != big.Exact {
		return nil, fmt.Errorf("timestamp too big")
	}

	t := time.Unix(0, i64).UTC()
	weekday := t.Weekday().String()
	return ast.String(weekday), nil
}

func int64ToJSONNumber(i int64) json.Number {
	return json.Number(strconv.FormatInt(i, 10))
}

func init() {
	RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos)
	RegisterFunctionalBuiltin1(ast.ParseRFC3339Nanos.Name, builtinTimeParseRFC3339Nanos)
	RegisterFunctionalBuiltin2(ast.ParseNanos.Name, builtinTimeParseNanos)
	RegisterFunctionalBuiltin1(ast.ParseDurationNanos.Name, builtinParseDurationNanos)
	RegisterFunctionalBuiltin1(ast.Date.Name, builtinDate)
	RegisterFunctionalBuiltin1(ast.Clock.Name, builtinClock)
	RegisterFunctionalBuiltin1(ast.Weekday.Name, builtinWeekday)
}
<commit_msg>topdown/time: deduplicate<commit_after>// Copyright 2017 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package topdown

import (
	"encoding/json"
	"fmt"
	"math/big"
	"strconv"
	"time"

	"github.com/open-policy-agent/opa/ast"
	"github.com/open-policy-agent/opa/topdown/builtins"
)

type nowKeyID string

var nowKey = nowKeyID("time.now_ns")

func builtinTimeNowNanos(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error {

	exist, ok := bctx.Cache.Get(nowKey)
	var now *ast.Term

	if !ok {
		curr := time.Now()
		now = ast.NewTerm(ast.Number(int64ToJSONNumber(curr.UnixNano())))
		bctx.Cache.Put(nowKey, now)
	} else {
		now = exist.(*ast.Term)
	}

	return iter(now)
}

func builtinTimeParseNanos(a, b ast.Value) (ast.Value, error) {

	format, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}

	value, err := builtins.StringOperand(b, 2)
	if err != nil {
		return nil, err
	}

	result, err := time.Parse(string(format), string(value))
	if err != nil {
		return nil, err
	}

	return ast.Number(int64ToJSONNumber(result.UnixNano())), nil
}

func builtinTimeParseRFC3339Nanos(a ast.Value) (ast.Value, error) {

	value, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}

	result, err := time.Parse(time.RFC3339, string(value))
	if err != nil {
		return nil, err
	}

	return ast.Number(int64ToJSONNumber(result.UnixNano())), nil
}
func builtinParseDurationNanos(a ast.Value) (ast.Value, error) {

	duration, err := builtins.StringOperand(a, 1)
	if err != nil {
		return nil, err
	}
	value, err := time.ParseDuration(string(duration))
	if err != nil {
		return nil, err
	}
	return ast.Number(int64ToJSONNumber(int64(value))), nil
}

func builtinDate(a ast.Value) (ast.Value, error) {
	t, err := utcTime(a)
	if err != nil {
		return nil, err
	}
	year, month, day := t.Date()
	result := ast.Array{ast.IntNumberTerm(year), ast.IntNumberTerm(int(month)), ast.IntNumberTerm(day)}
	return result, nil
}

func builtinClock(a ast.Value) (ast.Value, error) {
	t, err := utcTime(a)
	if err != nil {
		return nil, err
	}
	hour, minute, second := t.Clock()
	result := ast.Array{ast.IntNumberTerm(hour), ast.IntNumberTerm(minute), ast.IntNumberTerm(second)}
	return result, nil
}

func builtinWeekday(a ast.Value) (ast.Value, error) {
	t, err := utcTime(a)
	if err != nil {
		return nil, err
	}
	weekday := t.Weekday().String()
	return ast.String(weekday), nil
}

func utcTime(a ast.Value) (time.Time, error) {
	value, err := builtins.NumberOperand(a, 1)
	if err != nil {
		return time.Time{}, err
	}

	f := builtins.NumberToFloat(value)
	i64, acc := f.Int64()
	if acc != big.Exact {
		return time.Time{}, fmt.Errorf("timestamp too big")
	}

	return time.Unix(0, i64).UTC(), nil
}

func int64ToJSONNumber(i int64) json.Number {
	return json.Number(strconv.FormatInt(i, 10))
}

func init() {
	RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos)
	RegisterFunctionalBuiltin1(ast.ParseRFC3339Nanos.Name, builtinTimeParseRFC3339Nanos)
	RegisterFunctionalBuiltin2(ast.ParseNanos.Name, builtinTimeParseNanos)
	RegisterFunctionalBuiltin1(ast.ParseDurationNanos.Name, builtinParseDurationNanos)
	RegisterFunctionalBuiltin1(ast.Date.Name, builtinDate)
	RegisterFunctionalBuiltin1(ast.Clock.Name, builtinClock)
	RegisterFunctionalBuiltin1(ast.Weekday.Name, builtinWeekday)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"regexp"
	"strings"
	"time"
)

const INTERVAL_PERIOD time.Duration = 5 * time.Minute
const HOUR_TO_TICK int = 0
const MINUTE_TO_TICK int = 0
const SECOND_TO_TICK int = 1
const SEPARATOR = "|"

// I would prefer 🕖, but it’s not available in most fonts
const ROBOT_BLOCK_IDENTIFIER = "ꜰ"

var regexTomorrow = regexp.MustCompile(`(?i)\smorgen:?\s`)
var regexToday = regexp.MustCompile(`(?i)\sheute:?\s`)

func TopicChanger() {
	ticker := time.NewTicker(INTERVAL_PERIOD)
	for {
		<-ticker.C
		setTopic("#chaos-hd")
	}
}

func setTopic(channel string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("most likely coding error: %v", r)
		}
	}()

	topic := TopicGet(channel)
	newtopic := insertNextEvent(topic)
	newtopic = advanceDates(newtopic)

	if topic == newtopic {
		return
	}

	log.Printf("%s OLD TOPIC: %s", channel, topic)
	log.Printf("%s NEW TOPIC: %s", channel, newtopic)

	Topic(channel, newtopic)
}

func advanceDates(topic string) string {
	parts := splitTopic(topic)
	new := []string{}

	dateToday := time.Now()
	dateTomorrow := time.Now().AddDate(0, 0, 1)

	for _, part := range parts {
		if strings.Contains(part, dateToday.Format("2006-01-02")) {
			part = strings.Replace(part, dateToday.Format("2006-01-02"), "HEUTE ("+dateToday.Format("02.Jan")+")", -1)
			new = append(new, part)

		} else if strings.Contains(part, dateTomorrow.Format("2006-01-02")) {
			part = strings.Replace(part, dateTomorrow.Format("2006-01-02"), "MORGEN ("+dateTomorrow.Format("02.Jan")+")", -1)
			new = append(new, part)

		} else if regexTomorrow.MatchString(part) {
			// tomorrow → today
			match := regexTomorrow.FindStringSubmatch(part)[0]
			r := " heute"
			if strings.HasSuffix(match, ": ") {
				r += ":"
			}
			r += " "

			if strings.HasPrefix(match, " MOR") {
				r = strings.ToUpper(r)
			}

			n := regexTomorrow.ReplaceAllString(part, r)
			new = append(new, n)

		} else if regexToday.MatchString(part) {
			// today → (remove)

		} else {
			// keep
			new = append(new, part)
		}
	}
	return joinTopic(new)
}

func insertNextEvent(topic string) string {
	event := " " + ROBOT_BLOCK_IDENTIFIER + " " + getNextEventString() + " "

	parts := splitTopic(topic)

	eventIdx := -1
	for i, part := range parts {
		if strings.Contains(part, ROBOT_BLOCK_IDENTIFIER) {
			eventIdx = i
			break
		}
	}

	if eventIdx < 0 {
		parts = append(parts, event)
	} else {
		parts[eventIdx] = event
	}

	return joinTopic(parts)
}

func splitTopic(topic string) []string {
	return strings.Split(" "+topic+" ", SEPARATOR)
}

func joinTopic(parts []string) string {
	return strings.TrimSpace(strings.Join(parts, SEPARATOR))
}

// stores all required data for the next event to accurately
// describe it everyone who listens.
type event struct {
	Stammtisch bool
	Override   sql.NullString
	Location   sql.NullString
	Date       time.Time
	Topic      sql.NullString
}

// retrieves the next event from the database and parses it into
// an “event”. Returns nil if the DB connection or query fails.
// Function is defined in this way so it may easily be overwritten
// when testing.
var getNextEvent = func() *event {
	db, err := sqlx.Connect("postgres", "dbname=nnev user=anon host=/var/run/postgresql sslmode=disable")
	if err != nil {
		log.Println(err)
		return nil
	}

	defer db.Close()

	evt := event{}
	err = db.Get(&evt, `
		SELECT stammtisch, override, location, termine.date, topic
		FROM termine
		LEFT JOIN vortraege
		ON termine.date = vortraege.date
		WHERE termine.date >= current_date
		ORDER BY termine.date ASC
		LIMIT 1`)

	if err != nil {
		log.Println(err)
		return nil
	}

	return &evt
}

// converts an event (retrieved from the database) into a condensed
// single string in human readable form
func getNextEventString() string {
	evt := getNextEvent()
	if evt == nil {
		return "SQL Error, see logs"
	}

	t := evt.Date.Format("2006-01-02") + ": "

	if toStr(evt.Override) != "" {
		t += "Ausnahmsweise: " + toStr(evt.Override)

	} else if evt.Stammtisch {
		t += "Stammtisch @ " + strOrDefault(toStr(evt.Location), "TBA")
		t += " https://www.noname-ev.de/yarpnarp.html"
		t += " bitte zu/absagen"

	} else {
		t += "c¼h: " + strOrDefault(toStr(evt.Topic), "noch keine ◉︵◉")
	}

	return strings.TrimSpace(t)
}

// returns the first argument “str”, unless it is empty. If so,
// it will instead return the second argument “def”.
func strOrDefault(str string, def string) string {
	if str == "" {
		return def
	} else {
		return str
	}
}

func toStr(ns sql.NullString) string {
	if ns.Valid {
		return ns.String
	} else {
		return ""
	}
}
<commit_msg>try to work around DST/timezone issues<commit_after>package main

import (
	"database/sql"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"regexp"
	"strings"
	"time"
)

const INTERVAL_PERIOD time.Duration = 5 * time.Minute
const HOUR_TO_TICK int = 0
const MINUTE_TO_TICK int = 0
const SECOND_TO_TICK int = 1
const SEPARATOR = "|"

// I would prefer 🕖, but it’s not available in most fonts
const ROBOT_BLOCK_IDENTIFIER = "ꜰ"

var regexTomorrow = regexp.MustCompile(`(?i)\smorgen:?\s`)
var regexToday = regexp.MustCompile(`(?i)\sheute:?\s`)

func TopicChanger() {
	ticker := time.NewTicker(INTERVAL_PERIOD)
	for {
		setTopic("#chaos-hd")
		<-ticker.C
	}
}

func setTopic(channel string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("most likely coding error: %v", r)
		}
	}()

	topic := TopicGet(channel)
	newtopic := insertNextEvent(topic)
	newtopic = advanceDates(newtopic)

	if topic == newtopic {
		return
	}

	log.Printf("%s OLD TOPIC: %s", channel, topic)
	log.Printf("%s NEW TOPIC: %s", channel, newtopic)

	Topic(channel, newtopic)
}

func advanceDates(topic string) string {
	parts := splitTopic(topic)
	new := []string{}

	dateToday := time.Now()
	dateTomorrow := time.Now().AddDate(0, 0, 1)

	for _, part := range parts {
		if strings.Contains(part, dateToday.Format("2006-01-02")) {
			part = strings.Replace(part, dateToday.Format("2006-01-02"), "HEUTE ("+dateToday.Format("02.Jan")+")", -1)
			new = append(new, part)

		} else if strings.Contains(part, dateTomorrow.Format("2006-01-02")) {
			part = strings.Replace(part, dateTomorrow.Format("2006-01-02"), "MORGEN ("+dateTomorrow.Format("02.Jan")+")", -1)
			new = append(new, part)

		} else if regexTomorrow.MatchString(part) {
			// tomorrow → today
			match := regexTomorrow.FindStringSubmatch(part)[0]
			r := " heute"
			if strings.HasSuffix(match, ": ") {
				r += ":"
			}
			r += " "

			if strings.HasPrefix(match, " MOR") {
				r = strings.ToUpper(r)
			}

			n := regexTomorrow.ReplaceAllString(part, r)
			new = append(new, n)

		} else if regexToday.MatchString(part) {
			// today → (remove)

		} else {
			// keep
			new = append(new, part)
		}
	}
	return joinTopic(new)
}

func insertNextEvent(topic string) string {
	event := " " + ROBOT_BLOCK_IDENTIFIER + " " + getNextEventString() + " "

	parts := splitTopic(topic)

	eventIdx := -1
	for i, part := range parts {
		if strings.Contains(part, ROBOT_BLOCK_IDENTIFIER) {
			eventIdx = i
			break
		}
	}

	if eventIdx < 0 {
		parts = append(parts, event)
	} else {
		parts[eventIdx] = event
	}

	return joinTopic(parts)
}

func splitTopic(topic string) []string {
	return strings.Split(" "+topic+" ", SEPARATOR)
}

func joinTopic(parts []string) string {
	return strings.TrimSpace(strings.Join(parts, SEPARATOR))
}

// stores all required data for the next event to accurately
// describe it everyone who listens.
type event struct {
	Stammtisch bool
	Override   sql.NullString
	Location   sql.NullString
	Date       time.Time
	Topic      sql.NullString
}

// retrieves the next event from the database and parses it into
// an “event”. Returns nil if the DB connection or query fails.
// Function is defined in this way so it may easily be overwritten
// when testing.
var getNextEvent = func() *event {
	db, err := sqlx.Connect("postgres", "dbname=nnev user=anon host=/var/run/postgresql sslmode=disable")
	if err != nil {
		log.Println(err)
		return nil
	}

	defer db.Close()

	evt := event{}
	err = db.Get(&evt, `
		SELECT stammtisch, override, location, termine.date, topic
		FROM termine
		LEFT JOIN vortraege
		ON termine.date = vortraege.date
		WHERE termine.date >= $1
		ORDER BY termine.date ASC
		LIMIT 1`, time.Now().Format("2006-01-02"))

	if err != nil {
		log.Println(err)
		return nil
	}

	if *verbose {
		log.Printf("event from SQL: %v", evt)
	}

	return &evt
}

// converts an event (retrieved from the database) into a condensed
// single string in human readable form
func getNextEventString() string {
	evt := getNextEvent()
	if evt == nil {
		return "SQL Error, see logs"
	}

	t := evt.Date.Format("2006-01-02") + ": "

	if toStr(evt.Override) != "" {
		t += "Ausnahmsweise: " + toStr(evt.Override)

	} else if evt.Stammtisch {
		t += "Stammtisch @ " + strOrDefault(toStr(evt.Location), "TBA")
		t += " https://www.noname-ev.de/yarpnarp.html"
		t += " bitte zu/absagen"

	} else {
		t += "c¼h: " + strOrDefault(toStr(evt.Topic), "noch keine ◉︵◉")
	}

	return strings.TrimSpace(t)
}

// returns the first argument “str”, unless it is empty. If so,
// it will instead return the second argument “def”.
func strOrDefault(str string, def string) string {
	if str == "" {
		return def
	} else {
		return str
	}
}

func toStr(ns sql.NullString) string {
	if ns.Valid {
		return ns.String
	} else {
		return ""
	}
}
<|endoftext|>


<commit_before>// +build windows

package mousetrap

import (
    "os"
    "fmt"
    "syscall"
    "strings"
    "unsafe"
)

const (
    th32cs_snapprocess uintptr = 0x2
)

type processEntry32 struct {
    dwSize uint32
    cntUsage uint32
    th32ProcessID uint32
    th32DefaultHeapID int
    th32ModuleID uint32
    cntThreads uint32
    th32ParentProcessID uint32
    pcPriClassBase int32
    dwFlags uint32
    szExeFile [0x00000104]byte
}

func getProcessEntry(pid int) (pe *processEntry32, err error) {
    kernel := syscall.MustLoadDLL("kernel32.dll")
    CreateToolhelp32Snapshot := kernel.MustFindProc("CreateToolhelp32Snapshot")
    Process32First := kernel.MustFindProc("Process32First")
    Process32Next := kernel.MustFindProc("Process32Next")
    CloseHandle := kernel.MustFindProc("CloseHandle")

    snapshot, _, e1 := CreateToolhelp32Snapshot.Call(th32cs_snapprocess, uintptr(0));
    if (snapshot == uintptr(syscall.InvalidHandle)) {
        err = fmt.Errorf("CreateToolhelp32Snapshot: %v", e1)
        return
    }

    var processEntry processEntry32
    processEntry.dwSize = uint32(unsafe.Sizeof(processEntry))
    ok, _ , e1 := Process32First.Call(snapshot, uintptr(unsafe.Pointer(&processEntry)))
    if ok == 0 {
        err = fmt.Errorf("Process32First: %v", e1)
        goto closeHandle;
    }

    for {
        if processEntry.th32ProcessID == uint32(pid) {
            pe = &processEntry
            goto closeHandle;
        }

        ok, _, e1 = Process32Next.Call(snapshot, uintptr(unsafe.Pointer(&processEntry)))
        if ok == 0 {
            err = fmt.Errorf("Process32Next: %v", e1)
            goto closeHandle;
        }
    }

closeHandle:
    CloseHandle.Call(snapshot);
    return
}

func getppid() (pid int, err error) {
    pe, err := getProcessEntry(os.Getpid())
    if err != nil {
        return
    }

    pid = int(pe.th32ParentProcessID)
    return
}

func InvokedFromCommandLine() (bool, error) {
    ppid, err := getppid()
    if err != nil {
        return true, err
    }

    pe, err := getProcessEntry(ppid)
    if err != nil {
        return true, err
    }

    fmt.Println(pe.szExeFile)
    var path string
    for i, b := range pe.szExeFile[:] {
        if b == 0 {
            path = string(pe.szExeFile[:i])
            break
        }
    }

    isExplorer := strings.HasSuffix(path, "explorer.exe")
    return !isExplorer, nil
}
<commit_msg>remove debugging output<commit_after>// +build windows

package mousetrap

import (
    "os"
    "fmt"
    "syscall"
    "strings"
    "unsafe"
)

const (
    th32cs_snapprocess uintptr = 0x2
)

type processEntry32 struct {
    dwSize uint32
    cntUsage uint32
    th32ProcessID uint32
    th32DefaultHeapID int
    th32ModuleID uint32
    cntThreads uint32
    th32ParentProcessID uint32
    pcPriClassBase int32
    dwFlags uint32
    szExeFile [0x00000104]byte
}

func getProcessEntry(pid int) (pe *processEntry32, err error) {
    kernel := syscall.MustLoadDLL("kernel32.dll")
    CreateToolhelp32Snapshot := kernel.MustFindProc("CreateToolhelp32Snapshot")
    Process32First := kernel.MustFindProc("Process32First")
    Process32Next := kernel.MustFindProc("Process32Next")
    CloseHandle := kernel.MustFindProc("CloseHandle")

    snapshot, _, e1 := CreateToolhelp32Snapshot.Call(th32cs_snapprocess, uintptr(0));
    if (snapshot == uintptr(syscall.InvalidHandle)) {
        err = fmt.Errorf("CreateToolhelp32Snapshot: %v", e1)
        return
    }

    var processEntry processEntry32
    processEntry.dwSize = uint32(unsafe.Sizeof(processEntry))
    ok, _ , e1 := Process32First.Call(snapshot, uintptr(unsafe.Pointer(&processEntry)))
    if ok == 0 {
        err = fmt.Errorf("Process32First: %v", e1)
        goto closeHandle;
    }

    for {
        if processEntry.th32ProcessID == uint32(pid) {
            pe = &processEntry
            goto closeHandle;
        }

        ok, _, e1 = Process32Next.Call(snapshot, uintptr(unsafe.Pointer(&processEntry)))
        if ok == 0 {
            err = fmt.Errorf("Process32Next: %v", e1)
            goto closeHandle;
        }
    }

closeHandle:
    CloseHandle.Call(snapshot);
    return
}

func getppid() (pid int, err error) {
    pe, err := getProcessEntry(os.Getpid())
    if err != nil {
        return
    }

    pid = int(pe.th32ParentProcessID)
    return
}

func InvokedFromCommandLine() (bool, error) {
    ppid, err := getppid()
    if err != nil {
        return true, err
    }

    pe, err := getProcessEntry(ppid)
    if err != nil {
        return true, err
    }

    var path string
    for i, b := range pe.szExeFile[:] {
        if b == 0 {
            path = string(pe.szExeFile[:i])
            break
        }
    }

    isExplorer := strings.HasSuffix(path, "explorer.exe")
    return !isExplorer, nil
}
<|endoftext|>


<commit_before>package tsa

const (
	ForwardWorker  = "forward-worker"
	RegisterWorker = "register-worker"

	LandWorker   = "land-worker"
	RetireWorker = "retire-worker"
	DeleteWorker = "delete-worker"

	ReportContainers      = "report-containers"
	ReportVolumes         = "report-volumes"
	ResourceActionMissing = "resource-type-missing"
)
<commit_msg>#4566: removed register-worker from tsa constants<commit_after>package tsa

const (
	ForwardWorker = "forward-worker"

	LandWorker   = "land-worker"
	RetireWorker = "retire-worker"
	DeleteWorker = "delete-worker"

	ReportContainers      = "report-containers"
	ReportVolumes         = "report-volumes"
	ResourceActionMissing = "resource-type-missing"
)
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bit

type Map struct {
	bits []uint64
	size int
	mask int
}

func NewMap(l, size int) *Map {
	return &Map{
		size: size,
		bits: make([]uint64, (l*size)/64),
	}
}

func NewMapFromRaw(bits []uint64, size int) *Map {
	mask := 1
	for i := 1; i < size; i++ {
		mask <<= 1
	}
	return &Map{
		size: size,
		bits: bits,
	}
}

func (m *Map) Set(i, val int) {
	i *= m.size
	pos := i / 64
	mask := (uint64(1) << uint(m.size)) - 1
	i %= 64
	m.bits[pos] = (m.bits[pos] & ^(mask << uint64(i))) | (uint64(val) << uint64(i))
}

func (m *Map) Get(i int) int {
	i *= m.size
	pos := i / 64
	mask := (uint64(1) << uint(m.size)) - 1
	i %= 64
	return int((m.bits[pos] >> uint64(i)) & mask)
}
<commit_msg>type/bit: clean up and allow for resizing<commit_after>// Copyright 2015 Matthew Collins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bit

import "fmt"

type Map struct {
	bits    []uint64
	BitSize int
	Length  int
}

func NewMap(l, size int) *Map {
	return &Map{
		BitSize: size,
		bits:    make([]uint64, (l*size)/64),
		Length:  l,
	}
}

func NewMapFromRaw(bits []uint64, size int) *Map {
	return &Map{
		BitSize: size,
		bits:    bits,
		Length:  (len(bits) * 64) / size,
	}
}

func (m *Map) ResizeBits(size int) *Map {
	n := NewMap(m.Length, size)
	for i := 0; i < m.Length; i++ {
		n.Set(i, m.Get(i))
	}
	return n
}

func (m *Map) Set(i, val int) {
	if val < 0 || val >= (int(1)<<uint(m.BitSize)) {
		panic(fmt.Sprintf("invalid value %d %d", val, m.BitSize))
	}
	i *= m.BitSize
	pos := i / 64
	mask := (uint64(1) << uint(m.BitSize)) - 1
	i %= 64
	m.bits[pos] = (m.bits[pos] & ^(mask << uint64(i))) | (uint64(val) << uint64(i))
}

func (m *Map) Get(i int) int {
	i *= m.BitSize
	pos := i / 64
	mask := (uint64(1) << uint(m.BitSize)) - 1
	i %= 64
	return int((m.bits[pos] >> uint64(i)) & mask)
}
<|endoftext|>


<commit_before>// Copyright 2015 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This services serves two feeds using news from "Plantão Empresas". One of
// the feeds is exclusive for FIIs and the other for all other news, excluding
// news related to FIIs.
package main

import (
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/gorilla/feeds"
	"github.com/tsuru/tsuru/db/storage"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

const (
	NewsURL = "http://www.bmfbovespa.com.br/agencia/corpo.asp?origem=exibir&id=%s"
	Limit   = 100
)

var (
	listenHTTP string
	regexpNews = regexp.MustCompile(`^/bovespa/(\d+)$`)
)

func init() {
	flag.StringVar(&listenHTTP, "listen", "127.0.0.1:7676", "address to listen to connections")
	flag.Parse()
}

type News struct {
	ID    string `bson:"_id"`
	Title string
	Date  time.Time
}

func (n *News) RedirectURL() string {
	return fmt.Sprintf(NewsURL, n.ID)
}

func (n *News) Path() string {
	return "/bovespa/" + n.ID
}

func collection() (*storage.Collection, error) {
	storage, err := storage.Open("localhost:27017", "bovespa_plantao_empresas")
	if err != nil {
		return nil, err
	}
	coll := storage.Collection("news")
	coll.EnsureIndex(mgo.Index{Key: []string{"title"}, Background: true, Sparse: true})
	coll.EnsureIndex(mgo.Index{Key: []string{"-date"}, Background: true, Sparse: true})
	coll.EnsureIndex(mgo.Index{Key: []string{"title", "-date"}, Background: true, Sparse: true})
	return coll, nil
}

func getFeed(query bson.M, id string, baseURL string) (*feeds.Feed, error) {
	if strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL[:len(baseURL)-1]
	}
	coll, err := collection()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var newsList []News
	err = coll.Find(query).Sort("-date").Limit(Limit).All(&newsList)
	if err != nil {
		return nil, err
	}
	location, _ := time.LoadLocation("America/Sao_Paulo")
	updated := time.Now()
	if len(newsList) > 0 {
		updated = newsList[0].Date.In(location)
	}
	feed := &feeds.Feed{
		Title:       "Bovespa - Plantão Empresas - " + id,
		Link:        &feeds.Link{Href: baseURL + "?w=" + id},
		Description: "Notícias sobre empresas listadas na Bovespa",
		Author:      &feeds.Author{Name: "Francisco Souza", Email: "f@souza.cc"},
		Created:     time.Date(2014, 3, 20, 10, 0, 0, 0, location),
		Updated:     updated,
	}
	for _, news := range newsList {
		item := feeds.Item{
			Id:          baseURL + news.Path(),
			Title:       news.Title,
			Link:        &feeds.Link{Href: baseURL + news.Path()},
			Description: news.Title,
			Author:      &feeds.Author{Name: "Bovespa", Email: "bovespa@bmfbovespa.com.br"},
			Created:     news.Date,
			Updated:     news.Date,
		}
		feed.Items = append(feed.Items, &item)
	}
	return feed, nil
}

func feedAll(w http.ResponseWriter, r *http.Request) {
	baseURL := "http://" + r.Host
	feed, err := getFeed(bson.M{"title": bson.M{"$regex": "^((?!fii))", "$options": "i"}}, "all", baseURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/xml")
	fmt.Fprint(w, atom)
}

func feedFIIs(w http.ResponseWriter, r *http.Request) {
	baseURL := "http://" + r.Host
	feed, err := getFeed(bson.M{"title": bson.M{"$regex": "^fii", "$options": "i"}}, "fii", baseURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/xml")
	fmt.Fprint(w, atom)
}

func redirectNews(w http.ResponseWriter, r *http.Request) {
	var newsID string
	var news News
	parts := regexpNews.FindStringSubmatch(r.URL.Path)
	if len(parts) > 1 {
		newsID = parts[1]
	} else {
		http.Error(w, "Page not found", http.StatusNotFound)
		return
	}
	coll, err := collection()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer coll.Close()
	err = coll.FindId(newsID).One(&news)
	if err == mgo.ErrNotFound {
		http.Error(w, "News not found", http.StatusNotFound)
		return
	} else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Location", news.RedirectURL())
	w.WriteHeader(http.StatusMovedPermanently)
}

func main() {
	http.Handle("/all.atom", http.HandlerFunc(feedAll))
	http.Handle("/fii.atom", http.HandlerFunc(feedFIIs))
	http.Handle("/", http.HandlerFunc(redirectNews))
	http.ListenAndServe(listenHTTP, nil)
}
<commit_msg>feed_plantao: properly set Content-Type of feeds<commit_after>// Copyright 2015 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This services serves two feeds using news from "Plantão Empresas". One of
// the feeds is exclusive for FIIs and the other for all other news, excluding
// news related to FIIs.
package main

import (
	"flag"
	"fmt"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/gorilla/feeds"
	"github.com/tsuru/tsuru/db/storage"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

const (
	NewsURL = "http://www.bmfbovespa.com.br/agencia/corpo.asp?origem=exibir&id=%s"
	Limit   = 100
)

var (
	listenHTTP string
	regexpNews = regexp.MustCompile(`^/bovespa/(\d+)$`)
)

func init() {
	flag.StringVar(&listenHTTP, "listen", "127.0.0.1:7676", "address to listen to connections")
	flag.Parse()
}

type News struct {
	ID    string `bson:"_id"`
	Title string
	Date  time.Time
}

func (n *News) RedirectURL() string {
	return fmt.Sprintf(NewsURL, n.ID)
}

func (n *News) Path() string {
	return "/bovespa/" + n.ID
}

func collection() (*storage.Collection, error) {
	storage, err := storage.Open("localhost:27017", "bovespa_plantao_empresas")
	if err != nil {
		return nil, err
	}
	coll := storage.Collection("news")
	coll.EnsureIndex(mgo.Index{Key: []string{"title"}, Background: true, Sparse: true})
	coll.EnsureIndex(mgo.Index{Key: []string{"-date"}, Background: true, Sparse: true})
	coll.EnsureIndex(mgo.Index{Key: []string{"title", "-date"}, Background: true, Sparse: true})
	return coll, nil
}

func getFeed(query bson.M, id string, baseURL string) (*feeds.Feed, error) {
	if strings.HasSuffix(baseURL, "/") {
		baseURL = baseURL[:len(baseURL)-1]
	}
	coll, err := collection()
	if err != nil {
		return nil, err
	}
	defer coll.Close()
	var newsList []News
	err = coll.Find(query).Sort("-date").Limit(Limit).All(&newsList)
	if err != nil {
		return nil, err
	}
	location, _ := time.LoadLocation("America/Sao_Paulo")
	updated := time.Now()
	if len(newsList) > 0 {
		updated = newsList[0].Date.In(location)
	}
	feed := &feeds.Feed{
		Title:       "Bovespa - Plantão Empresas - " + id,
		Link:        &feeds.Link{Href: baseURL + "?w=" + id},
		Description: "Notícias sobre empresas listadas na Bovespa",
		Author:      &feeds.Author{Name: "Francisco Souza", Email: "f@souza.cc"},
		Created:     time.Date(2014, 3, 20, 10, 0, 0, 0, location),
		Updated:     updated,
	}
	for _, news := range newsList {
		item := feeds.Item{
			Id:          baseURL + news.Path(),
			Title:       news.Title,
			Link:        &feeds.Link{Href: baseURL + news.Path()},
			Description: news.Title,
			Author:      &feeds.Author{Name: "Bovespa", Email: "bovespa@bmfbovespa.com.br"},
			Created:     news.Date,
			Updated:     news.Date,
		}
		feed.Items = append(feed.Items, &item)
	}
	return feed, nil
}

func feedAll(w http.ResponseWriter, r *http.Request) {
	baseURL := "http://" + r.Host
	feed, err := getFeed(bson.M{"title": bson.M{"$regex": "^((?!fii))", "$options": "i"}}, "all", baseURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/atom+xml")
	fmt.Fprint(w, atom)
}

func feedFIIs(w http.ResponseWriter, r *http.Request) {
	baseURL := "http://" + r.Host
	feed, err := getFeed(bson.M{"title": bson.M{"$regex": "^fii", "$options": "i"}}, "fii", baseURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	atom, err := feed.ToAtom()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Content-Type", "application/tom+xml")
	fmt.Fprint(w, atom)
}

func redirectNews(w http.ResponseWriter, r *http.Request) {
	var newsID string
	var news News
	parts := regexpNews.FindStringSubmatch(r.URL.Path)
	if len(parts) > 1 {
		newsID = parts[1]
	} else {
		http.Error(w, "Page not found", http.StatusNotFound)
		return
	}
	coll, err := collection()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer coll.Close()
	err = coll.FindId(newsID).One(&news)
	if err == mgo.ErrNotFound {
		http.Error(w, "News not found", http.StatusNotFound)
		return
	} else if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Add("Location", news.RedirectURL())
	w.WriteHeader(http.StatusMovedPermanently)
}

func main() {
	http.Handle("/all.atom", http.HandlerFunc(feedAll))
	http.Handle("/fii.atom", http.HandlerFunc(feedFIIs))
	http.Handle("/", http.HandlerFunc(redirectNews))
	http.ListenAndServe(listenHTTP, nil)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"
	"math"
	"os"
	"path"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	mfs    *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mfs = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mfs.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mfs.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) EmptyRoot() {
	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInRoot() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// Placeholder
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/",
					},
					Contents: "",
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderNotPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ListDirectoryTwice_NoChange() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

func (t *readOnlyTest) ListDirectoryTwice_Changed() {
	AssertTrue(false, "TODO")
}

// TODO(jacobsa): Inodes
// TODO(jacobsa): Error conditions
func (t *readOnlyTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<commit_msg>ReadOnlyTest.ListDirectoryTwice_Changed<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"
	"math"
	"os"
	"path"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx    context.Context
	bucket gcs.Bucket
	mfs    *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mfs = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mfs.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mfs.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mfs.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

func (t *fsTest) createObjects(objects []*gcsutil.ObjectInfo) error {
	_, err := gcsutil.CreateObjects(t.ctx, t.bucket, objects)
	return err
}

func (t *fsTest) createEmptyObjects(names []string) error {
	_, err := gcsutil.CreateEmptyObjects(t.ctx, t.bucket, names)
	return err
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) EmptyRoot() {
	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInRoot() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) EmptySubDirectory() {
	// Set up an empty directory placeholder called 'bar'.
	AssertEq(nil, t.createEmptyObjects([]string{"bar/"}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "bar"))
	AssertEq(nil, err)

	ExpectThat(entries, ElementsAre())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// Placeholder
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/",
					},
					Contents: "",
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ContentsInSubDirectory_PlaceholderNotPresent() {
	// Set up contents.
	AssertEq(
		nil,
		t.createObjects(
			[]*gcsutil.ObjectInfo{
				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/foo",
					},
					Contents: "taco",
				},

				// Directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/bar/",
					},
				},

				// File
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/baz",
					},
					Contents: "burrito",
				},

				// File in sub-directory
				&gcsutil.ObjectInfo{
					Attrs: storage.ObjectAttrs{
						Name: "dir/qux/asdf",
					},
					Contents: "",
				},
			}))

	// ReadDir
	entries, err := ioutil.ReadDir(path.Join(t.mfs.Dir(), "dir"))
	AssertEq(nil, err)

	AssertEq(4, len(entries))
	var e os.FileInfo

	// bar
	e = entries[0]
	ExpectEq("bar", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())

	// baz
	e = entries[1]
	ExpectEq("baz", e.Name())
	ExpectEq(len("burrito"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// foo
	e = entries[2]
	ExpectEq("foo", e.Name())
	ExpectEq(len("taco"), e.Size())
	ExpectEq(os.FileMode(0400), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectFalse(e.IsDir())

	// qux
	e = entries[3]
	ExpectEq("qux", e.Name())
	ExpectEq(0, e.Size())
	ExpectEq(os.ModeDir|os.FileMode(0500), e.Mode())
	ExpectLt(math.Abs(time.Since(e.ModTime()).Seconds()), 30)
	ExpectTrue(e.IsDir())
}

func (t *readOnlyTest) ListDirectoryTwice_NoChange() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

func (t *readOnlyTest) ListDirectoryTwice_Changed() {
	// Set up initial contents.
	AssertEq(
		nil,
		t.createEmptyObjects([]string{
			"foo",
			"bar",
		}))

	// List once.
	entries, err := ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("bar", entries[0].Name())
	ExpectEq("foo", entries[1].Name())

	// Add "baz" and remove "bar".
	AssertEq(nil, t.bucket.DeleteObject(t.ctx, "bar"))
	AssertEq(nil, t.createEmptyObjects([]string{"baz"}))

	// List again.
	entries, err = ioutil.ReadDir(t.mfs.Dir())
	AssertEq(nil, err)

	AssertEq(2, len(entries))
	ExpectEq("baz", entries[0].Name())
	ExpectEq("foo", entries[1].Name())
}

// TODO(jacobsa): Inodes
// TODO(jacobsa): Error conditions
func (t *readOnlyTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		op.localFile, err = makeLocalFile(ctx, bucket, name, srcGeneration)
		if err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() {
	panic("TODO")
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up a local temporary file for the given generation of the given object.
// Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Fetch its contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}
<commit_msg>Refactored local file ensuring.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() {
	panic("TODO")
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up a local temporary file for the given generation of the given object.
// Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Fetch its contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>