<commit_before>package sqlobjectstore

import (
	"database/sql"
	"encoding/json"
	"strings"
	"time"

	// required for sqlite3
	_ "github.com/mattn/go-sqlite3"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/migration"
	"nimona.io/pkg/object"
	"nimona.io/pkg/objectstore"
)

// nolint: lll
var migrations = []string{
	`CREATE TABLE IF NOT EXISTS Objects (Hash TEXT NOT NULL PRIMARY KEY);`,
	`ALTER TABLE Objects ADD Type TEXT;`,
	`ALTER TABLE Objects ADD Body TEXT;`,
	`ALTER TABLE Objects ADD RootHash TEXT;`,
	`ALTER TABLE Objects ADD TTL INT;`,
	`ALTER TABLE Objects ADD Created INT;`,
	`ALTER TABLE Objects ADD LastAccessed INT;`,
	`ALTER TABLE Objects ADD SignerPublicKey TEXT;`,
	`ALTER TABLE Objects ADD AuthorPublicKey TEXT;`,
	`ALTER TABLE Objects RENAME AuthorPublicKey TO OwnerPublicKey;`,
	`ALTER TABLE Objects RENAME SignerPublicKey TO _DeprecatedSignerPublicKey;`,
	`CREATE INDEX Created_idx ON Objects(Created);`,
	`CREATE INDEX TTL_LastAccessed_idx ON Objects(TTL, LastAccessed);`,
	`CREATE INDEX Type_RootHash_OwnerPublicKey_idx ON Objects(Type, RootHash, OwnerPublicKey);`,
	`CREATE INDEX RootHash_idx ON Objects(RootHash);`,
	`CREATE INDEX RootHash_TTL_idx ON Objects(RootHash, TTL);`,
	`CREATE INDEX Hash_LastAccessed_idx ON Objects(Hash, LastAccessed);`,
}

type Store struct {
	db *sql.DB
}

func New(
	db *sql.DB,
) (*Store, error) {
	ndb := &Store{
		db: db,
	}

	// run migrations
	if err := migration.Up(db, migrations...); err != nil {
		return nil, errors.Wrap(err, errors.New("failed to run migrations"))
	}

	// Initialize the garbage collector in the background to run every minute
	go func() {
		for {
			ndb.gc() // nolint
			time.Sleep(1 * time.Minute)
		}
	}()

	return ndb, nil
}

func (st *Store) Close() error {
	return st.db.Close()
}

func (st *Store) Get(
	hash object.Hash,
) (object.Object, error) {
	obj := object.Object{}

	// get the object
	stmt, err := st.db.Prepare("SELECT Body FROM Objects WHERE Hash=?")
	if err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	row := stmt.QueryRow(hash.String())

	m := map[string]interface{}{}
	data := []byte{}

	if err := row.Scan(&data); err != nil {
		return obj, errors.Wrap(
			err,
			objectstore.ErrNotFound,
		)
	}

	if err := json.Unmarshal(data, &m); err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not unmarshal data"),
		)
	}

	obj = object.FromMap(m)

	// update the last accessed column
	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed=? WHERE Hash=?")
	if err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		time.Now().Unix(),
		hash.String(),
	); err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not update last access"),
		)
	}

	return obj, nil
}

func (st *Store) GetByStream(
	streamRootHash object.Hash,
) (object.ReadCloser, error) {
	return st.Filter(
		FilterByStreamHash(streamRootHash),
	)
}

func (st *Store) GetByType(
	objectType string,
) (object.ReadCloser, error) {
	return st.Filter(
		FilterByObjectType(objectType),
	)
}

func (st *Store) Put(
	obj object.Object,
) error {
	return st.PutWithTimeout(obj, 0)
}

func (st *Store) PutWithTimeout(
	obj object.Object,
	ttl time.Duration,
) error {
	// TODO(geoah) why replace?
	stmt, err := st.db.Prepare(`
	REPLACE INTO Objects (
		Hash,
		Type,
		RootHash,
		OwnerPublicKey,
		Body,
		Created,
		LastAccessed,
		TTl
	) VALUES (
		?, ?, ?, ?, ?, ?, ?, ?
	) ON CONFLICT (Hash) DO UPDATE SET
		LastAccessed=?
	`)
	if err != nil {
		return errors.Wrap(err,
			errors.New("could not prepare insert to objects table"))
	}
	defer stmt.Close() // nolint: errcheck

	body, err := json.Marshal(obj.ToMap())
	if err != nil {
		return errors.Wrap(err, errors.New("could not marshal object"))
	}

	objectType := obj.GetType()
	objectHash := obj.Hash().String()
	streamHash := obj.GetStream().String()
	// TODO support multiple owners
	ownerPublicKey := ""
	if !obj.GetOwner().IsEmpty() {
		ownerPublicKey = obj.GetOwner().String()
	}

	// if the object doesn't belong to a stream, we need to set the stream
	// to the object's hash.
	// This should allow queries to consider the root object part of the stream.
	if streamHash == "" {
		streamHash = objectHash
	}

	_, err = stmt.Exec(
		objectHash,
		objectType,
		streamHash,
		ownerPublicKey,
		body,
		time.Now().Unix(),
		time.Now().Unix(),
		ttl,
		time.Now().Unix(),
	)
	if err != nil {
		return errors.Wrap(err, errors.New("could not insert to objects table"))
	}

	return nil
}

func (st *Store) GetRelations(
	parent object.Hash,
) ([]object.Hash, error) {
	stmt, err := st.db.Prepare("SELECT Hash FROM Objects WHERE RootHash=?")
	if err != nil {
		return nil, errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query(parent.String())
	if err != nil {
		return nil, errors.Wrap(err, errors.New("could not query"))
	}
	defer rows.Close() // nolint: errcheck

	hashList := []object.Hash{}

	for rows.Next() {
		data := ""
		if err := rows.Scan(&data); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		hashList = append(hashList, object.Hash(data))
	}

	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed=? WHERE RootHash=?",
	)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		time.Now().Unix(),
		parent.String(),
	); err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not update last access"),
		)
	}

	return hashList, nil
}

func (st *Store) UpdateTTL(
	hash object.Hash,
	minutes int,
) error {
	stmt, err := st.db.Prepare(`UPDATE Objects SET TTL=? WHERE RootHash=?`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(minutes, hash.String()); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not update last access and ttl"),
		)
	}

	return nil
}

func (st *Store) Remove(
	hash object.Hash,
) error {
	stmt, err := st.db.Prepare(`
	DELETE FROM Objects
	WHERE Hash=?`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(
		hash.String(),
	); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not delete object"),
		)
	}

	return nil
}

func (st *Store) gc() error {
	stmt, err := st.db.Prepare(`
	DELETE FROM Objects WHERE
	  TTL > 0 AND
	  datetime(LastAccessed + TTL * 60, 'unixepoch') < datetime ('now');
	`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not gc delete objects"),
		)
	}

	return nil
}

func (st *Store) Filter(
	lookupOptions ...LookupOption,
) (object.ReadCloser, error) {
	options := newLookupOptions(lookupOptions...)

	where := "WHERE 1 "
	whereArgs := []interface{}{}

	if len(options.Lookups.ObjectHashes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.ObjectHashes))[1:]
		where += "AND Hash IN (" + qs + ") "
		whereArgs = append(whereArgs, ahtoai(options.Lookups.ObjectHashes)...)
	}

	if len(options.Lookups.ContentTypes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.ContentTypes))[1:]
		where += "AND Type IN (" + qs + ") "
		whereArgs = append(whereArgs, astoai(options.Lookups.ContentTypes)...)
	}

	if len(options.Lookups.StreamHashes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.StreamHashes))[1:]
		where += "AND RootHash IN (" + qs + ") "
		whereArgs = append(whereArgs, ahtoai(options.Lookups.StreamHashes)...)
	}

	if len(options.Lookups.Owners) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.Owners))[1:]
		where += "AND OwnerPublicKey IN (" + qs + ") "
		whereArgs = append(whereArgs, aktoai(options.Lookups.Owners)...)
	}

	where += "ORDER BY Created ASC"

	// get the object
	// nolint: gosec
	stmt, err := st.db.Prepare("SELECT Hash FROM Objects " + where)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare statement"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query(whereArgs...)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not query"),
		)
	}
	defer rows.Close() // nolint: errcheck

	hashes := []string{}
	hashesForUpdate := []interface{}{}

	errorChan := make(chan error)
	objectsChan := make(chan *object.Object)
	closeChan := make(chan struct{})

	reader := object.NewReadCloser(
		context.TODO(),
		objectsChan,
		errorChan,
		closeChan,
	)

	for rows.Next() {
		hash := ""
		if err := rows.Scan(&hash); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		hashes = append(hashes, hash)
		hashesForUpdate = append(hashesForUpdate, hash)
	}

	if len(hashes) == 0 {
		return nil, objectstore.ErrNotFound
	}

	// update the last accessed column
	updateQs := strings.Repeat(",?", len(hashes))[1:]
	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed = ? " +
			"WHERE Hash IN (" + updateQs + ")",
	)
	if err != nil {
		return nil, err
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		append([]interface{}{time.Now().Unix()}, hashesForUpdate...)...,
	); err != nil {
		return nil, err
	}

	go func() {
		defer close(objectsChan)
		defer close(errorChan)
		for _, hash := range hashes {
			o, err := st.Get(object.Hash(hash))
			if err != nil {
				errorChan <- err
				return
			}
			select {
			case <-closeChan:
				return
			case objectsChan <- &o:
				// all good
			}
		}
	}()

	return reader, nil
}

func (st *Store) GetPinned() ([]object.Hash, error) {
	stmt, err := st.db.Prepare("SELECT Hash FROM Objects WHERE TTL = 0")
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare statement"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query()
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not query"),
		)
	}
	defer rows.Close() // nolint: errcheck

	hs := []object.Hash{}
	for rows.Next() {
		h := ""
		if err := rows.Scan(&h); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		if h != "" {
			hs = append(hs, object.Hash(h))
		}
	}

	return hs, nil
}

func astoai(ah []string) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h
	}
	return as
}

func ahtoai(ah []object.Hash) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h.String()
	}
	return as
}

func aktoai(ah []crypto.PublicKey) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h.String()
	}
	return as
}
<commit_msg>fix(sqlobjectstore): GetPinned should only return simple objects or stream roots<commit_after>package sqlobjectstore

import (
	"database/sql"
	"encoding/json"
	"strings"
	"time"

	// required for sqlite3
	_ "github.com/mattn/go-sqlite3"

	"nimona.io/pkg/context"
	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/migration"
	"nimona.io/pkg/object"
	"nimona.io/pkg/objectstore"
)

// nolint: lll
var migrations = []string{
	`CREATE TABLE IF NOT EXISTS Objects (Hash TEXT NOT NULL PRIMARY KEY);`,
	`ALTER TABLE Objects ADD Type TEXT;`,
	`ALTER TABLE Objects ADD Body TEXT;`,
	`ALTER TABLE Objects ADD RootHash TEXT;`,
	`ALTER TABLE Objects ADD TTL INT;`,
	`ALTER TABLE Objects ADD Created INT;`,
	`ALTER TABLE Objects ADD LastAccessed INT;`,
	`ALTER TABLE Objects ADD SignerPublicKey TEXT;`,
	`ALTER TABLE Objects ADD AuthorPublicKey TEXT;`,
	`ALTER TABLE Objects RENAME AuthorPublicKey TO OwnerPublicKey;`,
	`ALTER TABLE Objects RENAME SignerPublicKey TO _DeprecatedSignerPublicKey;`,
	`CREATE INDEX Created_idx ON Objects(Created);`,
	`CREATE INDEX TTL_LastAccessed_idx ON Objects(TTL, LastAccessed);`,
	`CREATE INDEX Type_RootHash_OwnerPublicKey_idx ON Objects(Type, RootHash, OwnerPublicKey);`,
	`CREATE INDEX RootHash_idx ON Objects(RootHash);`,
	`CREATE INDEX RootHash_TTL_idx ON Objects(RootHash, TTL);`,
	`CREATE INDEX Hash_LastAccessed_idx ON Objects(Hash, LastAccessed);`,
}

type Store struct {
	db *sql.DB
}

func New(
	db *sql.DB,
) (*Store, error) {
	ndb := &Store{
		db: db,
	}

	// run migrations
	if err := migration.Up(db, migrations...); err != nil {
		return nil, errors.Wrap(err, errors.New("failed to run migrations"))
	}

	// Initialize the garbage collector in the background to run every minute
	go func() {
		for {
			ndb.gc() // nolint
			time.Sleep(1 * time.Minute)
		}
	}()

	return ndb, nil
}

func (st *Store) Close() error {
	return st.db.Close()
}

func (st *Store) Get(
	hash object.Hash,
) (object.Object, error) {
	obj := object.Object{}

	// get the object
	stmt, err := st.db.Prepare("SELECT Body FROM Objects WHERE Hash=?")
	if err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	row := stmt.QueryRow(hash.String())

	m := map[string]interface{}{}
	data := []byte{}

	if err := row.Scan(&data); err != nil {
		return obj, errors.Wrap(
			err,
			objectstore.ErrNotFound,
		)
	}

	if err := json.Unmarshal(data, &m); err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not unmarshal data"),
		)
	}

	obj = object.FromMap(m)

	// update the last accessed column
	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed=? WHERE Hash=?")
	if err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		time.Now().Unix(),
		hash.String(),
	); err != nil {
		return obj, errors.Wrap(
			err,
			errors.New("could not update last access"),
		)
	}

	return obj, nil
}

func (st *Store) GetByStream(
	streamRootHash object.Hash,
) (object.ReadCloser, error) {
	return st.Filter(
		FilterByStreamHash(streamRootHash),
	)
}

func (st *Store) GetByType(
	objectType string,
) (object.ReadCloser, error) {
	return st.Filter(
		FilterByObjectType(objectType),
	)
}

func (st *Store) Put(
	obj object.Object,
) error {
	return st.PutWithTimeout(obj, 0)
}

func (st *Store) PutWithTimeout(
	obj object.Object,
	ttl time.Duration,
) error {
	// TODO(geoah) why replace?
	stmt, err := st.db.Prepare(`
	REPLACE INTO Objects (
		Hash,
		Type,
		RootHash,
		OwnerPublicKey,
		Body,
		Created,
		LastAccessed,
		TTl
	) VALUES (
		?, ?, ?, ?, ?, ?, ?, ?
	) ON CONFLICT (Hash) DO UPDATE SET
		LastAccessed=?
	`)
	if err != nil {
		return errors.Wrap(err,
			errors.New("could not prepare insert to objects table"))
	}
	defer stmt.Close() // nolint: errcheck

	body, err := json.Marshal(obj.ToMap())
	if err != nil {
		return errors.Wrap(err, errors.New("could not marshal object"))
	}

	objectType := obj.GetType()
	objectHash := obj.Hash().String()
	streamHash := obj.GetStream().String()
	// TODO support multiple owners
	ownerPublicKey := ""
	if !obj.GetOwner().IsEmpty() {
		ownerPublicKey = obj.GetOwner().String()
	}

	// if the object doesn't belong to a stream, we need to set the stream
	// to the object's hash.
	// This should allow queries to consider the root object part of the stream.
	if streamHash == "" {
		streamHash = objectHash
	}

	_, err = stmt.Exec(
		objectHash,
		objectType,
		streamHash,
		ownerPublicKey,
		body,
		time.Now().Unix(),
		time.Now().Unix(),
		ttl,
		time.Now().Unix(),
	)
	if err != nil {
		return errors.Wrap(err, errors.New("could not insert to objects table"))
	}

	return nil
}

func (st *Store) GetRelations(
	parent object.Hash,
) ([]object.Hash, error) {
	stmt, err := st.db.Prepare("SELECT Hash FROM Objects WHERE RootHash=?")
	if err != nil {
		return nil, errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query(parent.String())
	if err != nil {
		return nil, errors.Wrap(err, errors.New("could not query"))
	}
	defer rows.Close() // nolint: errcheck

	hashList := []object.Hash{}

	for rows.Next() {
		data := ""
		if err := rows.Scan(&data); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		hashList = append(hashList, object.Hash(data))
	}

	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed=? WHERE RootHash=?",
	)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare query"),
		)
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		time.Now().Unix(),
		parent.String(),
	); err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not update last access"),
		)
	}

	return hashList, nil
}

func (st *Store) UpdateTTL(
	hash object.Hash,
	minutes int,
) error {
	stmt, err := st.db.Prepare(`UPDATE Objects SET TTL=? WHERE RootHash=?`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(minutes, hash.String()); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not update last access and ttl"),
		)
	}

	return nil
}

func (st *Store) Remove(
	hash object.Hash,
) error {
	stmt, err := st.db.Prepare(`
	DELETE FROM Objects
	WHERE Hash=?`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(
		hash.String(),
	); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not delete object"),
		)
	}

	return nil
}

func (st *Store) gc() error {
	stmt, err := st.db.Prepare(`
	DELETE FROM Objects WHERE
	  TTL > 0 AND
	  datetime(LastAccessed + TTL * 60, 'unixepoch') < datetime ('now');
	`)
	if err != nil {
		return errors.Wrap(err, errors.New("could not prepare query"))
	}
	defer stmt.Close() // nolint: errcheck

	if _, err := stmt.Exec(); err != nil {
		return errors.Wrap(
			err,
			errors.New("could not gc delete objects"),
		)
	}

	return nil
}

func (st *Store) Filter(
	lookupOptions ...LookupOption,
) (object.ReadCloser, error) {
	options := newLookupOptions(lookupOptions...)

	where := "WHERE 1 "
	whereArgs := []interface{}{}

	if len(options.Lookups.ObjectHashes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.ObjectHashes))[1:]
		where += "AND Hash IN (" + qs + ") "
		whereArgs = append(whereArgs, ahtoai(options.Lookups.ObjectHashes)...)
	}

	if len(options.Lookups.ContentTypes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.ContentTypes))[1:]
		where += "AND Type IN (" + qs + ") "
		whereArgs = append(whereArgs, astoai(options.Lookups.ContentTypes)...)
	}

	if len(options.Lookups.StreamHashes) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.StreamHashes))[1:]
		where += "AND RootHash IN (" + qs + ") "
		whereArgs = append(whereArgs, ahtoai(options.Lookups.StreamHashes)...)
	}

	if len(options.Lookups.Owners) > 0 {
		qs := strings.Repeat(",?", len(options.Lookups.Owners))[1:]
		where += "AND OwnerPublicKey IN (" + qs + ") "
		whereArgs = append(whereArgs, aktoai(options.Lookups.Owners)...)
	}

	where += "ORDER BY Created ASC"

	// get the object
	// nolint: gosec
	stmt, err := st.db.Prepare("SELECT Hash FROM Objects " + where)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare statement"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query(whereArgs...)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not query"),
		)
	}
	defer rows.Close() // nolint: errcheck

	hashes := []string{}
	hashesForUpdate := []interface{}{}

	errorChan := make(chan error)
	objectsChan := make(chan *object.Object)
	closeChan := make(chan struct{})

	reader := object.NewReadCloser(
		context.TODO(),
		objectsChan,
		errorChan,
		closeChan,
	)

	for rows.Next() {
		hash := ""
		if err := rows.Scan(&hash); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		hashes = append(hashes, hash)
		hashesForUpdate = append(hashesForUpdate, hash)
	}

	if len(hashes) == 0 {
		return nil, objectstore.ErrNotFound
	}

	// update the last accessed column
	updateQs := strings.Repeat(",?", len(hashes))[1:]
	istmt, err := st.db.Prepare(
		"UPDATE Objects SET LastAccessed = ? " +
			"WHERE Hash IN (" + updateQs + ")",
	)
	if err != nil {
		return nil, err
	}
	defer istmt.Close() // nolint: errcheck

	if _, err := istmt.Exec(
		append([]interface{}{time.Now().Unix()}, hashesForUpdate...)...,
	); err != nil {
		return nil, err
	}

	go func() {
		defer close(objectsChan)
		defer close(errorChan)
		for _, hash := range hashes {
			o, err := st.Get(object.Hash(hash))
			if err != nil {
				errorChan <- err
				return
			}
			select {
			case <-closeChan:
				return
			case objectsChan <- &o:
				// all good
			}
		}
	}()

	return reader, nil
}

const (
	pinnedQuery = "SELECT Hash FROM Objects WHERE TTL = 0 AND Hash = RootHash"
)

func (st *Store) GetPinned() ([]object.Hash, error) {
	stmt, err := st.db.Prepare(pinnedQuery)
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not prepare statement"),
		)
	}
	defer stmt.Close() // nolint: errcheck

	rows, err := stmt.Query()
	if err != nil {
		return nil, errors.Wrap(
			err,
			errors.New("could not query"),
		)
	}
	defer rows.Close() // nolint: errcheck

	hs := []object.Hash{}
	for rows.Next() {
		h := ""
		if err := rows.Scan(&h); err != nil {
			return nil, errors.Wrap(
				err,
				objectstore.ErrNotFound,
			)
		}
		if h != "" {
			hs = append(hs, object.Hash(h))
		}
	}

	return hs, nil
}

func astoai(ah []string) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h
	}
	return as
}

func ahtoai(ah []object.Hash) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h.String()
	}
	return as
}

func aktoai(ah []crypto.PublicKey) []interface{} {
	as := make([]interface{}, len(ah))
	for i, h := range ah {
		as[i] = h.String()
	}
	return as
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"

	"github.com/openshift/origin/pkg/monitor/monitorapi"
	"k8s.io/apimachinery/pkg/util/sets"
)

func testServerAvailability(owner, locator string, events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	errDuration, errMessages, _ := monitorapi.BackendDisruptionSeconds(locator, events)

	testName := fmt.Sprintf("[%s] %s should be available throughout the test", owner, locator)
	successTest := &junitapi.JUnitTestCase{
		Name:     testName,
		Duration: jobRunDuration.Seconds(),
	}
	if errDuration > 0 {
		test := &junitapi.JUnitTestCase{
			Name:     testName,
			Duration: jobRunDuration.Seconds(),
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s was failing for %s seconds (test duration: %s)", locator, errDuration.Round(time.Second), jobRunDuration.Round(time.Second)),
			},
			SystemOut: strings.Join(errMessages, "\n"),
		}
		// Return *two* tests results to pretend this is a flake not to fail whole testsuite.
		return []*junitapi.JUnitTestCase{test, successTest}

	} else {
		successTest.SystemOut = fmt.Sprintf("%s was failing for %s seconds (test duration: %s)", locator, errDuration.Round(time.Second), jobRunDuration.Round(time.Second))
		return []*junitapi.JUnitTestCase{successTest}
	}
}

func testAllAPIAvailability(events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	allAPIServerLocators := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		if strings.HasSuffix(backend, "-api") {
			allAPIServerLocators.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, apiServerLocator := range allAPIServerLocators.List() {
		ret = append(ret, testServerAvailability("sig-api-machinery", apiServerLocator, allDisruptionEventsIntervals, jobRunDuration)...)
	}

	return ret
}

func testAllIngressAvailability(events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	allAPIServerLocators := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		if strings.HasPrefix(backend, "ingress-") {
			allAPIServerLocators.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, apiServerLocator := range allAPIServerLocators.List() {
		ret = append(ret, testServerAvailability("sig-network-edge", apiServerLocator, allDisruptionEventsIntervals, jobRunDuration)...)
	}

	return ret
}

func testPodNodeNameIsImmutable(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
	const testName = "[sig-api-machinery] the pod.spec.nodeName field is immutable, once set cannot be changed"

	failures := []string{}
	for _, event := range events {
		if strings.Contains(event.Message, "pod once assigned to a node must stay on it") {
			failures = append(failures, fmt.Sprintf("%v %v", event.Locator, event.Message))
		}
	}
	if len(failures) == 0 {
		return []*junitapi.JUnitTestCase{{Name: testName}}
	}

	return []*junitapi.JUnitTestCase{{
		Name:      testName,
		SystemOut: strings.Join(failures, "\n"),
		FailureOutput: &junitapi.FailureOutput{
			Output: fmt.Sprintf("Please report in https://bugzilla.redhat.com/show_bug.cgi?id=2042657\n\n%d pods had their immutable field (spec.nodeName) changed\n\n%v", len(failures), strings.Join(failures, "\n")),
		},
	}}
}

func testMultipleSingleSecondAvailabilityFailure(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
	const multipleFailuresTestPrefix = "[sig-network] there should be nearly zero single second disruptions for "
	const manyFailureTestPrefix = "[sig-network] there should be reasonably few single second disruptions for "

	allServers := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		switch {
		case strings.HasPrefix(backend, "ingress-"):
			allServers.Insert(eventInterval.Locator)
		case strings.HasSuffix(backend, "-api"):
			allServers.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, serverLocator := range allServers.List() {
		disruptionEvents := events.Filter(
			monitorapi.And(
				isOneSecondEvent,
				monitorapi.IsEventForLocator(serverLocator),
				monitorapi.IsErrorEvent,
			),
		)

		multipleFailuresTestName := multipleFailuresTestPrefix + serverLocator
		manyFailuresTestName := manyFailureTestPrefix + serverLocator
		multipleFailuresPass := &junitapi.JUnitTestCase{
			Name:      multipleFailuresTestName,
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		manyFailuresPass := &junitapi.JUnitTestCase{
			Name:      manyFailuresTestName,
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		multipleFailuresFail := &junitapi.JUnitTestCase{
			Name: multipleFailuresTestName,
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s had %v single second disruptions", serverLocator, len(disruptionEvents)),
			},
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		manyFailuresFail := &junitapi.JUnitTestCase{
			Name: manyFailuresTestName,
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s had %v single second disruptions", serverLocator, len(disruptionEvents)),
			},
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}

		switch {
		case len(disruptionEvents) < 20:
			ret = append(ret, multipleFailuresPass, manyFailuresPass)

		case len(disruptionEvents) > 20: // chosen to be big enough that we should not hit this unless something is weird
			ret = append(ret, multipleFailuresFail, manyFailuresPass)

		case len(disruptionEvents) > 50: // chosen to be big enough that we should not hit this unless something is really really wrong
			ret = append(ret, multipleFailuresFail, manyFailuresFail)
		}
	}

	return ret
}

func isOneSecondEvent(eventInterval monitorapi.EventInterval) bool {
	duration := eventInterval.To.Sub(eventInterval.From)
	switch {
	case duration <= 0:
		return false
	case duration > time.Second:
		return false
	default:
		return true
	}
}
<commit_msg>don't trigger single second disruption on contiguous disruption<commit_after>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"

	"github.com/openshift/origin/pkg/monitor/monitorapi"
	"k8s.io/apimachinery/pkg/util/sets"
)

func testServerAvailability(owner, locator string, events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	errDuration, errMessages, _ := monitorapi.BackendDisruptionSeconds(locator, events)

	testName := fmt.Sprintf("[%s] %s should be available throughout the test", owner, locator)
	successTest := &junitapi.JUnitTestCase{
		Name:     testName,
		Duration: jobRunDuration.Seconds(),
	}
	if errDuration > 0 {
		test := &junitapi.JUnitTestCase{
			Name:     testName,
			Duration: jobRunDuration.Seconds(),
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s was failing for %s seconds (test duration: %s)", locator, errDuration.Round(time.Second), jobRunDuration.Round(time.Second)),
			},
			SystemOut: strings.Join(errMessages, "\n"),
		}
		// Return *two* tests results to pretend this is a flake not to fail whole testsuite.
		return []*junitapi.JUnitTestCase{test, successTest}

	} else {
		successTest.SystemOut = fmt.Sprintf("%s was failing for %s seconds (test duration: %s)", locator, errDuration.Round(time.Second), jobRunDuration.Round(time.Second))
		return []*junitapi.JUnitTestCase{successTest}
	}
}

func testAllAPIAvailability(events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	allAPIServerLocators := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		if strings.HasSuffix(backend, "-api") {
			allAPIServerLocators.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, apiServerLocator := range allAPIServerLocators.List() {
		ret = append(ret, testServerAvailability("sig-api-machinery", apiServerLocator, allDisruptionEventsIntervals, jobRunDuration)...)
	}

	return ret
}

func testAllIngressAvailability(events monitorapi.Intervals, jobRunDuration time.Duration) []*junitapi.JUnitTestCase {
	allAPIServerLocators := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		if strings.HasPrefix(backend, "ingress-") {
			allAPIServerLocators.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, apiServerLocator := range allAPIServerLocators.List() {
		ret = append(ret, testServerAvailability("sig-network-edge", apiServerLocator, allDisruptionEventsIntervals, jobRunDuration)...)
	}

	return ret
}

func testPodNodeNameIsImmutable(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
	const testName = "[sig-api-machinery] the pod.spec.nodeName field is immutable, once set cannot be changed"

	failures := []string{}
	for _, event := range events {
		if strings.Contains(event.Message, "pod once assigned to a node must stay on it") {
			failures = append(failures, fmt.Sprintf("%v %v", event.Locator, event.Message))
		}
	}
	if len(failures) == 0 {
		return []*junitapi.JUnitTestCase{{Name: testName}}
	}

	return []*junitapi.JUnitTestCase{{
		Name:      testName,
		SystemOut: strings.Join(failures, "\n"),
		FailureOutput: &junitapi.FailureOutput{
			Output: fmt.Sprintf("Please report in https://bugzilla.redhat.com/show_bug.cgi?id=2042657\n\n%d pods had their immutable field (spec.nodeName) changed\n\n%v", len(failures), strings.Join(failures, "\n")),
		},
	}}
}

func testMultipleSingleSecondAvailabilityFailure(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
	const multipleFailuresTestPrefix = "[sig-network] there should be nearly zero single second disruptions for "
	const manyFailureTestPrefix = "[sig-network] there should be reasonably few single second disruptions for "

	allServers := sets.String{}
	allDisruptionEventsIntervals := events.Filter(monitorapi.IsDisruptionEvent)
	for _, eventInterval := range allDisruptionEventsIntervals {
		backend := monitorapi.DisruptionFrom(monitorapi.LocatorParts(eventInterval.Locator))
		switch {
		case strings.HasPrefix(backend, "ingress-"):
			allServers.Insert(eventInterval.Locator)
		case strings.HasSuffix(backend, "-api"):
			allServers.Insert(eventInterval.Locator)
		}
	}

	ret := []*junitapi.JUnitTestCase{}
	for _, serverLocator := range allServers.List() {
		allDisruptionEvents := events.Filter(
			monitorapi.And(
				monitorapi.IsEventForLocator(serverLocator),
				monitorapi.IsErrorEvent,
			),
		)

		disruptionEvents := monitorapi.Intervals{}
		for i, interval := range allDisruptionEvents {
			if !isOneSecondEvent(interval) {
				continue
			}
			if i > 0 {
				prev := allDisruptionEvents[i-1]
				// if the previous disruption interval for this backend is within one second of when this one started,
				// then we're looking at a contiguous outage that is longer than one second.
				// this can happen when we have contiguous failures for different reasons.
				if prev.To.Add(1 * time.Second).After(interval.From) {
					continue
				}
			}
			if i < len(allDisruptionEvents)-1 {
				next := allDisruptionEvents[i+1]
				// if the next disruption interval for this backend is within one second of when this one ended,
				// then we're looking at a contiguous outage that is longer than one second.
				// this can happen when we have contiguous failures for different reasons.
				if interval.To.Add(1 * time.Second).After(next.From) {
					continue
				}
			}

			disruptionEvents = append(disruptionEvents, allDisruptionEvents[i])
		}

		multipleFailuresTestName := multipleFailuresTestPrefix + serverLocator
		manyFailuresTestName := manyFailureTestPrefix + serverLocator
		multipleFailuresPass := &junitapi.JUnitTestCase{
			Name:      multipleFailuresTestName,
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		manyFailuresPass := &junitapi.JUnitTestCase{
			Name:      manyFailuresTestName,
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		multipleFailuresFail := &junitapi.JUnitTestCase{
			Name: multipleFailuresTestName,
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s had %v single second disruptions", serverLocator, len(disruptionEvents)),
			},
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}
		manyFailuresFail := &junitapi.JUnitTestCase{
			Name: manyFailuresTestName,
			FailureOutput: &junitapi.FailureOutput{
				Output: fmt.Sprintf("%s had %v single second disruptions", serverLocator, len(disruptionEvents)),
			},
			SystemOut: strings.Join(disruptionEvents.Strings(), "\n"),
		}

		switch {
		case len(disruptionEvents) < 20:
			ret = append(ret, multipleFailuresPass, manyFailuresPass)

		case len(disruptionEvents) > 20: // chosen to be big enough that we should not hit this unless something is weird
			ret = append(ret, multipleFailuresFail, manyFailuresPass)

		case len(disruptionEvents) > 50: // chosen to be big enough that we should not hit this unless something is really really wrong
			ret = append(ret, multipleFailuresFail, manyFailuresFail)
		}
	}

	return ret
}

func isOneSecondEvent(eventInterval monitorapi.EventInterval) bool {
	duration := eventInterval.To.Sub(eventInterval.From)
	switch {
	case duration <= 0:
		return false
	case duration > time.Second:
		return false
	default:
		return true
	}
}
<|endoftext|>


<commit_before>package chained

import (
	"bufio"
	"fmt"
	"net"
	"net/http"

	"strings"
)

// Dialer is an implementation of proxy.Dialer that proxies traffic via an
// upstream server proxy.  Its Dial function uses DialServer to dial the server
// proxy and then issues a CONNECT request to instruct the server to connect to
// the destination at the specified network and addr.
type Dialer struct {
	// DialServer: function that dials the upstream server proxy
	DialServer func() (net.Conn, error)

	// OnRequest: optional function that gets called on every CONNECT request to
	// the server and is allowed to modify the http.Request before it passes to
	// the server.
	OnRequest func(req *http.Request)
}

// Dial implements the method from proxy.Dialer
func (d *Dialer) Dial(network, addr string) (net.Conn, error) {
	conn, err := d.DialServer()
	if err != nil {
		return nil, fmt.Errorf("Unable to dial server: %s", err)
	}
	err = d.sendCONNECT(network, addr, conn)
	if err != nil {
		conn.Close()
		return nil, err
	}
	return conn, nil
}

// Close implements the method from proxy.Dialer
func (d *Dialer) Close() error {
	return nil
}

func (d *Dialer) sendCONNECT(network, addr string, conn net.Conn) error {
	if !strings.Contains(network, "tcp") {
		return fmt.Errorf("%s connections are not supported, only tcp is supported", network)
	}

	req, err := buildCONNECTRequest(addr, d.OnRequest)
	if err != nil {
		return fmt.Errorf("Unable to construct CONNECT request: %s", err)
	}
	err = req.Write(conn)
	if err != nil {
		return fmt.Errorf("Unable to write CONNECT request: %s", err)
	}

	r := bufio.NewReader(conn)
	err = checkCONNECTResponse(r, req)
	return err
}

func checkCONNECTResponse(r *bufio.Reader, req *http.Request) error {
	resp, err := http.ReadResponse(r, req)
	if err != nil {
		return fmt.Errorf("Error reading CONNECT response: %s", err)
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("Bad status code on CONNECT response: %d", resp.StatusCode)
	}
	return nil
}

func buildCONNECTRequest(addr string, onRequest func(req *http.Request)) (*http.Request, error) {
	req, err := http.NewRequest(CONNECT, addr, nil)
	if err != nil {
		return nil, err
	}
	req.Host = addr
	if onRequest != nil {
		onRequest(req)
	}
	return req, nil
}
<commit_msg>Reordered code<commit_after>package chained

import (
	"bufio"
	"fmt"
	"net"
	"net/http"

	"strings"
)

// Dialer is an implementation of proxy.Dialer that proxies traffic via an
// upstream server proxy.  Its Dial function uses DialServer to dial the server
// proxy and then issues a CONNECT request to instruct the server to connect to
// the destination at the specified network and addr.
type Dialer struct {
	// DialServer: function that dials the upstream server proxy
	DialServer func() (net.Conn, error)

	// OnRequest: optional function that gets called on every CONNECT request to
	// the server and is allowed to modify the http.Request before it passes to
	// the server.
	OnRequest func(req *http.Request)
}

// Dial implements the method from proxy.Dialer
func (d *Dialer) Dial(network, addr string) (net.Conn, error) {
	conn, err := d.DialServer()
	if err != nil {
		return nil, fmt.Errorf("Unable to dial server: %s", err)
	}
	err = d.sendCONNECT(network, addr, conn)
	if err != nil {
		conn.Close()
		return nil, err
	}
	return conn, nil
}

// Close implements the method from proxy.Dialer
func (d *Dialer) Close() error {
	return nil
}

func (d *Dialer) sendCONNECT(network, addr string, conn net.Conn) error {
	if !strings.Contains(network, "tcp") {
		return fmt.Errorf("%s connections are not supported, only tcp is supported", network)
	}

	req, err := buildCONNECTRequest(addr, d.OnRequest)
	if err != nil {
		return fmt.Errorf("Unable to construct CONNECT request: %s", err)
	}
	err = req.Write(conn)
	if err != nil {
		return fmt.Errorf("Unable to write CONNECT request: %s", err)
	}

	r := bufio.NewReader(conn)
	err = checkCONNECTResponse(r, req)
	return err
}

func buildCONNECTRequest(addr string, onRequest func(req *http.Request)) (*http.Request, error) {
	req, err := http.NewRequest(CONNECT, addr, nil)
	if err != nil {
		return nil, err
	}
	req.Host = addr
	if onRequest != nil {
		onRequest(req)
	}
	return req, nil
}

func checkCONNECTResponse(r *bufio.Reader, req *http.Request) error {
	resp, err := http.ReadResponse(r, req)
	if err != nil {
		return fmt.Errorf("Error reading CONNECT response: %s", err)
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("Bad status code on CONNECT response: %d", resp.StatusCode)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ident

import (
	"github.com/m3db/m3x/checked"
	"github.com/m3db/m3x/context"
	"github.com/m3db/m3x/pool"
)

const (
	defaultCapacityOptions    = 16
	defaultMaxCapacityOptions = 32
)

// PoolOptions is a set of pooling options.
type PoolOptions struct {
	IDPoolOptions           pool.ObjectPoolOptions
	TagsPoolOptions         pool.ObjectPoolOptions
	TagsCapacity            int
	TagsMaxCapacity         int
	TagsIteratorPoolOptions pool.ObjectPoolOptions
}

func (o PoolOptions) defaultsIfNotSet() PoolOptions {
	if o.IDPoolOptions == nil {
		o.IDPoolOptions = pool.NewObjectPoolOptions()
	}
	if o.TagsPoolOptions == nil {
		o.TagsPoolOptions = pool.NewObjectPoolOptions()
	}
	if o.TagsCapacity == 0 {
		o.TagsCapacity = defaultCapacityOptions
	}
	if o.TagsMaxCapacity == 0 {
		o.TagsMaxCapacity = defaultMaxCapacityOptions
	}
	if o.TagsIteratorPoolOptions == nil {
		o.TagsIteratorPoolOptions = pool.NewObjectPoolOptions()
	}
	return o
}

// NewPool constructs a new simple Pool.
func NewPool(
	bytesPool pool.CheckedBytesPool,
	opts PoolOptions,
) Pool {
	opts = opts.defaultsIfNotSet()

	p := &simplePool{
		bytesPool: bytesPool,
		pool:      pool.NewObjectPool(opts.IDPoolOptions),
		tagArrayPool: newTagArrayPool(tagArrayPoolOpts{
			Options:     opts.TagsPoolOptions,
			Capacity:    opts.TagsCapacity,
			MaxCapacity: opts.TagsMaxCapacity,
		}),
		itersPool: pool.NewObjectPool(opts.TagsIteratorPoolOptions),
	}
	p.pool.Init(func() interface{} {
		return &id{pool: p}
	})
	p.tagArrayPool.Init()
	p.itersPool.Init(func() interface{} {
		return newTagSliceIter(Tags{}, p)
	})

	return p
}

type simplePool struct {
	bytesPool    pool.CheckedBytesPool
	pool         pool.ObjectPool
	tagArrayPool tagArrayPool
	itersPool    pool.ObjectPool
}

func (p *simplePool) GetBinaryID(ctx context.Context, v checked.Bytes) ID {
	id := p.BinaryID(v)
	ctx.RegisterFinalizer(id)
	return id
}

func (p *simplePool) BinaryID(v checked.Bytes) ID {
	id := p.pool.Get().(*id)
	v.IncRef()
	id.pool, id.data = p, v
	return id
}

func (p *simplePool) GetBinaryTag(
	ctx context.Context,
	name checked.Bytes,
	value checked.Bytes,
) Tag {
	return Tag{
		Name:  TagName(p.GetBinaryID(ctx, name)),
		Value: TagValue(p.GetBinaryID(ctx, value)),
	}
}

func (p *simplePool) BinaryTag(
	name checked.Bytes,
	value checked.Bytes,
) Tag {
	return Tag{
		Name:  TagName(p.BinaryID(name)),
		Value: TagValue(p.BinaryID(value)),
	}
}

func (p *simplePool) GetStringID(ctx context.Context, v string) ID {
	id := p.StringID(v)
	ctx.RegisterFinalizer(id)
	return id
}

func (p *simplePool) StringID(v string) ID {
	data := p.bytesPool.Get(len(v))
	data.IncRef()
	data.AppendAll([]byte(v))
	data.DecRef()

	return p.BinaryID(data)
}

func (p *simplePool) GetTagsIterator(c context.Context) TagsIterator {
	iter := p.itersPool.Get().(*tagSliceIter)
	c.RegisterCloser(iter)
	return iter
}

func (p *simplePool) TagsIterator() TagsIterator {
	return p.itersPool.Get().(*tagSliceIter)
}

func (p *simplePool) Tags() Tags {
	return Tags{
		values: p.tagArrayPool.Get(),
		pool:   p,
	}
}

func (p *simplePool) Put(v ID) {
	p.pool.Put(v)
}

func (p *simplePool) PutTag(t Tag) {
	p.Put(t.Name)
	p.Put(t.Value)
}

func (p *simplePool) PutTags(t Tags) {
	p.tagArrayPool.Put(t.values)
}

func (p *simplePool) PutTagsIterator(iter TagsIterator) {
	iter.Reset(Tags{})
	p.itersPool.Put(iter)
}

func (p *simplePool) GetStringTag(ctx context.Context, name string, value string) Tag {
	return Tag{
		Name:  TagName(p.GetStringID(ctx, name)),
		Value: TagValue(p.GetStringID(ctx, value)),
	}
}

func (p *simplePool) StringTag(name string, value string) Tag {
	return Tag{
		Name:  TagName(p.StringID(name)),
		Value: TagValue(p.StringID(value)),
	}
}

func (p *simplePool) Clone(existing ID) ID {
	id := p.pool.Get().(*id)

	// NB(rartoul): Do not modify this function without careful
	// benchmarking on a hot production workload. When we tried to
	// introduce a helper function for the lines below we saw no
	// discrepancy in micro-benchmarks, but heavy perf degradation in production.
	data := existing.Bytes()
	newData := p.bytesPool.Get(len(data))
	newData.IncRef()
	newData.AppendAll(data)

	id.pool, id.data = p, newData

	return id
}

func (p *simplePool) CloneTag(t Tag) Tag {
	return Tag{
		Name:  p.Clone(t.Name),
		Value: p.Clone(t.Value),
	}
}

func (p *simplePool) CloneTags(t Tags) Tags {
	tags := p.tagArrayPool.Get()[:0]
	for _, tag := range t.Values() {
		tags = append(tags, p.CloneTag(tag))
	}
	return Tags{
		values: tags,
		pool:   p,
	}
}
<commit_msg>remove invalid comment (#210)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ident

import (
	"github.com/m3db/m3x/checked"
	"github.com/m3db/m3x/context"
	"github.com/m3db/m3x/pool"
)

const (
	defaultCapacityOptions    = 16
	defaultMaxCapacityOptions = 32
)

// PoolOptions is a set of pooling options.
type PoolOptions struct {
	IDPoolOptions           pool.ObjectPoolOptions
	TagsPoolOptions         pool.ObjectPoolOptions
	TagsCapacity            int
	TagsMaxCapacity         int
	TagsIteratorPoolOptions pool.ObjectPoolOptions
}

func (o PoolOptions) defaultsIfNotSet() PoolOptions {
	if o.IDPoolOptions == nil {
		o.IDPoolOptions = pool.NewObjectPoolOptions()
	}
	if o.TagsPoolOptions == nil {
		o.TagsPoolOptions = pool.NewObjectPoolOptions()
	}
	if o.TagsCapacity == 0 {
		o.TagsCapacity = defaultCapacityOptions
	}
	if o.TagsMaxCapacity == 0 {
		o.TagsMaxCapacity = defaultMaxCapacityOptions
	}
	if o.TagsIteratorPoolOptions == nil {
		o.TagsIteratorPoolOptions = pool.NewObjectPoolOptions()
	}
	return o
}

// NewPool constructs a new simple Pool.
func NewPool(
	bytesPool pool.CheckedBytesPool,
	opts PoolOptions,
) Pool {
	opts = opts.defaultsIfNotSet()

	p := &simplePool{
		bytesPool: bytesPool,
		pool:      pool.NewObjectPool(opts.IDPoolOptions),
		tagArrayPool: newTagArrayPool(tagArrayPoolOpts{
			Options:     opts.TagsPoolOptions,
			Capacity:    opts.TagsCapacity,
			MaxCapacity: opts.TagsMaxCapacity,
		}),
		itersPool: pool.NewObjectPool(opts.TagsIteratorPoolOptions),
	}
	p.pool.Init(func() interface{} {
		return &id{pool: p}
	})
	p.tagArrayPool.Init()
	p.itersPool.Init(func() interface{} {
		return newTagSliceIter(Tags{}, p)
	})

	return p
}

type simplePool struct {
	bytesPool    pool.CheckedBytesPool
	pool         pool.ObjectPool
	tagArrayPool tagArrayPool
	itersPool    pool.ObjectPool
}

func (p *simplePool) GetBinaryID(ctx context.Context, v checked.Bytes) ID {
	id := p.BinaryID(v)
	ctx.RegisterFinalizer(id)
	return id
}

func (p *simplePool) BinaryID(v checked.Bytes) ID {
	id := p.pool.Get().(*id)
	v.IncRef()
	id.pool, id.data = p, v
	return id
}

func (p *simplePool) GetBinaryTag(
	ctx context.Context,
	name checked.Bytes,
	value checked.Bytes,
) Tag {
	return Tag{
		Name:  TagName(p.GetBinaryID(ctx, name)),
		Value: TagValue(p.GetBinaryID(ctx, value)),
	}
}

func (p *simplePool) BinaryTag(
	name checked.Bytes,
	value checked.Bytes,
) Tag {
	return Tag{
		Name:  TagName(p.BinaryID(name)),
		Value: TagValue(p.BinaryID(value)),
	}
}

func (p *simplePool) GetStringID(ctx context.Context, v string) ID {
	id := p.StringID(v)
	ctx.RegisterFinalizer(id)
	return id
}

func (p *simplePool) StringID(v string) ID {
	data := p.bytesPool.Get(len(v))
	data.IncRef()
	data.AppendAll([]byte(v))
	data.DecRef()

	return p.BinaryID(data)
}

func (p *simplePool) GetTagsIterator(c context.Context) TagsIterator {
	iter := p.itersPool.Get().(*tagSliceIter)
	c.RegisterCloser(iter)
	return iter
}

func (p *simplePool) TagsIterator() TagsIterator {
	return p.itersPool.Get().(*tagSliceIter)
}

func (p *simplePool) Tags() Tags {
	return Tags{
		values: p.tagArrayPool.Get(),
		pool:   p,
	}
}

func (p *simplePool) Put(v ID) {
	p.pool.Put(v)
}

func (p *simplePool) PutTag(t Tag) {
	p.Put(t.Name)
	p.Put(t.Value)
}

func (p *simplePool) PutTags(t Tags) {
	p.tagArrayPool.Put(t.values)
}

func (p *simplePool) PutTagsIterator(iter TagsIterator) {
	iter.Reset(Tags{})
	p.itersPool.Put(iter)
}

func (p *simplePool) GetStringTag(ctx context.Context, name string, value string) Tag {
	return Tag{
		Name:  TagName(p.GetStringID(ctx, name)),
		Value: TagValue(p.GetStringID(ctx, value)),
	}
}

func (p *simplePool) StringTag(name string, value string) Tag {
	return Tag{
		Name:  TagName(p.StringID(name)),
		Value: TagValue(p.StringID(value)),
	}
}

func (p *simplePool) Clone(existing ID) ID {
	var (
		id      = p.pool.Get().(*id)
		data    = existing.Bytes()
		newData = p.bytesPool.Get(len(data))
	)

	newData.IncRef()
	newData.AppendAll(data)

	id.pool, id.data = p, newData

	return id
}

func (p *simplePool) CloneTag(t Tag) Tag {
	return Tag{
		Name:  p.Clone(t.Name),
		Value: p.Clone(t.Value),
	}
}

func (p *simplePool) CloneTags(t Tags) Tags {
	tags := p.tagArrayPool.Get()[:0]
	for _, tag := range t.Values() {
		tags = append(tags, p.CloneTag(tag))
	}
	return Tags{
		values: tags,
		pool:   p,
	}
}
<|endoftext|>


<commit_before>package memory

import (
	"strings"
	"sync"

	"github.com/grafana/metrictank/stats"
	lru "github.com/hashicorp/golang-lru"
	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

var (
	// metric idx.memory.find-cache.hit is a counter of findCache hits
	findCacheHit = stats.NewCounterRate32("idx.memory.find-cache.hit")
	// metric idx.memory.find-cache.hit is a counter of findCache misses
	findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.miss")
)

type FindCache struct {
	cache map[uint32]*lru.Cache
	size  int
	sync.RWMutex
	newSeries map[uint32]chan struct{}
}

func NewFindCache(size int) *FindCache {
	fc := &FindCache{
		cache:     make(map[uint32]*lru.Cache),
		size:      size,
		newSeries: make(map[uint32]chan struct{}),
	}
	return fc
}

func (c *FindCache) Get(orgId uint32, pattern string) ([]*Node, bool) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		findCacheMiss.Inc()
		return nil, false
	}
	nodes, ok := cache.Get(pattern)
	if !ok {
		return nil, ok
	}
	findCacheHit.Inc()
	return nodes.([]*Node), ok
}

func (c *FindCache) Add(orgId uint32, pattern string, nodes []*Node) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	var err error
	if !ok {
		cache, err = lru.New(c.size)
		if err != nil {
			log.Errorf("memory-idx: findCache failed to create lru. err=%s", err)
			return
		}
		c.Lock()
		c.cache[orgId] = cache
		c.newSeries[orgId] = make(chan struct{}, 100)
		c.Unlock()
	}
	cache.Add(pattern, nodes)
}

func (c *FindCache) Purge(orgId uint32) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		return
	}
	cache.Purge()
}

func (c *FindCache) InvalidateFor(orgId uint32, path string) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok || cache.Len() < 1 {
		return
	}

	select {
	case c.newSeries[orgId] <- struct{}{}:
	default:
		for i := 0; i < len(c.newSeries[orgId]); i++ {
			select {
			case <-c.newSeries[orgId]:
			default:
			}
		}
		c.Purge(orgId)
		return
	}

	tree := &Tree{
		Items: map[string]*Node{
			"": {
				Path:     "",
				Children: make([]string, 0),
				Defs:     make([]schema.MKey, 0),
			},
		},
	}
	pos := strings.Index(path, ".")
	prevPos := 0
	for {
		branch := path[:pos]
		// add as child of parent branch
		tree.Items[path[:prevPos]].Children = []string{branch}

		// create this branch/leaf
		tree.Items[branch] = &Node{
			Path: branch,
		}
		if branch == path {
			tree.Items[branch].Defs = []schema.MKey{{}}
			break
		}
		prevPos = pos
		pos = pos + strings.Index(path[pos+1:], ".")
	}

	for _, k := range cache.Keys() {
		matches, err := find(tree, k.(string))
		if err != nil {
			log.Errorf("memory-idx: checking if new series matches expressions in findCache. series=%s expr=%s", path, k)
			continue
		}
		if len(matches) > 0 {
			cache.Remove(k)
		}
	}
	select {
	case <-c.newSeries[orgId]:
	default:
	}
}
<commit_msg>disable the findCache for 1min when a flood of new series is received.<commit_after>package memory

import (
	"strings"
	"sync"
	"time"

	"github.com/grafana/metrictank/stats"
	lru "github.com/hashicorp/golang-lru"
	"github.com/raintank/schema"
	log "github.com/sirupsen/logrus"
)

var (
	// metric idx.memory.find-cache.hit is a counter of findCache hits
	findCacheHit = stats.NewCounterRate32("idx.memory.find-cache.hit")
	// metric idx.memory.find-cache.hit is a counter of findCache misses
	findCacheMiss = stats.NewCounterRate32("idx.memory.find-cache.miss")
)

type FindCache struct {
	cache map[uint32]*lru.Cache
	size  int
	sync.RWMutex
	newSeries map[uint32]chan struct{}
	backoff   map[uint32]time.Time
}

func NewFindCache(size int) *FindCache {
	fc := &FindCache{
		cache:     make(map[uint32]*lru.Cache),
		size:      size,
		newSeries: make(map[uint32]chan struct{}),
		backoff:   make(map[uint32]time.Time),
	}
	return fc
}

func (c *FindCache) Get(orgId uint32, pattern string) ([]*Node, bool) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		findCacheMiss.Inc()
		return nil, false
	}
	nodes, ok := cache.Get(pattern)
	if !ok {
		return nil, ok
	}
	findCacheHit.Inc()
	return nodes.([]*Node), ok
}

func (c *FindCache) Add(orgId uint32, pattern string, nodes []*Node) {
	c.RLock()
	cache, ok := c.cache[orgId]
	t := c.backoff[orgId]
	c.RUnlock()
	var err error
	if !ok {
		// dont init the cache if we are in backoff mode.
		if time.Until(t) > 0 {
			return
		}
		cache, err = lru.New(c.size)
		if err != nil {
			log.Errorf("memory-idx: findCache failed to create lru. err=%s", err)
			return
		}
		c.Lock()
		c.cache[orgId] = cache
		c.newSeries[orgId] = make(chan struct{}, 100)
		c.Unlock()
	}
	cache.Add(pattern, nodes)
}

func (c *FindCache) Purge(orgId uint32) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok {
		return
	}
	cache.Purge()
}

func (c *FindCache) InvalidateFor(orgId uint32, path string) {
	c.RLock()
	cache, ok := c.cache[orgId]
	c.RUnlock()
	if !ok || cache.Len() < 1 {
		return
	}

	select {
	case c.newSeries[orgId] <- struct{}{}:
	default:
		c.Lock()
		c.backoff[orgId] = time.Now().Add(time.Minute)
		delete(c.cache, orgId)
		c.Unlock()
		for i := 0; i < len(c.newSeries[orgId]); i++ {
			select {
			case <-c.newSeries[orgId]:
			default:
			}
		}

		return
	}

	tree := &Tree{
		Items: map[string]*Node{
			"": {
				Path:     "",
				Children: make([]string, 0),
				Defs:     make([]schema.MKey, 0),
			},
		},
	}
	pos := strings.Index(path, ".")
	prevPos := 0
	for {
		branch := path[:pos]
		// add as child of parent branch
		tree.Items[path[:prevPos]].Children = []string{branch}

		// create this branch/leaf
		tree.Items[branch] = &Node{
			Path: branch,
		}
		if branch == path {
			tree.Items[branch].Defs = []schema.MKey{{}}
			break
		}
		prevPos = pos
		pos = pos + strings.Index(path[pos+1:], ".")
	}

	for _, k := range cache.Keys() {
		matches, err := find(tree, k.(string))
		if err != nil {
			log.Errorf("memory-idx: checking if new series matches expressions in findCache. series=%s expr=%s", path, k)
			continue
		}
		if len(matches) > 0 {
			cache.Remove(k)
		}
	}
	select {
	case <-c.newSeries[orgId]:
	default:
	}
}
<|endoftext|>


<commit_before>/*
news-maily - Open-Source Newsletter Mail System

Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/

Copyright 2015 Filip Nikolovski

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package utils has general utilities that are shared between the other packages.
package utils
<commit_msg>Update utils doc<commit_after>// Package utils has general utilities that are shared between the other packages.
package utils
<|endoftext|>


<commit_before>package helpers

import (
	"bufio"
	"fmt"
	"net"
	"os"
)

// LinesToIPList processes a list of IP addresses or networks in CIDR format.
// Returning a list of all possible IP addresses.
func LinesToIPList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIP(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, fmt.Errorf("%s is not an IP Address or CIDR Network", line)
		}
	}
	return ipList, nil
}

// increases an IP by a single address.
func increaseIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// ReadFileLines returns all the lines in a file.
func ReadFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if scanner.Text() == "" {
			continue
		}
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}
<commit_msg>Add the ability to parse dashed networks<commit_after>package helpers

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"strings"
)

// LinesToIPList processes a list of IP addresses or networks in CIDR format.
// Returning a list of all possible IP addresses.
func LinesToIPList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIP(ip) {
				ipList = append(ipList, ip.String())
			}
		} else if strings.Contains(line, "-") {
			splitIP := strings.SplitN(line, "-", 2)
			ip := net.ParseIP(splitIP[0])
			endIP := net.ParseIP(splitIP[1])
			if endIP != nil {
				if !isStartingIPLower(ip, endIP) {
					return ipList, fmt.Errorf("%s is greater than %s", ip.String(), endIP.String())
				}
				ipList = append(ipList, ip.String())
				for !ip.Equal(endIP) {
					increaseIP(ip)
					ipList = append(ipList, ip.String())
				}
			} else {
				ipOct := strings.SplitN(ip.String(), ".", 4)
				endIP := net.ParseIP(ipOct[0] + "." + ipOct[1] + "." + ipOct[2] + "." + splitIP[1])
				if endIP != nil {
					if !isStartingIPLower(ip, endIP) {
						return ipList, fmt.Errorf("%s is greater than %s", ip.String(), endIP.String())
					}
					ipList = append(ipList, ip.String())
					for !ip.Equal(endIP) {
						increaseIP(ip)
						ipList = append(ipList, ip.String())
					}
				} else {
					return ipList, fmt.Errorf("%s is not an IP Address or CIDR Network", line)
				}
			}
		} else {
			return ipList, fmt.Errorf("%s is not an IP Address or CIDR Network", line)
		}
	}
	return ipList, nil
}

// increases an IP by a single address.
func increaseIP(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

func isStartingIPLower(start, end net.IP) bool {
	if len(start) != len(end) {
		return false
	}
	for i := range start {
		if start[i] > end[i] {
			return false
		}
	}
	return true
}

// ReadFileLines returns all the lines in a file.
func ReadFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if scanner.Text() == "" {
			continue
		}
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}
<|endoftext|>


<commit_before>// Copyright (c) 2012 The gocql Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The uuid package can be used to generate and parse universally unique
// identifiers, a standardized format in the form of a 128 bit number.
//
// http://tools.ietf.org/html/rfc4122

// Package uuid provides functions to create time-based UUIDs.
package uuid // import "github.com/influxdata/influxdb/uuid"

import (
	"crypto/rand"
	"net"
	"sync/atomic"
	"time"
)

// UUID - unique identifier type representing a 128 bit number
type UUID [16]byte

var timeBase = time.Date(1582, time.October, 15, 0, 0, 0, 0, time.UTC).Unix()
var hardwareAddr [6]byte = hwAddrFunc()
var clockSeq uint32

func hwAddrFunc() [6]byte {
	u := [6]byte{}

	ifaces, err := net.Interfaces()
	if err != nil {
		rand.Reader.Read(u[:])
		return u
	}
	for _, iface := range ifaces {
		if len(iface.HardwareAddr) >= 6 {
			copy(u[:], iface.HardwareAddr)
			return u
		}
	}
	rand.Reader.Read(u[:])
	return u
}

// TimeUUID generates a new time based UUID (version 1) using the current
// time as the timestamp.
func TimeUUID() UUID {
	return FromTime(time.Now())
}

// FromTime generates a new time based UUID (version 1) as described in
// RFC 4122. This UUID contains the MAC address of the node that generated
// the UUID, the given timestamp and a sequence number.
func FromTime(aTime time.Time) UUID {
	var u UUID

	utcTime := aTime.In(time.UTC)
	t := uint64(utcTime.Unix()-timeBase)*10000000 + uint64(utcTime.Nanosecond()/100)
	u[0], u[1], u[2], u[3] = byte(t>>24), byte(t>>16), byte(t>>8), byte(t)
	u[4], u[5] = byte(t>>40), byte(t>>32)
	u[6], u[7] = byte(t>>56)&0x0F, byte(t>>48)

	clock := atomic.AddUint32(&clockSeq, 1)
	u[8] = byte(clock >> 8)
	u[9] = byte(clock)

	copy(u[10:], hardwareAddr[:])

	u[6] |= 0x10 // set version to 1 (time based uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant

	return u
}

// String returns the UUID in it's canonical form, a 32 digit hexadecimal
// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34}
	const hexString = "0123456789abcdef"
	r := make([]byte, 36)
	for i, b := range u {
		r[offsets[i]] = hexString[b>>4]
		r[offsets[i]+1] = hexString[b&0xF]
	}
	r[8] = '-'
	r[13] = '-'
	r[18] = '-'
	r[23] = '-'
	return string(r)

}
<commit_msg>golint uuid package<commit_after>// Copyright (c) 2012 The gocql Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// The uuid package can be used to generate and parse universally unique
// identifiers, a standardized format in the form of a 128 bit number.
//
// http://tools.ietf.org/html/rfc4122

// Package uuid provides functions to create time-based UUIDs.
package uuid // import "github.com/influxdata/influxdb/uuid"

import (
	"crypto/rand"
	"net"
	"sync/atomic"
	"time"
)

// UUID - unique identifier type representing a 128 bit number
type UUID [16]byte

var timeBase = time.Date(1582, time.October, 15, 0, 0, 0, 0, time.UTC).Unix()
var hardwareAddr = hwAddrFunc()
var clockSeq uint32

func hwAddrFunc() [6]byte {
	u := [6]byte{}

	ifaces, err := net.Interfaces()
	if err != nil {
		rand.Reader.Read(u[:])
		return u
	}
	for _, iface := range ifaces {
		if len(iface.HardwareAddr) >= 6 {
			copy(u[:], iface.HardwareAddr)
			return u
		}
	}
	rand.Reader.Read(u[:])
	return u
}

// TimeUUID generates a new time based UUID (version 1) using the current
// time as the timestamp.
func TimeUUID() UUID {
	return FromTime(time.Now())
}

// FromTime generates a new time based UUID (version 1) as described in
// RFC 4122. This UUID contains the MAC address of the node that generated
// the UUID, the given timestamp and a sequence number.
func FromTime(aTime time.Time) UUID {
	var u UUID

	utcTime := aTime.In(time.UTC)
	t := uint64(utcTime.Unix()-timeBase)*10000000 + uint64(utcTime.Nanosecond()/100)
	u[0], u[1], u[2], u[3] = byte(t>>24), byte(t>>16), byte(t>>8), byte(t)
	u[4], u[5] = byte(t>>40), byte(t>>32)
	u[6], u[7] = byte(t>>56)&0x0F, byte(t>>48)

	clock := atomic.AddUint32(&clockSeq, 1)
	u[8] = byte(clock >> 8)
	u[9] = byte(clock)

	copy(u[10:], hardwareAddr[:])

	u[6] |= 0x10 // set version to 1 (time based uuid)
	u[8] &= 0x3F // clear variant
	u[8] |= 0x80 // set to IETF variant

	return u
}

// String returns the UUID in it's canonical form, a 32 digit hexadecimal
// number in the form of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34}
	const hexString = "0123456789abcdef"
	r := make([]byte, 36)
	for i, b := range u {
		r[offsets[i]] = hexString[b>>4]
		r[offsets[i]+1] = hexString[b&0xF]
	}
	r[8] = '-'
	r[13] = '-'
	r[18] = '-'
	r[23] = '-'
	return string(r)

}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"

	"github.com/dockpit/lang"
	"github.com/dockpit/pit/config"
	"github.com/dockpit/pit/contract"
	"github.com/dockpit/state"
)

var SpecFilename = "dockpit.json"

// CLI Command interface
type C interface {
	Name() string
	Description() string
	Usage() string
	Run(c *cli.Context) (*template.Template, interface{}, error)
	Action() func(ctx *cli.Context)
	Flags() []cli.Flag
}

type cmd struct {
	out io.Writer
}

func newCmd(out io.Writer) *cmd {

	//if no specific output, use stdout
	if out == nil {
		out = os.Stdout
	}

	//redirect default log ouput
	log.SetOutput(out)

	return &cmd{out}
}

func (c *cmd) Run(ctx *cli.Context) (*template.Template, interface{}, error) {
	return nil, nil, fmt.Errorf("Command '%s' is not yet implemented", ctx.Command.Name)
}

func (c *cmd) DockerFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{Name: "docker, d", Value: "", Usage: fmt.Sprintf("The Docker host location, defaults to reading from DOCKER_HOST environment variable.")},
		cli.StringFlag{Name: "docker-cert, c", Value: "", Usage: fmt.Sprintf("The Docker Certificates Path location, defaults to reading from DOCKER_CERT_PATH environment variable.")},
	}
}

func (c *cmd) BuildStatesFlags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".manifest", "states")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "states, s", Value: wd, Usage: fmt.Sprintf(" Specify where to look for states.")},
	}
}

func (c *cmd) LoadConfig() (config.C, error) {

	//get working dir
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	l := config.NewLoader(wd)
	conf, err := l.Load()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Could not find dockpit configuration file (dockpit.json) in '%s'", wd)
		}
		return nil, err
	}

	return conf, err
}

func (c *cmd) ParseExampleFlags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".manifest", "examples")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "examples, e", Value: wd, Usage: fmt.Sprintf(" Specify where to look for examples.")},
	}
}

func (c *cmd) DockerClient(ctx *cli.Context) (*docker.Client, error) {

	//fetch required args
	host, cert, err := c.DockerHostCertArguments(ctx)
	if err != nil {
		return nil, err
	}

	//parse docker host addr as url
	hurl, err := url.Parse(host)
	if err != nil {
		return nil, err
	}

	//change to http connection
	hurl.Scheme = "https"

	//return client
	return docker.NewTLSClient(hurl.String(), filepath.Join(cert, "cert.pem"), filepath.Join(cert, "key.pem"), filepath.Join(cert, "ca.pem"))

}

func (c *cmd) DockerHostCertArguments(ctx *cli.Context) (string, string, error) {

	//try docker host retrieval
	host := strings.TrimSpace(ctx.String("docker"))
	if host == "" {
		host = os.Getenv("DOCKER_HOST")
		if host == "" {
			return "", "", fmt.Errorf("Could not retrieve DOCKER_HOST, not provided as option and not in env")
		}
	}

	//try docker cert retrieval
	cert := strings.TrimSpace(ctx.String("docker-cert"))
	if cert == "" {
		cert = os.Getenv("DOCKER_CERT_PATH")
		if cert == "" {
			return "", "", fmt.Errorf("Could not retrieve DOCKER_CERT_PATH, not provided as option and not in env")
		}
	}

	return host, cert, nil
}

func (c *cmd) StateManager(ctx *cli.Context) (*state.Manager, error) {

	//path to state context folders
	path := strings.TrimSpace(ctx.String("states"))
	host, cert, err := c.DockerHostCertArguments(ctx)
	if err != nil {
		return nil, err
	}

	//create state manager
	m, err := state.NewManager(host, cert, path)
	if err != nil {
		return nil, err
	}

	return m, nil
}

func (c *cmd) ExamplesPath(ctx *cli.Context) string {
	return strings.TrimSpace(ctx.String("examples"))
}

func (c *cmd) ParseExamples(ctx *cli.Context) (contract.C, error) {

	//retrieve path
	path := c.ExamplesPath(ctx)

	//parse the spec
	p := lang.NewParser(path)
	cd, err := p.Parse()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Failed to open .manifest/examples in '%s', is this a Dockpit project?", path)
		}

		return nil, fmt.Errorf("Parsing error: %s", err)
	}

	//create contract from data
	contract, err := contract.NewContract(cd)
	if err != nil {
		return nil, fmt.Errorf("Failed to create contract from parsed data: %s", err)
	}

	return contract, nil
}

func (c *cmd) templated(fn func(c *cli.Context) (*template.Template, interface{}, error)) func(ctx *cli.Context) {
	return func(ctx *cli.Context) {
		t, data, err := fn(ctx)
		if err != nil {
			log.Println(err, ", Command: '", ctx.Command.Name, "' Args: ", ctx.Args())

			//@todo set exit code to non-zero?
			return
		}

		err = t.Execute(c.out, data)
		if err != nil {
			log.Fatal("Template Error: ", err)
		}
	}
}
<commit_msg>state manager now has a config instance<commit_after>package command

import (
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"

	"github.com/dockpit/lang"
	"github.com/dockpit/pit/config"
	"github.com/dockpit/pit/contract"
	"github.com/dockpit/state"
)

var SpecFilename = "dockpit.json"

// CLI Command interface
type C interface {
	Name() string
	Description() string
	Usage() string
	Run(c *cli.Context) (*template.Template, interface{}, error)
	Action() func(ctx *cli.Context)
	Flags() []cli.Flag
}

type cmd struct {
	out io.Writer
}

func newCmd(out io.Writer) *cmd {

	//if no specific output, use stdout
	if out == nil {
		out = os.Stdout
	}

	//redirect default log ouput
	log.SetOutput(out)

	return &cmd{out}
}

func (c *cmd) Run(ctx *cli.Context) (*template.Template, interface{}, error) {
	return nil, nil, fmt.Errorf("Command '%s' is not yet implemented", ctx.Command.Name)
}

func (c *cmd) DockerFlags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{Name: "docker, d", Value: "", Usage: fmt.Sprintf("The Docker host location, defaults to reading from DOCKER_HOST environment variable.")},
		cli.StringFlag{Name: "docker-cert, c", Value: "", Usage: fmt.Sprintf("The Docker Certificates Path location, defaults to reading from DOCKER_CERT_PATH environment variable.")},
	}
}

func (c *cmd) BuildStatesFlags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".manifest", "states")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "states, s", Value: wd, Usage: fmt.Sprintf(" Specify where to look for states.")},
	}
}

func (c *cmd) LoadConfig() (config.C, error) {

	//get working dir
	wd, err := os.Getwd()
	if err != nil {
		return nil, err
	}

	l := config.NewLoader(wd)
	conf, err := l.Load()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Could not find dockpit configuration file (dockpit.json) in '%s'", wd)
		}
		return nil, err
	}

	return conf, err
}

func (c *cmd) ParseExampleFlags() []cli.Flag {

	//get working dir
	wd, err := os.Getwd()
	if err == nil {
		wd = filepath.Join(wd, ".manifest", "examples")
	} else {
		wd = fmt.Sprintf("[%s]", err.Error())
	}

	return []cli.Flag{
		cli.StringFlag{Name: "examples, e", Value: wd, Usage: fmt.Sprintf(" Specify where to look for examples.")},
	}
}

func (c *cmd) DockerClient(ctx *cli.Context) (*docker.Client, error) {

	//fetch required args
	host, cert, err := c.DockerHostCertArguments(ctx)
	if err != nil {
		return nil, err
	}

	//parse docker host addr as url
	hurl, err := url.Parse(host)
	if err != nil {
		return nil, err
	}

	//change to http connection
	hurl.Scheme = "https"

	//return client
	return docker.NewTLSClient(hurl.String(), filepath.Join(cert, "cert.pem"), filepath.Join(cert, "key.pem"), filepath.Join(cert, "ca.pem"))

}

func (c *cmd) DockerHostCertArguments(ctx *cli.Context) (string, string, error) {

	//try docker host retrieval
	host := strings.TrimSpace(ctx.String("docker"))
	if host == "" {
		host = os.Getenv("DOCKER_HOST")
		if host == "" {
			return "", "", fmt.Errorf("Could not retrieve DOCKER_HOST, not provided as option and not in env")
		}
	}

	//try docker cert retrieval
	cert := strings.TrimSpace(ctx.String("docker-cert"))
	if cert == "" {
		cert = os.Getenv("DOCKER_CERT_PATH")
		if cert == "" {
			return "", "", fmt.Errorf("Could not retrieve DOCKER_CERT_PATH, not provided as option and not in env")
		}
	}

	return host, cert, nil
}

func (c *cmd) StateManager(ctx *cli.Context) (*state.Manager, error) {

	//path to state context folders
	path := strings.TrimSpace(ctx.String("states"))
	host, cert, err := c.DockerHostCertArguments(ctx)
	if err != nil {
		return nil, err
	}

	conf, err := c.LoadConfig()
	if err != nil {
		return nil, err
	}

	//create state manager
	m, err := state.NewManager(host, cert, path, conf)
	if err != nil {
		return nil, err
	}

	return m, nil
}

func (c *cmd) ExamplesPath(ctx *cli.Context) string {
	return strings.TrimSpace(ctx.String("examples"))
}

func (c *cmd) ParseExamples(ctx *cli.Context) (contract.C, error) {

	//retrieve path
	path := c.ExamplesPath(ctx)

	//parse the spec
	p := lang.NewParser(path)
	cd, err := p.Parse()
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("Failed to open .manifest/examples in '%s', is this a Dockpit project?", path)
		}

		return nil, fmt.Errorf("Parsing error: %s", err)
	}

	//create contract from data
	contract, err := contract.NewContract(cd)
	if err != nil {
		return nil, fmt.Errorf("Failed to create contract from parsed data: %s", err)
	}

	return contract, nil
}

func (c *cmd) templated(fn func(c *cli.Context) (*template.Template, interface{}, error)) func(ctx *cli.Context) {
	return func(ctx *cli.Context) {
		t, data, err := fn(ctx)
		if err != nil {
			log.Println(err, ", Command: '", ctx.Command.Name, "' Args: ", ctx.Args())

			//@todo set exit code to non-zero?
			return
		}

		err = t.Execute(c.out, data)
		if err != nil {
			log.Fatal("Template Error: ", err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Kurt Jung (Gmail: kurt.w.jung)
//
// Permission to use, copy, modify, and distribute this software for any purpose
// with or without fee is hereby granted, provided that the above copyright notice
// and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
// FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.

// Package example provides some helper routines for the test packages of
// gofpdf and its various contributed packages located beneath the contrib
// directory.
package example

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jung-kurt/gofpdf"
)

var gofpdfDir string

func init() {
	setRoot()
	gofpdf.SetDefaultCompression(false)
	gofpdf.SetDefaultCatalogSort(true)
	gofpdf.SetDefaultCreationDate(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
}

// Assign the relative path to the gofpdfDir directory based on current working
// directory
func setRoot() {
	wdStr, err := os.Getwd()
	if err == nil {
		gofpdfDir = ""
		list := strings.Split(filepath.ToSlash(wdStr), "/")
		for j := len(list) - 1; j >= 0 && list[j] != "gofpdf"; j-- {
			gofpdfDir = filepath.Join(gofpdfDir, "..")
		}
	} else {
		panic(err)
	}
}

// ImageFile returns a qualified filename in which the path to the image
// directory is prepended to the specified filename.
func ImageFile(fileStr string) string {
	return filepath.Join(gofpdfDir, "image", fileStr)
}

// FontDir returns the path to the font directory.
func FontDir() string {
	return filepath.Join(gofpdfDir, "font")
}

// FontFile returns a qualified filename in which the path to the font
// directory is prepended to the specified filename.
func FontFile(fileStr string) string {
	return filepath.Join(FontDir(), fileStr)
}

// TextFile returns a qualified filename in which the path to the text
// directory is prepended to the specified filename.
func TextFile(fileStr string) string {
	return filepath.Join(gofpdfDir, "text", fileStr)
}

// PdfDir returns the path to the PDF output directory.
func PdfDir() string {
	return filepath.Join(gofpdfDir, "pdf")
}

// PdfFile returns a qualified filename in which the path to the PDF output
// directory is prepended to the specified filename.
func PdfFile(fileStr string) string {
	return filepath.Join(PdfDir(), fileStr)
}

// Filename returns a qualified filename in which the example PDF directory
// path is prepended and the suffix ".pdf" is appended to the specified
// filename.
func Filename(baseStr string) string {
	return PdfFile(baseStr + ".pdf")
}

// referenceCompare compares the specified file with the file's reference copy
// located in the 'reference' subdirectory. All bytes of the two files are
// compared except for the value of the /CreationDate field in the PDF. This
// function succeeds if both files are equivalent except for their
// /CreationDate values or if the reference file does not exist.
func referenceCompare(fileStr string) (err error) {
	var refFileStr, refDirStr, dirStr, baseFileStr string
	dirStr, baseFileStr = filepath.Split(fileStr)
	refDirStr = filepath.Join(dirStr, "reference")
	err = os.MkdirAll(refDirStr, 0755)
	if err == nil {
		refFileStr = filepath.Join(refDirStr, baseFileStr)
		err = gofpdf.ComparePDFFiles(fileStr, refFileStr)
	}
	return
}

// Summary generates a predictable report for use by test examples. If the
// specified error is nil, the filename delimiters are normalized and the
// filename printed to standard output with a success message. If the specified
// error is not nil, its String() value is printed to standard output.
func Summary(err error, fileStr string) {
	if err == nil {
		err = referenceCompare(fileStr)
	}
	if err == nil {
		fileStr = filepath.ToSlash(fileStr)
		fmt.Printf("Successfully generated %s\n", fileStr)
	} else {
		fmt.Println(err)
	}
}
<commit_msg>Updated example<commit_after>// Copyright (c) 2015 Kurt Jung (Gmail: kurt.w.jung)
//
// Permission to use, copy, modify, and distribute this software for any purpose
// with or without fee is hereby granted, provided that the above copyright notice
// and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
// FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.

// Package example provides some helper routines for the test packages of
// gofpdf and its various contributed packages located beneath the contrib
// directory.
package example

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jung-kurt/gofpdf"
)

var gofpdfDir string

func init() {
	setRoot()
	gofpdf.SetDefaultCompression(false)
	gofpdf.SetDefaultCatalogSort(true)
	gofpdf.SetDefaultCreationDate(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC))
}

// Assign the relative path to the gofpdfDir directory based on current working
// directory
func setRoot() {
	wdStr, err := os.Getwd()
	if err == nil {
		gofpdfDir = ""
		list := strings.Split(filepath.ToSlash(wdStr), "/")
		for j := len(list) - 1; j >= 0 && list[j] != "gofpdf"; j-- {
			gofpdfDir = filepath.Join(gofpdfDir, "..")
		}
	} else {
		panic(err)
	}
}

// ImageFile returns a qualified filename in which the path to the image
// directory is prepended to the specified filename.
func ImageFile(fileStr string) string {
	return filepath.Join(gofpdfDir, "image", fileStr)
}

// FontDir returns the path to the font directory.
func FontDir() string {
	return filepath.Join(gofpdfDir, "font")
}

// FontFile returns a qualified filename in which the path to the font
// directory is prepended to the specified filename.
func FontFile(fileStr string) string {
	return filepath.Join(FontDir(), fileStr)
}

// TextFile returns a qualified filename in which the path to the text
// directory is prepended to the specified filename.
func TextFile(fileStr string) string {
	return filepath.Join(gofpdfDir, "text", fileStr)
}

// PdfDir returns the path to the PDF output directory.
func PdfDir() string {
	return filepath.Join(gofpdfDir, "pdf")
}

// PdfFile returns a qualified filename in which the path to the PDF output
// directory is prepended to the specified filename.
func PdfFile(fileStr string) string {
	return filepath.Join(PdfDir(), fileStr)
}

// Filename returns a qualified filename in which the example PDF directory
// path is prepended and the suffix ".pdf" is appended to the specified
// filename.
func Filename(baseStr string) string {
	return PdfFile(baseStr + ".pdf")
}

// referenceCompare compares the specified file with the file's reference copy
// located in the 'reference' subdirectory. All bytes of the two files are
// compared except for the value of the /CreationDate field in the PDF. This
// function succeeds if both files are equivalent except for their
// /CreationDate values or if the reference file does not exist.
func referenceCompare(fileStr string) (err error) {
	var refFileStr, refDirStr, dirStr, baseFileStr string
	dirStr, baseFileStr = filepath.Split(fileStr)
	refDirStr = filepath.Join(dirStr, "reference")
	err = os.MkdirAll(refDirStr, 0755)
	if err == nil {
		refFileStr = filepath.Join(refDirStr, baseFileStr)
		err = gofpdf.ComparePDFFiles(fileStr, refFileStr, true)
	}
	return
}

// Summary generates a predictable report for use by test examples. If the
// specified error is nil, the filename delimiters are normalized and the
// filename printed to standard output with a success message. If the specified
// error is not nil, its String() value is printed to standard output.
func Summary(err error, fileStr string) {
	if err == nil {
		err = referenceCompare(fileStr)
	}
	if err == nil {
		fileStr = filepath.ToSlash(fileStr)
		fmt.Printf("Successfully generated %s\n", fileStr)
	} else {
		fmt.Println(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gensupport

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"time"
)

// Hook is the type of a function that is called once before each HTTP request
// that is sent by a generated API.  It returns a function that is called after
// the request returns.
// Hooks are not called if the context is nil.
type Hook func(ctx context.Context, req *http.Request) func(resp *http.Response)

var hooks []Hook

// RegisterHook registers a Hook to be called before each HTTP request by a
// generated API.  Hooks are called in the order they are registered.  Each
// hook can return a function; if it is non-nil, it is called after the HTTP
// request returns.  These functions are called in the reverse order.
// RegisterHook should not be called concurrently with itself or SendRequest.
func RegisterHook(h Hook) {
	hooks = append(hooks, h)
}

// SendRequest sends a single HTTP request using the given client.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
	if _, ok := req.Header["Accept-Encoding"]; ok {
		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
	}
	if ctx == nil {
		return client.Do(req)
	}
	// Call hooks in order of registration, store returned funcs.
	post := make([]func(resp *http.Response), len(hooks))
	for i, h := range hooks {
		fn := h(ctx, req)
		post[i] = fn
	}

	// Send request.
	resp, err := send(ctx, client, req)

	// Call returned funcs in reverse order.
	for i := len(post) - 1; i >= 0; i-- {
		if fn := post[i]; fn != nil {
			fn(resp)
		}
	}
	return resp, err
}

func send(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	resp, err := client.Do(req.WithContext(ctx))
	// If we got an error, and the context has been canceled,
	// the context's error is probably more useful.
	if err != nil {
		select {
		case <-ctx.Done():
			err = ctx.Err()
		default:
		}
	}
	return resp, err
}

// SendRequestWithRetry sends a single HTTP request using the given client,
// with retries if a retryable error is returned.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequestWithRetry(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
	if _, ok := req.Header["Accept-Encoding"]; ok {
		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
	}
	if ctx == nil {
		return client.Do(req)
	}
	// Call hooks in order of registration, store returned funcs.
	post := make([]func(resp *http.Response), len(hooks))
	for i, h := range hooks {
		fn := h(ctx, req)
		post[i] = fn
	}

	// Send request with retry.
	resp, err := sendAndRetry(ctx, client, req)

	// Call returned funcs in reverse order.
	for i := len(post) - 1; i >= 0; i-- {
		if fn := post[i]; fn != nil {
			fn(resp)
		}
	}
	return resp, err
}

func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}

	var resp *http.Response
	var err error

	// Loop to retry the request, up to the context deadline.
	var pause time.Duration
	bo := backoff()

	for {
		select {
		case <-ctx.Done():
			// If we got an error, and the context has been canceled,
			// the context's error is probably more useful.
			if err == nil {
				err = ctx.Err()
			}
			return resp, err
		case <-time.After(pause):
		}

		resp, err = client.Do(req.WithContext(ctx))

		var status int
		if resp != nil {
			status = resp.StatusCode
		}

		// Check if we can retry the request. A retry can only be done if the error
		// is retryable and the request body can be re-created using GetBody (this
		// will not be possible if the body was unbuffered).
		if req.GetBody == nil || !shouldRetry(status, err) {
			break
		}
		var errBody error
		req.Body, errBody = req.GetBody()
		if errBody != nil {
			break
		}

		pause = bo.Pause()
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}
	return resp, err
}

// DecodeResponse decodes the body of res into target. If there is no body,
// target is unchanged.
func DecodeResponse(target interface{}, res *http.Response) error {
	if res.StatusCode == http.StatusNoContent {
		return nil
	}
	return json.NewDecoder(res.Body).Decode(target)
}
<commit_msg>chore(internal/gensupport): remove Hook (#772)<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gensupport

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"time"
)

// SendRequest sends a single HTTP request using the given client.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
	if _, ok := req.Header["Accept-Encoding"]; ok {
		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
	}
	if ctx == nil {
		return client.Do(req)
	}
	return send(ctx, client, req)
}

func send(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	resp, err := client.Do(req.WithContext(ctx))
	// If we got an error, and the context has been canceled,
	// the context's error is probably more useful.
	if err != nil {
		select {
		case <-ctx.Done():
			err = ctx.Err()
		default:
		}
	}
	return resp, err
}

// SendRequestWithRetry sends a single HTTP request using the given client,
// with retries if a retryable error is returned.
// If ctx is non-nil, it calls all hooks, then sends the request with
// req.WithContext, then calls any functions returned by the hooks in
// reverse order.
func SendRequestWithRetry(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
	if _, ok := req.Header["Accept-Encoding"]; ok {
		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
	}
	if ctx == nil {
		return client.Do(req)
	}
	return sendAndRetry(ctx, client, req)
}

func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}

	var resp *http.Response
	var err error

	// Loop to retry the request, up to the context deadline.
	var pause time.Duration
	bo := backoff()

	for {
		select {
		case <-ctx.Done():
			// If we got an error, and the context has been canceled,
			// the context's error is probably more useful.
			if err == nil {
				err = ctx.Err()
			}
			return resp, err
		case <-time.After(pause):
		}

		resp, err = client.Do(req.WithContext(ctx))

		var status int
		if resp != nil {
			status = resp.StatusCode
		}

		// Check if we can retry the request. A retry can only be done if the error
		// is retryable and the request body can be re-created using GetBody (this
		// will not be possible if the body was unbuffered).
		if req.GetBody == nil || !shouldRetry(status, err) {
			break
		}
		var errBody error
		req.Body, errBody = req.GetBody()
		if errBody != nil {
			break
		}

		pause = bo.Pause()
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
	}
	return resp, err
}

// DecodeResponse decodes the body of res into target. If there is no body,
// target is unchanged.
func DecodeResponse(target interface{}, res *http.Response) error {
	if res.StatusCode == http.StatusNoContent {
		return nil
	}
	return json.NewDecoder(res.Body).Decode(target)
}
<|endoftext|>