<commit_before>package boltq

import (
	"bytes"
	"encoding/binary"

	"github.com/boltdb/bolt"
)

/*
DataStore adds composite key support to BoltDB
*/
type DataStore struct {
	*bolt.DB
}

/*
Store places the value in the datastore. The collection key is used for the root bucket
of the storage while the composite key is used as the path to the final storage loaction
*/
func (ds DataStore) Store(collection []byte, compositeKey [][]byte, value []byte) error {

	err := ds.Update(func(tx *bolt.Tx) error {
		return TxStore(tx, collection, compositeKey, value)
	})

	return err
}

/*
TxStore places the value in the datastore that tx came from. The collection key is used
for the root bucket of the storage while the composite key is used as the path to the
final storage loaction
*/
func TxStore(tx *bolt.Tx, collection []byte, compositeKey [][]byte, value []byte) error {
	b, err := tx.CreateBucketIfNotExists(collection)
	lastKeyIndex := len(compositeKey) - 1
	for i := 0; err == nil && i < lastKeyIndex; i += 1 {
		b, err = b.CreateBucketIfNotExists(compositeKey[i])
	}
	if err == nil {
		err = b.Put(compositeKey[lastKeyIndex], value)
	}
	return err
}

/*
Index records the composite key associated with the value for use in index queries
*/
func (ds DataStore) Index(collection, index, value []byte, compositeKey [][]byte) error {
	err := ds.Update(func(tx *bolt.Tx) error {
		return TxIndex(tx, collection, index, value, compositeKey)
	})

	return err
}

/*
TxDeleteIndex deletes the specified index
*/
func (ds DataStore) DeleteIndex(collection, index []byte) error {
	err := ds.Update(func(tx *bolt.Tx) error {
		return TxDeleteIndex(tx, collection, index)
	})
	return err
}

/*
createIndexKey creates a key used to access the bucket that stores index values
*/
func createIndexKey(collection, index []byte) []byte {
	indexKey := make([]byte, 0, len(collection)+len(index))
	indexKey = append(indexKey, collection...)
	return append(indexKey, index...)
}

/*
TxDeleteIndex deletes the specified index in the given transaction
*/
func TxDeleteIndex(tx *bolt.Tx, collection, index []byte) error {
	indexKey := createIndexKey(collection, index)
	return tx.DeleteBucket(indexKey)
}

/*
Index records the composite key associated with the value for use in index queries
*/
func TxIndex(tx *bolt.Tx, collection, index, value []byte, compositeKey [][]byte) error {
	indexKey := createIndexKey(collection, index)
	b, err := tx.CreateBucketIfNotExists(indexKey)
	if err == nil {
		var serializedKey []byte
		var serializedSet []byte
		serializedKey = SerializeComposite(compositeKey)
		if err == nil {
			serializedSet, err = addToSetValue(b.Get(value), serializedKey)
			if err == nil {
				b.Put(value, serializedSet)
			}
		}
	}

	return err
}

/*
addToSetValue adds value to a serialized set
a serialized version of the updated set is returned
*/
func addToSetValue(serializedSet, value []byte) ([]byte, error) {
	var err error
	var rval []byte
	var set map[string]bool
	var keys []string
	if serializedSet == nil {
		set = make(map[string]bool)
	} else {
		set, keys, err = deserializeSet(serializedSet)
	}
	if err == nil {
		valStr := string(value)
		_, exists := set[valStr]
		if !exists {
			set[valStr] = true
			keys = append(keys, valStr)
		}
		rval = serializeSet(set, keys)
	}

	return rval, err
}

/*
serializeSet serializes an ordered set to bytes
*/
func serializeSet(set map[string]bool, keys []string) []byte {
	sizeBuff := make([]byte, 8)
	var buff bytes.Buffer
	for _, keyStr := range keys {
		key := []byte(keyStr)
		size := len(key)
		buff.Grow(size + 8)
		binary.PutVarint(sizeBuff, int64(size))
		buff.Write(sizeBuff)
		buff.Write(key)
	}
	return buff.Bytes()
}

/*
deserializeSet deserializes an ordered set from bytes
the set is returned as a map along with the key ordering
*/
func deserializeSet(serialized []byte) (map[string]bool, []string, error) {
	var err error
	rval := make(map[string]bool)
	var keys []string
	sizeBuff := make([]byte, 8)
	buff := bytes.NewBuffer(serialized)
	count := 0
	for count < len(serialized) {
		var length int
		length, err = buff.Read(sizeBuff)
		if err == nil {
			count += length
			size, _ := binary.Varint(sizeBuff)
			key := make([]byte, size)
			length, err = buff.Read(key)
			count += length
			keyStr := string(key)
			_, exists := rval[keyStr]
			if !exists {
				keys = append(keys, keyStr)
				rval[keyStr] = true
			}
		}
	}
	return rval, keys, err
}

/*
serializeComposite serializes the composite key to be stored in an index
*/
func SerializeComposite(compositeKey [][]byte) []byte {
	sizeBuff := make([]byte, 8)
	var buff bytes.Buffer
	for _, key := range compositeKey {
		size := len(key)
		buff.Grow(size + 8)
		binary.PutVarint(sizeBuff, int64(size))
		buff.Write(sizeBuff)
		buff.Write(key)
	}
	return buff.Bytes()
}

/*
deserializeComposite deserializes a composite key that was stored in an index
*/
func DeserializeComposite(serialized []byte) ([][]byte, error) {
	var err error
	var rval [][]byte
	sizeBuff := make([]byte, 8)
	buff := bytes.NewBuffer(serialized)
	count := 0
	for count < len(serialized) {
		var length int
		length, err = buff.Read(sizeBuff)
		if err == nil {
			count += length
			size, _ := binary.Varint(sizeBuff)
			key := make([]byte, size)
			length, err = buff.Read(key)
			count += length
			rval = append(rval, key)
		}
	}
	return rval, err
}

/*
qState is composed of data that represents the state at a specific level of a query
*/
type qState struct {
	b     *bolt.Bucket
	terms []*Term
	level int
}

/*
next gets the next state level for the query
*/
func (s qState) next(b *bolt.Bucket) qState {
	return qState{b, s.terms, s.level + 1}
}

/*
TxQuery executes the query in the provided transaction
*/
func TxQuery(tx *bolt.Tx, query *Query) ([][]byte, error) {
	var rval [][]byte
	var err error

	b := tx.Bucket(query.Collection)
	if b != nil {
		state := qState{b, query.Terms, 0}
		rval, err = getByTerm(state, rval)
	}

	return rval, err
}

/*
getByTerm recursively collects query results accoring to the term of the current query level
*/
func getByTerm(state qState, results [][]byte) ([][]byte, error) {
	var err error
	var term *Term
	if state.level >= len(state.terms) {
		term = Any()
	} else {
		term = state.terms[state.level]
	}

	switch term.op {
	case ANY:
		results, err = getAny(state, results)
	case EQ:
		results, err = getEq(state, results)
	case RANGE:
		results, err = getRange(state, results)
	case IN:
		results, err = getIn(state, results)
	case NOT:
		results, err = getNot(state, results)
	}

	return results, err
}

/*
getVal gets the results stored at the key/val pair which could be a bucket or item.
A nil val signifies that key points to nested bucket
*/
func getVal(state qState, key, val []byte, results [][]byte) ([][]byte, error) {
	var err error
	if val != nil {
		results = append(results, val)
	} else {
		nested := state.b.Bucket(key)
		results, err = getByTerm(state.next(nested), results)
	}
	return results, err
}

/*
getAny gets all results at the current level of the query
*/
func getAny(state qState, results [][]byte) ([][]byte, error) {
	var err error
	c := state.b.Cursor()
	for key, val := c.First(); key != nil; key, val = c.Next() {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getEq get results for a specific key at the current level
*/
func getEq(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]
	queryKey := term.keys[0]
	c := state.b.Cursor()
	key, val := c.Seek(queryKey)

	if bytes.Equal(key, queryKey) {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getRange gets results for a range of keys at the current level
*/
func getRange(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]
	lower := term.keys[0]
	upper := term.keys[1]
	c := state.b.Cursor()
	limit, _ := c.Seek(upper)
	key, val := c.Seek(lower)
	for ; key != nil && !bytes.Equal(key, limit); key, val = c.Next() {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getIn gets results for a set of keys at the curent level
*/
func getIn(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]

	c := state.b.Cursor()
	for _, queryKey := range term.keys {
		key, val := c.Seek(queryKey)

		if bytes.Equal(key, queryKey) {
			results, err = getVal(state, key, val, results)
		}
	}

	return results, err
}

/*
getNot gets results at the current level whose keys are not in the term
*/
func getNot(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]

	c := state.b.Cursor()
	for key, val := c.First(); key != nil; key, val = c.Next() {
		if !in(key, term.keys) {
			results, err = getVal(state, key, val, results)
		}
	}

	return results, err
}

/*
in returns true if val is in arr
*/
func in(val []byte, arr [][]byte) bool {
	rval := false
	for _, entry := range arr {
		if bytes.Equal(val, entry) {
			rval = true
			break
		}
	}
	return rval
}

/*
TxIndexMatchAny queries for items using the given index and the values that were indexed
*/
func TxIndexMatchAny(tx *bolt.Tx, collection, index []byte, values ...[]byte) ([][]byte, error) {
	var err error
	var rval [][]byte

	indexKey := createIndexKey(collection, index)
	b := tx.Bucket(indexKey)

	if b != nil {
		for i := 0; err == nil && i < len(values); i += 1 {
			value := values[i]
			serializedSet := b.Get(value)
			if serializedSet != nil {
				var keys []string
				_, keys, err = deserializeSet(serializedSet)
				if err == nil {
					for _, keyStr := range keys {
						key := []byte(keyStr)
						compositeKey, err := DeserializeComposite(key)
						if err != nil {
							return rval, err
						}
						rval, err = collectResults(tx, collection, compositeKey, rval)
						if err != nil {
							return rval, err
						}
					}
				}
			}
		}
	}

	return rval, err
}

/*
TxIndexMatchAll queries for items using the given index and the values that were indexed
*/
func TxIndexMatchAll(tx *bolt.Tx, collection, index []byte, values ...[]byte) ([][]byte, error) {
	var err error
	var rval [][]byte

	indexKey := createIndexKey(collection, index)
	b := tx.Bucket(indexKey)

	if b != nil {
		var matched map[string]bool
		for i := 0; err == nil && i < len(values); i += 1 {
			value := values[i]
			serializedSet := b.Get(value)
			if serializedSet != nil {
				var set map[string]bool
				set, _, err = deserializeSet(serializedSet)
				if err == nil {
					if matched == nil {
						matched = set
					} else {
						for key, _ := range matched {
							inSet, _ := set[key]
							if !inSet {
								matched[key] = false
							}
						}
					}
				}
			}
		}
		for keyStr, inSet := range matched {
			if err == nil && inSet {
				key := []byte(keyStr)
				compositeKey, err := DeserializeComposite(key)
				if err == nil {
					rval, err = collectResults(tx, collection, compositeKey, rval)
				}
			}
		}
	}

	return rval, err
}

/*
collectResults appends values found at composite key to index query results
*/
func collectResults(tx *bolt.Tx, collection []byte, compositeKey, results [][]byte) ([][]byte, error) {
	var err error

	q := NewQuery(collection, EqAll(compositeKey)...)
	res, err := TxQuery(tx, q)
	for j := 0; err == nil && j < len(res); j += 1 {
		results = append(results, res...)
	}

	return results, err
}
<commit_msg>added delete function<commit_after>package boltq

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"github.com/boltdb/bolt"
)

/*
DataStore adds composite key support to BoltDB
*/
type DataStore struct {
	*bolt.DB
}

/*
Store places the value in the datastore. The collection key is used for the root bucket
of the storage while the composite key is used as the path to the final storage loaction
*/
func (ds DataStore) Store(collection []byte, compositeKey [][]byte, value []byte) error {

	err := ds.Update(func(tx *bolt.Tx) error {
		return TxStore(tx, collection, compositeKey, value)
	})

	return err
}

/*
TxStore places the value in the datastore that tx came from. The collection key is used
for the root bucket of the storage while the composite key is used as the path to the
final storage loaction
*/
func TxStore(tx *bolt.Tx, collection []byte, compositeKey [][]byte, value []byte) error {
	b, err := tx.CreateBucketIfNotExists(collection)
	lastKeyIndex := len(compositeKey) - 1
	for i := 0; err == nil && i < lastKeyIndex; i += 1 {
		b, err = b.CreateBucketIfNotExists(compositeKey[i])
	}
	if err == nil {
		err = b.Put(compositeKey[lastKeyIndex], value)
	}
	return err
}

/*
Index records the composite key associated with the value for use in index queries
*/
func (ds DataStore) Index(collection, index, value []byte, compositeKey [][]byte) error {
	err := ds.Update(func(tx *bolt.Tx) error {
		return TxIndex(tx, collection, index, value, compositeKey)
	})

	return err
}

/*
TxDeleteIndex deletes the specified index
*/
func (ds DataStore) DeleteIndex(collection, index []byte) error {
	err := ds.Update(func(tx *bolt.Tx) error {
		return TxDeleteIndex(tx, collection, index)
	})
	return err
}

/*
createIndexKey creates a key used to access the bucket that stores index values
*/
func createIndexKey(collection, index []byte) []byte {
	indexKey := make([]byte, 0, len(collection)+len(index))
	indexKey = append(indexKey, collection...)
	return append(indexKey, index...)
}

/*
TxDeleteIndex deletes the specified index in the given transaction
*/
func TxDeleteIndex(tx *bolt.Tx, collection, index []byte) error {
	indexKey := createIndexKey(collection, index)
	return tx.DeleteBucket(indexKey)
}

/*
Index records the composite key associated with the value for use in index queries
*/
func TxIndex(tx *bolt.Tx, collection, index, value []byte, compositeKey [][]byte) error {
	indexKey := createIndexKey(collection, index)
	b, err := tx.CreateBucketIfNotExists(indexKey)
	if err == nil {
		var serializedKey []byte
		var serializedSet []byte
		serializedKey = SerializeComposite(compositeKey)
		if err == nil {
			serializedSet, err = addToSetValue(b.Get(value), serializedKey)
			if err == nil {
				b.Put(value, serializedSet)
			}
		}
	}

	return err
}

/*
addToSetValue adds value to a serialized set
a serialized version of the updated set is returned
*/
func addToSetValue(serializedSet, value []byte) ([]byte, error) {
	var err error
	var rval []byte
	var set map[string]bool
	var keys []string
	if serializedSet == nil {
		set = make(map[string]bool)
	} else {
		set, keys, err = deserializeSet(serializedSet)
	}
	if err == nil {
		valStr := string(value)
		_, exists := set[valStr]
		if !exists {
			set[valStr] = true
			keys = append(keys, valStr)
		}
		rval = serializeSet(set, keys)
	}

	return rval, err
}

/*
serializeSet serializes an ordered set to bytes
*/
func serializeSet(set map[string]bool, keys []string) []byte {
	sizeBuff := make([]byte, 8)
	var buff bytes.Buffer
	for _, keyStr := range keys {
		key := []byte(keyStr)
		size := len(key)
		buff.Grow(size + 8)
		binary.PutVarint(sizeBuff, int64(size))
		buff.Write(sizeBuff)
		buff.Write(key)
	}
	return buff.Bytes()
}

/*
deserializeSet deserializes an ordered set from bytes
the set is returned as a map along with the key ordering
*/
func deserializeSet(serialized []byte) (map[string]bool, []string, error) {
	var err error
	rval := make(map[string]bool)
	var keys []string
	sizeBuff := make([]byte, 8)
	buff := bytes.NewBuffer(serialized)
	count := 0
	for count < len(serialized) {
		var length int
		length, err = buff.Read(sizeBuff)
		if err == nil {
			count += length
			size, _ := binary.Varint(sizeBuff)
			key := make([]byte, size)
			length, err = buff.Read(key)
			count += length
			keyStr := string(key)
			_, exists := rval[keyStr]
			if !exists {
				keys = append(keys, keyStr)
				rval[keyStr] = true
			}
		}
	}
	return rval, keys, err
}

/*
serializeComposite serializes the composite key to be stored in an index
*/
func SerializeComposite(compositeKey [][]byte) []byte {
	sizeBuff := make([]byte, 8)
	var buff bytes.Buffer
	for _, key := range compositeKey {
		size := len(key)
		buff.Grow(size + 8)
		binary.PutVarint(sizeBuff, int64(size))
		buff.Write(sizeBuff)
		buff.Write(key)
	}
	return buff.Bytes()
}

/*
deserializeComposite deserializes a composite key that was stored in an index
*/
func DeserializeComposite(serialized []byte) ([][]byte, error) {
	var err error
	var rval [][]byte
	sizeBuff := make([]byte, 8)
	buff := bytes.NewBuffer(serialized)
	count := 0
	for count < len(serialized) {
		var length int
		length, err = buff.Read(sizeBuff)
		if err == nil {
			count += length
			size, _ := binary.Varint(sizeBuff)
			key := make([]byte, size)
			length, err = buff.Read(key)
			count += length
			rval = append(rval, key)
		}
	}
	return rval, err
}

/*
qState is composed of data that represents the state at a specific level of a query
*/
type qState struct {
	b     *bolt.Bucket
	terms []*Term
	level int
}

/*
next gets the next state level for the query
*/
func (s qState) next(b *bolt.Bucket) qState {
	return qState{b, s.terms, s.level + 1}
}

/*
TxQuery executes the query in the provided transaction
*/
func TxQuery(tx *bolt.Tx, query *Query) ([][]byte, error) {
	var rval [][]byte
	var err error

	b := tx.Bucket(query.Collection)
	if b != nil {
		state := qState{b, query.Terms, 0}
		rval, err = getByTerm(state, rval)
	}

	return rval, err
}

/*
getByTerm recursively collects query results accoring to the term of the current query level
*/
func getByTerm(state qState, results [][]byte) ([][]byte, error) {
	var err error
	var term *Term
	if state.level >= len(state.terms) {
		term = Any()
	} else {
		term = state.terms[state.level]
	}

	switch term.op {
	case ANY:
		results, err = getAny(state, results)
	case EQ:
		results, err = getEq(state, results)
	case RANGE:
		results, err = getRange(state, results)
	case IN:
		results, err = getIn(state, results)
	case NOT:
		results, err = getNot(state, results)
	}

	return results, err
}

/*
getVal gets the results stored at the key/val pair which could be a bucket or item.
A nil val signifies that key points to nested bucket
*/
func getVal(state qState, key, val []byte, results [][]byte) ([][]byte, error) {
	var err error
	if val != nil {
		results = append(results, val)
	} else {
		nested := state.b.Bucket(key)
		results, err = getByTerm(state.next(nested), results)
	}
	return results, err
}

/*
getAny gets all results at the current level of the query
*/
func getAny(state qState, results [][]byte) ([][]byte, error) {
	var err error
	c := state.b.Cursor()
	for key, val := c.First(); key != nil; key, val = c.Next() {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getEq get results for a specific key at the current level
*/
func getEq(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]
	queryKey := term.keys[0]
	c := state.b.Cursor()
	key, val := c.Seek(queryKey)

	if bytes.Equal(key, queryKey) {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getRange gets results for a range of keys at the current level
*/
func getRange(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]
	lower := term.keys[0]
	upper := term.keys[1]
	c := state.b.Cursor()
	limit, _ := c.Seek(upper)
	key, val := c.Seek(lower)
	for ; key != nil && !bytes.Equal(key, limit); key, val = c.Next() {
		results, err = getVal(state, key, val, results)
	}

	return results, err
}

/*
getIn gets results for a set of keys at the curent level
*/
func getIn(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]

	c := state.b.Cursor()
	for _, queryKey := range term.keys {
		key, val := c.Seek(queryKey)

		if bytes.Equal(key, queryKey) {
			results, err = getVal(state, key, val, results)
		}
	}

	return results, err
}

/*
getNot gets results at the current level whose keys are not in the term
*/
func getNot(state qState, results [][]byte) ([][]byte, error) {
	var err error

	term := state.terms[state.level]

	c := state.b.Cursor()
	for key, val := c.First(); key != nil; key, val = c.Next() {
		if !in(key, term.keys) {
			results, err = getVal(state, key, val, results)
		}
	}

	return results, err
}

/*
in returns true if val is in arr
*/
func in(val []byte, arr [][]byte) bool {
	rval := false
	for _, entry := range arr {
		if bytes.Equal(val, entry) {
			rval = true
			break
		}
	}
	return rval
}

/*
TxIndexMatchAny queries for items using the given index and the values that were indexed
*/
func TxIndexMatchAny(tx *bolt.Tx, collection, index []byte, values ...[]byte) ([][]byte, error) {
	var err error
	var rval [][]byte

	indexKey := createIndexKey(collection, index)
	b := tx.Bucket(indexKey)

	if b != nil {
		for i := 0; err == nil && i < len(values); i += 1 {
			value := values[i]
			serializedSet := b.Get(value)
			if serializedSet != nil {
				var keys []string
				_, keys, err = deserializeSet(serializedSet)
				if err == nil {
					for _, keyStr := range keys {
						key := []byte(keyStr)
						compositeKey, err := DeserializeComposite(key)
						if err != nil {
							return rval, err
						}
						rval, err = collectResults(tx, collection, compositeKey, rval)
						if err != nil {
							return rval, err
						}
					}
				}
			}
		}
	}

	return rval, err
}

/*
TxIndexMatchAll queries for items using the given index and the values that were indexed
*/
func TxIndexMatchAll(tx *bolt.Tx, collection, index []byte, values ...[]byte) ([][]byte, error) {
	var err error
	var rval [][]byte

	indexKey := createIndexKey(collection, index)
	b := tx.Bucket(indexKey)

	if b != nil {
		var matched map[string]bool
		for i := 0; err == nil && i < len(values); i += 1 {
			value := values[i]
			serializedSet := b.Get(value)
			if serializedSet != nil {
				var set map[string]bool
				set, _, err = deserializeSet(serializedSet)
				if err == nil {
					if matched == nil {
						matched = set
					} else {
						for key, _ := range matched {
							inSet, _ := set[key]
							if !inSet {
								matched[key] = false
							}
						}
					}
				}
			}
		}
		for keyStr, inSet := range matched {
			if err == nil && inSet {
				key := []byte(keyStr)
				compositeKey, err := DeserializeComposite(key)
				if err == nil {
					rval, err = collectResults(tx, collection, compositeKey, rval)
				}
			}
		}
	}

	return rval, err
}

/*
collectResults appends values found at composite key to index query results
*/
func collectResults(tx *bolt.Tx, collection []byte, compositeKey, results [][]byte) ([][]byte, error) {
	var err error

	q := NewQuery(collection, EqAll(compositeKey)...)
	res, err := TxQuery(tx, q)
	for j := 0; err == nil && j < len(res); j += 1 {
		results = append(results, res...)
	}

	return results, err
}

/* TODO support query deletes */

func TxDelete(tx *bolt.Tx, collection []byte, keys ...[]byte) (err error) {
	b := tx.Bucket(collection)
	if b != nil {
		err = recursiveDelete(b, keys)
	}
	return
}

func recursiveDelete(b *bolt.Bucket, keys [][]byte) (err error) {
	numKeys := len(keys)
	if b == nil || numKeys < 1 {
		return
	}
	c := b.Cursor()
	k, v := c.Seek(keys[0])
	if k != nil && bytes.Equal(k, keys[0]) {
		/* return key matched, we found something */
		if v == nil {
			/* nil value signals that key is nested bucket */
			if numKeys == 1 {
				/* no more keys to filter on, prune bucket tree */
				err = b.DeleteBucket(k)
			} else {
				nextBucket := b.Bucket(k)
				err = recursiveDelete(nextBucket, keys[1:])
			}
		} else {
			if numKeys == 1 {
				err = b.Delete(k)
			} else {
				err = fmt.Errorf("More keys specified than nested buckets available")
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Broadcast()
		this.ccond.L.Unlock()
		time.Sleep(5 * time.Microsecond)
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			//fmt.Println("read wait")
			this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Broadcast()
		this.pcond.L.Unlock()
		time.Sleep(5 * time.Microsecond)
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			//fmt.Println("write wait")
			this.ccond.Broadcast()
			this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
		} else {
			break
		}

	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		b := make([]byte, 5)
		n, err := r.Read(b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		total_tmp := remlen_tmp + int64(1) + int64(m)

		write_bytes = make([]byte, 0, total_tmp)
		write_bytes = append(write_bytes, b[0:m+1]...)
		nlen := int64(0)
		times := 0
		cnt_ := 32
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			len := int64(cnt_)
			if tmpm < int64(cnt_) {
				len = tmpm
			}
			//var b_ []byte
			/*if tmpm < int64(cnt_) {
				b_ = make([]byte, tmpm)
			} else {
				b_ = make([]byte, cnt_)
			}*/

			//b_ := make([]byte, remlen)
			//n, err = r.Read(b_[0:])

			n, err = r.Read(write_bytes[total_tmp-tmpm : total_tmp-tmpm+len])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>修改 buffer.go process.go sendrecv.go buffer modify sleep 5 Microsecond modify buffre.go readfrom sleep 5毫秒 n, err = r.Read(write_bytes[total_tmp-tmpm : total_tmp-tmpm+len])<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	this.ccond.L.Lock()
	defer func() {
		this.pcond.Broadcast()
		this.ccond.L.Unlock()
		time.Sleep(5 * time.Microsecond)
	}()
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			//fmt.Println("read wait")
			this.pcond.Broadcast()
			this.ccond.Wait()
		} else {
			break
		}
	}
	index := readIndex & this.mask //替代求模
	p = this.buf[index]
	this.buf[index] = nil
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	this.pcond.L.Lock()
	defer func() {
		this.ccond.Broadcast()
		this.pcond.L.Unlock()
		time.Sleep(5 * time.Microsecond)
	}()
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			//fmt.Println("write wait")
			this.ccond.Broadcast()
			this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
		} else {
			break
		}

	}
	index := writeIndex & this.mask //替代求模
	this.buf[index] = in
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		b := make([]byte, 5)
		n, err := r.Read(b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		total_tmp := remlen_tmp + int64(1) + int64(m)

		write_bytes = make([]byte, 0, total_tmp)
		write_bytes = append(write_bytes, b[0:m+1]...)
		nlen := int64(0)
		times := 0
		cnt_ := 32
		start := int64(1) + int64(m)
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			len := int64(cnt_)
			if tmpm < int64(cnt_) {
				len = tmpm
			}
			//var b_ []byte
			/*if tmpm < int64(cnt_) {
				b_ = make([]byte, tmpm)
			} else {
				b_ = make([]byte, cnt_)
			}*/

			//b_ := make([]byte, remlen)
			//n, err = r.Read(b_[0:])

			n, err = r.Read(write_bytes[start+nlen : start+nlen+len])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"runtime"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond

	b []byte //readfrom中的临时数组，为反复使用不必创建新数组
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		b:          make([]byte, 5),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	//this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	//this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	/*this.ccond.L.Lock()
	defer func() {
		//this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
		//time.Sleep(3 * time.Millisecond)
	}()*/
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			//this.pcond.Signal()
			//this.pcond.Broadcast()
			//this.ccond.Wait()
			//runtime.Gosched()
			time.Sleep(5 * time.Millisecond)
		} else {
			break
		}
		//time.Sleep(500 * time.Microsecond)
		//time.Sleep(1 * time.Millisecond)
	}
	index := readIndex & this.mask //替代求模
	this.ccond.L.Lock()
	p = this.buf[index]
	this.buf[index] = nil
	this.ccond.L.Unlock()
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	/*this.pcond.L.Lock()
	defer func() {
		//this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
		//time.Sleep(3 * time.Millisecond)
	}()*/
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			//this.ccond.Signal()
			//this.ccond.Broadcast()
			//this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
			//runtime.Gosched()
			time.Sleep(5 * time.Millisecond)
		} else {
			break
		}
		//time.Sleep(500 * time.Microsecond)
		//time.Sleep(1 * time.Millisecond)
	}
	index := writeIndex & this.mask //替代求模
	this.pcond.L.Lock()
	this.buf[index] = in
	this.pcond.L.Unlock()
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

/**
修改 尽量减少数据的创建
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		//b := make([]byte, 5)
		n, err := r.Read(this.b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(this.b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if this.b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(this.b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		start_ := int64(1) + int64(m)
		total_tmp := remlen_tmp + start_

		write_bytes = make([]byte, total_tmp)
		copy(write_bytes[0:m+1], this.b[0:m+1])
		nlen := int64(0)
		times := 0
		cnt_ := int64(32)
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			b_ := write_bytes[(start_ + nlen):]
			if tmpm > cnt_ {
				b_ = write_bytes[(start_ + nlen):(start_ + nlen + cnt_)]
			}

			//b_ := make([]byte, remlen)
			n, err = r.Read(b_[0:])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<commit_msg>修改 buffer.go process.go sendrecv.go xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 读写时修改锁定方式<commit_after>// Copyright (c) 2014 The SurgeMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package service

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"
)

var (
	bufcnt            int64
	DefaultBufferSize int64

	DeviceInBufferSize  int64
	DeviceOutBufferSize int64

	MasterInBufferSize  int64
	MasterOutBufferSize int64
)

type sequence struct {
	// The current position of the producer or consumer
	cursor,

	// The previous known position of the consumer (if producer) or producer (if consumer)
	gate,

	// These are fillers to pad the cache line, which is generally 64 bytes
	p2, p3, p4, p5, p6, p7 int64
}

func newSequence() *sequence {
	return &sequence{}
}

func (this *sequence) get() int64 {
	return atomic.LoadInt64(&this.cursor)
}

func (this *sequence) set(seq int64) {
	atomic.StoreInt64(&this.cursor, seq)
}

type buffer struct {
	id int64

	readIndex  int64 //读序号
	writeIndex int64 //写序号
	buf        []*[]byte

	size int64
	mask int64

	done int64

	pcond *sync.Cond
	ccond *sync.Cond

	b []byte //readfrom中的临时数组，为反复使用不必创建新数组
}

func newBuffer(size int64) (*buffer, error) {
	if size < 0 {
		return nil, bufio.ErrNegativeCount
	}

	if size == 0 {
		size = DefaultBufferSize
	}

	if !powerOfTwo64(size) {
		fmt.Printf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
		return nil, fmt.Errorf("Size must be power of two. Try %d.", roundUpPowerOfTwo64(size))
	}

	return &buffer{
		id:         atomic.AddInt64(&bufcnt, 1),
		readIndex:  int64(0),
		writeIndex: int64(0),
		buf:        make([]*[]byte, size),
		size:       size,
		mask:       size - 1,
		pcond:      sync.NewCond(new(sync.Mutex)),
		ccond:      sync.NewCond(new(sync.Mutex)),
		b:          make([]byte, 5),
	}, nil
}

/**
获取当前读序号
*/
func (this *buffer) GetCurrentReadIndex() int64 {
	return atomic.LoadInt64(&this.readIndex)
}

/**
获取当前写序号
*/
func (this *buffer) GetCurrentWriteIndex() int64 {
	return atomic.LoadInt64(&this.writeIndex)
}

func (this *buffer) ID() int64 {
	return this.id
}

func (this *buffer) Close() error {
	atomic.StoreInt64(&this.done, 1)

	this.pcond.L.Lock()
	//this.ccond.Signal()
	//this.ccond.Broadcast()
	this.pcond.L.Unlock()

	this.ccond.L.Lock()
	//this.pcond.Signal()
	//this.pcond.Broadcast()
	this.ccond.L.Unlock()

	return nil
}

/**
读取ringbuffer指定的buffer指针，返回该指针并清空ringbuffer该位置存在的指针内容，以及将读序号加1
*/
func (this *buffer) ReadBuffer() (p *[]byte, ok bool) {
	/*this.ccond.L.Lock()
	defer func() {
		//this.pcond.Signal()
		//this.pcond.Broadcast()
		this.ccond.L.Unlock()
		//time.Sleep(3 * time.Millisecond)
	}()*/
	ok = false
	p = nil
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return nil, false
		}
		writeIndex = this.GetCurrentWriteIndex()
		if readIndex >= writeIndex {
			//this.pcond.Signal()
			//this.pcond.Broadcast()
			//this.ccond.Wait()
			//runtime.Gosched()
			time.Sleep(5 * time.Millisecond)
		} else {
			break
		}
		//time.Sleep(500 * time.Microsecond)
		//time.Sleep(1 * time.Millisecond)
	}
	index := readIndex & this.mask //替代求模
	this.ccond.L.Lock()
	p = this.buf[index]
	this.buf[index] = nil
	this.ccond.L.Unlock()
	atomic.AddInt64(&this.readIndex, int64(1))
	if p != nil {
		ok = true
	}
	return p, ok
}

/**
写入ringbuffer指针，以及将写序号加1
*/
func (this *buffer) WriteBuffer(in *[]byte) (ok bool) {
	/*this.pcond.L.Lock()
	defer func() {
		//this.ccond.Signal()
		//this.ccond.Broadcast()
		this.pcond.L.Unlock()
		//time.Sleep(3 * time.Millisecond)
	}()*/
	ok = false
	readIndex := this.GetCurrentReadIndex()
	writeIndex := this.GetCurrentWriteIndex()
	for {
		if this.isDone() {
			return false
		}
		readIndex = this.GetCurrentReadIndex()
		if writeIndex >= readIndex && writeIndex-readIndex >= this.size {
			//this.ccond.Signal()
			//this.ccond.Broadcast()
			//this.pcond.Wait()
			//time.Sleep(1 * time.Millisecond)
			//runtime.Gosched()
			time.Sleep(5 * time.Millisecond)
		} else {
			break
		}
		//time.Sleep(500 * time.Microsecond)
		//time.Sleep(1 * time.Millisecond)
	}
	index := writeIndex & this.mask //替代求模
	this.pcond.L.Lock()
	this.buf[index] = in
	this.pcond.L.Unlock()
	atomic.AddInt64(&this.writeIndex, int64(1))
	ok = true
	return ok
}

/**
修改 尽量减少数据的创建
*/
func (this *buffer) ReadFrom(r io.Reader) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		time.Sleep(5 * time.Millisecond)
		if this.isDone() {
			return total, io.EOF
		}

		var write_bytes []byte

		//b := make([]byte, 5)
		n, err := r.Read(this.b[0:1])
		if err != nil {
			return total, io.EOF
		}
		total += int64(n)
		max_cnt := 1
		for {
			if this.isDone() {
				return total, io.EOF
			}
			// If we have read 5 bytes and still not done, then there's a problem.
			if max_cnt > 4 {
				return 0, fmt.Errorf("sendrecv/peekMessageSize: 4th byte of remaining length has continuation bit set")
			}
			_, err := r.Read(this.b[max_cnt:(max_cnt + 1)])

			//fmt.Println(b)
			if err != nil {
				return total, err
			}
			if this.b[max_cnt] >= 0x80 {
				max_cnt++
			} else {
				break
			}
		}
		remlen, m := binary.Uvarint(this.b[1 : max_cnt+1])
		remlen_tmp := int64(remlen)
		start_ := int64(1) + int64(m)
		total_tmp := remlen_tmp + start_

		write_bytes = make([]byte, total_tmp)
		copy(write_bytes[0:m+1], this.b[0:m+1])
		nlen := int64(0)
		times := 0
		cnt_ := int64(32)
		for nlen < remlen_tmp {
			if this.isDone() {
				return total, io.EOF
			}
			if times > 100 {
				return total, io.EOF
			} else {
				times = 0
			}
			times++
			tmpm := remlen_tmp - nlen

			b_ := write_bytes[(start_ + nlen):]
			if tmpm > cnt_ {
				b_ = write_bytes[(start_ + nlen):(start_ + nlen + cnt_)]
			}

			//b_ := make([]byte, remlen)
			n, err = r.Read(b_[0:])

			if err != nil {
				/*Log.Errorc(func() string {
					return fmt.Sprintf("从conn读取数据失败(%s)(0)", err)
				})
				time.Sleep(5 * time.Millisecond)
				continue*/
				return total, err
			}
			//write_bytes = append(write_bytes, b_[0:]...)
			nlen += int64(n)
			total += int64(n)
		}

		ok := this.WriteBuffer(&write_bytes)

		if !ok {
			return total, errors.New("write ringbuffer failed")
		}
	}
}

func (this *buffer) WriteTo(w io.Writer) (int64, error) {
	defer this.Close()

	total := int64(0)

	for {
		if this.isDone() {
			return total, io.EOF
		}

		p, ok := this.ReadBuffer()
		if !ok {
			return total, errors.New("read buffer failed")
		}
		// There's some data, let's process it first
		if len(*p) > 0 {
			n, err := w.Write(*p)
			total += int64(n)
			//Log.Debugc(func() string{ return fmt.Sprintf("Wrote %d bytes, totaling %d bytes", n, total)})

			if err != nil {
				return total, err
			}
		}

	}
}

func (this *buffer) isDone() bool {
	if atomic.LoadInt64(&this.done) == 1 {
		return true
	}

	return false
}

func ringCopy(dst, src []byte, start int64) int {
	n := len(src)

	i, l := 0, 0

	for n > 0 {
		l = copy(dst[start:], src[i:])
		i += l
		n -= l

		if n > 0 {
			start = 0
		}
	}

	return i
}

func powerOfTwo64(n int64) bool {
	return n != 0 && (n&(n-1)) == 0
}

func roundUpPowerOfTwo64(n int64) int64 {
	n--
	n |= n >> 1
	n |= n >> 2
	n |= n >> 4
	n |= n >> 8
	n |= n >> 16
	n |= n >> 32
	n++

	return n
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package writer

import (
	"time"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/quad"
)

func init() {
	graph.RegisterWriter("single", NewSingleReplication)
}

type Single struct {
	currentID  graph.PrimaryKey
	qs         graph.QuadStore
	ignoreOpts graph.IgnoreOpts
}

func NewSingleReplication(qs graph.QuadStore, opts graph.Options) (graph.QuadWriter, error) {
	var (
		ignoreMissing   bool
		ignoreDuplicate bool
		err             error
	)

	if graph.IgnoreMissing {
		ignoreMissing = true
	} else {
		ignoreMissing, _, err = opts.BoolKey("ignore_missing")
		if err != nil {
			return nil, err
		}
	}

	if graph.IgnoreDuplicates {
		ignoreDuplicate = true
	} else {
		ignoreDuplicate, _, err = opts.BoolKey("ignore_duplicate")
		if err != nil {
			return nil, err
		}
	}

	return &Single{
		currentID: qs.Horizon(),
		qs:        qs,
		ignoreOpts: graph.IgnoreOpts{
			IgnoreDup:     ignoreDuplicate,
			IgnoreMissing: ignoreMissing,
		},
	}, nil
}

func (s *Single) AddQuad(q quad.Quad) error {
	deltas := make([]graph.Delta, 1)
	deltas[0] = graph.Delta{
		ID:        s.currentID.Next(),
		Quad:      q,
		Action:    graph.Add,
		Timestamp: time.Now(),
	}
	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

func (s *Single) AddQuadSet(set []quad.Quad) error {
	deltas := make([]graph.Delta, len(set))
	for i, q := range set {
		deltas[i] = graph.Delta{
			ID:        s.currentID.Next(),
			Quad:      q,
			Action:    graph.Add,
			Timestamp: time.Now(),
		}
	}

	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

func (s *Single) RemoveQuad(q quad.Quad) error {
	deltas := make([]graph.Delta, 1)
	deltas[0] = graph.Delta{
		ID:        s.currentID.Next(),
		Quad:      q,
		Action:    graph.Delete,
		Timestamp: time.Now(),
	}
	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

// RemoveNode removes all quads with the given value
func (s *Single) RemoveNode(v quad.Value) error {
	gv := s.qs.ValueOf(v)
	if gv == nil {
		return nil
	}
	var deltas []graph.Delta
	// TODO(dennwc): QuadStore may remove node without iterations. Consider optional interface for this.
	for _, d := range []quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label} {
		it := s.qs.QuadIterator(d, gv)
		for it.Next() {
			deltas = append(deltas, graph.Delta{
				ID:        s.currentID.Next(),
				Quad:      s.qs.Quad(it.Result()),
				Action:    graph.Delete,
				Timestamp: time.Now(),
			})
		}
		it.Close()
	}
	return s.qs.ApplyDeltas(deltas, graph.IgnoreOpts{IgnoreMissing: true})
}

func (s *Single) Close() error {
	// Nothing to clean up locally.
	return nil
}

func (s *Single) ApplyTransaction(t *graph.Transaction) error {
	ts := time.Now()
	for i := 0; i < len(t.Deltas); i++ {
		t.Deltas[i].ID = s.currentID.Next()
		t.Deltas[i].Timestamp = ts
	}
	return s.qs.ApplyDeltas(t.Deltas, s.ignoreOpts)
}
<commit_msg>writer: add a direct constructor for single replication<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package writer

import (
	"time"

	"github.com/cayleygraph/cayley/graph"
	"github.com/cayleygraph/cayley/quad"
)

func init() {
	graph.RegisterWriter("single", NewSingleReplication)
}

type Single struct {
	currentID  graph.PrimaryKey
	qs         graph.QuadStore
	ignoreOpts graph.IgnoreOpts
}

func NewSingle(qs graph.QuadStore, opts graph.IgnoreOpts) (graph.QuadWriter, error) {
	return &Single{
		currentID:  qs.Horizon(),
		qs:         qs,
		ignoreOpts: opts,
	}, nil
}

func NewSingleReplication(qs graph.QuadStore, opts graph.Options) (graph.QuadWriter, error) {
	var (
		ignoreMissing   bool
		ignoreDuplicate bool
		err             error
	)

	if graph.IgnoreMissing {
		ignoreMissing = true
	} else {
		ignoreMissing, _, err = opts.BoolKey("ignore_missing")
		if err != nil {
			return nil, err
		}
	}

	if graph.IgnoreDuplicates {
		ignoreDuplicate = true
	} else {
		ignoreDuplicate, _, err = opts.BoolKey("ignore_duplicate")
		if err != nil {
			return nil, err
		}
	}

	return NewSingle(qs, graph.IgnoreOpts{
		IgnoreMissing: ignoreMissing,
		IgnoreDup:     ignoreDuplicate,
	})
}

func (s *Single) AddQuad(q quad.Quad) error {
	deltas := make([]graph.Delta, 1)
	deltas[0] = graph.Delta{
		ID:        s.currentID.Next(),
		Quad:      q,
		Action:    graph.Add,
		Timestamp: time.Now(),
	}
	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

func (s *Single) AddQuadSet(set []quad.Quad) error {
	deltas := make([]graph.Delta, len(set))
	for i, q := range set {
		deltas[i] = graph.Delta{
			ID:        s.currentID.Next(),
			Quad:      q,
			Action:    graph.Add,
			Timestamp: time.Now(),
		}
	}

	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

func (s *Single) RemoveQuad(q quad.Quad) error {
	deltas := make([]graph.Delta, 1)
	deltas[0] = graph.Delta{
		ID:        s.currentID.Next(),
		Quad:      q,
		Action:    graph.Delete,
		Timestamp: time.Now(),
	}
	return s.qs.ApplyDeltas(deltas, s.ignoreOpts)
}

// RemoveNode removes all quads with the given value
func (s *Single) RemoveNode(v quad.Value) error {
	gv := s.qs.ValueOf(v)
	if gv == nil {
		return nil
	}
	var deltas []graph.Delta
	// TODO(dennwc): QuadStore may remove node without iterations. Consider optional interface for this.
	for _, d := range []quad.Direction{quad.Subject, quad.Predicate, quad.Object, quad.Label} {
		it := s.qs.QuadIterator(d, gv)
		for it.Next() {
			deltas = append(deltas, graph.Delta{
				ID:        s.currentID.Next(),
				Quad:      s.qs.Quad(it.Result()),
				Action:    graph.Delete,
				Timestamp: time.Now(),
			})
		}
		it.Close()
	}
	return s.qs.ApplyDeltas(deltas, graph.IgnoreOpts{IgnoreMissing: true})
}

func (s *Single) Close() error {
	// Nothing to clean up locally.
	return nil
}

func (s *Single) ApplyTransaction(t *graph.Transaction) error {
	ts := time.Now()
	for i := 0; i < len(t.Deltas); i++ {
		t.Deltas[i].ID = s.currentID.Next()
		t.Deltas[i].Timestamp = ts
	}
	return s.qs.ApplyDeltas(t.Deltas, s.ignoreOpts)
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"io"
	"io/ioutil"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	"github.com/jbenet/go-ipfs/blocks"
	cmds "github.com/jbenet/go-ipfs/commands"
	u "github.com/jbenet/go-ipfs/util"
)

type Block struct {
	Key    string
	Length int
}

var blockCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Manipulate raw IPFS blocks",
		ShortDescription: `
'ipfs block' is a plumbing command used to manipulate raw ipfs blocks.
Reads from stdin or writes to stdout, and <key> is a base58 encoded
multihash.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"get": blockGetCmd,
		"put": blockPutCmd,
	},
}

var blockGetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Get a raw IPFS block",
		ShortDescription: `
'ipfs block get' is a plumbing command for retreiving raw ipfs blocks.
It outputs to stdout, and <key> is a base58 encoded multihash.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("key", true, false, "The base58 multihash of an existing block to get"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		key, ok := req.Arguments()[0].(string)
		if !ok {
			return nil, u.ErrCast()
		}

		if !u.IsValidHash(key) {
			return nil, cmds.Error{"Not a valid hash", cmds.ErrClient}
		}

		h, err := mh.FromB58String(key)
		if err != nil {
			return nil, err
		}

		k := u.Key(h)
		ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
		b, err := n.Blocks.GetBlock(ctx, k)
		if err != nil {
			return nil, err
		}
		log.Debugf("BlockGet key: '%q'", b.Key())

		return bytes.NewReader(b.Data), nil
	},
}

var blockPutCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Stores input as an IPFS block",
		ShortDescription: `
ipfs block put is a plumbing command for storing raw ipfs blocks.
It reads from stdin, and <key> is a base58 encoded multihash.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("data", true, false, "The data to be stored as an IPFS block").EnableStdin(),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		in, ok := req.Arguments()[0].(io.Reader)
		if !ok {
			return nil, u.ErrCast()
		}

		data, err := ioutil.ReadAll(in)
		if err != nil {
			return nil, err
		}

		b := blocks.NewBlock(data)
		log.Debugf("BlockPut key: '%q'", b.Key())

		k, err := n.Blocks.AddBlock(b)
		if err != nil {
			return nil, err
		}

		return &Block{
			Key:    k.String(),
			Length: len(data),
		}, nil
	},
	Type: &Block{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			block := res.Output().(*Block)
			return []byte(block.Key + "\n"), nil
		},
	},
}
<commit_msg>core/commands2: Fixed 'block' command for req.Files() API<commit_after>package commands

import (
	"bytes"
	"io/ioutil"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	"github.com/jbenet/go-ipfs/blocks"
	cmds "github.com/jbenet/go-ipfs/commands"
	u "github.com/jbenet/go-ipfs/util"
)

type Block struct {
	Key    string
	Length int
}

var blockCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Manipulate raw IPFS blocks",
		ShortDescription: `
'ipfs block' is a plumbing command used to manipulate raw ipfs blocks.
Reads from stdin or writes to stdout, and <key> is a base58 encoded
multihash.
`,
	},

	Subcommands: map[string]*cmds.Command{
		"get": blockGetCmd,
		"put": blockPutCmd,
	},
}

var blockGetCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Get a raw IPFS block",
		ShortDescription: `
'ipfs block get' is a plumbing command for retreiving raw ipfs blocks.
It outputs to stdout, and <key> is a base58 encoded multihash.
`,
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("key", true, false, "The base58 multihash of an existing block to get"),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		key, ok := req.Arguments()[0].(string)
		if !ok {
			return nil, u.ErrCast()
		}

		if !u.IsValidHash(key) {
			return nil, cmds.Error{"Not a valid hash", cmds.ErrClient}
		}

		h, err := mh.FromB58String(key)
		if err != nil {
			return nil, err
		}

		k := u.Key(h)
		ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
		b, err := n.Blocks.GetBlock(ctx, k)
		if err != nil {
			return nil, err
		}
		log.Debugf("BlockGet key: '%q'", b.Key())

		return bytes.NewReader(b.Data), nil
	},
}

var blockPutCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Stores input as an IPFS block",
		ShortDescription: `
ipfs block put is a plumbing command for storing raw ipfs blocks.
It reads from stdin, and <key> is a base58 encoded multihash.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("data", true, false, "The data to be stored as an IPFS block").EnableStdin(),
	},
	Run: func(req cmds.Request) (interface{}, error) {
		n, err := req.Context().GetNode()
		if err != nil {
			return nil, err
		}

		file, err := req.Files().NextFile()
		if err != nil {
			return nil, err
		}

		data, err := ioutil.ReadAll(file)
		if err != nil {
			return nil, err
		}

		err = file.Close()
		if err != nil {
			return nil, err
		}

		b := blocks.NewBlock(data)
		log.Debugf("BlockPut key: '%q'", b.Key())

		k, err := n.Blocks.AddBlock(b)
		if err != nil {
			return nil, err
		}

		return &Block{
			Key:    k.String(),
			Length: len(data),
		}, nil
	},
	Type: &Block{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) ([]byte, error) {
			block := res.Output().(*Block)
			return []byte(block.Key + "\n"), nil
		},
	},
}
<|endoftext|>


<commit_before>// Package errcheck is the library used to implement the errcheck command-line tool.
//
// Note: The API of this package has not been finalized and may change at any point.
package errcheck

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"regexp"

	"code.google.com/p/go.tools/go/types"
	"honnef.co/go/importer"
)

var (
	// ErrNoGoFiles is returned when CheckPackage is run on a package with no Go source files
	ErrNoGoFiles = errors.New("package contains no go source files")
)

// UncheckedErrors is returned from the CheckPackage function if the package contains
// any unchecked errors.
type UncheckedErrors struct {
	// Errors is a list of all the unchecked errors in the package.
	// Printing an error reports its position within the file and the contents of the line.
	Errors []error
}

func (e UncheckedErrors) Error() string {
	return fmt.Sprintf("%d unchecked errors", len(e.Errors))
}

// CheckPackage checks a package at pkgPath for errors.
// ignore is a map of package names to regular expressions. Identifiers from a package are
// checked against its regular expressions and if any of the expressions match the call
// is not checked.
// If blank is true then assignments to the blank identifier are also considered to be
// ignored errors.
func CheckPackage(pkgPath string, ignore map[string]*regexp.Regexp, blank bool) error {
	pkg, err := newPackage(pkgPath)
	if err != nil {
		return err
	}

	return checkPackage(pkg, ignore, blank)
}

// package_ represents a single Go package
type package_ struct {
	path     string
	fset     *token.FileSet
	astFiles []*ast.File
	files    map[string]file
}

// newPackage creates a package_ from the Go files in path
func newPackage(path string) (package_, error) {
	p := package_{path: path, fset: token.NewFileSet()}
	pkg, err := findPackage(path)
	if err != nil {
		return p, fmt.Errorf("could not find package: %s", err)
	}
	fileNames := getFiles(pkg)

	if len(fileNames) == 0 {
		return p, ErrNoGoFiles
	}

	p.astFiles = make([]*ast.File, len(fileNames))
	p.files = make(map[string]file, len(fileNames))

	for i, fileName := range fileNames {
		f, err := parseFile(p.fset, fileName)
		if err != nil {
			return p, fmt.Errorf("could not parse %s: %s", fileName, err)
		}
		p.files[fileName] = f
		p.astFiles[i] = f.ast
	}

	return p, nil
}

// typedPackage is like package_ but with type information
type typedPackage struct {
	package_
	callTypes map[ast.Expr]types.TypeAndValue
	identObjs map[*ast.Ident]types.Object
}

// typeCheck creates a typedPackage from a package_
func typeCheck(p package_) (typedPackage, error) {
	tp := typedPackage{
		package_:  p,
		callTypes: make(map[ast.Expr]types.TypeAndValue),
		identObjs: make(map[*ast.Ident]types.Object),
	}

	info := types.Info{
		Types:   tp.callTypes,
		Objects: tp.identObjs,
	}
	imp := importer.New()
	// Preliminary cgo support.
	// https://github.com/kisielk/errcheck/issues/16#issuecomment-26436917
	imp.Config = importer.Config{UseGcFallback: true}
	context := types.Config{Import: imp.Import}

	_, err := context.Check(p.path, p.fset, p.astFiles, &info)
	return tp, err
}

// file represents a single Go source file
type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

func parseFile(fset *token.FileSet, fileName string) (f file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}

// checker implements the errcheck algorithm
type checker struct {
	pkg    typedPackage
	ignore map[string]*regexp.Regexp
	blank  bool

	errors []error
}

type uncheckedError struct {
	pos  token.Position
	line []byte
}

func (e uncheckedError) Error() string {
	return fmt.Sprintf("%s\t%s", e.pos, e.line)
}

func (c *checker) ignoreCall(call *ast.CallExpr) bool {
	// Try to get an identifier.
	// Currently only supports simple expressions:
	//     1. f()
	//     2. x.y.f()
	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		// eg: *ast.SliceExpr, *ast.IndexExpr
	}

	if id == nil {
		return false
	}

	// If we got an identifier for the function, see if it is ignored

	if re, ok := c.ignore[""]; ok && re.MatchString(id.Name) {
		return true
	}

	if obj := c.pkg.identObjs[id]; obj != nil {
		if pkg := obj.Pkg(); pkg != nil {
			if re, ok := c.ignore[pkg.Path()]; ok {
				return re.MatchString(id.Name)
			}
		}
	}

	return false
}

// errorsByArg returns a slice s such that
// len(s) == number of return types of call
// s[i] == true iff return type at position i from left is an error type
func (c *checker) errorsByArg(call *ast.CallExpr) []bool {
	switch t := c.pkg.callTypes[call].Type.(type) {
	case *types.Named:
		// Single return
		return []bool{isErrorType(t.Obj())}
	case *types.Tuple:
		// Multiple returns
		s := make([]bool, t.Len())
		for i := 0; i < t.Len(); i++ {
			nt, ok := t.At(i).Type().(*types.Named)
			s[i] = ok && isErrorType(nt.Obj())
		}
		return s
	}
	return nil
}

func (c *checker) callReturnsError(call *ast.CallExpr) bool {
	if c.isRecover(call) {
		return true
	}
	for _, isError := range c.errorsByArg(call) {
		if isError {
			return true
		}
	}
	return false
}

// isRecover returns true if the given CallExpr is a call to the built-in recover() function.
func (c *checker) isRecover(call *ast.CallExpr) bool {
	if fun, ok := call.Fun.(*ast.Ident); ok {
		if _, ok := c.pkg.identObjs[fun].(*types.Builtin); ok {
			return fun.Name == "recover"
		}
	}
	return false
}

func (c *checker) addErrorAtPosition(position token.Pos) {
	pos := c.pkg.fset.Position(position)
	line := bytes.TrimSpace(c.pkg.files[pos.Filename].lines[pos.Line-1])
	c.errors = append(c.errors, uncheckedError{pos, line})
}

func (c *checker) Visit(node ast.Node) ast.Visitor {
	switch stmt := node.(type) {
	case *ast.ExprStmt:
		if call, ok := stmt.X.(*ast.CallExpr); ok {
			if !c.ignoreCall(call) && c.callReturnsError(call) {
				c.addErrorAtPosition(call.Lparen)
			}
		}
	case *ast.GoStmt:
		if !c.ignoreCall(stmt.Call) && c.callReturnsError(stmt.Call) {
			c.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.DeferStmt:
		if !c.ignoreCall(stmt.Call) && c.callReturnsError(stmt.Call) {
			c.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.AssignStmt:
		if !c.blank {
			break
		}
		if len(stmt.Rhs) == 1 {
			// single value on rhs; check against lhs identifiers
			if call, ok := stmt.Rhs[0].(*ast.CallExpr); ok {
				if c.ignoreCall(call) {
					break
				}
				isError := c.errorsByArg(call)
				for i := 0; i < len(stmt.Lhs); i++ {
					if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
						// We shortcut calls to recover() because errorsByArg can't
						// check its return types for errors since it returns interface{}.
						if id.Name == "_" && (c.isRecover(call) || isError[i]) {
							c.addErrorAtPosition(id.NamePos)
						}
					}
				}
			}
		} else {
			// multiple value on rhs; in this case a call can't return
			// multiple values. Assume len(stmt.Lhs) == len(stmt.Rhs)
			for i := 0; i < len(stmt.Lhs); i++ {
				if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
					if call, ok := stmt.Rhs[i].(*ast.CallExpr); ok {
						if c.ignoreCall(call) {
							continue
						}
						if id.Name == "_" && c.callReturnsError(call) {
							c.addErrorAtPosition(id.NamePos)
						}
					}
				}
			}
		}
	default:
	}
	return c
}

func checkPackage(pkg package_, ignore map[string]*regexp.Regexp, blank bool) error {
	tp, err := typeCheck(pkg)
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	visitor := &checker{tp, ignore, blank, []error{}}
	for _, astFile := range pkg.astFiles {
		ast.Walk(visitor, astFile)
	}

	if len(visitor.errors) > 0 {
		return UncheckedErrors{visitor.errors}
	}
	return nil
}

type obj interface {
	Pkg() *types.Package
	Name() string
}

func isErrorType(v obj) bool {
	return v.Pkg() == nil && v.Name() == "error"
}
<commit_msg>Update to new go/types API.<commit_after>// Package errcheck is the library used to implement the errcheck command-line tool.
//
// Note: The API of this package has not been finalized and may change at any point.
package errcheck

import (
	"bytes"
	"errors"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"regexp"

	"code.google.com/p/go.tools/go/types"
	"honnef.co/go/importer"
)

var (
	// ErrNoGoFiles is returned when CheckPackage is run on a package with no Go source files
	ErrNoGoFiles = errors.New("package contains no go source files")
)

// UncheckedErrors is returned from the CheckPackage function if the package contains
// any unchecked errors.
type UncheckedErrors struct {
	// Errors is a list of all the unchecked errors in the package.
	// Printing an error reports its position within the file and the contents of the line.
	Errors []error
}

func (e UncheckedErrors) Error() string {
	return fmt.Sprintf("%d unchecked errors", len(e.Errors))
}

// CheckPackage checks a package at pkgPath for errors.
// ignore is a map of package names to regular expressions. Identifiers from a package are
// checked against its regular expressions and if any of the expressions match the call
// is not checked.
// If blank is true then assignments to the blank identifier are also considered to be
// ignored errors.
func CheckPackage(pkgPath string, ignore map[string]*regexp.Regexp, blank bool) error {
	pkg, err := newPackage(pkgPath)
	if err != nil {
		return err
	}

	return checkPackage(pkg, ignore, blank)
}

// package_ represents a single Go package
type package_ struct {
	path     string
	fset     *token.FileSet
	astFiles []*ast.File
	files    map[string]file
}

// newPackage creates a package_ from the Go files in path
func newPackage(path string) (package_, error) {
	p := package_{path: path, fset: token.NewFileSet()}
	pkg, err := findPackage(path)
	if err != nil {
		return p, fmt.Errorf("could not find package: %s", err)
	}
	fileNames := getFiles(pkg)

	if len(fileNames) == 0 {
		return p, ErrNoGoFiles
	}

	p.astFiles = make([]*ast.File, len(fileNames))
	p.files = make(map[string]file, len(fileNames))

	for i, fileName := range fileNames {
		f, err := parseFile(p.fset, fileName)
		if err != nil {
			return p, fmt.Errorf("could not parse %s: %s", fileName, err)
		}
		p.files[fileName] = f
		p.astFiles[i] = f.ast
	}

	return p, nil
}

// typedPackage is like package_ but with type information
type typedPackage struct {
	package_
	callTypes map[ast.Expr]types.TypeAndValue
	defs      map[*ast.Ident]types.Object
	uses      map[*ast.Ident]types.Object
}

// typeCheck creates a typedPackage from a package_
func typeCheck(p package_) (typedPackage, error) {
	tp := typedPackage{
		package_:  p,
		callTypes: make(map[ast.Expr]types.TypeAndValue),
		defs:      make(map[*ast.Ident]types.Object),
		uses:      make(map[*ast.Ident]types.Object),
	}

	info := types.Info{
		Types: tp.callTypes,
		Defs:  tp.defs,
		Uses:  tp.uses,
	}
	imp := importer.New()
	// Preliminary cgo support.
	// https://github.com/kisielk/errcheck/issues/16#issuecomment-26436917
	imp.Config = importer.Config{UseGcFallback: true}
	context := types.Config{Import: imp.Import}

	_, err := context.Check(p.path, p.fset, p.astFiles, &info)
	return tp, err
}

// file represents a single Go source file
type file struct {
	fset  *token.FileSet
	name  string
	ast   *ast.File
	lines [][]byte
}

func parseFile(fset *token.FileSet, fileName string) (f file, err error) {
	rd, err := os.Open(fileName)
	if err != nil {
		return f, err
	}
	defer rd.Close()

	data, err := ioutil.ReadAll(rd)
	if err != nil {
		return f, err
	}

	astFile, err := parser.ParseFile(fset, fileName, bytes.NewReader(data), parser.ParseComments)
	if err != nil {
		return f, fmt.Errorf("could not parse: %s", err)
	}

	lines := bytes.Split(data, []byte("\n"))
	f = file{fset: fset, name: fileName, ast: astFile, lines: lines}
	return f, nil
}

// checker implements the errcheck algorithm
type checker struct {
	pkg    typedPackage
	ignore map[string]*regexp.Regexp
	blank  bool

	errors []error
}

type uncheckedError struct {
	pos  token.Position
	line []byte
}

func (e uncheckedError) Error() string {
	return fmt.Sprintf("%s\t%s", e.pos, e.line)
}

func (c *checker) ignoreCall(call *ast.CallExpr) bool {
	// Try to get an identifier.
	// Currently only supports simple expressions:
	//     1. f()
	//     2. x.y.f()
	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		// eg: *ast.SliceExpr, *ast.IndexExpr
	}

	if id == nil {
		return false
	}

	// If we got an identifier for the function, see if it is ignored

	if re, ok := c.ignore[""]; ok && re.MatchString(id.Name) {
		return true
	}

	if obj := c.pkg.uses[id]; obj != nil {
		if pkg := obj.Pkg(); pkg != nil {
			if re, ok := c.ignore[pkg.Path()]; ok {
				return re.MatchString(id.Name)
			}
		}
	}

	return false
}

// errorsByArg returns a slice s such that
// len(s) == number of return types of call
// s[i] == true iff return type at position i from left is an error type
func (c *checker) errorsByArg(call *ast.CallExpr) []bool {
	switch t := c.pkg.callTypes[call].Type.(type) {
	case *types.Named:
		// Single return
		return []bool{isErrorType(t.Obj())}
	case *types.Tuple:
		// Multiple returns
		s := make([]bool, t.Len())
		for i := 0; i < t.Len(); i++ {
			nt, ok := t.At(i).Type().(*types.Named)
			s[i] = ok && isErrorType(nt.Obj())
		}
		return s
	}
	return nil
}

func (c *checker) callReturnsError(call *ast.CallExpr) bool {
	if c.isRecover(call) {
		return true
	}
	for _, isError := range c.errorsByArg(call) {
		if isError {
			return true
		}
	}
	return false
}

// isRecover returns true if the given CallExpr is a call to the built-in recover() function.
func (c *checker) isRecover(call *ast.CallExpr) bool {
	if fun, ok := call.Fun.(*ast.Ident); ok {
		if _, ok := c.pkg.uses[fun].(*types.Builtin); ok {
			return fun.Name == "recover"
		}
	}
	return false
}

func (c *checker) addErrorAtPosition(position token.Pos) {
	pos := c.pkg.fset.Position(position)
	line := bytes.TrimSpace(c.pkg.files[pos.Filename].lines[pos.Line-1])
	c.errors = append(c.errors, uncheckedError{pos, line})
}

func (c *checker) Visit(node ast.Node) ast.Visitor {
	switch stmt := node.(type) {
	case *ast.ExprStmt:
		if call, ok := stmt.X.(*ast.CallExpr); ok {
			if !c.ignoreCall(call) && c.callReturnsError(call) {
				c.addErrorAtPosition(call.Lparen)
			}
		}
	case *ast.GoStmt:
		if !c.ignoreCall(stmt.Call) && c.callReturnsError(stmt.Call) {
			c.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.DeferStmt:
		if !c.ignoreCall(stmt.Call) && c.callReturnsError(stmt.Call) {
			c.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.AssignStmt:
		if !c.blank {
			break
		}
		if len(stmt.Rhs) == 1 {
			// single value on rhs; check against lhs identifiers
			if call, ok := stmt.Rhs[0].(*ast.CallExpr); ok {
				if c.ignoreCall(call) {
					break
				}
				isError := c.errorsByArg(call)
				for i := 0; i < len(stmt.Lhs); i++ {
					if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
						// We shortcut calls to recover() because errorsByArg can't
						// check its return types for errors since it returns interface{}.
						if id.Name == "_" && (c.isRecover(call) || isError[i]) {
							c.addErrorAtPosition(id.NamePos)
						}
					}
				}
			}
		} else {
			// multiple value on rhs; in this case a call can't return
			// multiple values. Assume len(stmt.Lhs) == len(stmt.Rhs)
			for i := 0; i < len(stmt.Lhs); i++ {
				if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
					if call, ok := stmt.Rhs[i].(*ast.CallExpr); ok {
						if c.ignoreCall(call) {
							continue
						}
						if id.Name == "_" && c.callReturnsError(call) {
							c.addErrorAtPosition(id.NamePos)
						}
					}
				}
			}
		}
	default:
	}
	return c
}

func checkPackage(pkg package_, ignore map[string]*regexp.Regexp, blank bool) error {
	tp, err := typeCheck(pkg)
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	visitor := &checker{tp, ignore, blank, []error{}}
	for _, astFile := range pkg.astFiles {
		ast.Walk(visitor, astFile)
	}

	if len(visitor.errors) > 0 {
		return UncheckedErrors{visitor.errors}
	}
	return nil
}

type obj interface {
	Pkg() *types.Package
	Name() string
}

func isErrorType(v obj) bool {
	return v.Pkg() == nil && v.Name() == "error"
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
)

func initTestDb(t *testing.T) *sql.DB {
	user := os.Getenv("DB_USER")
	pass := os.Getenv("DB_PASS")
	db_name := os.Getenv("DB_TEST_DB")
	if len(db_name) == 0 {
		t.Fatal("Must specify a db so we don't accidentally overwrite anything.")
	}
	db, err := sql.Open("postgres", fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, db_name))
	if err != nil {
		t.Fatal("Couldn't connect to DB.", err)
	}
	return db
}

func TestGetUser_Exists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	_, err := db.Exec(`INSERT INTO auth_user VALUES (
                   1, 'username',
                   'first', 'last',
                   'em@a.il', 'pass',
                   TRUE, TRUE, TRUE, now(), now()
                 )`)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Exec("DELETE from auth_user")

	u, err := dbc.GetUser(1)
	if err != nil {
		t.Fatal(err)
	}

	if u.Id != 1 {
		t.Fatal("Incorrect user id.")
	}
	if u.username != "username" {
		t.Fatal("Incorrect username.")
	}
	if u.Email != "em@a.il" {
		t.Fatal("Incorrect email.")
	}
}

func testGetUser_NoExists(t *testing.T) {

}

func testGetUserRepos_None(t *testing.T) {

}

func testGetUserRepos_One(t *testing.T) {
}

func testGetUserRepos_Many(t *testing.T) {
}

func testGetRepoActivity_OtherRepo(t *testing.T) {

}

func testGetRepoActivity_OtherUser(t *testing.T) {

}

func testGetRepoActivity_Old(t *testing.T) {

}

func testGetRepoActivity_AlreadySent(t *testing.T) {

}

func testGetRepoActivity_NeverSent(t *testing.T) {

}

func testMarkUserRepoSent(t *testing.T) {

}
<commit_msg>factor out data creation into helper methods. Test user fetching.<commit_after>package main

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
)

func initTestDb(t *testing.T) *sql.DB {
	user := os.Getenv("DB_USER")
	pass := os.Getenv("DB_PASS")
	db_name := os.Getenv("DB_TEST_DB")
	if len(db_name) == 0 {
		t.Fatal("Must specify a db so we don't accidentally overwrite anything.")
	}
	db, err := sql.Open("postgres", fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, db_name))
	if err != nil {
		t.Fatal("Couldn't connect to DB.", err)
	}
	return db
}

func make_user(id int, username, email string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO auth_user VALUES (
                   $1, $2,
                   'first', 'last',
                   $3, 'pass',
                   TRUE, TRUE, TRUE, now(), now()
                 )`, id, username, email)
	if err != nil {
		t.Fatal("Unable to create user.", err)
	}
}

func make_repo(id int, user, project string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_repo VALUES (
                   $1, $2, $3, NULL, NULL
                 )`, id, user, project)
	if err != nil {
		t.Fatal("Unable to create repo.", err)
	}
}

func make_userprofile(id, uid int, time_interval string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile
                 (id, user_id, max_time_interval_between_emails) VALUES (
                   $1, $2, $3
                 )`, id, uid, time_interval)
	if err != nil {
		t.Fatal("Unable to create userprofile.", err)
	}
}

func make_userprofile_repo(id, userprofile_id, repo_id int, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile_repos
                 (id, userprofile_id, repo_id) VALUES (
                   $1, $2, $3
                 )`, id, userprofile_id, repo_id)
	if err != nil {
		t.Fatal("Unable to create userprofile_repo.", err)
	}
}

func TestGetUser_Exists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	defer db.Exec("DELETE from auth_user")
	make_user(1, "username", "em@a.il", db, t)

	u, err := dbc.GetUser(1)
	if err != nil {
		t.Fatal(err)
	}

	if u.Id != 1 {
		t.Fatal("Incorrect user id.")
		return
	}
	if u.username != "username" {
		t.Fatal("Incorrect username.")
		return
	}
	if u.Email != "em@a.il" {
		t.Fatal("Incorrect email.")
		return
	}
}

func TestGetUser_NoExists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	_, err := dbc.GetUser(1)
	if err == nil {
		t.Fatal("No error when accessing invalid user id.")
		return
	}
}

func TestGetUserRepos_None(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()
	uid := 1

	defer db.Exec("DELETE FROM auth_user;")
	make_user(uid, "username", "em@a.il", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile;")
	make_userprofile(2, uid, "D", db, t) // Daily, should really be enum or something.
	defer db.Exec("DELETE FROM streamer_repo;")
	make_repo(3, "user", "repo", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile_repos;")
	make_userprofile_repo(4, 2, 3, db, t)

	ur, err := dbc.GetUserRepos(uid)
	if err != nil {
		t.Fatal("Can't get user's repos.", err)
		return
	}
	if len(ur) != 1 {
		fmt.Println(ur)
		t.Fatal("Got user's repos, but expected len 1, but got ", len(ur))
		return
	}
}

func testGetUserRepos_One(t *testing.T) {
}

func testGetUserRepos_Many(t *testing.T) {
}

func testGetRepoActivity_OtherRepo(t *testing.T) {

}

func testGetRepoActivity_OtherUser(t *testing.T) {

}

func testGetRepoActivity_Old(t *testing.T) {

}

func testGetRepoActivity_AlreadySent(t *testing.T) {

}

func testGetRepoActivity_NeverSent(t *testing.T) {

}

func testMarkUserRepoSent(t *testing.T) {

}
<|endoftext|>


<commit_before>package gophercloud

import "strings"

// ServiceClient stores details required to interact with a specific service API implemented by a provider.
// Generally, you'll acquire these by calling the appropriate `New` method on a ProviderClient.
type ServiceClient struct {
	// Provider is a reference to the provider that implements this service.
	Provider *ProviderClient

	// Endpoint is the base URL of the service's API, acquired from a service catalog.
	// It MUST end with a /.
	Endpoint string

	// ResourceBase is the base URL shared by the resources within a service's API. It should include
	// the API version and, like Endpoint, MUST end with a / if set. If not set, the Endpoint is used
	// as-is, instead.
	ResourceBase string
}

// ResourceBaseURL returns the base URL of any resources used by this service. It MUST end with a /.
func (client *ServiceClient) ResourceBaseURL() string {
	if client.ResourceBase != "" {
		return client.ResourceBase
	}
	return client.Endpoint
}

// ServiceURL constructs a URL for a resource belonging to this provider.
func (client *ServiceClient) ServiceURL(parts ...string) string {
	return client.ResourceBaseURL() + strings.Join(parts, "/")
}

// AuthenticatedHeaders returns a collection of HTTP request headers that mark a request as
// belonging to the currently authenticated user.
func (client *ServiceClient) AuthenticatedHeaders() map[string]string {
	return client.Provider.AuthenticatedHeaders()
}
<commit_msg>Embed the ProviderClient struct in ServiceClient.<commit_after>package gophercloud

import "strings"

// ServiceClient stores details required to interact with a specific service API implemented by a provider.
// Generally, you'll acquire these by calling the appropriate `New` method on a ProviderClient.
type ServiceClient struct {
	// ProviderClient is a reference to the provider that implements this service.
	*ProviderClient

	// Endpoint is the base URL of the service's API, acquired from a service catalog.
	// It MUST end with a /.
	Endpoint string

	// ResourceBase is the base URL shared by the resources within a service's API. It should include
	// the API version and, like Endpoint, MUST end with a / if set. If not set, the Endpoint is used
	// as-is, instead.
	ResourceBase string
}

// ResourceBaseURL returns the base URL of any resources used by this service. It MUST end with a /.
func (client *ServiceClient) ResourceBaseURL() string {
	if client.ResourceBase != "" {
		return client.ResourceBase
	}
	return client.Endpoint
}

// ServiceURL constructs a URL for a resource belonging to this provider.
func (client *ServiceClient) ServiceURL(parts ...string) string {
	return client.ResourceBaseURL() + strings.Join(parts, "/")
}
<|endoftext|>