<commit_before>package content

import (
	"bytes"
	"context"
	"crypto/aes"
	cryptorand "crypto/rand"
	"encoding/hex"
	"io"
	"sync"
	"time"

	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/buf"
	"github.com/kopia/kopia/internal/gather"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/encryption"
	"github.com/kopia/kopia/repo/hashing"
)

// lockFreeManager contains parts of Manager state that can be accessed without locking
type lockFreeManager struct {
	// this one is not lock-free
	Stats Stats

	st             blob.Storage
	Format         FormattingOptions
	CachingOptions CachingOptions

	indexBlobManager  indexBlobManager
	contentCache      contentCache
	metadataCache     contentCache
	committedContents *committedContentIndex

	checkInvariantsOnUnlock bool

	writeFormatVersion int32 // format version to write

	maxPackSize       int
	hasher            hashing.HashFunc
	encryptor         encryption.Encryptor
	minPreambleLength int
	maxPreambleLength int
	paddingUnit       int
	timeNow           func() time.Time

	repositoryFormatBytes []byte

	encryptionBufferPool *buf.Pool
}

func (bm *lockFreeManager) maybeEncryptContentDataForPacking(output *gather.WriteBuffer, data []byte, contentID ID) error {
	var hashOutput [maxHashSize]byte

	iv, err := getPackedContentIV(hashOutput[:], contentID)
	if err != nil {
		return errors.Wrapf(err, "unable to get packed content IV for %q", contentID)
	}

	b := bm.encryptionBufferPool.Allocate(len(data) + bm.encryptor.MaxOverhead())
	defer b.Release()

	cipherText, err := bm.encryptor.Encrypt(b.Data[:0], data, iv)
	if err != nil {
		return errors.Wrap(err, "unable to encrypt")
	}

	bm.Stats.encrypted(len(data))

	output.Append(cipherText)

	return nil
}

func writeRandomBytesToBuffer(b *gather.WriteBuffer, count int) error {
	var rnd [defaultPaddingUnit]byte

	if _, err := io.ReadFull(cryptorand.Reader, rnd[0:count]); err != nil {
		return err
	}

	b.Append(rnd[0:count])

	return nil
}

func (bm *lockFreeManager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo, bool, error) {
	nextSleepTime := 100 * time.Millisecond //nolint:gomnd

	for i := 0; i < indexLoadAttempts; i++ {
		if err := ctx.Err(); err != nil {
			return nil, false, err
		}

		if i > 0 {
			bm.indexBlobManager.flushCache()
			log(ctx).Debugf("encountered NOT_FOUND when loading, sleeping %v before retrying #%v", nextSleepTime, i)
			time.Sleep(nextSleepTime)
			nextSleepTime *= 2
		}

		indexBlobs, err := bm.indexBlobManager.listIndexBlobs(ctx, false)
		if err != nil {
			return nil, false, err
		}

		err = bm.tryLoadPackIndexBlobsUnlocked(ctx, indexBlobs)
		if err == nil {
			var indexBlobIDs []blob.ID
			for _, b := range indexBlobs {
				indexBlobIDs = append(indexBlobIDs, b.BlobID)
			}

			var updated bool

			updated, err = bm.committedContents.use(ctx, indexBlobIDs)
			if err != nil {
				return nil, false, err
			}

			return indexBlobs, updated, nil
		}

		if err != blob.ErrBlobNotFound {
			return nil, false, err
		}
	}

	return nil, false, errors.Errorf("unable to load pack indexes despite %v retries", indexLoadAttempts)
}

func (bm *lockFreeManager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, indexBlobs []IndexBlobInfo) error {
	ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(ctx, indexBlobs)
	if err != nil {
		return err
	}

	if len(ch) == 0 {
		return nil
	}

	log(ctx).Debugf("downloading %v new index blobs (%v bytes)...", len(ch), unprocessedIndexesSize)

	var wg sync.WaitGroup

	errch := make(chan error, parallelFetches)

	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()

			for indexBlobID := range ch {
				data, err := bm.indexBlobManager.getIndexBlob(ctx, indexBlobID)
				if err != nil {
					errch <- err
					return
				}

				if err := bm.committedContents.addContent(ctx, indexBlobID, data, false); err != nil {
					errch <- errors.Wrap(err, "unable to add to committed content cache")
					return
				}
			}
		}()
	}

	wg.Wait()
	close(errch)

	// Propagate async errors, if any.
	for err := range errch {
		return err
	}

	log(ctx).Debugf("Index contents downloaded.")

	return nil
}

// unprocessedIndexBlobsUnlocked returns a closed channel filled with content IDs that are not in committedContents cache.
func (bm *lockFreeManager) unprocessedIndexBlobsUnlocked(ctx context.Context, contents []IndexBlobInfo) (resultCh <-chan blob.ID, totalSize int64, err error) {
	ch := make(chan blob.ID, len(contents))
	defer close(ch)

	for _, c := range contents {
		has, err := bm.committedContents.cache.hasIndexBlobID(ctx, c.BlobID)
		if err != nil {
			return nil, 0, err
		}

		if has {
			log(ctx).Debugf("index blob %q already in cache, skipping", c.BlobID)
			continue
		}

		ch <- c.BlobID
		totalSize += c.Length
	}

	return ch, totalSize, nil
}

// ValidatePrefix returns an error if a given prefix is invalid.
func ValidatePrefix(prefix ID) error {
	switch len(prefix) {
	case 0:
		return nil
	case 1:
		if prefix[0] >= 'g' && prefix[0] <= 'z' {
			return nil
		}
	}

	return errors.Errorf("invalid prefix, must be a empty or single letter between 'g' and 'z'")
}

func (bm *lockFreeManager) getCacheForContentID(id ID) contentCache {
	if id.HasPrefix() {
		return bm.metadataCache
	}

	return bm.contentCache
}

func (bm *lockFreeManager) getContentDataUnlocked(ctx context.Context, pp *pendingPackInfo, bi *Info) ([]byte, error) {
	var payload []byte

	if pp != nil && pp.packBlobID == bi.PackBlobID {
		payload = pp.currentPackData.AppendSectionTo(nil, int(bi.PackOffset), int(bi.Length))
	} else {
		var err error

		payload, err = bm.getCacheForContentID(bi.ID).getContent(ctx, cacheKey(bi.ID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length))
		if err != nil {
			return nil, err
		}
	}

	bm.Stats.readContent(len(payload))

	var hashBuf [maxHashSize]byte

	iv, err := getPackedContentIV(hashBuf[:], bi.ID)
	if err != nil {
		return nil, err
	}

	decrypted, err := bm.decryptAndVerify(payload, iv)
	if err != nil {
		return nil, errors.Wrapf(err, "invalid checksum at %v offset %v length %v", bi.PackBlobID, bi.PackOffset, len(payload))
	}

	return decrypted, nil
}

func (bm *lockFreeManager) decryptAndVerify(encrypted, iv []byte) ([]byte, error) {
	decrypted, err := bm.encryptor.Decrypt(nil, encrypted, iv)
	if err != nil {
		return nil, errors.Wrap(err, "decrypt")
	}

	bm.Stats.decrypted(len(decrypted))

	if bm.encryptor.IsAuthenticated() {
		// already verified
		return decrypted, nil
	}

	// Since the encryption key is a function of data, we must be able to generate exactly the same key
	// after decrypting the content. This serves as a checksum.
	return decrypted, bm.verifyChecksum(decrypted, iv)
}

func (bm *lockFreeManager) preparePackDataContent(ctx context.Context, pp *pendingPackInfo) (packIndexBuilder, error) {
	formatLog(ctx).Debugf("preparing content data with %v items (contents %v)", len(pp.currentPackItems), pp.currentPackData.Length())

	packFileIndex := packIndexBuilder{}
	haveContent := false

	for _, info := range pp.currentPackItems {
		if info.PackBlobID == pp.packBlobID {
			haveContent = true
		}

		packFileIndex.Add(info)
	}

	if len(packFileIndex) == 0 {
		return nil, nil
	}

	if !haveContent {
		// we wrote pack preamble but no actual content, revert it
		pp.currentPackData.Reset()
		return packFileIndex, nil
	}

	if pp.finalized {
		return packFileIndex, nil
	}

	pp.finalized = true

	if bm.paddingUnit > 0 {
		if missing := bm.paddingUnit - (pp.currentPackData.Length() % bm.paddingUnit); missing > 0 {
			if err := writeRandomBytesToBuffer(pp.currentPackData, missing); err != nil {
				return nil, errors.Wrap(err, "unable to prepare content postamble")
			}
		}
	}

	err := bm.writePackFileIndexRecoveryData(pp.currentPackData, packFileIndex)

	return packFileIndex, err
}

// IndexBlobs returns the list of active index blobs.
func (bm *lockFreeManager) IndexBlobs(ctx context.Context, includeInactive bool) ([]IndexBlobInfo, error) {
	return bm.indexBlobManager.listIndexBlobs(ctx, includeInactive)
}

func getPackedContentIV(output []byte, contentID ID) ([]byte, error) {
	n, err := hex.Decode(output, []byte(contentID[len(contentID)-(aes.BlockSize*2):]))
	if err != nil {
		return nil, err
	}

	return output[0:n], nil
}

func (bm *lockFreeManager) writePackFileNotLocked(ctx context.Context, packFile blob.ID, data gather.Bytes) error {
	bm.Stats.wroteContent(data.Length())

	return bm.st.PutBlob(ctx, packFile, data)
}

func (bm *lockFreeManager) hashData(output, data []byte) []byte {
	// Hash the content and compute encryption key.
	contentID := bm.hasher(output, data)
	bm.Stats.hashedContent(len(data))

	return contentID
}

func (bm *lockFreeManager) verifyChecksum(data, contentID []byte) error {
	var hashOutput [maxHashSize]byte

	expected := bm.hasher(hashOutput[:0], data)
	expected = expected[len(expected)-aes.BlockSize:]

	if !bytes.HasSuffix(contentID, expected) {
		bm.Stats.foundInvalidContent()
		return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected)
	}

	bm.Stats.foundValidContent()

	return nil
}

// CreateHashAndEncryptor returns new hashing and encrypting functions based on
// the specified formatting options
func CreateHashAndEncryptor(f *FormattingOptions) (hashing.HashFunc, encryption.Encryptor, error) {
	h, err := hashing.CreateHashFunc(f)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create hash")
	}

	e, err := encryption.CreateEncryptor(f)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create encryptor")
	}

	contentID := h(nil, nil)

	_, err = e.Encrypt(nil, nil, contentID)
	if err != nil {
		return nil, nil, errors.Wrap(err, "invalid encryptor")
	}

	return h, e, nil
}
<commit_msg>content: emit warning whenever the number of index blobs gets too high, which is an indication that maintenance has not run in a long time<commit_after>package content

import (
	"bytes"
	"context"
	"crypto/aes"
	cryptorand "crypto/rand"
	"encoding/hex"
	"io"
	"sync"
	"time"

	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/buf"
	"github.com/kopia/kopia/internal/gather"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/encryption"
	"github.com/kopia/kopia/repo/hashing"
)

const indexBlobCompactionWarningThreshold = 100

// lockFreeManager contains parts of Manager state that can be accessed without locking
type lockFreeManager struct {
	// this one is not lock-free
	Stats Stats

	st             blob.Storage
	Format         FormattingOptions
	CachingOptions CachingOptions

	indexBlobManager  indexBlobManager
	contentCache      contentCache
	metadataCache     contentCache
	committedContents *committedContentIndex

	checkInvariantsOnUnlock bool

	writeFormatVersion int32 // format version to write

	maxPackSize       int
	hasher            hashing.HashFunc
	encryptor         encryption.Encryptor
	minPreambleLength int
	maxPreambleLength int
	paddingUnit       int
	timeNow           func() time.Time

	repositoryFormatBytes []byte

	encryptionBufferPool *buf.Pool
}

func (bm *lockFreeManager) maybeEncryptContentDataForPacking(output *gather.WriteBuffer, data []byte, contentID ID) error {
	var hashOutput [maxHashSize]byte

	iv, err := getPackedContentIV(hashOutput[:], contentID)
	if err != nil {
		return errors.Wrapf(err, "unable to get packed content IV for %q", contentID)
	}

	b := bm.encryptionBufferPool.Allocate(len(data) + bm.encryptor.MaxOverhead())
	defer b.Release()

	cipherText, err := bm.encryptor.Encrypt(b.Data[:0], data, iv)
	if err != nil {
		return errors.Wrap(err, "unable to encrypt")
	}

	bm.Stats.encrypted(len(data))

	output.Append(cipherText)

	return nil
}

func writeRandomBytesToBuffer(b *gather.WriteBuffer, count int) error {
	var rnd [defaultPaddingUnit]byte

	if _, err := io.ReadFull(cryptorand.Reader, rnd[0:count]); err != nil {
		return err
	}

	b.Append(rnd[0:count])

	return nil
}

func (bm *lockFreeManager) loadPackIndexesUnlocked(ctx context.Context) ([]IndexBlobInfo, bool, error) {
	nextSleepTime := 100 * time.Millisecond //nolint:gomnd

	for i := 0; i < indexLoadAttempts; i++ {
		if err := ctx.Err(); err != nil {
			return nil, false, err
		}

		if i > 0 {
			bm.indexBlobManager.flushCache()
			log(ctx).Debugf("encountered NOT_FOUND when loading, sleeping %v before retrying #%v", nextSleepTime, i)
			time.Sleep(nextSleepTime)
			nextSleepTime *= 2
		}

		indexBlobs, err := bm.indexBlobManager.listIndexBlobs(ctx, false)
		if err != nil {
			return nil, false, err
		}

		err = bm.tryLoadPackIndexBlobsUnlocked(ctx, indexBlobs)
		if err == nil {
			var indexBlobIDs []blob.ID
			for _, b := range indexBlobs {
				indexBlobIDs = append(indexBlobIDs, b.BlobID)
			}

			var updated bool

			updated, err = bm.committedContents.use(ctx, indexBlobIDs)
			if err != nil {
				return nil, false, err
			}

			if len(indexBlobs) > indexBlobCompactionWarningThreshold {
				log(ctx).Warningf("Found too many index blobs (%v), this may result in degraded performance.\n\nPlease ensure periodic repository maintenance is enabled or run 'kopia maintenance'.", len(indexBlobs))
			}

			return indexBlobs, updated, nil
		}

		if err != blob.ErrBlobNotFound {
			return nil, false, err
		}
	}

	return nil, false, errors.Errorf("unable to load pack indexes despite %v retries", indexLoadAttempts)
}

func (bm *lockFreeManager) tryLoadPackIndexBlobsUnlocked(ctx context.Context, indexBlobs []IndexBlobInfo) error {
	ch, unprocessedIndexesSize, err := bm.unprocessedIndexBlobsUnlocked(ctx, indexBlobs)
	if err != nil {
		return err
	}

	if len(ch) == 0 {
		return nil
	}

	log(ctx).Debugf("downloading %v new index blobs (%v bytes)...", len(ch), unprocessedIndexesSize)

	var wg sync.WaitGroup

	errch := make(chan error, parallelFetches)

	for i := 0; i < parallelFetches; i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()

			for indexBlobID := range ch {
				data, err := bm.indexBlobManager.getIndexBlob(ctx, indexBlobID)
				if err != nil {
					errch <- err
					return
				}

				if err := bm.committedContents.addContent(ctx, indexBlobID, data, false); err != nil {
					errch <- errors.Wrap(err, "unable to add to committed content cache")
					return
				}
			}
		}()
	}

	wg.Wait()
	close(errch)

	// Propagate async errors, if any.
	for err := range errch {
		return err
	}

	log(ctx).Debugf("Index contents downloaded.")

	return nil
}

// unprocessedIndexBlobsUnlocked returns a closed channel filled with content IDs that are not in committedContents cache.
func (bm *lockFreeManager) unprocessedIndexBlobsUnlocked(ctx context.Context, contents []IndexBlobInfo) (resultCh <-chan blob.ID, totalSize int64, err error) {
	ch := make(chan blob.ID, len(contents))
	defer close(ch)

	for _, c := range contents {
		has, err := bm.committedContents.cache.hasIndexBlobID(ctx, c.BlobID)
		if err != nil {
			return nil, 0, err
		}

		if has {
			log(ctx).Debugf("index blob %q already in cache, skipping", c.BlobID)
			continue
		}

		ch <- c.BlobID
		totalSize += c.Length
	}

	return ch, totalSize, nil
}

// ValidatePrefix returns an error if a given prefix is invalid.
func ValidatePrefix(prefix ID) error {
	switch len(prefix) {
	case 0:
		return nil
	case 1:
		if prefix[0] >= 'g' && prefix[0] <= 'z' {
			return nil
		}
	}

	return errors.Errorf("invalid prefix, must be a empty or single letter between 'g' and 'z'")
}

func (bm *lockFreeManager) getCacheForContentID(id ID) contentCache {
	if id.HasPrefix() {
		return bm.metadataCache
	}

	return bm.contentCache
}

func (bm *lockFreeManager) getContentDataUnlocked(ctx context.Context, pp *pendingPackInfo, bi *Info) ([]byte, error) {
	var payload []byte

	if pp != nil && pp.packBlobID == bi.PackBlobID {
		payload = pp.currentPackData.AppendSectionTo(nil, int(bi.PackOffset), int(bi.Length))
	} else {
		var err error

		payload, err = bm.getCacheForContentID(bi.ID).getContent(ctx, cacheKey(bi.ID), bi.PackBlobID, int64(bi.PackOffset), int64(bi.Length))
		if err != nil {
			return nil, err
		}
	}

	bm.Stats.readContent(len(payload))

	var hashBuf [maxHashSize]byte

	iv, err := getPackedContentIV(hashBuf[:], bi.ID)
	if err != nil {
		return nil, err
	}

	decrypted, err := bm.decryptAndVerify(payload, iv)
	if err != nil {
		return nil, errors.Wrapf(err, "invalid checksum at %v offset %v length %v", bi.PackBlobID, bi.PackOffset, len(payload))
	}

	return decrypted, nil
}

func (bm *lockFreeManager) decryptAndVerify(encrypted, iv []byte) ([]byte, error) {
	decrypted, err := bm.encryptor.Decrypt(nil, encrypted, iv)
	if err != nil {
		return nil, errors.Wrap(err, "decrypt")
	}

	bm.Stats.decrypted(len(decrypted))

	if bm.encryptor.IsAuthenticated() {
		// already verified
		return decrypted, nil
	}

	// Since the encryption key is a function of data, we must be able to generate exactly the same key
	// after decrypting the content. This serves as a checksum.
	return decrypted, bm.verifyChecksum(decrypted, iv)
}

func (bm *lockFreeManager) preparePackDataContent(ctx context.Context, pp *pendingPackInfo) (packIndexBuilder, error) {
	formatLog(ctx).Debugf("preparing content data with %v items (contents %v)", len(pp.currentPackItems), pp.currentPackData.Length())

	packFileIndex := packIndexBuilder{}
	haveContent := false

	for _, info := range pp.currentPackItems {
		if info.PackBlobID == pp.packBlobID {
			haveContent = true
		}

		packFileIndex.Add(info)
	}

	if len(packFileIndex) == 0 {
		return nil, nil
	}

	if !haveContent {
		// we wrote pack preamble but no actual content, revert it
		pp.currentPackData.Reset()
		return packFileIndex, nil
	}

	if pp.finalized {
		return packFileIndex, nil
	}

	pp.finalized = true

	if bm.paddingUnit > 0 {
		if missing := bm.paddingUnit - (pp.currentPackData.Length() % bm.paddingUnit); missing > 0 {
			if err := writeRandomBytesToBuffer(pp.currentPackData, missing); err != nil {
				return nil, errors.Wrap(err, "unable to prepare content postamble")
			}
		}
	}

	err := bm.writePackFileIndexRecoveryData(pp.currentPackData, packFileIndex)

	return packFileIndex, err
}

// IndexBlobs returns the list of active index blobs.
func (bm *lockFreeManager) IndexBlobs(ctx context.Context, includeInactive bool) ([]IndexBlobInfo, error) {
	return bm.indexBlobManager.listIndexBlobs(ctx, includeInactive)
}

func getPackedContentIV(output []byte, contentID ID) ([]byte, error) {
	n, err := hex.Decode(output, []byte(contentID[len(contentID)-(aes.BlockSize*2):]))
	if err != nil {
		return nil, err
	}

	return output[0:n], nil
}

func (bm *lockFreeManager) writePackFileNotLocked(ctx context.Context, packFile blob.ID, data gather.Bytes) error {
	bm.Stats.wroteContent(data.Length())

	return bm.st.PutBlob(ctx, packFile, data)
}

func (bm *lockFreeManager) hashData(output, data []byte) []byte {
	// Hash the content and compute encryption key.
	contentID := bm.hasher(output, data)
	bm.Stats.hashedContent(len(data))

	return contentID
}

func (bm *lockFreeManager) verifyChecksum(data, contentID []byte) error {
	var hashOutput [maxHashSize]byte

	expected := bm.hasher(hashOutput[:0], data)
	expected = expected[len(expected)-aes.BlockSize:]

	if !bytes.HasSuffix(contentID, expected) {
		bm.Stats.foundInvalidContent()
		return errors.Errorf("invalid checksum for blob %x, expected %x", contentID, expected)
	}

	bm.Stats.foundValidContent()

	return nil
}

// CreateHashAndEncryptor returns new hashing and encrypting functions based on
// the specified formatting options
func CreateHashAndEncryptor(f *FormattingOptions) (hashing.HashFunc, encryption.Encryptor, error) {
	h, err := hashing.CreateHashFunc(f)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create hash")
	}

	e, err := encryption.CreateEncryptor(f)
	if err != nil {
		return nil, nil, errors.Wrap(err, "unable to create encryptor")
	}

	contentID := h(nil, nil)

	_, err = e.Encrypt(nil, nil, contentID)
	if err != nil {
		return nil, nil, errors.Wrap(err, "invalid encryptor")
	}

	return h, e, nil
}
<|endoftext|>


<commit_before>package rabbus

import (
	"sync"
	"time"

	"github.com/rafaeljesus/retry-go"
	"github.com/sony/gobreaker"
	"github.com/streadway/amqp"
)

const (
	// Transient means higher throughput but messages will not be restored on broker restart.
	Transient uint8 = 1
	// Persistent messages will be restored to durable queues and lost on non-durable queues during server restart.
	Persistent uint8 = 2
	// ContentTypeJSON define json content type
	ContentTypeJSON string = "application/json"
	// ContentTypePlain define plain text content type
	ContentTypePlain string = "plain/text"
)

// Rabbus exposes a interface for emitting and listening for messages.
type Rabbus interface {
	// EmitAsync emits a message to RabbitMQ, but does not wait for the response from broker.
	EmitAsync() chan<- Message
	// EmitErr returns an error if encoding payload fails, or if after circuit breaker is open or retries attempts exceed.
	EmitErr() <-chan error
	// EmitOk returns true when the message was sent.
	EmitOk() <-chan struct{}
	// Listen to a message from RabbitMQ, returns
	// an error if exchange, queue name and function handler not passed or if an error occurred while creating
	// amqp consumer.
	Listen(ListenConfig) (chan ConsumerMessage, error)
	// Close attempt to close channel and connection.
	Close()
}

// Config carries the variables to tune a newly started rabbus.
type Config struct {
	// Dsn is the amqp url address.
	Dsn string
	// Durable indicates of the queue will survive broker restarts. Default to true.
	Durable bool
	// Attempts is the max number of retries on broker outages.
	Attempts int
	// Sleep is the sleep time of the retry mechanism.
	Sleep time.Duration
	// Interval is the cyclic period of the closed state for CircuitBreaker to clear the internal counts,
	// If Interval is 0, CircuitBreaker doesn't clear the internal counts during the closed state.
	Interval time.Duration
	// Timeout is the period of the open state, after which the state of CircuitBreaker becomes half-open.
	// If Timeout is 0, the timeout value of CircuitBreaker is set to 60 seconds.
	Timeout time.Duration
	// Threshold when a threshold of failures has been reached, future calls to the broker will not run.
	// During this state, the circuit breaker will periodically allow the calls to run and, if it is successful,
	// will start running the function again. Default value is 5.
	Threshold uint32
	// OnStateChange is called whenever the state of CircuitBreaker changes.
	OnStateChange func(name, from, to string)
	// Qos controls how many messages or how many bytes will be consumed before receiving delivery acks
	Qos Qos
}

// Qos controls how many messages or how many bytes the server will try to keep on the network for consumers before receiving delivery acks.
type Qos struct {
	PrefetchCount int
	PrefetchSize  int
	Global        bool
}

// Message carries fields for sending messages.
type Message struct {
	// Exchange the exchange name.
	Exchange string
	// Kind the exchange type.
	Kind string
	// Key the routing key name.
	Key string
	// Payload the message payload.
	Payload []byte
	// DeliveryMode indicates if the is Persistent or Transient.
	DeliveryMode uint8
	// ContentType the message content-type.
	ContentType string
	// Headers the message application headers
	Headers map[string]interface{}
}

// ListenConfig carries fields for listening messages.
type ListenConfig struct {
	// Exchange the exchange name.
	Exchange string
	// Kind the exchange type.
	Kind string
	// Key the routing key name.
	Key string
	// PassiveExchange determines a passive exchange connection it uses
	// amqp's ExchangeDeclarePassive instead the default ExchangeDeclare
	PassiveExchange bool
	// Queue the queue name
	Queue string
}

// Delivery wraps amqp.Delivery struct
type Delivery struct {
	amqp.Delivery
}

type rabbus struct {
	sync.RWMutex
	conn       *amqp.Connection
	ch         *amqp.Channel
	breaker    *gobreaker.CircuitBreaker
	emit       chan Message
	emitErr    chan error
	emitOk     chan struct{}
	config     Config
	exDeclared map[string]struct{}
}

// NewRabbus returns a new Rabbus configured with the
// variables from the config parameter, or returning an non-nil err
// if an error occurred while creating connection and channel.
func NewRabbus(c Config) (Rabbus, error) {
	conn, err := amqp.Dial(c.Dsn)
	if err != nil {
		return nil, err
	}

	ch, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	err = ch.Qos(c.Qos.PrefetchCount, c.Qos.PrefetchSize, c.Qos.Global)
	if err != nil {
		return nil, err
	}

	if c.Threshold == 0 {
		c.Threshold = 5
	}

	st := gobreaker.Settings{
		Name:     "Rabbus",
		Interval: c.Interval,
		Timeout:  c.Timeout,
		ReadyToTrip: func(counts gobreaker.Counts) bool {
			return counts.ConsecutiveFailures > c.Threshold
		},
		OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) {
			c.OnStateChange(name, from.String(), to.String())
		},
	}

	r := &rabbus{
		conn:       conn,
		ch:         ch,
		breaker:    gobreaker.NewCircuitBreaker(st),
		emit:       make(chan Message),
		emitErr:    make(chan error),
		emitOk:     make(chan struct{}),
		config:     c,
		exDeclared: make(map[string]struct{}),
	}

	go r.register()
	go notifyClose(c.Dsn, r)

	rab := r

	return rab, nil
}

// EmitAsync emits a message to RabbitMQ, but does not wait for the response from broker.
func (r *rabbus) EmitAsync() chan<- Message {
	return r.emit
}

// EmitErr returns an error if encoding payload fails, or if after circuit breaker is open or retries attempts exceed.
func (r *rabbus) EmitErr() <-chan error {
	return r.emitErr
}

// EmitOk returns true when the message was sent.
func (r *rabbus) EmitOk() <-chan struct{} {
	return r.emitOk
}

// Listen to a message from RabbitMQ, returns
// an error if exchange, queue name and function handler not passed or if an error occurred while creating
// amqp consumer.
func (r *rabbus) Listen(c ListenConfig) (chan ConsumerMessage, error) {
	if c.Exchange == "" {
		return nil, ErrMissingExchange
	}

	if c.Kind == "" {
		return nil, ErrMissingKind
	}

	if c.Queue == "" {
		return nil, ErrMissingQueue
	}

	if c.PassiveExchange {
		if err := r.ch.ExchangeDeclarePassive(c.Exchange, c.Kind, r.config.Durable, false, false, false, nil); err != nil {
			return nil, err
		}
	} else {
		if err := r.ch.ExchangeDeclare(c.Exchange, c.Kind, r.config.Durable, false, false, false, nil); err != nil {
			return nil, err
		}
	}

	q, err := r.ch.QueueDeclare(c.Queue, r.config.Durable, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	if err := r.ch.QueueBind(q.Name, c.Key, c.Exchange, false, nil); err != nil {
		return nil, err
	}

	msgs, err := r.ch.Consume(q.Name, "", false, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	messages := make(chan ConsumerMessage, 256)
	go func(msgs <-chan amqp.Delivery, messages chan ConsumerMessage) {
		for m := range msgs {
			messages <- newConsumerMessage(m)
		}
	}(msgs, messages)

	return messages, nil
}

// Close attempt to close channel and connection.
func (r *rabbus) Close() {
	r.ch.Close()
	r.conn.Close()
}

func (r *rabbus) register() {
	for m := range r.emit {
		r.produce(m)
	}
}

func (r *rabbus) produce(m Message) {
	if _, ok := r.exDeclared[m.Exchange]; !ok {
		if err := r.ch.ExchangeDeclare(m.Exchange, m.Kind, r.config.Durable, false, false, false, nil); err != nil {
			r.emitErr <- err
			return
		}
		r.exDeclared[m.Exchange] = struct{}{}
	}

	if m.ContentType == "" {
		m.ContentType = ContentTypeJSON
	}

	if m.DeliveryMode == 0 {
		m.DeliveryMode = Persistent
	}

	if _, err := r.breaker.Execute(func() (interface{}, error) {
		return nil, retry.Do(func() error {
			return r.ch.Publish(m.Exchange, m.Key, false, false, amqp.Publishing{
				Headers:         amqp.Table(m.Headers),
				ContentType:     m.ContentType,
				ContentEncoding: "UTF-8",
				DeliveryMode:    m.DeliveryMode,
				Timestamp:       time.Now(),
				Body:            m.Payload,
			})
		}, r.config.Attempts, r.config.Sleep)
	}); err != nil {
		r.emitErr <- err
		return
	}

	r.emitOk <- struct{}{}
}

func notifyClose(dsn string, r *rabbus) {
	err := <-r.conn.NotifyClose(make(chan *amqp.Error))
	if err != nil {
		for {
			time.Sleep(time.Second * 2)
			conn, err := amqp.Dial(dsn)
			if err != nil {
				continue
			}

			ch, err := conn.Channel()
			if err != nil {
				continue
			}

			r.Lock()
			defer r.Unlock()
			r.conn = conn
			r.ch = ch

			go notifyClose(dsn, r)

			break
		}
	}
}
<commit_msg>refactor: add global config for passive exchange<commit_after>package rabbus

import (
	"sync"
	"time"

	"github.com/rafaeljesus/retry-go"
	"github.com/sony/gobreaker"
	"github.com/streadway/amqp"
)

const (
	// Transient means higher throughput but messages will not be restored on broker restart.
	Transient uint8 = 1
	// Persistent messages will be restored to durable queues and lost on non-durable queues during server restart.
	Persistent uint8 = 2
	// ContentTypeJSON define json content type
	ContentTypeJSON string = "application/json"
	// ContentTypePlain define plain text content type
	ContentTypePlain string = "plain/text"
)

// Rabbus exposes a interface for emitting and listening for messages.
type Rabbus interface {
	// EmitAsync emits a message to RabbitMQ, but does not wait for the response from broker.
	EmitAsync() chan<- Message
	// EmitErr returns an error if encoding payload fails, or if after circuit breaker is open or retries attempts exceed.
	EmitErr() <-chan error
	// EmitOk returns true when the message was sent.
	EmitOk() <-chan struct{}
	// Listen to a message from RabbitMQ, returns
	// an error if exchange, queue name and function handler not passed or if an error occurred while creating
	// amqp consumer.
	Listen(ListenConfig) (chan ConsumerMessage, error)
	// Close attempt to close channel and connection.
	Close()
}

// Config carries the variables to tune a newly started rabbus.
type Config struct {
	// Dsn is the amqp url address.
	Dsn string
	// Durable indicates of the queue will survive broker restarts. Default to true.
	Durable bool
	// Attempts is the max number of retries on broker outages.
	Attempts int
	// Sleep is the sleep time of the retry mechanism.
	Sleep time.Duration
	// Interval is the cyclic period of the closed state for CircuitBreaker to clear the internal counts,
	// If Interval is 0, CircuitBreaker doesn't clear the internal counts during the closed state.
	Interval time.Duration
	// PassiveExchange forces passive connection with all exchanges using
	// amqp's ExchangeDeclarePassive instead the default ExchangeDeclare
	PassiveExchange bool
	// Timeout is the period of the open state, after which the state of CircuitBreaker becomes half-open.
	// If Timeout is 0, the timeout value of CircuitBreaker is set to 60 seconds.
	Timeout time.Duration
	// Threshold when a threshold of failures has been reached, future calls to the broker will not run.
	// During this state, the circuit breaker will periodically allow the calls to run and, if it is successful,
	// will start running the function again. Default value is 5.
	Threshold uint32
	// OnStateChange is called whenever the state of CircuitBreaker changes.
	OnStateChange func(name, from, to string)
	// Qos controls how many messages or how many bytes will be consumed before receiving delivery acks
	Qos Qos
}

// Qos controls how many messages or how many bytes the server will try to keep on the network for consumers before receiving delivery acks.
type Qos struct {
	PrefetchCount int
	PrefetchSize  int
	Global        bool
}

// Message carries fields for sending messages.
type Message struct {
	// Exchange the exchange name.
	Exchange string
	// Kind the exchange type.
	Kind string
	// Key the routing key name.
	Key string
	// Payload the message payload.
	Payload []byte
	// DeliveryMode indicates if the is Persistent or Transient.
	DeliveryMode uint8
	// ContentType the message content-type.
	ContentType string
	// Headers the message application headers
	Headers map[string]interface{}
}

// ListenConfig carries fields for listening messages.
type ListenConfig struct {
	// Exchange the exchange name.
	Exchange string
	// Kind the exchange type.
	Kind string
	// Key the routing key name.
	Key string
	// PassiveExchange determines a passive exchange connection it uses
	// amqp's ExchangeDeclarePassive instead the default ExchangeDeclare
	PassiveExchange bool
	// Queue the queue name
	Queue string
}

// Delivery wraps amqp.Delivery struct
type Delivery struct {
	amqp.Delivery
}

type rabbus struct {
	sync.RWMutex
	conn       *amqp.Connection
	ch         *amqp.Channel
	breaker    *gobreaker.CircuitBreaker
	emit       chan Message
	emitErr    chan error
	emitOk     chan struct{}
	config     Config
	exDeclared map[string]struct{}
}

// NewRabbus returns a new Rabbus configured with the
// variables from the config parameter, or returning an non-nil err
// if an error occurred while creating connection and channel.
func NewRabbus(c Config) (Rabbus, error) {
	conn, err := amqp.Dial(c.Dsn)
	if err != nil {
		return nil, err
	}

	ch, err := conn.Channel()
	if err != nil {
		return nil, err
	}

	err = ch.Qos(c.Qos.PrefetchCount, c.Qos.PrefetchSize, c.Qos.Global)
	if err != nil {
		return nil, err
	}

	if c.Threshold == 0 {
		c.Threshold = 5
	}

	st := gobreaker.Settings{
		Name:     "Rabbus",
		Interval: c.Interval,
		Timeout:  c.Timeout,
		ReadyToTrip: func(counts gobreaker.Counts) bool {
			return counts.ConsecutiveFailures > c.Threshold
		},
		OnStateChange: func(name string, from gobreaker.State, to gobreaker.State) {
			c.OnStateChange(name, from.String(), to.String())
		},
	}

	r := &rabbus{
		conn:       conn,
		ch:         ch,
		breaker:    gobreaker.NewCircuitBreaker(st),
		emit:       make(chan Message),
		emitErr:    make(chan error),
		emitOk:     make(chan struct{}),
		config:     c,
		exDeclared: make(map[string]struct{}),
	}

	go r.register()
	go notifyClose(c.Dsn, r)

	rab := r

	return rab, nil
}

// EmitAsync emits a message to RabbitMQ, but does not wait for the response from broker.
func (r *rabbus) EmitAsync() chan<- Message {
	return r.emit
}

// EmitErr returns an error if encoding payload fails, or if after circuit breaker is open or retries attempts exceed.
func (r *rabbus) EmitErr() <-chan error {
	return r.emitErr
}

// EmitOk returns true when the message was sent.
func (r *rabbus) EmitOk() <-chan struct{} {
	return r.emitOk
}

// Listen to a message from RabbitMQ, returns
// an error if exchange, queue name and function handler not passed or if an error occurred while creating
// amqp consumer.
func (r *rabbus) Listen(c ListenConfig) (chan ConsumerMessage, error) {
	if c.Exchange == "" {
		return nil, ErrMissingExchange
	}

	if c.Kind == "" {
		return nil, ErrMissingKind
	}

	if c.Queue == "" {
		return nil, ErrMissingQueue
	}

	if err := declareExchange(r.ch, c.Exchange, c.Kind, r.config); err != nil {
		return nil, err
	}
	r.exDeclared[c.Exchange] = struct{}{}

	q, err := r.ch.QueueDeclare(c.Queue, r.config.Durable, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	if err := r.ch.QueueBind(q.Name, c.Key, c.Exchange, false, nil); err != nil {
		return nil, err
	}

	msgs, err := r.ch.Consume(q.Name, "", false, false, false, false, nil)
	if err != nil {
		return nil, err
	}

	messages := make(chan ConsumerMessage, 256)
	go func(msgs <-chan amqp.Delivery, messages chan ConsumerMessage) {
		for m := range msgs {
			messages <- newConsumerMessage(m)
		}
	}(msgs, messages)

	return messages, nil
}

// Close attempt to close channel and connection.
func (r *rabbus) Close() {
	r.ch.Close()
	r.conn.Close()
}

func (r *rabbus) register() {
	for m := range r.emit {
		r.produce(m)
	}
}

func (r *rabbus) produce(m Message) {
	if _, ok := r.exDeclared[m.Exchange]; !ok {
		err := declareExchange(r.ch, m.Exchange, m.Kind, r.config)
		if err != nil {
			r.emitErr <- err
			return
		}
		r.exDeclared[m.Exchange] = struct{}{}
	}

	if m.ContentType == "" {
		m.ContentType = ContentTypeJSON
	}

	if m.DeliveryMode == 0 {
		m.DeliveryMode = Persistent
	}

	if _, err := r.breaker.Execute(func() (interface{}, error) {
		return nil, retry.Do(func() error {
			return r.ch.Publish(m.Exchange, m.Key, false, false, amqp.Publishing{
				Headers:         amqp.Table(m.Headers),
				ContentType:     m.ContentType,
				ContentEncoding: "UTF-8",
				DeliveryMode:    m.DeliveryMode,
				Timestamp:       time.Now(),
				Body:            m.Payload,
			})
		}, r.config.Attempts, r.config.Sleep)
	}); err != nil {
		r.emitErr <- err
		return
	}

	r.emitOk <- struct{}{}
}

func notifyClose(dsn string, r *rabbus) {
	err := <-r.conn.NotifyClose(make(chan *amqp.Error))
	if err != nil {
		for {
			time.Sleep(time.Second * 2)
			conn, err := amqp.Dial(dsn)
			if err != nil {
				continue
			}

			ch, err := conn.Channel()
			if err != nil {
				continue
			}

			r.Lock()
			defer r.Unlock()
			r.conn = conn
			r.ch = ch

			go notifyClose(dsn, r)

			break
		}
	}
}

func declareExchange(ch *amqp.Channel, ex, kind string, config Config) error {
	if config.PassiveExchange {
		if err := ch.ExchangeDeclarePassive(ex, kind, config.Durable, false, false, false, nil); err != nil {
			return err
		}
	} else {
		if err := ch.ExchangeDeclare(ex, kind, config.Durable, false, false, false, nil); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package dbutil

import (
	"github.com/vbatts/imgsrv/hash"
	"github.com/vbatts/imgsrv/types"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"strings"
)

const (
	DEFAULT_DB_NAME = "filesrv"
)

type Util struct {
	Seed    string // mongo host seed to Dial into
	User    string // mongo credentials, if needed
	Pass    string // mongo credentials, if needed
	DbName  string // mongo database name, if needed
	Session *mgo.Session
	FileDb  *mgo.Database
	Gfs     *mgo.GridFS
}

func (u *Util) Init() error {
	var err error
	u.Session, err = mgo.Dial(u.Seed)
	if err != nil {
		return err
	}

	if len(u.DbName) > 0 {
		u.FileDb = u.Session.DB(u.DbName)
	} else {
		u.FileDb = u.Session.DB(DEFAULT_DB_NAME)
	}

	if len(u.User) > 0 && len(u.Pass) > 0 {
		err = u.FileDb.Login(u.User, u.Pass)
		if err != nil {
			return err
		}
	}
	u.Gfs = u.FileDb.GridFS("fs")
	return nil
}

func (u Util) Close() {
	u.Session.Close()
}

/*
pass through for GridFs
*/
func (u Util) Open(filename string) (file *mgo.GridFile, err error) {
	return u.Gfs.Open(strings.ToLower(filename))
}

/*
pass through for GridFs
*/
func (u Util) Create(filename string) (file *mgo.GridFile, err error) {
	return u.Gfs.Create(strings.ToLower(filename))
}

/*
pass through for GridFs
*/
func (u Util) Remove(filename string) (err error) {
	return u.Gfs.Remove(strings.ToLower(filename))
}

/*
Find files by their MD5 checksum
*/
func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
match for file name
*/
func (u Util) FindFilesByName(filename string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Get all the files.

pass -1 for all files
*/
func (u Util) GetFiles(limit int) (files []types.File, err error) {
	//files = []types.File{}
	if limit == -1 {
		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
	} else {
		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
	}
	return files, err
}

/*
Count the filename matches
*/
func (u Util) CountFiles(filename string) (count int, err error) {
	query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
	return query.Count()
}

/*
Get one file back, by searching by file name
*/
func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file)
	if err != nil {
		return this_file, err
	}
	return this_file, nil
}

func (u Util) GetFileRandom() (this_file types.File, err error) {
	r := hash.Rand64()
	err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file)
	if err != nil {
		return this_file, err
	}
	if len(this_file.Md5) == 0 {
		err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file)
	}
	if err != nil {
		return this_file, err
	}
	return this_file, nil
}

/*
Check whether this types.File filename is on Mongo
*/
func (u Util) HasFileByFilename(filename string) (exists bool, err error) {
	c, err := u.CountFiles(filename)
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

func (u Util) HasFileByMd5(md5 string) (exists bool, err error) {
	c, err := u.Gfs.Find(bson.M{"md5": md5}).Count()
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) {
	c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

/*
get a list of file extensions and their frequency count
*/
func (u Util) GetExtensions() (kp []types.IdCount, err error) {
	job := &mgo.MapReduce{
		Map: `
    function() {
        if (!this.filename) {
          return;
        }

        s = this.filename.split(".")
        ext = s[s.length - 1] // get the last segment of the split
        emit(ext,1);
    }
    `,
		Reduce: `
    function(previous, current) {
      var count = 0;

      for (index in current) {
        count += current[index];
      }

      return count;
    }
    `,
	}
	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
		return kp, err
	}
	// Less than effecient, but cleanest place to put this
	for i := range kp {
		kp[i].Root = "ext" // for extension. Maps to /ext/
	}
	return kp, nil
}

/*
get a list of keywords and their frequency count
*/
func (u Util) GetKeywords() (kp []types.IdCount, err error) {
	job := &mgo.MapReduce{
		Map: `
    function() {
        if (!this.metadata.keywords) {
          return;
        }

        for (index in this.metadata.keywords) {
          emit(this.metadata.keywords[index], 1);
        }
    }
    `,
		Reduce: `
    function(previous, current) {
      var count = 0;

      for (index in current) {
        count += current[index];
      }

      return count;
    }
    `,
	}
	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
		return kp, err
	}
	// Less than effecient, but cleanest place to put this
	for i := range kp {
		kp[i].Root = "k" // for keyword. Maps to /k/
	}
	return kp, nil
}
<commit_msg>the All listing is broken<commit_after>package dbutil

import (
	"github.com/vbatts/imgsrv/hash"
	"github.com/vbatts/imgsrv/types"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"strings"
)

const (
	DEFAULT_DB_NAME = "filesrv"
)

type Util struct {
	Seed    string // mongo host seed to Dial into
	User    string // mongo credentials, if needed
	Pass    string // mongo credentials, if needed
	DbName  string // mongo database name, if needed
	Session *mgo.Session
	FileDb  *mgo.Database
	Gfs     *mgo.GridFS
}

func (u *Util) Init() error {
	var err error
	u.Session, err = mgo.Dial(u.Seed)
	if err != nil {
		return err
	}

	if len(u.DbName) > 0 {
		u.FileDb = u.Session.DB(u.DbName)
	} else {
		u.FileDb = u.Session.DB(DEFAULT_DB_NAME)
	}

	if len(u.User) > 0 && len(u.Pass) > 0 {
		err = u.FileDb.Login(u.User, u.Pass)
		if err != nil {
			return err
		}
	}
	u.Gfs = u.FileDb.GridFS("fs")
	return nil
}

func (u Util) Close() {
	u.Session.Close()
}

/*
pass through for GridFs
*/
func (u Util) Open(filename string) (file *mgo.GridFile, err error) {
	return u.Gfs.Open(strings.ToLower(filename))
}

/*
pass through for GridFs
*/
func (u Util) Create(filename string) (file *mgo.GridFile, err error) {
	return u.Gfs.Create(strings.ToLower(filename))
}

/*
pass through for GridFs
*/
func (u Util) Remove(filename string) (err error) {
	return u.Gfs.Remove(strings.ToLower(filename))
}

/*
Find files by their MD5 checksum
*/
func (u Util) FindFilesByMd5(md5 string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"md5": md5}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
match for file name
*/
func (u Util) FindFilesByName(filename string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": filename}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByPatt(filename_pat string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": bson.M{"$regex": filename_pat, "$options": "i"}}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Case-insensitive pattern match for file name
*/
func (u Util) FindFilesByKeyword(keyword string) (files []types.File, err error) {
	err = u.Gfs.Find(bson.M{"metadata.keywords": strings.ToLower(keyword)}).Sort("-metadata.timestamp").All(&files)
	return files, err
}

/*
Get all the files.

pass -1 for all files
*/
func (u Util) GetFiles(limit int) (files []types.File, err error) {
	//files = []types.File{}
	if limit == -1 {
		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").All(&files)
	} else {
		err = u.Gfs.Find(nil).Sort("-metadata.timestamp").Limit(limit).All(&files)
	}
	return files, err
}

/*
Count the filename matches
*/
func (u Util) CountFiles(filename string) (count int, err error) {
	query := u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)})
	return query.Count()
}

/*
Get one file back, by searching by file name
*/
func (u Util) GetFileByFilename(filename string) (this_file types.File, err error) {
	err = u.Gfs.Find(bson.M{"filename": strings.ToLower(filename)}).One(&this_file)
	if err != nil {
		return this_file, err
	}
	return this_file, nil
}

func (u Util) GetFileRandom() (this_file types.File, err error) {
	r := hash.Rand64()
	err = u.Gfs.Find(bson.M{"random": bson.M{"$gt": r}}).One(&this_file)
	if err != nil {
		return this_file, err
	}
	if len(this_file.Md5) == 0 {
		err = u.Gfs.Find(bson.M{"random": bson.M{"$lt": r}}).One(&this_file)
	}
	if err != nil {
		return this_file, err
	}
	return this_file, nil
}

/*
Check whether this types.File filename is on Mongo
*/
func (u Util) HasFileByFilename(filename string) (exists bool, err error) {
	c, err := u.CountFiles(filename)
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

func (u Util) HasFileByMd5(md5 string) (exists bool, err error) {
	c, err := u.Gfs.Find(bson.M{"md5": md5}).Count()
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

func (u Util) HasFileByKeyword(keyword string) (exists bool, err error) {
	c, err := u.Gfs.Find(bson.M{"metadata": bson.M{"keywords": strings.ToLower(keyword)}}).Count()
	if err != nil {
		return false, err
	}
	exists = (c > 0)
	return exists, nil
}

/*
get a list of file extensions and their frequency count
*/
func (u Util) GetExtensions() (kp []types.IdCount, err error) {
	job := &mgo.MapReduce{
		Map: `
    function() {
        if (!this.filename) {
          return;
        }

        s = this.filename.split(".")
        ext = s[s.length - 1] // get the last segment of the split
        emit(ext,1);
    }
    `,
		Reduce: `
    function(previous, current) {
      var count = 0;

      for (index in current) {
        count += current[index];
      }

      return count;
    }
    `,
	}
	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
		return kp, err
	}
	// Less than effecient, but cleanest place to put this
	for i := range kp {
		kp[i].Root = "ext" // for extension. Maps to /ext/
	}
	return kp, nil
}

/*
get a list of keywords and their frequency count
*/
func (u Util) GetKeywords() (kp []types.IdCount, err error) {
	job := &mgo.MapReduce{
		Map: `
    function() {
        if (!this.metadata.keywords) {
          return;
        }

        for (index in this.metadata.keywords) {
          emit(this.metadata.keywords[index], 1);
        }
    }
    `,
		Reduce: `
    function(previous, current) {
      var count = 0;

      for (index in current) {
        count += current[index];
      }

      return count;
    }
    `,
	}
	if _, err := u.Gfs.Find(nil).MapReduce(job, &kp); err != nil {
		return kp, err
	}
	// Less than effecient, but cleanest place to put this
	for i := range kp {
		kp[i].Root = "k" // for keyword. Maps to /k/
	}
	return kp, nil
}
<|endoftext|>


<commit_before>package debounce_test

import (
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/bep/debounce"
	"github.com/fortytw2/leaktest"
)

func TestDebounce(t *testing.T) {
	defer leaktest.Check(t)()

	counter1 := 0

	f1 := func() {
		counter1++
	}

	counter2 := 0

	f2 := func() {
		counter2++
	}

	f3 := func() {
		counter2 += 2
	}

	debounced, shutdown := debounce.New(100 * time.Millisecond)

	for i := 0; i < 3; i++ {
		for j := 0; j < 10; j++ {
			debounced(f1)
		}

		time.Sleep(200 * time.Millisecond)
	}

	for i := 0; i < 4; i++ {
		for j := 0; j < 10; j++ {
			debounced(f2)
		}
		for j := 0; j < 10; j++ {
			debounced(f3)
		}

		time.Sleep(200 * time.Millisecond)
	}

	close(shutdown)

	<-time.After(200 * time.Millisecond)

	if counter1 != 3 {
		t.Error("Expected count 3, was", counter1)
	}

	if counter2 != 8 {
		t.Error("Expected count 8, was", counter2)
	}

}

func TestDebounceInParallel(t *testing.T) {
	defer leaktest.Check(t)()

	var counter uint64

	f := func() {
		atomic.AddUint64(&counter, 1)
	}

	debounced, shutdown := debounce.New(100 * time.Millisecond)

	var wg sync.WaitGroup

	for i := 0; i < 20; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			debouncedInner, shutdown := debounce.New(100 * time.Millisecond)
			for j := 0; j < 10; j++ {
				debouncedInner(f)
				debounced(f)
			}
			time.Sleep(150 * time.Millisecond)
			close(shutdown)
		}()
	}
	wg.Wait()

	close(shutdown)

	<-time.After(200 * time.Millisecond)

	if counter != 21 {
		t.Error("Expected count 21, was", counter)
	}
}

func TestDebounceCloseEarly(t *testing.T) {
	defer leaktest.Check(t)()

	counter := 0

	f := func() {
		counter++
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	debounced(f)

	close(finish)

	<-time.After(200 * time.Millisecond)

	if counter != 0 {
		t.Error("Expected count 0, was", counter)
	}

}

func BenchmarkDebounce(b *testing.B) {
	counter := 0

	f := func() {
		counter++
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		debounced(f)
	}
	close(finish)
	if counter != 0 {
		b.Fatal("Expected count 0, was", counter)
	}
}

func ExampleNew() {
	counter := 0

	f := func() {
		counter++
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	for i := 0; i < 3; i++ {
		for j := 0; j < 10; j++ {
			debounced(f)
		}

		time.Sleep(200 * time.Millisecond)
	}

	close(finish)

	<-time.After(200 * time.Millisecond)

	fmt.Println("Counter is", counter)
	// Output: Counter is 3
}
<commit_msg>testing: Fix Go 1.10-reported data races in tests<commit_after>package debounce_test

import (
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/bep/debounce"
	"github.com/fortytw2/leaktest"
)

func TestDebounce(t *testing.T) {
	defer leaktest.Check(t)()

	var (
		counter1 uint64
		counter2 uint64
	)

	f1 := func() {
		atomic.AddUint64(&counter1, 1)
	}

	f2 := func() {
		atomic.AddUint64(&counter2, 1)
	}

	f3 := func() {
		atomic.AddUint64(&counter2, 2)
	}

	debounced, shutdown := debounce.New(100 * time.Millisecond)

	for i := 0; i < 3; i++ {
		for j := 0; j < 10; j++ {
			debounced(f1)
		}

		time.Sleep(200 * time.Millisecond)
	}

	for i := 0; i < 4; i++ {
		for j := 0; j < 10; j++ {
			debounced(f2)
		}
		for j := 0; j < 10; j++ {
			debounced(f3)
		}

		time.Sleep(200 * time.Millisecond)
	}

	close(shutdown)

	<-time.After(200 * time.Millisecond)

	c1 := int(atomic.LoadUint64(&counter1))
	c2 := int(atomic.LoadUint64(&counter2))
	if c1 != 3 {
		t.Error("Expected count 3, was", c1)
	}
	if c2 != 8 {
		t.Error("Expected count 8, was", c2)
	}
}

func TestDebounceInParallel(t *testing.T) {
	defer leaktest.Check(t)()

	var counter uint64

	f := func() {
		atomic.AddUint64(&counter, 1)
	}

	debounced, shutdown := debounce.New(100 * time.Millisecond)

	var wg sync.WaitGroup

	for i := 0; i < 20; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			debouncedInner, shutdown := debounce.New(100 * time.Millisecond)
			for j := 0; j < 10; j++ {
				debouncedInner(f)
				debounced(f)
			}
			time.Sleep(150 * time.Millisecond)
			close(shutdown)
		}()
	}
	wg.Wait()

	close(shutdown)

	<-time.After(200 * time.Millisecond)

	c := int(atomic.LoadUint64(&counter))
	if c != 21 {
		t.Error("Expected count 21, was", c)
	}
}

func TestDebounceCloseEarly(t *testing.T) {
	defer leaktest.Check(t)()

	var counter uint64

	f := func() {
		atomic.AddUint64(&counter, 1)
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	debounced(f)

	close(finish)

	<-time.After(200 * time.Millisecond)

	c := int(atomic.LoadUint64(&counter))
	if c != 0 {
		t.Error("Expected count 0, was", c)
	}

}

func BenchmarkDebounce(b *testing.B) {
	var counter uint64

	f := func() {
		atomic.AddUint64(&counter, 1)
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		debounced(f)
	}
	close(finish)
	c := int(atomic.LoadUint64(&counter))
	if c != 0 {
		b.Fatal("Expected count 0, was", c)
	}
}

func ExampleNew() {
	var counter uint64

	f := func() {
		atomic.AddUint64(&counter, 1)
	}

	debounced, finish := debounce.New(100 * time.Millisecond)

	for i := 0; i < 3; i++ {
		for j := 0; j < 10; j++ {
			debounced(f)
		}

		time.Sleep(200 * time.Millisecond)
	}

	close(finish)

	<-time.After(200 * time.Millisecond)

	c := int(atomic.LoadUint64(&counter))

	fmt.Println("Counter is", c)
	// Output: Counter is 3
}
<|endoftext|>


<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package decode

import (
	"fmt"
	"log"
	"math"
)

// PacketConfig specifies packet-specific radio configuration.
type PacketConfig struct {
	DataRate int

	BlockSize, BlockSize2    int
	ChipLength, SymbolLength int
	SampleRate               int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	Preamble                       string

	BufferLength int

	CenterFreq uint32
}

func (cfg PacketConfig) Decimate(decimation int) PacketConfig {
	cfg.BlockSize /= decimation
	cfg.BlockSize2 /= decimation
	cfg.ChipLength /= decimation
	cfg.SymbolLength /= decimation
	cfg.SampleRate /= decimation
	cfg.DataRate /= decimation

	cfg.PreambleLength /= decimation
	cfg.PacketLength /= decimation

	cfg.BufferLength /= decimation

	return cfg
}

func (d Decoder) Log() {
	if d.Decimation != 1 {
		log.Printf("BlockSize: %d|%d\n", d.Cfg.BlockSize, d.DecCfg.BlockSize)
		log.Println("CenterFreq:", d.Cfg.CenterFreq)
		log.Printf("SampleRate: %d|%d\n", d.Cfg.SampleRate, d.DecCfg.SampleRate)
		log.Printf("DataRate: %d|%d\n", d.Cfg.DataRate, d.DecCfg.DataRate)
		log.Printf("ChipLength: %d|%d\n", d.Cfg.ChipLength, d.DecCfg.ChipLength)
		log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
		log.Printf("PreambleLength: %d|%d\n", d.Cfg.PreambleLength, d.DecCfg.PreambleLength)
		log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
		log.Printf("PacketLength: %d|%d\n", d.Cfg.PacketLength, d.DecCfg.PacketLength)
		log.Println("Preamble:", d.Cfg.Preamble)

		if d.Cfg.ChipLength%d.Decimation != 0 {
			log.Println("Warning: decimated symbol length is non-integral, sensitivity may be poor")
		}

		if d.DecCfg.ChipLength < 3 {
			log.Fatal("Error: illegal decimation factor, choose a smaller factor")
		}

		return
	}

	log.Println("CenterFreq:", d.Cfg.CenterFreq)
	log.Println("SampleRate:", d.Cfg.SampleRate)
	log.Println("DataRate:", d.Cfg.DataRate)
	log.Println("ChipLength:", d.Cfg.ChipLength)
	log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
	log.Println("PreambleLength:", d.Cfg.PreambleLength)
	log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
	log.Println("PacketLength:", d.Cfg.PacketLength)
	log.Println("Preamble:", d.Cfg.Preamble)
}

// Decoder contains buffers and radio configuration.
type Decoder struct {
	Cfg PacketConfig

	Decimation int
	DecCfg     PacketConfig

	Signal    []float64
	Filtered  []float64
	Quantized []byte

	csum  []float64
	demod Demodulator

	preamble []byte
	slices   [][]byte

	preambleFinder *byteFinder

	pkt []byte
}

// Create a new decoder with the given packet configuration.
func NewDecoder(cfg PacketConfig, decimation int) (d Decoder) {
	d.Cfg = cfg

	d.Cfg.SymbolLength = d.Cfg.ChipLength << 1
	d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.ChipLength

	d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength
	d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength

	d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength)
	d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1

	d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize

	d.Decimation = decimation
	d.DecCfg = d.Cfg.Decimate(d.Decimation)

	// Allocate necessary buffers.
	d.Signal = make([]float64, d.DecCfg.BlockSize+d.DecCfg.SymbolLength)
	d.Filtered = make([]float64, d.DecCfg.BlockSize)
	d.Quantized = make([]byte, d.DecCfg.BufferLength)

	d.csum = make([]float64, len(d.Signal)+1)

	// Calculate magnitude lookup table specified by -fastmag flag.
	d.demod = NewMagLUT()

	// Pre-calculate a byte-slice version of the preamble for searching.
	d.preamble = make([]byte, d.Cfg.PreambleSymbols)
	for idx := range d.Cfg.Preamble {
		if d.Cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	// Slice quantized sample buffer to make searching for the preamble more
	// memory local.
	d.slices = make([][]byte, d.DecCfg.SymbolLength)

	symbolsPerBlock := d.DecCfg.BlockSize/d.DecCfg.SymbolLength + d.DecCfg.PreambleSymbols
	for symbolOffset := range d.slices {
		d.slices[symbolOffset] = make([]byte, symbolsPerBlock)
	}

	d.preambleFinder = makeByteFinder(d.preamble)

	// Signal up to the final stage is 1-bit per byte. Allocate a buffer to
	// store packed version 8-bits per byte.
	d.pkt = make([]byte, (d.DecCfg.PacketSymbols+7)>>3)

	return
}

// Decode accepts a sample block and performs various DSP techniques to extract a packet.
func (d Decoder) Decode(input []byte) []int {
	// Shift buffers to append new block.
	copy(d.Signal, d.Signal[d.DecCfg.BlockSize:])
	copy(d.Quantized, d.Quantized[d.DecCfg.BlockSize:])

	// Compute the magnitude of the new block.
	d.demod.Execute(input, d.Signal[d.DecCfg.SymbolLength:])

	// Perform matched filter on new block.
	d.Filter(d.Signal, d.Filtered)

	// Perform bit-decision on new block.
	Quantize(d.Filtered[d.DecCfg.SymbolLength:], d.Quantized[d.DecCfg.PacketLength:])

	// Pack the quantized signal into slices for searching.
	d.Transpose(d.Quantized)

	// Return a list of indexes the preamble exists at.
	return d.Search()
}

// A Demodulator knows how to demodulate an array of uint8 IQ samples into an
// array of float64 samples.
type Demodulator interface {
	Execute([]byte, []float64)
}

// Default Magnitude Lookup Table
type MagLUT []float64

// Pre-computes normalized squares with most common DC offset for rtl-sdr dongles.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = (127.5 - float64(idx)) / 127.5
		lut[idx] *= lut[idx]
	}
	return
}

// Calculates complex magnitude on given IQ stream writing result to output.
func (lut MagLUT) Execute(input []byte, output []float64) {
	decIdx := 0
	dec := (len(input) / len(output))

	for idx := 0; decIdx < len(output); idx += dec {
		output[decIdx] = lut[input[idx]] + lut[input[idx+1]]
		decIdx++
	}
}

// Matched filter for Manchester coded signals. Output signal's sign at each
// sample determines the bit-value since Manchester symbols have odd symmetry.
func (d Decoder) Filter(input, output []float64) {
	// Computing the cumulative summation over the signal simplifies
	// filtering to the difference of a pair of subtractions.
	var sum float64
	for idx, v := range input {
		sum += v
		d.csum[idx+1] = sum
	}

	// Filter result is difference of summation of lower and upper symbols.
	lower := d.csum[d.DecCfg.ChipLength:]
	upper := d.csum[d.DecCfg.SymbolLength:]
	for idx, l := range lower[:len(output)] {
		output[idx] = (l - d.csum[idx]) - (upper[idx] - l)
	}

	return
}

// Bit-value is determined by the sign of each sample after filtering.
func Quantize(input []float64, output []byte) {
	// Should optimize out to memclr operation.
	for idx := range output {
		output[idx] = 0
	}

	// Only set necessary bits. Previous method set all bits regardless of value.
	for idx, val := range input {
		if val > 0 {
			output[idx] = 1
		}
	}

	return
}

// Transpose quantized signal into slices such that the first rank represents
// sample offsets and the second represents the value of each symbol from the
// given offset.
//
// Transforms:
// <--Sym1--><--Sym2--><--Sym3--><--Sym4--><--Sym5--><--Sym6--><--Sym7--><--Sym8-->
// <12345678><12345678><12345678><12345678><12345678><12345678><12345678><12345678>
// to:
// <11111111><22222222><33333333><44444444><55555555><66666666><77777777><88888888>
func (d *Decoder) Transpose(input []byte) {
	for symbolOffset, slice := range d.slices {
		for symbolIdx := range slice {
			slice[symbolIdx] = input[symbolIdx*d.DecCfg.SymbolLength+symbolOffset]
		}
	}

	return
}

// For each sample offset look for the preamble. Return a list of indexes the
// preamble is found at. Indexes are absolute in the unsliced quantized
// buffer.
func (d *Decoder) Search() (indexes []int) {
	for symbolOffset, slice := range d.slices {
		offset := 0
		idx := 0
		for {
			idx = d.preambleFinder.next(slice[offset:])
			if idx != -1 {
				indexes = append(indexes, (offset+idx)*d.DecCfg.SymbolLength+symbolOffset)
				offset += idx + 1
			} else {
				break
			}
		}
	}

	return
}

// Given a list of indeces the preamble exists at, sample the appropriate bits
// of the signal's bit-decision. Pack bits of each index into an array of byte
// arrays and return.
func (d Decoder) Slice(indices []int) (pkts [][]byte) {
	// We will likely find multiple instances of the message so only keep
	// track of unique instances.
	seen := make(map[string]bool)

	// For each of the indices the preamble exists at.
	for _, qIdx := range indices {
		// Check that we're still within the first sample block. We'll catch
		// the message on the next sample block otherwise.
		if qIdx > d.DecCfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.DecCfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.DecCfg.SymbolLength)]
		}

		// Store the packet in the seen map and append to the packet list.
		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<commit_msg>Minor optimizations in Transpose and Quantize.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2015 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package decode

import (
	"fmt"
	"log"
	"math"
)

// PacketConfig specifies packet-specific radio configuration.
type PacketConfig struct {
	DataRate int

	BlockSize, BlockSize2    int
	ChipLength, SymbolLength int
	SampleRate               int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	Preamble                       string

	BufferLength int

	CenterFreq uint32
}

func (cfg PacketConfig) Decimate(decimation int) PacketConfig {
	cfg.BlockSize /= decimation
	cfg.BlockSize2 /= decimation
	cfg.ChipLength /= decimation
	cfg.SymbolLength /= decimation
	cfg.SampleRate /= decimation
	cfg.DataRate /= decimation

	cfg.PreambleLength /= decimation
	cfg.PacketLength /= decimation

	cfg.BufferLength /= decimation

	return cfg
}

func (d Decoder) Log() {
	if d.Decimation != 1 {
		log.Printf("BlockSize: %d|%d\n", d.Cfg.BlockSize, d.DecCfg.BlockSize)
		log.Println("CenterFreq:", d.Cfg.CenterFreq)
		log.Printf("SampleRate: %d|%d\n", d.Cfg.SampleRate, d.DecCfg.SampleRate)
		log.Printf("DataRate: %d|%d\n", d.Cfg.DataRate, d.DecCfg.DataRate)
		log.Printf("ChipLength: %d|%d\n", d.Cfg.ChipLength, d.DecCfg.ChipLength)
		log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
		log.Printf("PreambleLength: %d|%d\n", d.Cfg.PreambleLength, d.DecCfg.PreambleLength)
		log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
		log.Printf("PacketLength: %d|%d\n", d.Cfg.PacketLength, d.DecCfg.PacketLength)
		log.Println("Preamble:", d.Cfg.Preamble)

		if d.Cfg.ChipLength%d.Decimation != 0 {
			log.Println("Warning: decimated symbol length is non-integral, sensitivity may be poor")
		}

		if d.DecCfg.ChipLength < 3 {
			log.Fatal("Error: illegal decimation factor, choose a smaller factor")
		}

		return
	}

	log.Println("CenterFreq:", d.Cfg.CenterFreq)
	log.Println("SampleRate:", d.Cfg.SampleRate)
	log.Println("DataRate:", d.Cfg.DataRate)
	log.Println("ChipLength:", d.Cfg.ChipLength)
	log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols)
	log.Println("PreambleLength:", d.Cfg.PreambleLength)
	log.Println("PacketSymbols:", d.Cfg.PacketSymbols)
	log.Println("PacketLength:", d.Cfg.PacketLength)
	log.Println("Preamble:", d.Cfg.Preamble)
}

// Decoder contains buffers and radio configuration.
type Decoder struct {
	Cfg PacketConfig

	Decimation int
	DecCfg     PacketConfig

	Signal    []float64
	Filtered  []float64
	Quantized []byte

	csum  []float64
	demod Demodulator

	preamble []byte
	slices   [][]byte

	preambleFinder *byteFinder

	pkt []byte
}

// Create a new decoder with the given packet configuration.
func NewDecoder(cfg PacketConfig, decimation int) (d Decoder) {
	d.Cfg = cfg

	d.Cfg.SymbolLength = d.Cfg.ChipLength << 1
	d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.ChipLength

	d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength
	d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength

	d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength)
	d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1

	d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize

	d.Decimation = decimation
	d.DecCfg = d.Cfg.Decimate(d.Decimation)

	// Allocate necessary buffers.
	d.Signal = make([]float64, d.DecCfg.BlockSize+d.DecCfg.SymbolLength)
	d.Filtered = make([]float64, d.DecCfg.BlockSize)
	d.Quantized = make([]byte, d.DecCfg.BufferLength)

	d.csum = make([]float64, len(d.Signal)+1)

	// Calculate magnitude lookup table specified by -fastmag flag.
	d.demod = NewMagLUT()

	// Pre-calculate a byte-slice version of the preamble for searching.
	d.preamble = make([]byte, d.Cfg.PreambleSymbols)
	for idx := range d.Cfg.Preamble {
		if d.Cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	// Slice quantized sample buffer to make searching for the preamble more
	// memory local.
	d.slices = make([][]byte, d.DecCfg.SymbolLength)

	symbolsPerBlock := d.DecCfg.BlockSize/d.DecCfg.SymbolLength + d.DecCfg.PreambleSymbols
	for symbolOffset := range d.slices {
		d.slices[symbolOffset] = make([]byte, symbolsPerBlock)
	}

	d.preambleFinder = makeByteFinder(d.preamble)

	// Signal up to the final stage is 1-bit per byte. Allocate a buffer to
	// store packed version 8-bits per byte.
	d.pkt = make([]byte, (d.DecCfg.PacketSymbols+7)>>3)

	return
}

// Decode accepts a sample block and performs various DSP techniques to extract a packet.
func (d Decoder) Decode(input []byte) []int {
	// Shift buffers to append new block.
	copy(d.Signal, d.Signal[d.DecCfg.BlockSize:])
	copy(d.Quantized, d.Quantized[d.DecCfg.BlockSize:])

	// Compute the magnitude of the new block.
	d.demod.Execute(input, d.Signal[d.DecCfg.SymbolLength:])

	// Perform matched filter on new block.
	d.Filter(d.Signal, d.Filtered)

	// Perform bit-decision on new block.
	Quantize(d.Filtered[d.DecCfg.SymbolLength:], d.Quantized[d.DecCfg.PacketLength:])

	// Pack the quantized signal into slices for searching.
	d.Transpose(d.Quantized)

	// Return a list of indexes the preamble exists at.
	return d.Search()
}

// A Demodulator knows how to demodulate an array of uint8 IQ samples into an
// array of float64 samples.
type Demodulator interface {
	Execute([]byte, []float64)
}

// Default Magnitude Lookup Table
type MagLUT []float64

// Pre-computes normalized squares with most common DC offset for rtl-sdr dongles.
func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = (127.5 - float64(idx)) / 127.5
		lut[idx] *= lut[idx]
	}
	return
}

// Calculates complex magnitude on given IQ stream writing result to output.
func (lut MagLUT) Execute(input []byte, output []float64) {
	decIdx := 0
	dec := (len(input) / len(output))

	for idx := 0; decIdx < len(output); idx += dec {
		output[decIdx] = lut[input[idx]] + lut[input[idx+1]]
		decIdx++
	}
}

// Matched filter for Manchester coded signals. Output signal's sign at each
// sample determines the bit-value since Manchester symbols have odd symmetry.
func (d Decoder) Filter(input, output []float64) {
	// Computing the cumulative summation over the signal simplifies
	// filtering to the difference of a pair of subtractions.
	var sum float64
	for idx, v := range input {
		sum += v
		d.csum[idx+1] = sum
	}

	// Filter result is difference of summation of lower and upper symbols.
	lower := d.csum[d.DecCfg.ChipLength:]
	upper := d.csum[d.DecCfg.SymbolLength:]
	for idx, l := range lower[:len(output)] {
		output[idx] = (l - d.csum[idx]) - (upper[idx] - l)
	}

	return
}

// Bit-value is determined by the sign of each sample after filtering.
func Quantize(input []float64, output []byte) {
	// There's really not much we can optimize here as it depends pretty
	// heavily on the content of the input, which is unbiased on average.
	for idx, val := range input {
		if val > 0 {
			output[idx] = 1
		} else {
			output[idx] = 0
		}
	}

	return
}

// Transpose quantized signal into slices such that the first rank represents
// sample offsets and the second represents the value of each symbol from the
// given offset.
//
// Transforms:
// <--Sym1--><--Sym2--><--Sym3--><--Sym4--><--Sym5--><--Sym6--><--Sym7--><--Sym8-->
// <12345678><12345678><12345678><12345678><12345678><12345678><12345678><12345678>
// to:
// <11111111><22222222><33333333><44444444><55555555><66666666><77777777><88888888>
func (d *Decoder) Transpose(input []byte) {
	for symbolOffset, slice := range d.slices {
		symbolInInput := 0
		offsetInput := input[symbolOffset:]
		for symbolIdx := range slice {
			slice[symbolIdx] = offsetInput[symbolInInput]
			symbolInInput += d.DecCfg.SymbolLength
		}
	}

	return
}

// For each sample offset look for the preamble. Return a list of indexes the
// preamble is found at. Indexes are absolute in the unsliced quantized
// buffer.
func (d *Decoder) Search() (indexes []int) {
	for symbolOffset, slice := range d.slices {
		offset := 0
		idx := 0
		for {
			idx = d.preambleFinder.next(slice[offset:])
			if idx != -1 {
				indexes = append(indexes, (offset+idx)*d.DecCfg.SymbolLength+symbolOffset)
				offset += idx + 1
			} else {
				break
			}
		}
	}

	return
}

// Given a list of indeces the preamble exists at, sample the appropriate bits
// of the signal's bit-decision. Pack bits of each index into an array of byte
// arrays and return.
func (d Decoder) Slice(indices []int) (pkts [][]byte) {
	// We will likely find multiple instances of the message so only keep
	// track of unique instances.
	seen := make(map[string]bool)

	// For each of the indices the preamble exists at.
	for _, qIdx := range indices {
		// Check that we're still within the first sample block. We'll catch
		// the message on the next sample block otherwise.
		if qIdx > d.DecCfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.DecCfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.DecCfg.SymbolLength)]
		}

		// Store the packet in the seen map and append to the packet list.
		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/hex"
	"hash"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"

	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/internal/log"
	"github.com/coreos/ignition/internal/resource"
	internalUtil "github.com/coreos/ignition/internal/util"
)

const (
	DefaultDirectoryPermissions os.FileMode = 0755
	DefaultFilePermissions      os.FileMode = 0644
)

type FetchOp struct {
	Hash         hash.Hash
	Path         string
	Mode         os.FileMode
	Uid          int
	Gid          int
	Url          url.URL
	FetchOptions resource.FetchOptions
}

// newHashedReader returns a new ReadCloser that also writes to the provided hash.
func newHashedReader(reader io.ReadCloser, hasher hash.Hash) io.ReadCloser {
	return struct {
		io.Reader
		io.Closer
	}{
		Reader: io.TeeReader(reader, hasher),
		Closer: reader,
	}
}

// PrepareFetch converts a given logger, http client, and types.File into a
// FetchOp. This includes operations such as parsing the source URL, generating
// a hasher, and performing user/group name lookups. If an error is encountered,
// the issue will be logged and nil will be returned.
func (u Util) PrepareFetch(l *log.Logger, f types.File) *FetchOp {
	var err error
	var expectedSum []byte

	// explicitly ignoring the error here because the config should already be
	// validated by this point
	uri, _ := url.Parse(f.Contents.Source)

	hasher, err := GetHasher(f.Contents.Verification)
	if err != nil {
		l.Crit("Error verifying file %q: %v", f.Path, err)
		return nil
	}

	if hasher != nil {
		// explicitly ignoring the error here because the config should already
		// be validated by this point
		_, expectedSumString, _ := f.Contents.Verification.HashParts()
		expectedSum, err = hex.DecodeString(expectedSumString)
		if err != nil {
			l.Crit("Error parsing verification string %q: %v", expectedSumString, err)
			return nil
		}
	}

	f.User.ID, f.Group.ID = u.GetUserGroupID(l, f.User, f.Group)
	if f.User.ID == nil || f.Group.ID == nil {
		return nil
	}

	return &FetchOp{
		Path: f.Path,
		Hash: hasher,
		Mode: os.FileMode(f.Mode),
		Uid:  *f.User.ID,
		Gid:  *f.Group.ID,
		Url:  *uri,
		FetchOptions: resource.FetchOptions{
			Hash:        hasher,
			Compression: f.Contents.Compression,
			ExpectedSum: expectedSum,
		},
	}
}

func (u Util) WriteLink(s types.Link) error {
	path := u.JoinPath(s.Path)

	if err := MkdirForFile(path); err != nil {
		return err
	}

	if s.Hard {
		targetPath := u.JoinPath(s.Target)
		return os.Link(targetPath, path)
	}

	if err := os.Symlink(s.Target, path); err != nil {
		return err
	}

	if err := os.Chown(path, *s.User.ID, *s.Group.ID); err != nil {
		return err
	}

	return nil
}

// PerformFetch performs a fetch operation generated by PrepareFetch, retrieving
// the file and writing it to disk. Any encountered errors are returned.
func (u Util) PerformFetch(f *FetchOp) error {
	var err error

	path := u.JoinPath(string(f.Path))

	if err := MkdirForFile(path); err != nil {
		return err
	}

	// Create a temporary file in the same directory to ensure it's on the same filesystem
	var tmp *os.File
	if tmp, err = ioutil.TempFile(filepath.Dir(path), "tmp"); err != nil {
		return err
	}

	defer func() {
		tmp.Close()
		if err != nil {
			os.Remove(tmp.Name())
		}
	}()

	err = u.Fetcher.Fetch(f.Url, tmp, f.FetchOptions)
	if err != nil {
		u.Crit("Error fetching file %q: %v", f.Path, err)
		return err
	}

	// XXX(vc): Note that we assume to be operating on the file we just wrote, this is only guaranteed
	// by using syscall.Fchown() and syscall.Fchmod()

	// Ensure the ownership and mode are as requested (since WriteFile can be affected by sticky bit)
	if err = os.Chown(tmp.Name(), f.Uid, f.Gid); err != nil {
		return err
	}

	if err = os.Chmod(tmp.Name(), f.Mode); err != nil {
		return err
	}

	if err = os.Rename(tmp.Name(), path); err != nil {
		return err
	}

	return nil
}

func (u Util) GetUserGroupID(l *log.Logger, user types.NodeUser, group types.NodeGroup) (*int, *int) {
	if user.Name != "" {
		usr, err := u.userLookup(user.Name)
		if err != nil {
			l.Crit("No such user %q: %v", user.Name, err)
			return nil, nil
		}
		uid, err := strconv.ParseInt(usr.Uid, 0, 0)
		if err != nil {
			l.Crit("Couldn't parse uid %q: %v", usr.Uid, err)
			return nil, nil
		}
		tmp := int(uid)
		user.ID = &tmp
	}
	if group.Name != "" {
		g, err := u.groupLookup(group.Name)
		if err != nil {
			l.Crit("No such group %q: %v", group.Name, err)
			return nil, nil
		}
		gid, err := strconv.ParseInt(g.Gid, 0, 0)
		if err != nil {
			l.Crit("Couldn't parse gid %q: %v", g.Gid, err)
			return nil, nil
		}
		tmp := int(gid)
		group.ID = &tmp
	}

	if user.ID == nil {
		user.ID = internalUtil.IntToPtr(0)
	}
	if group.ID == nil {
		group.ID = internalUtil.IntToPtr(0)
	}

	return user.ID, group.ID
}

// MkdirForFile helper creates the directory components of path.
func MkdirForFile(path string) error {
	return os.MkdirAll(filepath.Dir(path), DefaultDirectoryPermissions)
}
<commit_msg>internal/exec/util/file: use lchown chown when working with links<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/hex"
	"hash"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"

	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/internal/log"
	"github.com/coreos/ignition/internal/resource"
	internalUtil "github.com/coreos/ignition/internal/util"
)

const (
	DefaultDirectoryPermissions os.FileMode = 0755
	DefaultFilePermissions      os.FileMode = 0644
)

type FetchOp struct {
	Hash         hash.Hash
	Path         string
	Mode         os.FileMode
	Uid          int
	Gid          int
	Url          url.URL
	FetchOptions resource.FetchOptions
}

// newHashedReader returns a new ReadCloser that also writes to the provided hash.
func newHashedReader(reader io.ReadCloser, hasher hash.Hash) io.ReadCloser {
	return struct {
		io.Reader
		io.Closer
	}{
		Reader: io.TeeReader(reader, hasher),
		Closer: reader,
	}
}

// PrepareFetch converts a given logger, http client, and types.File into a
// FetchOp. This includes operations such as parsing the source URL, generating
// a hasher, and performing user/group name lookups. If an error is encountered,
// the issue will be logged and nil will be returned.
func (u Util) PrepareFetch(l *log.Logger, f types.File) *FetchOp {
	var err error
	var expectedSum []byte

	// explicitly ignoring the error here because the config should already be
	// validated by this point
	uri, _ := url.Parse(f.Contents.Source)

	hasher, err := GetHasher(f.Contents.Verification)
	if err != nil {
		l.Crit("Error verifying file %q: %v", f.Path, err)
		return nil
	}

	if hasher != nil {
		// explicitly ignoring the error here because the config should already
		// be validated by this point
		_, expectedSumString, _ := f.Contents.Verification.HashParts()
		expectedSum, err = hex.DecodeString(expectedSumString)
		if err != nil {
			l.Crit("Error parsing verification string %q: %v", expectedSumString, err)
			return nil
		}
	}

	f.User.ID, f.Group.ID = u.GetUserGroupID(l, f.User, f.Group)
	if f.User.ID == nil || f.Group.ID == nil {
		return nil
	}

	return &FetchOp{
		Path: f.Path,
		Hash: hasher,
		Mode: os.FileMode(f.Mode),
		Uid:  *f.User.ID,
		Gid:  *f.Group.ID,
		Url:  *uri,
		FetchOptions: resource.FetchOptions{
			Hash:        hasher,
			Compression: f.Contents.Compression,
			ExpectedSum: expectedSum,
		},
	}
}

func (u Util) WriteLink(s types.Link) error {
	path := u.JoinPath(s.Path)

	if err := MkdirForFile(path); err != nil {
		return err
	}

	if s.Hard {
		targetPath := u.JoinPath(s.Target)
		return os.Link(targetPath, path)
	}

	if err := os.Symlink(s.Target, path); err != nil {
		return err
	}

	if err := os.Lchown(path, *s.User.ID, *s.Group.ID); err != nil {
		return err
	}

	return nil
}

// PerformFetch performs a fetch operation generated by PrepareFetch, retrieving
// the file and writing it to disk. Any encountered errors are returned.
func (u Util) PerformFetch(f *FetchOp) error {
	var err error

	path := u.JoinPath(string(f.Path))

	if err := MkdirForFile(path); err != nil {
		return err
	}

	// Create a temporary file in the same directory to ensure it's on the same filesystem
	var tmp *os.File
	if tmp, err = ioutil.TempFile(filepath.Dir(path), "tmp"); err != nil {
		return err
	}

	defer func() {
		tmp.Close()
		if err != nil {
			os.Remove(tmp.Name())
		}
	}()

	err = u.Fetcher.Fetch(f.Url, tmp, f.FetchOptions)
	if err != nil {
		u.Crit("Error fetching file %q: %v", f.Path, err)
		return err
	}

	// XXX(vc): Note that we assume to be operating on the file we just wrote, this is only guaranteed
	// by using syscall.Fchown() and syscall.Fchmod()

	// Ensure the ownership and mode are as requested (since WriteFile can be affected by sticky bit)
	if err = os.Chown(tmp.Name(), f.Uid, f.Gid); err != nil {
		return err
	}

	if err = os.Chmod(tmp.Name(), f.Mode); err != nil {
		return err
	}

	if err = os.Rename(tmp.Name(), path); err != nil {
		return err
	}

	return nil
}

func (u Util) GetUserGroupID(l *log.Logger, user types.NodeUser, group types.NodeGroup) (*int, *int) {
	if user.Name != "" {
		usr, err := u.userLookup(user.Name)
		if err != nil {
			l.Crit("No such user %q: %v", user.Name, err)
			return nil, nil
		}
		uid, err := strconv.ParseInt(usr.Uid, 0, 0)
		if err != nil {
			l.Crit("Couldn't parse uid %q: %v", usr.Uid, err)
			return nil, nil
		}
		tmp := int(uid)
		user.ID = &tmp
	}
	if group.Name != "" {
		g, err := u.groupLookup(group.Name)
		if err != nil {
			l.Crit("No such group %q: %v", group.Name, err)
			return nil, nil
		}
		gid, err := strconv.ParseInt(g.Gid, 0, 0)
		if err != nil {
			l.Crit("Couldn't parse gid %q: %v", g.Gid, err)
			return nil, nil
		}
		tmp := int(gid)
		group.ID = &tmp
	}

	if user.ID == nil {
		user.ID = internalUtil.IntToPtr(0)
	}
	if group.ID == nil {
		group.ID = internalUtil.IntToPtr(0)
	}

	return user.ID, group.ID
}

// MkdirForFile helper creates the directory components of path.
func MkdirForFile(path string) error {
	return os.MkdirAll(filepath.Dir(path), DefaultDirectoryPermissions)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"sync"

	"github.com/jacobsa/fuse/fuseops"
	"golang.org/x/net/context"
)

type Inode interface {
	// All methods below require the lock to be held unless otherwise documented.
	sync.Locker

	// Return the ID assigned to the inode.
	//
	// Does not require the lock to be held.
	ID() fuseops.InodeID

	// Return the name of the GCS object backing the inode. This may be "foo/bar"
	// for a file, or "foo/bar/" for a directory.
	//
	// Does not require the lock to be held.
	Name() string

	// Increment the lookup count for the inode. For use in fuse operations where
	// the kernel expects us to remember the inode.
	IncrementLookupCount()

	// Return up to date attributes for this inode.
	Attributes(ctx context.Context) (fuseops.InodeAttributes, error)

	// Decrement the lookup count for the inode by the given amount.
	//
	// If this method returns true, the lookup count has hit zero and the
	// Destroy() method should be called to release any local resources, perhaps
	// after releasing locks that should not be held while blocking.
	DecrementLookupCount(n uint64) (destroy bool)

	// Clean up any local resources used by the inode, putting it into an
	// indeterminate state where no method should be called except Unlock.
	//
	// This method may block. Errors are for logging purposes only.
	Destroy() (err error)
}

// An inode that is backed by a particular generation of a GCS object.
type GenerationBackedInode interface {
	Inode

	// Requires the inode lock.
	SourceGeneration() int64
}
<commit_msg>Changed GenerationBackedInode so that it returns a pair.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inode

import (
	"sync"

	"github.com/jacobsa/fuse/fuseops"
	"golang.org/x/net/context"
)

type Inode interface {
	// All methods below require the lock to be held unless otherwise documented.
	sync.Locker

	// Return the ID assigned to the inode.
	//
	// Does not require the lock to be held.
	ID() fuseops.InodeID

	// Return the name of the GCS object backing the inode. This may be "foo/bar"
	// for a file, or "foo/bar/" for a directory.
	//
	// Does not require the lock to be held.
	Name() string

	// Increment the lookup count for the inode. For use in fuse operations where
	// the kernel expects us to remember the inode.
	IncrementLookupCount()

	// Return up to date attributes for this inode.
	Attributes(ctx context.Context) (fuseops.InodeAttributes, error)

	// Decrement the lookup count for the inode by the given amount.
	//
	// If this method returns true, the lookup count has hit zero and the
	// Destroy() method should be called to release any local resources, perhaps
	// after releasing locks that should not be held while blocking.
	DecrementLookupCount(n uint64) (destroy bool)

	// Clean up any local resources used by the inode, putting it into an
	// indeterminate state where no method should be called except Unlock.
	//
	// This method may block. Errors are for logging purposes only.
	Destroy() (err error)
}

// An inode that is backed by a particular generation of a GCS object.
type GenerationBackedInode interface {
	Inode

	// Requires the inode lock.
	SourceGeneration() Generation
}

// A particular generation of a GCS object, consisting of both a GCS object
// generation number and meta-generation number. Lexicographically ordered on
// the two.
//
// Cf. https://cloud.google.com/storage/docs/generations-preconditions
type Generation struct {
	Object int64
	Meta   int64
}
<|endoftext|>


<commit_before>// Copyright 2015-2020 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file in the root of the source tree.

package properties

import (
	"fmt"

	"github.com/freetaxii/libstix2/timestamp"
)

// ----------------------------------------------------------------------
// Define Types
// ----------------------------------------------------------------------

/* ModifiedProperty - A property used by all STIX objects that captures the date
and time that the object was modified or changed. This property effectively
tracks the version of the object. */
type ModifiedProperty struct {
	Created  string `json:"created,omitempty"`
	Modified string `json:"modified,omitempty"`
}

// ----------------------------------------------------------------------
// Public Methods - ModifiedProperty - Setters
// ----------------------------------------------------------------------

/* SetModifiedToCurrentTime - This methods sets the object created time to the
current time */
func (o *ModifiedProperty) SetModifiedToCurrentTime() error {
	o.Created = timestamp.CurrentTime("milli")
	return nil
}

/* SetModified - This method takes in a timestamp in either time.Time or string
format and updates the modifed property with it. The value is stored as a
string, so if the value is in time.Time format, it will be converted to the
correct STIX timestamp format. */
func (o *ModifiedProperty) SetModified(t interface{}) error {
	ts, _ := timestamp.ToString(t, "milli")
	o.Modified = ts
	return nil
}

/* GetModified - This method will return the modified timestamp as a string. If
the value is the same as the created timestamp, then this object is the first
version of the object. */
func (o *ModifiedProperty) GetModified() string {
	return o.Modified
}

// ----------------------------------------------------------------------
// Public Methods - ModifiedProperty - Checks
// ----------------------------------------------------------------------

/* VerifyExists - This method will verify that the modified property on an object
is present if required. It will return a boolean, an integer that tracks the
number of problems found, and a slice of strings that contain the detailed
results, whether good or bad. */
func (o *ModifiedProperty) VerifyExists() (bool, int, []string) {
	problemsFound := 0
	resultDetails := make([]string, 1)

	if o.Modified == "" {
		problemsFound++
		resultDetails[0] = fmt.Sprintf("-- The modified property is required but missing")
		return false, problemsFound, resultDetails
	}

	resultDetails[0] = fmt.Sprintf("++ The modified property is required and is present")
	return true, problemsFound, resultDetails
}

/* Compare - This method will compare two properties to make sure they are the
same and will return a boolean, an integer that tracks the number of problems
found, and a slice of strings that contain the detailed results, whether good or
bad. */
func (o *ModifiedProperty) Compare(obj2 *ModifiedProperty) (bool, int, []string) {
	problemsFound := 0
	resultDetails := make([]string, 0)

	// Check Modified Value
	if o.Modified != obj2.Modified {
		problemsFound++
		str := fmt.Sprintf("-- The modified dates do not match: %s | %s", o.Modified, obj2.Modified)
		resultDetails = append(resultDetails, str)
	} else {
		str := fmt.Sprintf("++ The modified dates match: %s | %s", o.Modified, obj2.Modified)
		resultDetails = append(resultDetails, str)
	}

	if problemsFound > 0 {
		return false, problemsFound, resultDetails
	}

	return true, 0, resultDetails
}
<commit_msg>fixed error in type value collision<commit_after>// Copyright 2015-2020 Bret Jordan, All rights reserved.
//
// Use of this source code is governed by an Apache 2.0 license that can be
// found in the LICENSE file in the root of the source tree.

package properties

import (
	"fmt"

	"github.com/freetaxii/libstix2/timestamp"
)

// ----------------------------------------------------------------------
// Define Types
// ----------------------------------------------------------------------

/* ModifiedProperty - A property used by all STIX objects that captures the date
and time that the object was modified or changed. This property effectively
tracks the version of the object. */
type ModifiedProperty struct {
	Modified string `json:"modified,omitempty"`
}

// ----------------------------------------------------------------------
// Public Methods - ModifiedProperty - Setters
// ----------------------------------------------------------------------

/* SetModifiedToCurrentTime - This methods sets the object created time to the
current time */
func (o *ModifiedProperty) SetModifiedToCurrentTime() error {
	o.Modified = timestamp.CurrentTime("milli")
	return nil
}

/* SetModified - This method takes in a timestamp in either time.Time or string
format and updates the modifed property with it. The value is stored as a
string, so if the value is in time.Time format, it will be converted to the
correct STIX timestamp format. */
func (o *ModifiedProperty) SetModified(t interface{}) error {
	ts, _ := timestamp.ToString(t, "milli")
	o.Modified = ts
	return nil
}

/* GetModified - This method will return the modified timestamp as a string. If
the value is the same as the created timestamp, then this object is the first
version of the object. */
func (o *ModifiedProperty) GetModified() string {
	return o.Modified
}

// ----------------------------------------------------------------------
// Public Methods - ModifiedProperty - Checks
// ----------------------------------------------------------------------

/* VerifyExists - This method will verify that the modified property on an object
is present if required. It will return a boolean, an integer that tracks the
number of problems found, and a slice of strings that contain the detailed
results, whether good or bad. */
func (o *ModifiedProperty) VerifyExists() (bool, int, []string) {
	problemsFound := 0
	resultDetails := make([]string, 1)

	if o.Modified == "" {
		problemsFound++
		resultDetails[0] = fmt.Sprintf("-- The modified property is required but missing")
		return false, problemsFound, resultDetails
	}

	resultDetails[0] = fmt.Sprintf("++ The modified property is required and is present")
	return true, problemsFound, resultDetails
}

/* Compare - This method will compare two properties to make sure they are the
same and will return a boolean, an integer that tracks the number of problems
found, and a slice of strings that contain the detailed results, whether good or
bad. */
func (o *ModifiedProperty) Compare(obj2 *ModifiedProperty) (bool, int, []string) {
	problemsFound := 0
	resultDetails := make([]string, 0)

	// Check Modified Value
	if o.Modified != obj2.Modified {
		problemsFound++
		str := fmt.Sprintf("-- The modified dates do not match: %s | %s", o.Modified, obj2.Modified)
		resultDetails = append(resultDetails, str)
	} else {
		str := fmt.Sprintf("++ The modified dates match: %s | %s", o.Modified, obj2.Modified)
		resultDetails = append(resultDetails, str)
	}

	if problemsFound > 0 {
		return false, problemsFound, resultDetails
	}

	return true, 0, resultDetails
}
<|endoftext|>