<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"strings"
	"sync"
	"time"
)

// Fetches bags (tar files) from S3 receiving buckets and validates them.
type APTFetcher struct {
	Context             *context.Context
	BagValidationConfig *validation.BagValidationConfig
	FetchChannel        chan *models.IngestState
	ValidationChannel   chan *models.IngestState
	CleanupChannel      chan *models.IngestState
	RecordChannel       chan *models.IngestState
	WaitGroup           sync.WaitGroup
}

func NewAPTFetcher(_context *context.Context) *APTFetcher {
	fetcher := &APTFetcher{
		Context: _context,
	}

	// Load the config settings that describe how to validate
	// APTrust bags. We'll exit here if the config can't be
	// loaded or is invalid.
	fetcher.BagValidationConfig = LoadAPTrustBagValidationConfig(_context)

	// Set up buffered channels
	fetcherBufferSize := _context.Config.FetchWorker.NetworkConnections * 4
	workerBufferSize := _context.Config.FetchWorker.Workers * 10
	fetcher.FetchChannel = make(chan *models.IngestState, fetcherBufferSize)
	fetcher.ValidationChannel = make(chan *models.IngestState, workerBufferSize)
	fetcher.RecordChannel = make(chan *models.IngestState, workerBufferSize)
	fetcher.CleanupChannel = make(chan *models.IngestState, workerBufferSize)
	// Set up a limited number of go routines
	for i := 0; i < _context.Config.FetchWorker.NetworkConnections; i++ {
		go fetcher.fetch()
	}
	for i := 0; i < _context.Config.FetchWorker.Workers; i++ {
		go fetcher.validate()
		go fetcher.cleanup()
		go fetcher.record()
	}
	return fetcher
}

// This is the callback that NSQ workers use to handle messages from NSQ.
func (fetcher *APTFetcher) HandleMessage(message *nsq.Message) error {
	// Set up our IngestState. Most of this comes from Pharos;
	// some of it we have to build fresh.
	ingestState, err := GetIngestState(message, fetcher.Context, true)
	if err != nil {
		fetcher.Context.MessageLog.Error(err.Error())
		return err
	}
	ingestState.NSQMessage = message

	// If this item was queued more than once, and this process or any
	// other is currently working on it, just finish the message and
	// assume that the in-progress worker will take care of the original.
	if ingestState.WorkItem.Node != "" && ingestState.WorkItem.Pid != 0 {
		fetcher.Context.MessageLog.Info("Marking WorkItem %d (%s/%s) as finished "+
			"without doing any work, because this item is currently in process by "+
			"node %s, pid %s. WorkItem was last updated at %s.",
			ingestState.WorkItem.Id, ingestState.WorkItem.Bucket,
			ingestState.WorkItem.Name, ingestState.WorkItem.Node,
			ingestState.WorkItem.Pid, ingestState.WorkItem.UpdatedAt)
		message.Finish()
		return nil
	}

	// Save the state of this item in Pharos.
	RecordWorkItemState(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)

	// Tell Pharos that we've started to fetch this item.
	err = MarkWorkItemStarted(ingestState, fetcher.Context, constants.StageFetch,
		"Fetching bag from receiving bucket.")
	if err != nil {
		fetcher.Context.MessageLog.Error(err.Error())
		return err
	}

	// NSQ message autoresponse periodically tells the queue
	// that the message is still being processed. This doesn't
	// work for us in cases where we're fetching a file that's
	// 100GB+ in size. We need to manually Touch() NSQ periodically
	// to let the queue know that we're still actively working on
	// the message. Otherwise, NSQ thinks it timed out and sends
	// the message to a new worker.
	message.DisableAutoResponse()

	if fetcher.canSkipFetchAndValidate(ingestState) {
		fetcher.Context.MessageLog.Info("Sending %s/%s straight to record queue",
			ingestState.IngestManifest.S3Bucket, ingestState.IngestManifest.S3Key)
		fetcher.RecordChannel <- ingestState
	} else {

		// Reserve disk space to download this item, or requeue it
		// if we can't get the disk space.
		if !fetcher.reserveSpaceForDownload(ingestState) {
			message.Requeue(1 * time.Minute)
			return nil
		}

		// Start at fetch, which is the very beginning.
		// This may be the second or third attempt to ingest this bag.
		// If so, clear out old error message from previous attempts.
		ingestState.IngestManifest.FetchResult.ClearErrors()
		ingestState.IngestManifest.ValidateResult.ClearErrors()

		fetcher.Context.MessageLog.Info("Putting %s/%s straight to fetch queue",
			ingestState.IngestManifest.S3Bucket, ingestState.IngestManifest.S3Key)

		fetcher.FetchChannel <- ingestState
	}

	// Return no error, so NSQ knows we're OK.
	return nil
}

// -------------------------------------------------------------------------
// Step 1 of 4: Fetch
//
// fetch copies the file from S3 to our local staging area.
// If all goes well, the file will wind up in
// ingestState.IngestManifest.Object.IngestTarFilePath
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) fetch() {
	for ingestState := range fetcher.FetchChannel {
		// Tell NSQ we're working on this
		ingestState.TouchNSQ()

		ingestState.IngestManifest.FetchResult.Start()
		ingestState.IngestManifest.FetchResult.Attempted = true
		ingestState.IngestManifest.FetchResult.AttemptNumber += 1

		err := fetcher.downloadFile(ingestState)

		// Download may have taken 1 second or 3 hours.
		// Remind NSQ that we're still on this.
		ingestState.TouchNSQ()

		if err != nil {
			ingestState.IngestManifest.FetchResult.AddError(err.Error())
		}
		ingestState.IngestManifest.FetchResult.Finish()
		fetcher.ValidationChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 2 of 4: Validate
//
// Make sure the tar file is a valid bag.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) validate() {
	for ingestState := range fetcher.ValidationChannel {
		// Don't time us out, NSQ!
		ingestState.TouchNSQ()

		// Tell Pharos that we've started to validate item.
		// Let's NOT quit if there's an error here. In that case, Pharos
		// might not know that we're validating, but we can still proceed.
		// Restarting the whole fetch process would be expensive.
		MarkWorkItemStarted(ingestState, fetcher.Context, constants.StageValidate,
			"Validating bag.")

		// Set up a new validator to check this bag.
		var validationResult *validation.ValidationResult
		validator, err := validation.NewBagValidator(
			ingestState.IngestManifest.Object.IngestTarFilePath,
			fetcher.BagValidationConfig)
		if err != nil {
			// Could not create a BagValidator. Should this be fatal?
			ingestState.IngestManifest.ValidateResult.AddError(err.Error())
		} else {

			// Here's where bag validation actually happens. There's a lot
			// going on in this call, which can take anywhere from 2 seconds
			// to several hours to complete, depending on the size of the bag.
			// The most time-consuming part of the validation process is
			// calculating md5 and sha256 checksums on every file in the bag.
			// If the bag is 100GB+ in size, that takes a long time.
			validationResult = validator.Validate()

			// The validator creates its own WorkSummary, complete with
			// Start/Finish timestamps, error messages and everything.
			// Just copy that into our IngestManifest.
			ingestState.IngestManifest.ValidateResult = validationResult.ValidationSummary

			// NOTE that we are OVERWRITING the IntellectualObject here
			// with the much more complete version returned by the validator,
			// but we have to reset some basic data that's only available
			// in the current context.
			ingestState.IngestManifest.Object = validationResult.IntellectualObject
			SetBasicObjectInfo(ingestState, fetcher.Context)

			// If the bag is invalid, that's a fatal error. We should not do
			// any further processing on it.
			if validationResult.HasErrors() {
				ingestState.IngestManifest.ValidateResult.ErrorIsFatal = true
				ingestState.IngestManifest.ValidateResult.Retry = false
			}
		}
		ingestState.TouchNSQ()
		fetcher.CleanupChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 3 of 4: Cleanup (conditional)
//
// cleanup deletes the tar file we just downloaded, if we determine that
// something is wrong with it and there should be no further processing.
// If the bag is valid, we leave it in the staging area. The next process
// (store) will pick it up and copy files to S3 and Glacier.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) cleanup() {
	for ingestState := range fetcher.CleanupChannel {
		tarFile := ingestState.IngestManifest.Object.IngestTarFilePath
		hasErrors := (ingestState.IngestManifest.FetchResult.HasErrors() ||
			ingestState.IngestManifest.ValidateResult.HasErrors())
		if hasErrors && fileutil.FileExists(tarFile) {
			// Most likely bad md5 digest, but perhaps also a partial download.
			fetcher.Context.MessageLog.Info("Deleting due to download error: %s",
				tarFile)
			DeleteBagFromStaging(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)
		}
		fetcher.RecordChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 4 of 4: Record updates the WorkItem and WorkItemState in Pharos.
//
// record tells Pharos what's happened with this WorkItem,
// and it pushes the item into the next queue (validation)
// if necessary.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) record() {
	for ingestState := range fetcher.RecordChannel {

		// Copy JSON representation of the IngestManifest to Pharos
		// and to the JSON log.
		RecordWorkItemState(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)

		// Fatal errors, or too many recurring transient errors
		attemptNumber := ingestState.IngestManifest.FetchResult.AttemptNumber
		maxAttempts := fetcher.Context.Config.FetchWorker.MaxAttempts
		itsTimeToGiveUp := (ingestState.IngestManifest.HasFatalErrors() ||
			(ingestState.IngestManifest.HasErrors() && attemptNumber >= maxAttempts))

		if itsTimeToGiveUp {
			ingestState.FinishNSQ()
			MarkWorkItemFailed(ingestState, fetcher.Context)
		} else if ingestState.IngestManifest.HasErrors() {
			ingestState.RequeueNSQ(30000)
			MarkWorkItemRequeued(ingestState, fetcher.Context)
		} else {
			ingestState.FinishNSQ()
			MarkWorkItemSucceeded(ingestState, fetcher.Context, constants.StageStore)
			PushToQueue(ingestState, fetcher.Context, fetcher.Context.Config.StoreWorker.NsqTopic)
		}
	}
}

// Make sure we have space to download this item.
func (fetcher *APTFetcher) reserveSpaceForDownload(ingestState *models.IngestState) bool {
	okToDownload := false
	err := fetcher.Context.VolumeClient.Ping(500)
	if err == nil {
		path := ingestState.IngestManifest.Object.IngestTarFilePath
		ok, err := fetcher.Context.VolumeClient.Reserve(path, uint64(ingestState.WorkItem.Size))
		if err != nil {
			fetcher.Context.MessageLog.Warning("Volume service returned an error. "+
				"Will requeue bag %s/%s because we may not have enough space to download %d bytes.",
				ingestState.WorkItem.Bucket, ingestState.WorkItem.Name, ingestState.WorkItem.Size)
		} else if ok {
			// VolumeService says we have enough space for this.
			okToDownload = ok
		}
	} else {
		fetcher.Context.MessageLog.Warning("Volume service is not running or returned an error. "+
			"Continuing as if we have enough space to download %d bytes.",
			ingestState.WorkItem.Size)
		okToDownload = true
	}
	return okToDownload
}

// Returns true if we can skip fetch and validate. We can skip those
// steps if on a previous run we validated the bag, and it's still
// there in our working directory. This anticipates the case where
// we did those steps but were not able to update the WorkItem record
// in Pharos at the end of the fetch/validate process.
func (fetcher *APTFetcher) canSkipFetchAndValidate(ingestState *models.IngestState) bool {
	return (ingestState.WorkItem.Stage == constants.StageValidate &&
		ingestState.IngestManifest.ValidateResult.Finished() &&
		!ingestState.IngestManifest.HasFatalErrors() &&
		fileutil.FileExists(ingestState.IngestManifest.Object.IngestTarFilePath))
}

// Download the file, and update the IngestManifest while we're at it.
func (fetcher *APTFetcher) downloadFile(ingestState *models.IngestState) error {
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		ingestState.IngestManifest.S3Bucket,
		ingestState.IngestManifest.S3Key,
		ingestState.IngestManifest.Object.IngestTarFilePath,
		true,  // calculate md5 checksum on the entire tar file
		false, // calculate sha256 checksum on the entire tar file
	)

	// It's fairly common for very large bags to fail more than
	// once on transient network errors (e.g. "Connection reset by peer")
	// So we give this several tries.
	for i := 0; i < 10; i++ {
		downloader.Fetch()
		if downloader.ErrorMessage == "" {
			fetcher.Context.MessageLog.Info("Fetched %s/%s after %d attempts",
				ingestState.IngestManifest.S3Bucket,
				ingestState.IngestManifest.S3Key,
				i+1)
			break
		}
	}

	// Return now if we failed.
	if downloader.ErrorMessage != "" {
		return fmt.Errorf("Error fetching %s/%s: %v",
			ingestState.IngestManifest.S3Bucket,
			ingestState.IngestManifest.S3Key,
			downloader.ErrorMessage)
	}

	obj := ingestState.IngestManifest.Object
	obj.IngestSize = downloader.BytesCopied
	obj.IngestRemoteMd5 = *downloader.Response.ETag
	obj.IngestLocalMd5 = downloader.Md5Digest

	// The ETag for S3 object uploaded via single-part upload is
	// the file's md5 digest. For objects uploaded via multi-part
	// upload, the ETag is calculated differently and includes a
	// dash near the end, followed by the number of parts in the
	// multipart upload. We can't use that kind of ETag to verify
	// the md5 checksum that we calculated.
	obj.IngestMd5Verifiable = strings.Contains(downloader.Md5Digest, "-")
	if obj.IngestMd5Verifiable {
		obj.IngestMd5Verified = obj.IngestRemoteMd5 == obj.IngestLocalMd5
	}

	// If we got a bad checksum, note the error in the WorkSummary.
	if obj.IngestMd5Verifiable && !obj.IngestMd5Verified {
		ingestState.IngestManifest.FetchResult.AddError("Our md5 '%s' does not match S3 md5 '%s'",
			obj.IngestLocalMd5, obj.IngestRemoteMd5)
		ingestState.IngestManifest.FetchResult.ErrorIsFatal = true
	}

	return nil
}

// This is for direct testing without NSQ.
func (fetcher *APTFetcher) RunWithoutNsq(ingestState *models.IngestState) {
	fetcher.WaitGroup.Add(1)
	fetcher.FetchChannel <- ingestState
	fetcher.Context.MessageLog.Debug("Put %s into Fluctus channel", ingestState.IngestManifest.S3Key)
	fetcher.WaitGroup.Wait()
}
<commit_msg>Removed unused WaitGroup<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// Fetches bags (tar files) from S3 receiving buckets and validates them.
type APTFetcher struct {
	Context             *context.Context
	BagValidationConfig *validation.BagValidationConfig
	FetchChannel        chan *models.IngestState
	ValidationChannel   chan *models.IngestState
	CleanupChannel      chan *models.IngestState
	RecordChannel       chan *models.IngestState
}

func NewAPTFetcher(_context *context.Context) *APTFetcher {
	fetcher := &APTFetcher{
		Context: _context,
	}

	// Load the config settings that describe how to validate
	// APTrust bags. We'll exit here if the config can't be
	// loaded or is invalid.
	fetcher.BagValidationConfig = LoadAPTrustBagValidationConfig(_context)

	// Set up buffered channels
	fetcherBufferSize := _context.Config.FetchWorker.NetworkConnections * 4
	workerBufferSize := _context.Config.FetchWorker.Workers * 10
	fetcher.FetchChannel = make(chan *models.IngestState, fetcherBufferSize)
	fetcher.ValidationChannel = make(chan *models.IngestState, workerBufferSize)
	fetcher.RecordChannel = make(chan *models.IngestState, workerBufferSize)
	fetcher.CleanupChannel = make(chan *models.IngestState, workerBufferSize)
	// Set up a limited number of go routines
	for i := 0; i < _context.Config.FetchWorker.NetworkConnections; i++ {
		go fetcher.fetch()
	}
	for i := 0; i < _context.Config.FetchWorker.Workers; i++ {
		go fetcher.validate()
		go fetcher.cleanup()
		go fetcher.record()
	}
	return fetcher
}

// This is the callback that NSQ workers use to handle messages from NSQ.
func (fetcher *APTFetcher) HandleMessage(message *nsq.Message) error {
	// Set up our IngestState. Most of this comes from Pharos;
	// some of it we have to build fresh.
	ingestState, err := GetIngestState(message, fetcher.Context, true)
	if err != nil {
		fetcher.Context.MessageLog.Error(err.Error())
		return err
	}
	ingestState.NSQMessage = message

	// If this item was queued more than once, and this process or any
	// other is currently working on it, just finish the message and
	// assume that the in-progress worker will take care of the original.
	if ingestState.WorkItem.Node != "" && ingestState.WorkItem.Pid != 0 {
		fetcher.Context.MessageLog.Info("Marking WorkItem %d (%s/%s) as finished "+
			"without doing any work, because this item is currently in process by "+
			"node %s, pid %s. WorkItem was last updated at %s.",
			ingestState.WorkItem.Id, ingestState.WorkItem.Bucket,
			ingestState.WorkItem.Name, ingestState.WorkItem.Node,
			ingestState.WorkItem.Pid, ingestState.WorkItem.UpdatedAt)
		message.Finish()
		return nil
	}

	// Save the state of this item in Pharos.
	RecordWorkItemState(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)

	// Tell Pharos that we've started to fetch this item.
	err = MarkWorkItemStarted(ingestState, fetcher.Context, constants.StageFetch,
		"Fetching bag from receiving bucket.")
	if err != nil {
		fetcher.Context.MessageLog.Error(err.Error())
		return err
	}

	// NSQ message autoresponse periodically tells the queue
	// that the message is still being processed. This doesn't
	// work for us in cases where we're fetching a file that's
	// 100GB+ in size. We need to manually Touch() NSQ periodically
	// to let the queue know that we're still actively working on
	// the message. Otherwise, NSQ thinks it timed out and sends
	// the message to a new worker.
	message.DisableAutoResponse()

	if fetcher.canSkipFetchAndValidate(ingestState) {
		fetcher.Context.MessageLog.Info("Sending %s/%s straight to record queue",
			ingestState.IngestManifest.S3Bucket, ingestState.IngestManifest.S3Key)
		fetcher.RecordChannel <- ingestState
	} else {

		// Reserve disk space to download this item, or requeue it
		// if we can't get the disk space.
		if !fetcher.reserveSpaceForDownload(ingestState) {
			message.Requeue(1 * time.Minute)
			return nil
		}

		// Start at fetch, which is the very beginning.
		// This may be the second or third attempt to ingest this bag.
		// If so, clear out old error message from previous attempts.
		ingestState.IngestManifest.FetchResult.ClearErrors()
		ingestState.IngestManifest.ValidateResult.ClearErrors()

		fetcher.Context.MessageLog.Info("Putting %s/%s straight to fetch queue",
			ingestState.IngestManifest.S3Bucket, ingestState.IngestManifest.S3Key)

		fetcher.FetchChannel <- ingestState
	}

	// Return no error, so NSQ knows we're OK.
	return nil
}

// -------------------------------------------------------------------------
// Step 1 of 4: Fetch
//
// fetch copies the file from S3 to our local staging area.
// If all goes well, the file will wind up in
// ingestState.IngestManifest.Object.IngestTarFilePath
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) fetch() {
	for ingestState := range fetcher.FetchChannel {
		// Tell NSQ we're working on this
		ingestState.TouchNSQ()

		ingestState.IngestManifest.FetchResult.Start()
		ingestState.IngestManifest.FetchResult.Attempted = true
		ingestState.IngestManifest.FetchResult.AttemptNumber += 1

		err := fetcher.downloadFile(ingestState)

		// Download may have taken 1 second or 3 hours.
		// Remind NSQ that we're still on this.
		ingestState.TouchNSQ()

		if err != nil {
			ingestState.IngestManifest.FetchResult.AddError(err.Error())
		}
		ingestState.IngestManifest.FetchResult.Finish()
		fetcher.ValidationChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 2 of 4: Validate
//
// Make sure the tar file is a valid bag.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) validate() {
	for ingestState := range fetcher.ValidationChannel {
		// Don't time us out, NSQ!
		ingestState.TouchNSQ()

		// Tell Pharos that we've started to validate item.
		// Let's NOT quit if there's an error here. In that case, Pharos
		// might not know that we're validating, but we can still proceed.
		// Restarting the whole fetch process would be expensive.
		MarkWorkItemStarted(ingestState, fetcher.Context, constants.StageValidate,
			"Validating bag.")

		// Set up a new validator to check this bag.
		var validationResult *validation.ValidationResult
		validator, err := validation.NewBagValidator(
			ingestState.IngestManifest.Object.IngestTarFilePath,
			fetcher.BagValidationConfig)
		if err != nil {
			// Could not create a BagValidator. Should this be fatal?
			ingestState.IngestManifest.ValidateResult.AddError(err.Error())
		} else {

			// Here's where bag validation actually happens. There's a lot
			// going on in this call, which can take anywhere from 2 seconds
			// to several hours to complete, depending on the size of the bag.
			// The most time-consuming part of the validation process is
			// calculating md5 and sha256 checksums on every file in the bag.
			// If the bag is 100GB+ in size, that takes a long time.
			validationResult = validator.Validate()

			// The validator creates its own WorkSummary, complete with
			// Start/Finish timestamps, error messages and everything.
			// Just copy that into our IngestManifest.
			ingestState.IngestManifest.ValidateResult = validationResult.ValidationSummary

			// NOTE that we are OVERWRITING the IntellectualObject here
			// with the much more complete version returned by the validator,
			// but we have to reset some basic data that's only available
			// in the current context.
			ingestState.IngestManifest.Object = validationResult.IntellectualObject
			SetBasicObjectInfo(ingestState, fetcher.Context)

			// If the bag is invalid, that's a fatal error. We should not do
			// any further processing on it.
			if validationResult.HasErrors() {
				ingestState.IngestManifest.ValidateResult.ErrorIsFatal = true
				ingestState.IngestManifest.ValidateResult.Retry = false
			}
		}
		ingestState.TouchNSQ()
		fetcher.CleanupChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 3 of 4: Cleanup (conditional)
//
// cleanup deletes the tar file we just downloaded, if we determine that
// something is wrong with it and there should be no further processing.
// If the bag is valid, we leave it in the staging area. The next process
// (store) will pick it up and copy files to S3 and Glacier.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) cleanup() {
	for ingestState := range fetcher.CleanupChannel {
		tarFile := ingestState.IngestManifest.Object.IngestTarFilePath
		hasErrors := (ingestState.IngestManifest.FetchResult.HasErrors() ||
			ingestState.IngestManifest.ValidateResult.HasErrors())
		if hasErrors && fileutil.FileExists(tarFile) {
			// Most likely bad md5 digest, but perhaps also a partial download.
			fetcher.Context.MessageLog.Info("Deleting due to download error: %s",
				tarFile)
			DeleteBagFromStaging(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)
		}
		fetcher.RecordChannel <- ingestState
	}
}

// -------------------------------------------------------------------------
// Step 4 of 4: Record updates the WorkItem and WorkItemState in Pharos.
//
// record tells Pharos what's happened with this WorkItem,
// and it pushes the item into the next queue (validation)
// if necessary.
// -------------------------------------------------------------------------
func (fetcher *APTFetcher) record() {
	for ingestState := range fetcher.RecordChannel {

		// Copy JSON representation of the IngestManifest to Pharos
		// and to the JSON log.
		RecordWorkItemState(ingestState, fetcher.Context, ingestState.IngestManifest.FetchResult)

		// Fatal errors, or too many recurring transient errors
		attemptNumber := ingestState.IngestManifest.FetchResult.AttemptNumber
		maxAttempts := fetcher.Context.Config.FetchWorker.MaxAttempts
		itsTimeToGiveUp := (ingestState.IngestManifest.HasFatalErrors() ||
			(ingestState.IngestManifest.HasErrors() && attemptNumber >= maxAttempts))

		if itsTimeToGiveUp {
			ingestState.FinishNSQ()
			MarkWorkItemFailed(ingestState, fetcher.Context)
		} else if ingestState.IngestManifest.HasErrors() {
			ingestState.RequeueNSQ(30000)
			MarkWorkItemRequeued(ingestState, fetcher.Context)
		} else {
			ingestState.FinishNSQ()
			MarkWorkItemSucceeded(ingestState, fetcher.Context, constants.StageStore)
			PushToQueue(ingestState, fetcher.Context, fetcher.Context.Config.StoreWorker.NsqTopic)
		}
	}
}

// Make sure we have space to download this item.
func (fetcher *APTFetcher) reserveSpaceForDownload(ingestState *models.IngestState) bool {
	okToDownload := false
	err := fetcher.Context.VolumeClient.Ping(500)
	if err == nil {
		path := ingestState.IngestManifest.Object.IngestTarFilePath
		ok, err := fetcher.Context.VolumeClient.Reserve(path, uint64(ingestState.WorkItem.Size))
		if err != nil {
			fetcher.Context.MessageLog.Warning("Volume service returned an error. "+
				"Will requeue bag %s/%s because we may not have enough space to download %d bytes.",
				ingestState.WorkItem.Bucket, ingestState.WorkItem.Name, ingestState.WorkItem.Size)
		} else if ok {
			// VolumeService says we have enough space for this.
			okToDownload = ok
		}
	} else {
		fetcher.Context.MessageLog.Warning("Volume service is not running or returned an error. "+
			"Continuing as if we have enough space to download %d bytes.",
			ingestState.WorkItem.Size)
		okToDownload = true
	}
	return okToDownload
}

// Returns true if we can skip fetch and validate. We can skip those
// steps if on a previous run we validated the bag, and it's still
// there in our working directory. This anticipates the case where
// we did those steps but were not able to update the WorkItem record
// in Pharos at the end of the fetch/validate process.
func (fetcher *APTFetcher) canSkipFetchAndValidate(ingestState *models.IngestState) bool {
	return (ingestState.WorkItem.Stage == constants.StageValidate &&
		ingestState.IngestManifest.ValidateResult.Finished() &&
		!ingestState.IngestManifest.HasFatalErrors() &&
		fileutil.FileExists(ingestState.IngestManifest.Object.IngestTarFilePath))
}

// Download the file, and update the IngestManifest while we're at it.
func (fetcher *APTFetcher) downloadFile(ingestState *models.IngestState) error {
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		ingestState.IngestManifest.S3Bucket,
		ingestState.IngestManifest.S3Key,
		ingestState.IngestManifest.Object.IngestTarFilePath,
		true,  // calculate md5 checksum on the entire tar file
		false, // calculate sha256 checksum on the entire tar file
	)

	// It's fairly common for very large bags to fail more than
	// once on transient network errors (e.g. "Connection reset by peer")
	// So we give this several tries.
	for i := 0; i < 10; i++ {
		downloader.Fetch()
		if downloader.ErrorMessage == "" {
			fetcher.Context.MessageLog.Info("Fetched %s/%s after %d attempts",
				ingestState.IngestManifest.S3Bucket,
				ingestState.IngestManifest.S3Key,
				i+1)
			break
		}
	}

	// Return now if we failed.
	if downloader.ErrorMessage != "" {
		return fmt.Errorf("Error fetching %s/%s: %v",
			ingestState.IngestManifest.S3Bucket,
			ingestState.IngestManifest.S3Key,
			downloader.ErrorMessage)
	}

	obj := ingestState.IngestManifest.Object
	obj.IngestSize = downloader.BytesCopied
	obj.IngestRemoteMd5 = *downloader.Response.ETag
	obj.IngestLocalMd5 = downloader.Md5Digest

	// The ETag for S3 object uploaded via single-part upload is
	// the file's md5 digest. For objects uploaded via multi-part
	// upload, the ETag is calculated differently and includes a
	// dash near the end, followed by the number of parts in the
	// multipart upload. We can't use that kind of ETag to verify
	// the md5 checksum that we calculated.
	obj.IngestMd5Verifiable = strings.Contains(downloader.Md5Digest, "-")
	if obj.IngestMd5Verifiable {
		obj.IngestMd5Verified = obj.IngestRemoteMd5 == obj.IngestLocalMd5
	}

	// If we got a bad checksum, note the error in the WorkSummary.
	if obj.IngestMd5Verifiable && !obj.IngestMd5Verified {
		ingestState.IngestManifest.FetchResult.AddError("Our md5 '%s' does not match S3 md5 '%s'",
			obj.IngestLocalMd5, obj.IngestRemoteMd5)
		ingestState.IngestManifest.FetchResult.ErrorIsFatal = true
	}

	return nil
}
<|endoftext|>


<commit_before>package torrent

import (
	"math"
	"time"

	"github.com/cenkalti/rain/internal/mse"
	"github.com/cenkalti/rain/internal/peer"
)

// Stats contains statistics about Torrent.
type Stats struct {
	// Status of the torrent.
	Status Status
	// Contains the error message if torrent is stopped unexpectedly.
	Error  error
	Pieces struct {
		// Number of pieces that are checked when torrent is in "Verifying" state.
		Checked uint32
		// Number of pieces that we are downloaded successfully and verivied by hash check.
		Have uint32
		// Number of pieces that need to be downloaded. Some of them may be being downloaded.
		// Pieces that are being downloaded may counted as missing until they are downloaded and passed hash check.
		Missing uint32
		// Number of unique pieces available on swarm.
		// If this number is less then the number of total pieces, the download may never finish.
		Available uint32
		// Number of total pieces in torrent.
		Total uint32
	}
	Bytes struct {
		// Bytes that are downloaded and passed hash check.
		Completed int64
		// The number of bytes that is needed to complete all missing pieces.
		Incomplete int64
		// The number of total bytes of files in torrent.  Total = Completed + Incomplete
		Total int64
		// Downloaded is the number of bytes downloaded from swarm.
		// Because some pieces may be downloaded more than once, this number may be greater than completed bytes.
		Downloaded int64
		// BytesUploaded is the number of bytes uploaded to the swarm.
		Uploaded int64
		// Bytes downloaded due to duplicate/non-requested pieces.
		Wasted int64
		// Bytes allocated on storage.
		Allocated int64
	}
	Peers struct {
		// Number of peers that are connected, handshaked and ready to send and receive messages.
		Total int
		// Number of peers that have connected to us.
		Incoming int
		// Number of peers that we have connected to.
		Outgoing int
	}
	Handshakes struct {
		// Number of peers that are not handshaked yet.
		Total int
		// Number of incoming peers in handshake state.
		Incoming int
		// Number of outgoing peers in handshake state.
		Outgoing int
	}
	Addresses struct {
		// Total number of peer addresses that are ready to be connected.
		Total int
		// Peers found via trackers.
		Tracker int
		// Peers found via DHT node.
		DHT int
		// Peers found via peer exchange.
		PEX int
	}
	Downloads struct {
		// Number of active piece downloads.
		Total int
		// Number of pieces that are being downloaded normally.
		Running int
		// Number of pieces that are being downloaded too slow.
		Snubbed int
		// Number of piece downloads in choked state.
		Choked int
	}
	MetadataDownloads struct {
		// Number of active metadata downloads.
		Total int
		// Number of peers that uploading too slow.
		Snubbed int
		// Number of peers that are being downloaded normally.
		Running int
	}
	// Name can change after metadata is downloaded.
	Name string
	// Is private torrent?
	Private bool
	// Length of a single piece.
	PieceLength uint32
	// Duration while the torrent is in Seeding status.
	SeededFor time.Duration
	// Speed is calculated as 1-minute moving average.
	Speed struct {
		// Downloaded bytes per second.
		Download uint
		// Uploaded bytes per second.
		Upload uint
	}
	// Time remaining to complete download. nil value means infinity.
	ETA *time.Duration
}

func (t *torrent) stats() Stats {
	t.updateSeedDuration()

	var s Stats
	s.Status = t.status()
	s.Error = t.lastError
	s.Addresses.Total = t.addrList.Len()
	s.Addresses.Tracker = t.addrList.LenSource(peer.SourceTracker)
	s.Addresses.DHT = t.addrList.LenSource(peer.SourceDHT)
	s.Addresses.PEX = t.addrList.LenSource(peer.SourcePEX)
	s.Handshakes.Incoming = len(t.incomingHandshakers)
	s.Handshakes.Outgoing = len(t.outgoingHandshakers)
	s.Handshakes.Total = len(t.incomingHandshakers) + len(t.outgoingHandshakers)
	s.Peers.Total = len(t.peers)
	s.Peers.Incoming = len(t.incomingPeers)
	s.Peers.Outgoing = len(t.outgoingPeers)
	s.MetadataDownloads.Total = len(t.infoDownloaders)
	s.MetadataDownloads.Snubbed = len(t.infoDownloadersSnubbed)
	s.MetadataDownloads.Running = len(t.infoDownloaders) - len(t.infoDownloadersSnubbed)
	s.Downloads.Total = len(t.pieceDownloaders)
	s.Downloads.Snubbed = len(t.pieceDownloadersSnubbed)
	s.Downloads.Choked = len(t.pieceDownloadersChoked)
	s.Downloads.Running = len(t.pieceDownloaders) - len(t.pieceDownloadersChoked) - len(t.pieceDownloadersSnubbed)
	s.Pieces.Available = t.avaliablePieceCount()
	s.Bytes.Downloaded = t.resumerStats.BytesDownloaded
	s.Bytes.Uploaded = t.resumerStats.BytesUploaded
	s.Bytes.Wasted = t.resumerStats.BytesWasted
	s.SeededFor = t.resumerStats.SeededFor
	s.Bytes.Allocated = t.bytesAllocated
	s.Pieces.Checked = t.checkedPieces
	s.Speed.Download = uint(t.downloadSpeed.Rate())
	s.Speed.Upload = uint(t.uploadSpeed.Rate())

	if t.info != nil {
		s.Bytes.Total = t.info.TotalLength
		s.Bytes.Completed = t.bytesComplete()
		s.Bytes.Incomplete = s.Bytes.Total - s.Bytes.Completed

		s.Name = t.info.Name
		s.Private = (t.info.Private == 1)
		s.PieceLength = t.info.PieceLength
	} else {
		// Some trackers don't send any peer address if don't tell we have missing bytes.
		s.Bytes.Incomplete = math.MaxUint32

		s.Name = t.name
	}
	if t.bitfield != nil {
		s.Pieces.Total = t.bitfield.Len()
		s.Pieces.Have = t.bitfield.Count()
		s.Pieces.Missing = s.Pieces.Total - s.Pieces.Have
	}
	bps := int64(s.Speed.Download)
	if bps != 0 {
		eta := time.Duration(s.Bytes.Incomplete/bps) * time.Second
		switch {
		case eta > 8*time.Hour:
			eta = eta.Round(time.Hour)
		case eta > 4*time.Hour:
			eta = eta.Round(30 * time.Minute)
		case eta > 2*time.Hour:
			eta = eta.Round(15 * time.Minute)
		case eta > time.Hour:
			eta = eta.Round(5 * time.Minute)
		case eta > 30*time.Minute:
			eta = eta.Round(1 * time.Minute)
		case eta > 15*time.Minute:
			eta = eta.Round(30 * time.Second)
		case eta > 5*time.Minute:
			eta = eta.Round(15 * time.Second)
		case eta > time.Minute:
			eta = eta.Round(5 * time.Second)
		}
		s.ETA = &eta
	}
	return s
}

func (t *torrent) avaliablePieceCount() uint32 {
	if t.piecePicker == nil {
		return 0
	}
	return t.piecePicker.Available()
}

func (t *torrent) bytesComplete() int64 {
	if t.bitfield == nil || len(t.pieces) == 0 {
		return 0
	}
	n := int64(t.info.PieceLength) * int64(t.bitfield.Count())
	if t.bitfield.Test(t.bitfield.Len() - 1) {
		n -= int64(t.info.PieceLength)
		n += int64(t.pieces[t.bitfield.Len()-1].Length)
	}
	return n
}

func (t *torrent) getTrackers() []Tracker {
	var trackers []Tracker
	for _, an := range t.announcers {
		st := an.Stats()
		t := Tracker{
			URL:      an.Tracker.URL(),
			Status:   TrackerStatus(st.Status),
			Seeders:  st.Seeders,
			Leechers: st.Leechers,
			Error:    st.Error,
		}
		trackers = append(trackers, t)
	}
	return trackers
}

func (t *torrent) getPeers() []Peer {
	var peers []Peer
	for pe := range t.peers {
		var source PeerSource
		switch pe.Source {
		case peer.SourceTracker:
			source = SourceTracker
		case peer.SourceDHT:
			source = SourceDHT
		case peer.SourcePEX:
			source = SourcePEX
		case peer.SourceIncoming:
			source = SourceIncoming
		default:
			panic("unhandled peer source")
		}
		p := Peer{
			ID:                 pe.ID,
			Addr:               pe.Addr(),
			ConnectedAt:        pe.ConnectedAt,
			Downloading:        pe.Downloading,
			ClientInterested:   pe.ClientInterested,
			ClientChoking:      pe.ClientChoking,
			PeerInterested:     pe.PeerInterested,
			PeerChoking:        pe.PeerChoking,
			OptimisticUnchoked: pe.OptimisticUnchoked,
			Snubbed:            pe.Snubbed,
			EncryptedHandshake: pe.EncryptionCipher != 0,
			EncryptedStream:    pe.EncryptionCipher == mse.RC4,
			Source:             source,
			DownloadSpeed:      pe.DownloadSpeed(),
			UploadSpeed:        pe.UploadSpeed(),
		}
		peers = append(peers, p)
	}
	return peers
}

func (t *torrent) updateSeedDuration() {
	if t.status() != Seeding {
		t.seedDurationUpdatedAt = time.Time{}
		return
	}
	if t.seedDurationUpdatedAt.IsZero() {
		t.seedDurationUpdatedAt = time.Now()
		return
	}
	now := time.Now()
	t.resumerStats.SeededFor += now.Sub(t.seedDurationUpdatedAt)
	t.seedDurationUpdatedAt = now
}
<commit_msg>display eta only if downloading<commit_after>package torrent

import (
	"math"
	"time"

	"github.com/cenkalti/rain/internal/mse"
	"github.com/cenkalti/rain/internal/peer"
)

// Stats contains statistics about Torrent.
type Stats struct {
	// Status of the torrent.
	Status Status
	// Contains the error message if torrent is stopped unexpectedly.
	Error  error
	Pieces struct {
		// Number of pieces that are checked when torrent is in "Verifying" state.
		Checked uint32
		// Number of pieces that we are downloaded successfully and verivied by hash check.
		Have uint32
		// Number of pieces that need to be downloaded. Some of them may be being downloaded.
		// Pieces that are being downloaded may counted as missing until they are downloaded and passed hash check.
		Missing uint32
		// Number of unique pieces available on swarm.
		// If this number is less then the number of total pieces, the download may never finish.
		Available uint32
		// Number of total pieces in torrent.
		Total uint32
	}
	Bytes struct {
		// Bytes that are downloaded and passed hash check.
		Completed int64
		// The number of bytes that is needed to complete all missing pieces.
		Incomplete int64
		// The number of total bytes of files in torrent.  Total = Completed + Incomplete
		Total int64
		// Downloaded is the number of bytes downloaded from swarm.
		// Because some pieces may be downloaded more than once, this number may be greater than completed bytes.
		Downloaded int64
		// BytesUploaded is the number of bytes uploaded to the swarm.
		Uploaded int64
		// Bytes downloaded due to duplicate/non-requested pieces.
		Wasted int64
		// Bytes allocated on storage.
		Allocated int64
	}
	Peers struct {
		// Number of peers that are connected, handshaked and ready to send and receive messages.
		Total int
		// Number of peers that have connected to us.
		Incoming int
		// Number of peers that we have connected to.
		Outgoing int
	}
	Handshakes struct {
		// Number of peers that are not handshaked yet.
		Total int
		// Number of incoming peers in handshake state.
		Incoming int
		// Number of outgoing peers in handshake state.
		Outgoing int
	}
	Addresses struct {
		// Total number of peer addresses that are ready to be connected.
		Total int
		// Peers found via trackers.
		Tracker int
		// Peers found via DHT node.
		DHT int
		// Peers found via peer exchange.
		PEX int
	}
	Downloads struct {
		// Number of active piece downloads.
		Total int
		// Number of pieces that are being downloaded normally.
		Running int
		// Number of pieces that are being downloaded too slow.
		Snubbed int
		// Number of piece downloads in choked state.
		Choked int
	}
	MetadataDownloads struct {
		// Number of active metadata downloads.
		Total int
		// Number of peers that uploading too slow.
		Snubbed int
		// Number of peers that are being downloaded normally.
		Running int
	}
	// Name can change after metadata is downloaded.
	Name string
	// Is private torrent?
	Private bool
	// Length of a single piece.
	PieceLength uint32
	// Duration while the torrent is in Seeding status.
	SeededFor time.Duration
	// Speed is calculated as 1-minute moving average.
	Speed struct {
		// Downloaded bytes per second.
		Download uint
		// Uploaded bytes per second.
		Upload uint
	}
	// Time remaining to complete download. nil value means infinity.
	ETA *time.Duration
}

func (t *torrent) stats() Stats {
	t.updateSeedDuration()

	var s Stats
	s.Status = t.status()
	s.Error = t.lastError
	s.Addresses.Total = t.addrList.Len()
	s.Addresses.Tracker = t.addrList.LenSource(peer.SourceTracker)
	s.Addresses.DHT = t.addrList.LenSource(peer.SourceDHT)
	s.Addresses.PEX = t.addrList.LenSource(peer.SourcePEX)
	s.Handshakes.Incoming = len(t.incomingHandshakers)
	s.Handshakes.Outgoing = len(t.outgoingHandshakers)
	s.Handshakes.Total = len(t.incomingHandshakers) + len(t.outgoingHandshakers)
	s.Peers.Total = len(t.peers)
	s.Peers.Incoming = len(t.incomingPeers)
	s.Peers.Outgoing = len(t.outgoingPeers)
	s.MetadataDownloads.Total = len(t.infoDownloaders)
	s.MetadataDownloads.Snubbed = len(t.infoDownloadersSnubbed)
	s.MetadataDownloads.Running = len(t.infoDownloaders) - len(t.infoDownloadersSnubbed)
	s.Downloads.Total = len(t.pieceDownloaders)
	s.Downloads.Snubbed = len(t.pieceDownloadersSnubbed)
	s.Downloads.Choked = len(t.pieceDownloadersChoked)
	s.Downloads.Running = len(t.pieceDownloaders) - len(t.pieceDownloadersChoked) - len(t.pieceDownloadersSnubbed)
	s.Pieces.Available = t.avaliablePieceCount()
	s.Bytes.Downloaded = t.resumerStats.BytesDownloaded
	s.Bytes.Uploaded = t.resumerStats.BytesUploaded
	s.Bytes.Wasted = t.resumerStats.BytesWasted
	s.SeededFor = t.resumerStats.SeededFor
	s.Bytes.Allocated = t.bytesAllocated
	s.Pieces.Checked = t.checkedPieces
	s.Speed.Download = uint(t.downloadSpeed.Rate())
	s.Speed.Upload = uint(t.uploadSpeed.Rate())

	if t.info != nil {
		s.Bytes.Total = t.info.TotalLength
		s.Bytes.Completed = t.bytesComplete()
		s.Bytes.Incomplete = s.Bytes.Total - s.Bytes.Completed

		s.Name = t.info.Name
		s.Private = (t.info.Private == 1)
		s.PieceLength = t.info.PieceLength
	} else {
		// Some trackers don't send any peer address if don't tell we have missing bytes.
		s.Bytes.Incomplete = math.MaxUint32

		s.Name = t.name
	}
	if t.bitfield != nil {
		s.Pieces.Total = t.bitfield.Len()
		s.Pieces.Have = t.bitfield.Count()
		s.Pieces.Missing = s.Pieces.Total - s.Pieces.Have
	}
	if s.Status == Downloading {
		bps := int64(s.Speed.Download)
		if bps != 0 {
			eta := time.Duration(s.Bytes.Incomplete/bps) * time.Second
			switch {
			case eta > 8*time.Hour:
				eta = eta.Round(time.Hour)
			case eta > 4*time.Hour:
				eta = eta.Round(30 * time.Minute)
			case eta > 2*time.Hour:
				eta = eta.Round(15 * time.Minute)
			case eta > time.Hour:
				eta = eta.Round(5 * time.Minute)
			case eta > 30*time.Minute:
				eta = eta.Round(1 * time.Minute)
			case eta > 15*time.Minute:
				eta = eta.Round(30 * time.Second)
			case eta > 5*time.Minute:
				eta = eta.Round(15 * time.Second)
			case eta > time.Minute:
				eta = eta.Round(5 * time.Second)
			}
			s.ETA = &eta
		}
	}
	return s
}

func (t *torrent) avaliablePieceCount() uint32 {
	if t.piecePicker == nil {
		return 0
	}
	return t.piecePicker.Available()
}

func (t *torrent) bytesComplete() int64 {
	if t.bitfield == nil || len(t.pieces) == 0 {
		return 0
	}
	n := int64(t.info.PieceLength) * int64(t.bitfield.Count())
	if t.bitfield.Test(t.bitfield.Len() - 1) {
		n -= int64(t.info.PieceLength)
		n += int64(t.pieces[t.bitfield.Len()-1].Length)
	}
	return n
}

func (t *torrent) getTrackers() []Tracker {
	var trackers []Tracker
	for _, an := range t.announcers {
		st := an.Stats()
		t := Tracker{
			URL:      an.Tracker.URL(),
			Status:   TrackerStatus(st.Status),
			Seeders:  st.Seeders,
			Leechers: st.Leechers,
			Error:    st.Error,
		}
		trackers = append(trackers, t)
	}
	return trackers
}

func (t *torrent) getPeers() []Peer {
	var peers []Peer
	for pe := range t.peers {
		var source PeerSource
		switch pe.Source {
		case peer.SourceTracker:
			source = SourceTracker
		case peer.SourceDHT:
			source = SourceDHT
		case peer.SourcePEX:
			source = SourcePEX
		case peer.SourceIncoming:
			source = SourceIncoming
		default:
			panic("unhandled peer source")
		}
		p := Peer{
			ID:                 pe.ID,
			Addr:               pe.Addr(),
			ConnectedAt:        pe.ConnectedAt,
			Downloading:        pe.Downloading,
			ClientInterested:   pe.ClientInterested,
			ClientChoking:      pe.ClientChoking,
			PeerInterested:     pe.PeerInterested,
			PeerChoking:        pe.PeerChoking,
			OptimisticUnchoked: pe.OptimisticUnchoked,
			Snubbed:            pe.Snubbed,
			EncryptedHandshake: pe.EncryptionCipher != 0,
			EncryptedStream:    pe.EncryptionCipher == mse.RC4,
			Source:             source,
			DownloadSpeed:      pe.DownloadSpeed(),
			UploadSpeed:        pe.UploadSpeed(),
		}
		peers = append(peers, p)
	}
	return peers
}

func (t *torrent) updateSeedDuration() {
	if t.status() != Seeding {
		t.seedDurationUpdatedAt = time.Time{}
		return
	}
	if t.seedDurationUpdatedAt.IsZero() {
		t.seedDurationUpdatedAt = time.Now()
		return
	}
	now := time.Now()
	t.resumerStats.SeededFor += now.Sub(t.seedDurationUpdatedAt)
	t.seedDurationUpdatedAt = now
}
<|endoftext|>


<commit_before>package bradescoShopFacil

var apiResponse = `
{
	{{if eq .returnCode "0"}}
       "DigitableLine": "{{fmtDigitableLine (trim .digitableLine)}}",
		"Links": [{
			"href":"{{.url}}",
			"rel": "pdf",
			"method":"GET"
		}]
    {{else}}
     "Errors": [
		{
			"Code": "{{.returnCode}}",
			"Message": "{{.returnMessage}}"
		}
        ]
    {{end}}
}
`

func getAPIResponseBradescoShopFacil() string {
	return apiResponse
}
<commit_msg>:art: Includes new validation on return of bradescoShopFacil<commit_after>package bradescoShopFacil

var apiResponse = `
{
	{{if or (eq .returnCode "0") (eq .returnCode "93005999")}}
       "DigitableLine": "{{fmtDigitableLine (trim .digitableLine)}}",
		"Links": [{
			"href":"{{.url}}",
			"rel": "pdf",
			"method":"GET"
		}]
    {{else}}
     "Errors": [
		{
			"Code": "{{.returnCode}}",
			"Message": "{{.returnMessage}}"
		}
        ]
    {{end}}
}
`

func getAPIResponseBradescoShopFacil() string {
	return apiResponse
}
<|endoftext|>


<commit_before>package master

import (
	"net/http"
	"fmt"
	"sync"
	"strconv"
	"github.com/syndtr/goleveldb/leveldb/errors"
)

const (
	vidKey = "__vid"
	fidKey = "__fid"
)

type Master struct {
	Port           int

	VMStatusList   []*VolumeManagerStatus
	VStatusListMap map[int][]*VolumeStatus
	statusMutex    sync.RWMutex

	Server         *http.ServeMux
	serverMutex    sync.RWMutex

	fidMutex       sync.Mutex
	vidMutex       sync.Mutex

	Metadata       Metadata
	Replication    [3]int
}

func NewMaster() (*Master, error) {
	m := new(Master)
	m.Port = 8888
	m.VMStatusList = make([]*VolumeManagerStatus, 0, 1)
	m.VStatusListMap = make(map[int][]*VolumeStatus)

	m.Server = http.NewServeMux()
	m.Server.HandleFunc("/", m.masterEntry)

	m.Replication = [3]int{
		0, //number of replica in the same machine
		0, //number of replica in the different machine
		0, //number of replica in the different datacenter
	}
	return m, nil
}

func (m *Master)Start() {
	err := http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", m.Port), m.Server)
	if err != nil {
		panic(err)
	}
}

func (m *Master)Stop() {
	m.serverMutex.Lock()
	m.Metadata.Close()
}

func (m *Master)getWritableVolumes() ([]*VolumeStatus, error) {
	m.statusMutex.RLock()
	defer m.statusMutex.RUnlock()

	//map 迭代是随机的,所以不需要手动负载均衡
	for _, vStatusList := range m.VStatusListMap {
		if m.vStatusListIsValid(vStatusList) {
			return vStatusList, nil
		}
	}

	return nil, errors.New("can't find writable volumes")
}

func (m *Master)vStatusListIsValid(vStatusList []*VolumeStatus) bool {
	for _, vs := range vStatusList {
		if !vs.vmStatus.IsAlive() {
			return false
		}
	}

	if len(vStatusList) != 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
		return false
	}

	//TODO: check volume writable
	return true
}

func (m *Master)createVolumeWithReplication(vms *VolumeManagerStatus) error {
	m.statusMutex.RLock()
	defer m.statusMutex.RUnlock()

	if !vms.IsAlive() {
		return fmt.Errorf("%s:%d is dead", vms.AdminHost, vms.AdminPort)
		//}else if !vms.canCreateVolume() {
		//	return fmt.Errorf("%s:%d can't create volume", vms.AdminHost, vms.AdminPort)
	}

	temp := []*VolumeManagerStatus{vms}

	find0:
	for _, vms := range m.VMStatusList {
		if len(temp) == 1 + m.Replication[0] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine != vms_.Machine || vms.DataCenter != vms_.DataCenter {
					continue find0
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] {
		return errors.New("can't find enough 'same machine VM' to create volume")
	}

	find1:
	for _, vms := range m.VMStatusList {
		if len(temp) == 1 + m.Replication[0] + m.Replication[1] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine == vms_.Machine || vms.DataCenter != vms_.DataCenter {
					continue find1
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] + m.Replication[1] {
		return errors.New("can't find enough 'different machine but same datacenter VM' to create volume")
	}

	find2:
	for _, vms := range m.VMStatusList {
		if len(temp) == 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine == vms_.Machine || vms.DataCenter == vms_.DataCenter {
					continue find2
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
		return errors.New("can't find enough 'different machine and different datacenter VM' to create volume")
	}

	vid := m.generateVid()
	for _, vms := range temp {
		err := vms.createVolume(vid)
		if err != nil {
			return err
		}
	}

	vStatusList := make([]*VolumeStatus, 0, len(temp))
	for _, vms := range temp {
		for _, vs := range vms.VStatusList {
			if vs.Id == vid {
				vStatusList = append(vStatusList, vs)
				break
			}
		}
	}
	m.statusMutex.RUnlock()
	m.statusMutex.Lock()
	m.VStatusListMap[vid] = vStatusList
	m.statusMutex.Unlock()
	m.statusMutex.RLock()

	return nil
}

func (m *Master)generateVid() int {
	m.vidMutex.Lock()
	defer m.vidMutex.Unlock()

	value, err := m.Metadata.getConfig(vidKey)
	if err != nil {
		value = "0"
	}

	vid, _ := strconv.Atoi(value)
	for i := 0; i < 3; i++ {
		err = m.Metadata.setConfig(vidKey, strconv.Itoa(vid + 1))
		if err == nil {
			break
		}
	}
	if err != nil {
		panic(err)
	}

	return vid
}

func (m *Master)generateFid() uint64 {
	m.fidMutex.Lock()
	defer m.fidMutex.Unlock()

	value, err := m.Metadata.getConfig(fidKey)
	if err != nil {
		value = "0"
	}
	fid, _ := strconv.ParseUint(value, 10, 64)
	for i := 0; i < 3; i++ {
		err = m.Metadata.setConfig(fidKey, strconv.FormatUint(fid + 1, 10))
		if err == nil {
			break
		}
	}
	if err != nil {
		panic(err)
	}

	return fid
}
<commit_msg>创建volume 随机选取vm<commit_after>package master

import (
	"net/http"
	"fmt"
	"sync"
	"strconv"
	"github.com/syndtr/goleveldb/leveldb/errors"
	"math/rand"
)

const (
	vidKey = "__vid"
	fidKey = "__fid"
)

type Master struct {
	Port           int

	VMStatusList   []*VolumeManagerStatus
	VStatusListMap map[int][]*VolumeStatus
	statusMutex    sync.RWMutex

	Server         *http.ServeMux
	serverMutex    sync.RWMutex

	fidMutex       sync.Mutex
	vidMutex       sync.Mutex

	Metadata       Metadata
	Replication    [3]int
}

func NewMaster() (*Master, error) {
	m := new(Master)
	m.Port = 8888
	m.VMStatusList = make([]*VolumeManagerStatus, 0, 1)
	m.VStatusListMap = make(map[int][]*VolumeStatus)

	m.Server = http.NewServeMux()
	m.Server.HandleFunc("/", m.masterEntry)

	m.Replication = [3]int{
		0, //number of replica in the same machine
		0, //number of replica in the different machine
		0, //number of replica in the different datacenter
	}
	return m, nil
}

func (m *Master)Start() {
	err := http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", m.Port), m.Server)
	if err != nil {
		panic(err)
	}
}

func (m *Master)Stop() {
	m.serverMutex.Lock()
	m.Metadata.Close()
}

func (m *Master)getWritableVolumes() ([]*VolumeStatus, error) {
	m.statusMutex.RLock()
	defer m.statusMutex.RUnlock()

	//map 迭代是随机的,所以不需要手动负载均衡
	for _, vStatusList := range m.VStatusListMap {
		if m.vStatusListIsValid(vStatusList) {
			return vStatusList, nil
		}
	}

	return nil, errors.New("can't find writable volumes")
}

func (m *Master)vStatusListIsValid(vStatusList []*VolumeStatus) bool {
	for _, vs := range vStatusList {
		if !vs.vmStatus.IsAlive() {
			return false
		}
	}

	if len(vStatusList) != 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
		return false
	}

	//TODO: check volume writable
	return true
}

func (m *Master)createVolumeWithReplication(vms *VolumeManagerStatus) error {
	m.statusMutex.RLock()
	defer m.statusMutex.RUnlock()

	if !vms.IsAlive() {
		return fmt.Errorf("%s:%d is dead", vms.AdminHost, vms.AdminPort)
		//}else if !vms.canCreateVolume() {
		//	return fmt.Errorf("%s:%d can't create volume", vms.AdminHost, vms.AdminPort)
	}

	temp := []*VolumeManagerStatus{vms}

	VMStatusList := append(make([]*VolumeManagerStatus, 0, len(m.VMStatusList)), m.VMStatusList...)
	length := len(VMStatusList)
	for i := 0; i < length; i++ {
		a := rand.Intn(length)
		b := rand.Intn(length)
		VMStatusList[a], VMStatusList[b] = VMStatusList[b], VMStatusList[a]
	}

	find0:
	for _, vms := range VMStatusList {
		if len(temp) == 1 + m.Replication[0] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine != vms_.Machine || vms.DataCenter != vms_.DataCenter {
					continue find0
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] {
		return errors.New("can't find enough 'same machine VM' to create volume")
	}

	find1:
	for _, vms := range VMStatusList {
		if len(temp) == 1 + m.Replication[0] + m.Replication[1] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine == vms_.Machine || vms.DataCenter != vms_.DataCenter {
					continue find1
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] + m.Replication[1] {
		return errors.New("can't find enough 'different machine but same datacenter VM' to create volume")
	}

	find2:
	for _, vms := range VMStatusList {
		if len(temp) == 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
			break
		}
		if vms.IsAlive() && vms.canCreateVolume() {
			for _, vms_ := range temp {
				if vms == vms_ || vms.Machine == vms_.Machine || vms.DataCenter == vms_.DataCenter {
					continue find2
				}
			}
			temp = append(temp, vms)
		}
	}
	if len(temp) != 1 + m.Replication[0] + m.Replication[1] + m.Replication[2] {
		return errors.New("can't find enough 'different machine and different datacenter VM' to create volume")
	}

	vid := m.generateVid()
	for _, vms := range temp {
		err := vms.createVolume(vid)
		if err != nil {
			return err
		}
	}

	vStatusList := make([]*VolumeStatus, 0, len(temp))
	for _, vms := range temp {
		for _, vs := range vms.VStatusList {
			if vs.Id == vid {
				vStatusList = append(vStatusList, vs)
				break
			}
		}
	}
	m.statusMutex.RUnlock()
	m.statusMutex.Lock()
	m.VStatusListMap[vid] = vStatusList
	m.statusMutex.Unlock()
	m.statusMutex.RLock()

	return nil
}

func (m *Master)generateVid() int {
	m.vidMutex.Lock()
	defer m.vidMutex.Unlock()

	value, err := m.Metadata.getConfig(vidKey)
	if err != nil {
		value = "0"
	}

	vid, _ := strconv.Atoi(value)
	for i := 0; i < 3; i++ {
		err = m.Metadata.setConfig(vidKey, strconv.Itoa(vid + 1))
		if err == nil {
			break
		}
	}
	if err != nil {
		panic(err)
	}

	return vid
}

func (m *Master)generateFid() uint64 {
	m.fidMutex.Lock()
	defer m.fidMutex.Unlock()

	value, err := m.Metadata.getConfig(fidKey)
	if err != nil {
		value = "0"
	}
	fid, _ := strconv.ParseUint(value, 10, 64)
	for i := 0; i < 3; i++ {
		err = m.Metadata.setConfig(fidKey, strconv.FormatUint(fid + 1, 10))
		if err == nil {
			break
		}
	}
	if err != nil {
		panic(err)
	}

	return fid
}
<|endoftext|>


<commit_before>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// An AS112 blackhole DNS server. Similar to the one found in evldns.
// Also see https://www.as112.net/

package main

import (
	"github.com/miekg/dns"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

const SOA string = "@ SOA prisoner.iana.org. hostmaster.root-servers.org. 2002040800 1800 900 0604800 604800"

func NewRR(s string) dns.RR { r, _ := dns.NewRR(s); return r }

var zones = map[string]dns.RR{
	"10.in-addr.arpa.":      NewRR("$ORIGIN 10.in-addr.arpa.\n" + SOA),
	"254.169.in-addr.arpa.": NewRR("$ORIGIN 254.169.in-addr.arpa.\n" + SOA),
	"168.192.in-addr.arpa.": NewRR("$ORIGIN 168.192.in-addr.arpa.\n" + SOA),
	"16.172.in-addr.arpa.":  NewRR("$ORIGIN 16.172.in-addr.arpa.\n" + SOA),
	"17.172.in-addr.arpa.":  NewRR("$ORIGIN 17.172.in-addr.arpa.\n" + SOA),
	"18.172.in-addr.arpa.":  NewRR("$ORIGIN 18.172.in-addr.arpa.\n" + SOA),
	"19.172.in-addr.arpa.":  NewRR("$ORIGIN 19.172.in-addr.arpa.\n" + SOA),
	"20.172.in-addr.arpa.":  NewRR("$ORIGIN 20.172.in-addr.arpa.\n" + SOA),
	"21.172.in-addr.arpa.":  NewRR("$ORIGIN 21.172.in-addr.arpa.\n" + SOA),
	"22.172.in-addr.arpa.":  NewRR("$ORIGIN 22.172.in-addr.arpa.\n" + SOA),
	"23.172.in-addr.arpa.":  NewRR("$ORIGIN 23.172.in-addr.arpa.\n" + SOA),
	"24.172.in-addr.arpa.":  NewRR("$ORIGIN 24.172.in-addr.arpa.\n" + SOA),
	"25.172.in-addr.arpa.":  NewRR("$ORIGIN 25.172.in-addr.arpa.\n" + SOA),
	"26.172.in-addr.arpa.":  NewRR("$ORIGIN 26.172.in-addr.arpa.\n" + SOA),
	"27.172.in-addr.arpa.":  NewRR("$ORIGIN 27.172.in-addr.arpa.\n" + SOA),
	"28.172.in-addr.arpa.":  NewRR("$ORIGIN 28.172.in-addr.arpa.\n" + SOA),
	"29.172.in-addr.arpa.":  NewRR("$ORIGIN 29.172.in-addr.arpa.\n" + SOA),
	"30.172.in-addr.arpa.":  NewRR("$ORIGIN 30.172.in-addr.arpa.\n" + SOA),
	"31.172.in-addr.arpa.":  NewRR("$ORIGIN 31.172.in-addr.arpa.\n" + SOA),
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 4)
	for z, rr := range zones {
		rrx := rr.(*dns.SOA) // Some foo needed to created actual RR, on the not a reference
		dns.HandleFunc(z, func(w dns.ResponseWriter, r *dns.Msg) {
			m := new(dns.Msg)
			m.SetReply(r)
			m.Authoritative = true
			m.Ns = []dns.RR{rrx}
			w.WriteMsg(m)
		})
	}
	go func() {
		err := dns.ListenAndServe(":8053", "tcp", nil)
		if err != nil {
			log.Fatal("Failed to set tcp listener %s\n", err.Error())
		}
	}()
	go func() {
		err := dns.ListenAndServe(":8053", "udp", nil)
		if err != nil {
			log.Fatal("Failed to set udp listener %s\n", err.Error())
		}
	}()
	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
	for {
		select {
		case s := <-sig:
			log.Fatalf("Signal (%d) received, stopping\n", s)
		}
	}
}
<commit_msg>typo in comment # Last 5 commit messages # ---------------------- # 10ad5a4 Merge branch 'master' of github.com:miekg/dns [Miek Gieben] # ac0703b Handle MixCase queries again. [Miek Gieben] # dbef97a Add MixCase query test [Miek Gieben] # 7ef3106 Add panics to make older Go versions happy [Miek Gieben] # 980f5e9 Merge pull request #51 from r4um/master [Miek Gieben]<commit_after>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// An AS112 blackhole DNS server. Similar to the one found in evldns.
// Also see https://www.as112.net/

package main

import (
	"github.com/miekg/dns"
	"log"
	"os"
	"os/signal"
	"runtime"
	"syscall"
)

const SOA string = "@ SOA prisoner.iana.org. hostmaster.root-servers.org. 2002040800 1800 900 0604800 604800"

func NewRR(s string) dns.RR { r, _ := dns.NewRR(s); return r }

var zones = map[string]dns.RR{
	"10.in-addr.arpa.":      NewRR("$ORIGIN 10.in-addr.arpa.\n" + SOA),
	"254.169.in-addr.arpa.": NewRR("$ORIGIN 254.169.in-addr.arpa.\n" + SOA),
	"168.192.in-addr.arpa.": NewRR("$ORIGIN 168.192.in-addr.arpa.\n" + SOA),
	"16.172.in-addr.arpa.":  NewRR("$ORIGIN 16.172.in-addr.arpa.\n" + SOA),
	"17.172.in-addr.arpa.":  NewRR("$ORIGIN 17.172.in-addr.arpa.\n" + SOA),
	"18.172.in-addr.arpa.":  NewRR("$ORIGIN 18.172.in-addr.arpa.\n" + SOA),
	"19.172.in-addr.arpa.":  NewRR("$ORIGIN 19.172.in-addr.arpa.\n" + SOA),
	"20.172.in-addr.arpa.":  NewRR("$ORIGIN 20.172.in-addr.arpa.\n" + SOA),
	"21.172.in-addr.arpa.":  NewRR("$ORIGIN 21.172.in-addr.arpa.\n" + SOA),
	"22.172.in-addr.arpa.":  NewRR("$ORIGIN 22.172.in-addr.arpa.\n" + SOA),
	"23.172.in-addr.arpa.":  NewRR("$ORIGIN 23.172.in-addr.arpa.\n" + SOA),
	"24.172.in-addr.arpa.":  NewRR("$ORIGIN 24.172.in-addr.arpa.\n" + SOA),
	"25.172.in-addr.arpa.":  NewRR("$ORIGIN 25.172.in-addr.arpa.\n" + SOA),
	"26.172.in-addr.arpa.":  NewRR("$ORIGIN 26.172.in-addr.arpa.\n" + SOA),
	"27.172.in-addr.arpa.":  NewRR("$ORIGIN 27.172.in-addr.arpa.\n" + SOA),
	"28.172.in-addr.arpa.":  NewRR("$ORIGIN 28.172.in-addr.arpa.\n" + SOA),
	"29.172.in-addr.arpa.":  NewRR("$ORIGIN 29.172.in-addr.arpa.\n" + SOA),
	"30.172.in-addr.arpa.":  NewRR("$ORIGIN 30.172.in-addr.arpa.\n" + SOA),
	"31.172.in-addr.arpa.":  NewRR("$ORIGIN 31.172.in-addr.arpa.\n" + SOA),
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU() * 4)
	for z, rr := range zones {
		rrx := rr.(*dns.SOA) // Needed to create the actual RR, and not an reference.
		dns.HandleFunc(z, func(w dns.ResponseWriter, r *dns.Msg) {
			m := new(dns.Msg)
			m.SetReply(r)
			m.Authoritative = true
			m.Ns = []dns.RR{rrx}
			w.WriteMsg(m)
		})
	}
	go func() {
		err := dns.ListenAndServe(":8053", "tcp", nil)
		if err != nil {
			log.Fatal("Failed to set tcp listener %s\n", err.Error())
		}
	}()
	go func() {
		err := dns.ListenAndServe(":8053", "udp", nil)
		if err != nil {
			log.Fatal("Failed to set udp listener %s\n", err.Error())
		}
	}()
	sig := make(chan os.Signal)
	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
	for {
		select {
		case s := <-sig:
			log.Fatalf("Signal (%d) received, stopping\n", s)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"time"
)

type Person struct {
	Name  string
	Birth time.Time
}

func (p *Person) IsBirthday() bool {
	year, month, day := time.Now().Date()
	birthYear, birthMonth, birthDay := p.Birth.Date()
	return year == birthYear && month == birthMonth && day == birthDay
}

type People []Person

func (p People) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p People) Less(i, j int) bool {
	return p[i].Name < p[j].Name
}

func (p People) Len() int {
	return len(p)
}

// TODO: embedding

func main() {

}
<commit_msg>basic/structs_and_methods: add code that use Person.IsBirthday<commit_after>// Copyright 2013 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"time"
)

type Person struct {
	Name  string
	Birth time.Time
}

func (p *Person) IsBirthday() bool {
	year, month, day := time.Now().Date()
	birthYear, birthMonth, birthDay := p.Birth.Date()
	return year == birthYear && month == birthMonth && day == birthDay
}

type People []Person

func (p People) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func (p People) Less(i, j int) bool {
	return p[i].Name < p[j].Name
}

func (p People) Len() int {
	return len(p)
}

func main() {
	p := Person{Name: "Francisco", Birth: time.Now().Add(-24 * time.Hour)}
	fmt.Println(p.IsBirthday())
}
<|endoftext|>


<commit_before>package types

type Version struct {
	Version                 string `json:"version"`
	GrindVersionRequired    string `json:"grindVersionRequired"`
	GrindVersionRecommended string `json:"grindVersionRecommended"`
}

var CurrentVersion = Version{
	Version:                 "1.9.1",
	GrindVersionRequired:    "1.9.1",
	GrindVersionRecommended: "1.9.1",
}
<commit_msg>version bump<commit_after>package types

type Version struct {
	Version                 string `json:"version"`
	GrindVersionRequired    string `json:"grindVersionRequired"`
	GrindVersionRecommended string `json:"grindVersionRecommended"`
}

var CurrentVersion = Version{
	Version:                 "1.9.2",
	GrindVersionRequired:    "1.9.2",
	GrindVersionRecommended: "1.9.2",
}
<|endoftext|>


<commit_before>package types

type Version struct {
	Version                 string `json:"version"`
	GrindVersionRequired    string `json:"grindVersionRequired"`
	GrindVersionRecommended string `json:"grindVersionRecommended"`
}

var CurrentVersion = Version{
	Version:                 "1.9.7",
	GrindVersionRequired:    "1.9.7",
	GrindVersionRecommended: "1.9.7",
}
<commit_msg>version bump<commit_after>package types

type Version struct {
	Version                 string `json:"version"`
	GrindVersionRequired    string `json:"grindVersionRequired"`
	GrindVersionRecommended string `json:"grindVersionRecommended"`
}

var CurrentVersion = Version{
	Version:                 "1.9.8",
	GrindVersionRequired:    "1.9.8",
	GrindVersionRecommended: "1.9.8",
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	versionRe  = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch     string
	goos       string
	version    string = "v1"
	race       bool
	workingDir string

	installRoot   = "/opt/grafana"
	configRoot    = "/etc/grafana"
	grafanaLogDir = "/var/log/grafana"
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	ensureGoPath()
	readVersionFromPackageJson()

	log.Printf("Version: %s\n", version)

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&race, "race", race, "Use race detector")
	flag.Parse()

	if flag.NArg() == 0 {
		log.Println("Usage: go run build.go build")
		return
	}

	workingDir, _ = os.Getwd()

	for _, cmd := range flag.Args() {
		switch cmd {
		case "setup":
			setup()

		case "build":
			pkg := "."
			var tags []string
			clean()
			build(pkg, tags)

		case "test":
			test("./pkg/...")
			grunt("test")

		case "package":
			//verifyGitRepoIsClean()
			grunt("release", "--pkgVer="+version)
			createRpmAndDeb()

		case "latest":
			makeLatestDistCopies()

		case "clean":
			clean()

		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func makeLatestDistCopies() {
	runError("cp", "dist/grafana_"+version+"_amd64.deb", "dist/grafana_latest_amd64.deb")
	runError("cp", "dist/grafana-"+strings.Replace(version, "-", "_", 5)+"-1.x86_64.rpm", "dist/grafana-latest-1.x84_64.rpm")
	runError("cp", "dist/grafana-"+version+".x86_64.tar.gz", "dist/grafana-latest.x84_64.tar.gz")
}

func readVersionFromPackageJson() {
	reader, err := os.Open("package.json")
	if err != nil {
		log.Fatal("Failed to open package.json")
		return
	}
	defer reader.Close()

	jsonObj := map[string]interface{}{}
	jsonParser := json.NewDecoder(reader)

	if err := jsonParser.Decode(&jsonObj); err != nil {
		log.Fatal("Failed to decode package.json")
	}

	version = jsonObj["version"].(string)
}

func createRpmAndDeb() {
	packageRoot, _ := ioutil.TempDir("", "grafana-linux-pack")
	postInstallScriptPath, _ := ioutil.TempFile("", "postinstall")

	versionFolder := filepath.Join(packageRoot, installRoot, "versions", version)
	configDir := filepath.Join(packageRoot, configRoot)

	runError("mkdir", "-p", versionFolder)
	runError("mkdir", "-p", configDir)

	// copy sample ini file to /etc/opt/grafana
	configFile := filepath.Join(configDir, "grafana.ini")
	runError("cp", "conf/sample.ini", configFile)
	// copy release files
	runError("cp", "-a", filepath.Join(workingDir, "tmp")+"/.", versionFolder)

	GeneratePostInstallScript(postInstallScriptPath.Name())

	args := []string{
		"-s", "dir",
		"--description", "Grafana",
		"-C", packageRoot,
		"--vendor", "Grafana",
		"--url", "http://grafana.org",
		"--license", "Apache 2.0",
		"--maintainer", "contact@grafana.org",
		"--config-files", filepath.Join(configRoot, "grafana.ini"),
		"--after-install", postInstallScriptPath.Name(),
		"--name", "grafana",
		"--version", version,
		"-p", "./dist",
		".",
	}

	fmt.Println("Creating debian package")
	runPrint("fpm", append([]string{"-t", "deb"}, args...)...)

	fmt.Println("Creating redhat/centos package")
	runPrint("fpm", append([]string{"-t", "rpm"}, args...)...)
}

func GeneratePostInstallScript(path string) {
	content := `
rm -f $INSTALL_ROOT_DIR/current
ln -s $INSTALL_ROOT_DIR/versions/$VERSION/ $INSTALL_ROOT_DIR/current

if [ ! -L /etc/init.d/grafana ]; then
    ln -sfn $INSTALL_ROOT_DIR/current/scripts/init.sh /etc/init.d/grafana
fi

chmod +x /etc/init.d/grafana
if which update-rc.d > /dev/null 2>&1 ; then
   update-rc.d -f grafana remove
   update-rc.d grafana defaults
else
   chkconfig --add grafana
fi

if ! id grafana >/dev/null 2>&1; then
   useradd --system -U -M grafana
fi
chown -R -L grafana:grafana $INSTALL_ROOT_DIR
chmod -R a+rX $INSTALL_ROOT_DIR
mkdir -p $GRAFANA_LOG_DIR
chown -R -L grafana:grafana $GRAFANA_LOG_DIR
`
	content = strings.Replace(content, "$INSTALL_ROOT_DIR", installRoot, -1)
	content = strings.Replace(content, "$VERSION", version, -1)
	content = strings.Replace(content, "$GRAFANA_LOG_DIR", grafanaLogDir, -1)
	ioutil.WriteFile(path, []byte(content), 0644)
}

func verifyGitRepoIsClean() {
	rs, err := runError("git", "ls-files", "--modified")
	if err != nil {
		log.Fatalf("Failed to check if git tree was clean, %v, %v\n", string(rs), err)
		return
	}
	count := len(string(rs))
	if count > 0 {
		log.Fatalf("Git repository has modified files, aborting")
	}

	log.Println("Git repository is clean")
}

func ensureGoPath() {
	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
}

func ChangeWorkingDir(dir string) {
	os.Chdir(dir)
}

func grunt(params ...string) {
	runPrint("./node_modules/grunt-cli/bin/grunt", params...)
}

func setup() {
	runPrint("go", "get", "-v", "github.com/tools/godep")
	runPrint("go", "get", "-v", "github.com/mattn/go-sqlite3")
	runPrint("go", "install", "-v", "github.com/mattn/go-sqlite3")
}

func test(pkg string) {
	setBuildEnv()
	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
}

func build(pkg string, tags []string) {
	binary := "./bin/grafana"
	if goos == "windows" {
		binary += ".exe"
	}

	rmr(binary, binary+".md5")
	args := []string{"build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	if race {
		args = append(args, "-race")
	}

	args = append(args, "-o", binary)
	args = append(args, pkg)
	setBuildEnv()
	runPrint("go", args...)

	// Create an md5 checksum of the binary, to be included in the archive for
	// automatic upgrades.
	err := md5File(binary)
	if err != nil {
		log.Fatal(err)
	}
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.version '%s'", version))
	b.WriteString(fmt.Sprintf(" -X main.commit '%s'", getGitSha()))
	b.WriteString(fmt.Sprintf(" -X main.buildstamp %d", buildStamp()))
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr("dist")
	rmr("tmp")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch)))
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "armv") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Println("Warning: can't determine current dir:", err)
		log.Println("Build might not work as expected")
	}
	os.Setenv("GOPATH", fmt.Sprintf("%s%c%s", filepath.Join(wd, "Godeps", "_workspace"), os.PathListSeparator, os.Getenv("GOPATH")))
	log.Println("GOPATH=" + os.Getenv("GOPATH"))
}

func getGitSha() string {
	v, err := runError("git", "describe", "--always", "--dirty")
	if err != nil {
		return "unknown-dev"
	}
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs, err := runError("git", "show", "-s", "--format=%ct")
	if err != nil {
		return time.Now().Unix()
	}
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func run(cmd string, args ...string) []byte {
	bs, err := runError(cmd, args...)
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runError(cmd string, args ...string) ([]byte, error) {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		return nil, err
	}

	return bytes.TrimSpace(bs), nil
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func md5File(file string) error {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()

	h := md5.New()
	_, err = io.Copy(h, fd)
	if err != nil {
		return err
	}

	out, err := os.Create(file + ".md5")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
	if err != nil {
		return err
	}

	return out.Close()
}
<commit_msg>Updated build file<commit_after>// +build ignore

package main

import (
	"bytes"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	versionRe  = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch     string
	goos       string
	version    string = "v1"
	race       bool
	workingDir string

	installRoot   = "/opt/grafana"
	configRoot    = "/etc/grafana"
	grafanaLogDir = "/var/log/grafana"
)

const minGoVersion = 1.3

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	ensureGoPath()
	readVersionFromPackageJson()

	log.Printf("Version: %s\n", version)

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&race, "race", race, "Use race detector")
	flag.Parse()

	if flag.NArg() == 0 {
		log.Println("Usage: go run build.go build")
		return
	}

	workingDir, _ = os.Getwd()

	for _, cmd := range flag.Args() {
		switch cmd {
		case "setup":
			setup()

		case "build":
			pkg := "."
			var tags []string
			clean()
			build(pkg, tags)

		case "test":
			test("./pkg/...")
			grunt("test")

		case "package":
			//verifyGitRepoIsClean()
			grunt("release", "--pkgVer="+version)
			createRpmAndDeb()

		case "latest":
			makeLatestDistCopies()

		case "clean":
			clean()

		default:
			log.Fatalf("Unknown command %q", cmd)
		}
	}
}

func makeLatestDistCopies() {
	runError("cp", "dist/grafana_"+version+"_amd64.deb", "dist/grafana_latest_amd64.deb")
	runError("cp", "dist/grafana-"+strings.Replace(version, "-", "_", 5)+"-1.x86_64.rpm", "dist/grafana-latest-1.x84_64.rpm")
	runError("cp", "dist/grafana-"+version+".x86_64.tar.gz", "dist/grafana-latest.x84_64.tar.gz")
}

func readVersionFromPackageJson() {
	reader, err := os.Open("package.json")
	if err != nil {
		log.Fatal("Failed to open package.json")
		return
	}
	defer reader.Close()

	jsonObj := map[string]interface{}{}
	jsonParser := json.NewDecoder(reader)

	if err := jsonParser.Decode(&jsonObj); err != nil {
		log.Fatal("Failed to decode package.json")
	}

	version = jsonObj["version"].(string)
}

func createRpmAndDeb() {
	packageRoot, _ := ioutil.TempDir("", "grafana-linux-pack")
	postInstallScriptPath, _ := ioutil.TempFile("", "postinstall")

	versionFolder := filepath.Join(packageRoot, installRoot, "versions", version)
	configDir := filepath.Join(packageRoot, configRoot)

	runError("mkdir", "-p", versionFolder)
	runError("mkdir", "-p", configDir)

	// copy sample ini file to /etc/opt/grafana
	configFile := filepath.Join(configDir, "grafana.ini")
	runError("cp", "conf/sample.ini", configFile)
	// copy release files
	runError("cp", "-a", filepath.Join(workingDir, "tmp")+"/.", versionFolder)

	GeneratePostInstallScript(postInstallScriptPath.Name())

	args := []string{
		"-s", "dir",
		"--description", "Grafana",
		"-C", packageRoot,
		"--vendor", "Grafana",
		"--url", "http://grafana.org",
		"--license", "Apache 2.0",
		"--maintainer", "contact@grafana.org",
		"--config-files", filepath.Join(configRoot, "grafana.ini"),
		"--after-install", postInstallScriptPath.Name(),
		"--name", "grafana",
		"--version", version,
		"-p", "./dist",
		".",
	}

	fmt.Println("Creating debian package")
	runPrint("fpm", append([]string{"-t", "deb"}, args...)...)

	fmt.Println("Creating redhat/centos package")
	runPrint("fpm", append([]string{"-t", "rpm"}, args...)...)
}

func GeneratePostInstallScript(path string) {
	content := `
rm -f $INSTALL_ROOT_DIR/current
ln -s $INSTALL_ROOT_DIR/versions/$VERSION/ $INSTALL_ROOT_DIR/current

if [ ! -L /etc/init.d/grafana ]; then
    ln -sfn $INSTALL_ROOT_DIR/current/scripts/init.sh /etc/init.d/grafana
fi

chmod +x /etc/init.d/grafana
if which update-rc.d > /dev/null 2>&1 ; then
   update-rc.d -f grafana remove
   update-rc.d grafana defaults
else
   chkconfig --add grafana
fi

if ! id grafana >/dev/null 2>&1; then
   useradd --system -U -M grafana
fi
chown -R -L grafana:grafana $INSTALL_ROOT_DIR
chmod -R a+rX $INSTALL_ROOT_DIR
mkdir -p $GRAFANA_LOG_DIR
chown -R -L grafana:grafana $GRAFANA_LOG_DIR
`
	content = strings.Replace(content, "$INSTALL_ROOT_DIR", installRoot, -1)
	content = strings.Replace(content, "$VERSION", version, -1)
	content = strings.Replace(content, "$GRAFANA_LOG_DIR", grafanaLogDir, -1)
	ioutil.WriteFile(path, []byte(content), 0644)
}

func verifyGitRepoIsClean() {
	rs, err := runError("git", "ls-files", "--modified")
	if err != nil {
		log.Fatalf("Failed to check if git tree was clean, %v, %v\n", string(rs), err)
		return
	}
	count := len(string(rs))
	if count > 0 {
		log.Fatalf("Git repository has modified files, aborting")
	}

	log.Println("Git repository is clean")
}

func ensureGoPath() {
	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
}

func ChangeWorkingDir(dir string) {
	os.Chdir(dir)
}

func grunt(params ...string) {
	runPrint("./node_modules/grunt-cli/bin/grunt", params...)
}

func setup() {
	runPrint("go", "get", "-v", "github.com/tools/godep")
	// runPrint("go", "get", "-v", "github.com/mattn/go-sqlite3")
	// runPrint("go", "install", "-v", "github.com/mattn/go-sqlite3")
}

func test(pkg string) {
	setBuildEnv()
	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
}

func build(pkg string, tags []string) {
	binary := "./bin/grafana"
	if goos == "windows" {
		binary += ".exe"
	}

	rmr(binary, binary+".md5")
	args := []string{"build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	if race {
		args = append(args, "-race")
	}

	args = append(args, "-o", binary)
	args = append(args, pkg)
	setBuildEnv()
	runPrint("go", args...)

	// Create an md5 checksum of the binary, to be included in the archive for
	// automatic upgrades.
	err := md5File(binary)
	if err != nil {
		log.Fatal(err)
	}
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.version '%s'", version))
	b.WriteString(fmt.Sprintf(" -X main.commit '%s'", getGitSha()))
	b.WriteString(fmt.Sprintf(" -X main.buildstamp %d", buildStamp()))
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr("dist")
	rmr("tmp")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/grafana", goos, goarch)))
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "armv") {
		os.Setenv("GOARCH", "arm")
		os.Setenv("GOARM", goarch[4:])
	} else {
		os.Setenv("GOARCH", goarch)
	}
	if goarch == "386" {
		os.Setenv("GO386", "387")
	}
	wd, err := os.Getwd()
	if err != nil {
		log.Println("Warning: can't determine current dir:", err)
		log.Println("Build might not work as expected")
	}
	os.Setenv("GOPATH", fmt.Sprintf("%s%c%s", filepath.Join(wd, "Godeps", "_workspace"), os.PathListSeparator, os.Getenv("GOPATH")))
	log.Println("GOPATH=" + os.Getenv("GOPATH"))
}

func getGitSha() string {
	v, err := runError("git", "describe", "--always", "--dirty")
	if err != nil {
		return "unknown-dev"
	}
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs, err := runError("git", "show", "-s", "--format=%ct")
	if err != nil {
		return time.Now().Unix()
	}
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func run(cmd string, args ...string) []byte {
	bs, err := runError(cmd, args...)
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runError(cmd string, args ...string) ([]byte, error) {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		return nil, err
	}

	return bytes.TrimSpace(bs), nil
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func md5File(file string) error {
	fd, err := os.Open(file)
	if err != nil {
		return err
	}
	defer fd.Close()

	h := md5.New()
	_, err = io.Copy(h, fd)
	if err != nil {
		return err
	}

	out, err := os.Create(file + ".md5")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintf(out, "%x\n", h.Sum(nil))
	if err != nil {
		return err
	}

	return out.Close()
}
<|endoftext|>