<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/util/storage"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"github.com/satori/go.uuid"
	"time"
)

type DPNFixityChecker struct {
	Context            *context.Context
	LocalDPNRestClient *network.DPNRestClient
	ValidationChannel  chan *DPNRestoreHelper
	RecordChannel      chan *DPNRestoreHelper
	CleanupChannel     chan *DPNRestoreHelper
	// PreTestChannel is used in testing only to set some properties
	// on the helper/manifest. PreTestChannel should push directly into
	// the ValidationChannel.
	PreTestChannel chan *DPNRestoreHelper
	// PostTestChannel is for testing only. It allows us to inspect the
	// state of our helper and manifest when processing completes.
	PostTestChannel     chan *DPNRestoreHelper
	BagValidationConfig *validation.BagValidationConfig
}

// NewDPNFixityChecker creates a new DPNFixityChecker.
func NewDPNFixityChecker(_context *context.Context) (*DPNFixityChecker, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	checker := &DPNFixityChecker{
		Context:            _context,
		LocalDPNRestClient: localClient,
	}
	// LoadDPNBagValidationConfig is defined in dpn/workers/common.go
	checker.BagValidationConfig = LoadDPNBagValidationConfig(checker.Context)
	workerBufferSize := _context.Config.DPN.DPNFixityWorker.Workers * 4
	checker.ValidationChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.RecordChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.CleanupChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.PreTestChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.PostTestChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNPackageWorker.Workers; i++ {
		go checker.validate()
		go checker.record()
		go checker.cleanup()
	}
	return checker, nil
}

// HandleMessage is the NSQ message handler. The NSQ consumer will pass each
// message in the subscribed channel to this function.
func (checker *DPNFixityChecker) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	helper, err := NewDPNRestoreHelper(message, checker.Context,
		checker.LocalDPNRestClient, constants.ActionFixityCheck,
		"ValidationSummary")
	if err != nil {
		checker.Context.MessageLog.Error(err.Error())
		return err
	}
	helper.WorkSummary.ClearErrors()
	helper.WorkSummary.Start()
	helper.Manifest.DPNWorkItem.Status = constants.StatusStarted
	helper.Manifest.DPNWorkItem.Stage = constants.StageValidate
	helper.SaveDPNWorkItem()

	if helper.WorkSummary.HasErrors() {
		checker.Context.MessageLog.Error("Error setting up manifest for WorkItem %s: %s",
			string(message.Body), helper.WorkSummary.AllErrorsAsString())
		// No use proceeding...
		checker.CleanupChannel <- helper
		return fmt.Errorf(helper.WorkSummary.AllErrorsAsString())
	}
	if helper.Manifest.DPNWorkItem.IsCompletedOrCancelled() {
		checker.Context.MessageLog.Info("Skipping WorkItem %d because status is %s",
			helper.Manifest.DPNWorkItem.Id, helper.Manifest.DPNWorkItem.Status)
		checker.CleanupChannel <- helper
		return nil
	}

	if helper.Manifest.ExpectedFixityValue == "" {
		helper.WorkSummary.AddError("ExpectedFixityValue for bag %s "+
			"is missing from manifest. Cannot validate.", helper.Manifest.DPNBag.UUID)
		helper.WorkSummary.ErrorIsFatal = true
		checker.CleanupChannel <- helper
		return nil
	}

	if checker.PreTestChannel != nil {
		checker.PreTestChannel <- helper
	} else {
		checker.ValidationChannel <- helper
	}
	return nil
}

func (checker *DPNFixityChecker) validate() {
	for helper := range checker.ValidationChannel {
		checker.Context.MessageLog.Info("Validating %s", helper.Manifest.LocalPath)
		// Validation can take long time.
		// Ping NSQ immediately before and after,
		// so we don't time out.
		helper.Manifest.NsqMessage.Touch()
		checker.ValidateBag(helper)
		helper.Manifest.NsqMessage.Touch()

		if helper.WorkSummary.HasErrors() {
			checker.CleanupChannel <- helper
		} else {
			checker.RecordChannel <- helper
		}
	}
}

// DPN fixity check requires us to validate the entire bag and
// extract the sha256 checksum of tagmanifest-sha256.txt file.
// If the bag is valid, and the fixity value of the tag manifest
// matches what's in the DPN registry, the fixity check passes.
// DPN currently has no way of recording that a fixity check has
// failed, other than a human looking at the records. We can record
// the DPNWorkItem as failed in Pharos.
func (checker *DPNFixityChecker) ValidateBag(helper *DPNRestoreHelper) {
	validator, err := validation.NewValidator(helper.Manifest.LocalPath,
		checker.BagValidationConfig, false)
	if err != nil {
		helper.WorkSummary.AddError(err.Error())
		helper.WorkSummary.ErrorIsFatal = true
	} else {
		// Validation can take a long time for large bags.
		summary, err := validator.Validate()
		if err != nil {
			helper.WorkSummary.AddError(err.Error())
		} else {
			checker.Context.MessageLog.Info("Finished validating %s",
				helper.Manifest.LocalPath)
			for _, validationError := range summary.Errors {
				helper.WorkSummary.AddError(validationError)
			}
			if summary.ErrorIsFatal {
				helper.WorkSummary.ErrorIsFatal = summary.ErrorIsFatal
			}
			checker.getTagManifestChecksum(helper, validator)
		}
	}
}

// The validator records the results of its work in a BoltDB
// file because we often get bags with over 100,000 files.
// We can extract the tagmanifest-sha256.txt checksum from the BoltDB.
func (checker *DPNFixityChecker) getTagManifestChecksum(helper *DPNRestoreHelper, validator *validation.Validator) {
	fileIdentifier := fmt.Sprintf("%s/tagmanifest-sha256.txt", helper.Manifest.DPNBag.UUID)
	checker.Context.MessageLog.Info("Looking up %s in BoltDB %s",
		fileIdentifier, validator.DBName())
	db, err := storage.NewBoltDB(validator.DBName())
	if err != nil {
		helper.WorkSummary.AddError("Error opening BoltDB: %v", err)
		return
	}
	defer db.Close()
	gf, err := db.GetGenericFile(fileIdentifier)
	if err != nil {
		helper.WorkSummary.AddError("Error finding file %s in BoltDB: %v", fileIdentifier, err)
		return
	}
	// Record on the manifest the actual sha256 digest that the validator
	// just calculated for the tagmanifest-sha256.txt file.
	helper.Manifest.ActualFixityValue = gf.IngestSha256
	checker.Context.MessageLog.Info("Validator calculated checksum %s for file %s",
		gf.IngestSha256, fileIdentifier)

	// TODO: Delete BoltDB?
	// TODO: Delete bag? (though not during unit tests)
}

// Record this fixity check in our local DPN REST server
func (checker *DPNFixityChecker) record() {
	for helper := range checker.RecordChannel {
		checker.SaveFixityRecord(helper)
		checker.CleanupChannel <- helper
	}
}

// Update NSQ and Pharos on the status of this task
func (checker *DPNFixityChecker) cleanup() {
	for helper := range checker.CleanupChannel {
		helper.WorkSummary.Finish()
		if helper.WorkSummary.HasErrors() {
			checker.FinishWithError(helper)
		} else {
			checker.FinishWithSuccess(helper)
		}
		// For testing only. The test code creates the PostTestChannel.
		// When running in demo & production, this channel is nil.
		if checker.PostTestChannel != nil {
			checker.PostTestChannel <- helper
		}
	}
}

// Save the fixity record to the local DPN REST server.
func (checker *DPNFixityChecker) SaveFixityRecord(helper *DPNRestoreHelper) {
	// Create a FixityCheck record and save it with
	// checker.LocalDPNRestClient.FixityCheckCreate()
	if helper.Manifest.ExpectedFixityValue == "" {
		helper.WorkSummary.AddError("Cannot create DPN FixityCheck record because " +
			"because ExpectedFixityValue is missing from manifest.")
		return
	}
	if helper.Manifest.ActualFixityValue == "" {
		helper.WorkSummary.AddError("Cannot create DPN FixityCheck record because " +
			"because ActualFixityValue is missing from manifest.")
		return
	}
	if helper.Manifest.FixityCheck == nil {
		helper.Manifest.FixityCheck = &models.FixityCheck{
			FixityCheckId: uuid.NewV4().String(),
			Bag:           helper.Manifest.DPNBag.UUID,
			Node:          checker.Context.Config.DPN.LocalNode,
			Success:       helper.Manifest.ExpectedFixityValue == helper.Manifest.ActualFixityValue,
			FixityAt:      time.Now().UTC(),
		}
	}
	resp := checker.LocalDPNRestClient.FixityCheckCreate(helper.Manifest.FixityCheck)
	if resp.Error != nil {
		helper.WorkSummary.AddError("Error saving FixityCheck to DPN REST server: %v",
			resp.Error)
		return
	}
	helper.Manifest.FixityCheck.CreatedAt = resp.FixityCheck().CreatedAt
	helper.Manifest.FixityCheckSavedAt = time.Now().UTC()
	if helper.Manifest.CheckCompletedAndFailed() {
		helper.WorkSummary.AddError(
			"Fixity check completed, and fixity record %s was saved to DPN. "+
				"Actual fixity value %s does not match expected fixity %s",
			helper.Manifest.FixityCheck.FixityCheckId,
			helper.Manifest.ActualFixityValue, helper.Manifest.ExpectedFixityValue)
		helper.WorkSummary.ErrorIsFatal = true
	}
}

func (checker *DPNFixityChecker) FinishWithSuccess(helper *DPNRestoreHelper) {
	helper.Manifest.DPNWorkItem.ClearNodeAndPid()
	note := fmt.Sprintf("Fixity check complete. Saved to DPN with FixityCheckId %s",
		helper.Manifest.FixityCheck.FixityCheckId)
	helper.Manifest.DPNWorkItem.Note = &note
	// TODO: To repurpose this code to support restoration,
	// branch here. If DPNWorkItem.Task is fixity check,
	// set Status to Success. If Task is restore, set Stage
	// to StageRestoring, status to StatusPending, and push the
	// bag into the restoration queue, or do whatever else is
	// necessary to complete the restore process.
	helper.WorkSummary.Finish()
	helper.Manifest.DPNWorkItem.Stage = constants.StageValidate
	helper.Manifest.DPNWorkItem.Status = constants.StatusSuccess
	helper.SaveDPNWorkItem()
	helper.Manifest.NsqMessage.Finish()
}

func (checker *DPNFixityChecker) FinishWithError(helper *DPNRestoreHelper) {
	helper.Manifest.DPNWorkItem.ClearNodeAndPid()
	// Copy errors into the DPNWorkItem note, so we can see them in
	// the Pharos UI.
	errors := helper.WorkSummary.AllErrorsAsString()
	helper.Manifest.DPNWorkItem.Note = &errors
	checker.Context.MessageLog.Error(errors)
	helper.WorkSummary.Finish()
	if helper.WorkSummary.ErrorIsFatal {
		// Mark the DPNWorkItem as failed
		checker.Context.MessageLog.Error("Error for %s is fatal. Not requeueing.",
			helper.Manifest.DPNWorkItem.Identifier)
		helper.Manifest.DPNWorkItem.Status = constants.StatusFailed
		helper.Manifest.DPNWorkItem.Retry = false
		helper.SaveDPNWorkItem()
		helper.Manifest.NsqMessage.Finish()
	} else {
		// Transient errors. Retry DPNWorkItem.
		// MINUTES_BETWEEN_RETRIES is defined in dpn_s3_retriever.go
		helper.Manifest.DPNWorkItem.Retry = true
		helper.SaveDPNWorkItem()
		helper.Manifest.NsqMessage.Requeue(MINUTES_BETWEEN_RETRIES * time.Minute)
	}
}
<commit_msg>Don't instantiate Pre and Post test channels on fixity check worker.<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/util/storage"
	"github.com/APTrust/exchange/validation"
	"github.com/nsqio/go-nsq"
	"github.com/satori/go.uuid"
	"time"
)

type DPNFixityChecker struct {
	Context            *context.Context
	LocalDPNRestClient *network.DPNRestClient
	ValidationChannel  chan *DPNRestoreHelper
	RecordChannel      chan *DPNRestoreHelper
	CleanupChannel     chan *DPNRestoreHelper
	// PreTestChannel is used in testing only to set some properties
	// on the helper/manifest. PreTestChannel should push directly into
	// the ValidationChannel.
	PreTestChannel chan *DPNRestoreHelper
	// PostTestChannel is for testing only. It allows us to inspect the
	// state of our helper and manifest when processing completes.
	PostTestChannel     chan *DPNRestoreHelper
	BagValidationConfig *validation.BagValidationConfig
}

// NewDPNFixityChecker creates a new DPNFixityChecker.
func NewDPNFixityChecker(_context *context.Context) (*DPNFixityChecker, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	checker := &DPNFixityChecker{
		Context:            _context,
		LocalDPNRestClient: localClient,
	}
	// LoadDPNBagValidationConfig is defined in dpn/workers/common.go
	checker.BagValidationConfig = LoadDPNBagValidationConfig(checker.Context)
	workerBufferSize := _context.Config.DPN.DPNFixityWorker.Workers * 4
	checker.ValidationChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.RecordChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	checker.CleanupChannel = make(chan *DPNRestoreHelper, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNPackageWorker.Workers; i++ {
		go checker.validate()
		go checker.record()
		go checker.cleanup()
	}
	return checker, nil
}

// HandleMessage is the NSQ message handler. The NSQ consumer will pass each
// message in the subscribed channel to this function.
func (checker *DPNFixityChecker) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	helper, err := NewDPNRestoreHelper(message, checker.Context,
		checker.LocalDPNRestClient, constants.ActionFixityCheck,
		"ValidationSummary")
	if err != nil {
		checker.Context.MessageLog.Error(err.Error())
		return err
	}
	helper.WorkSummary.ClearErrors()
	helper.WorkSummary.Start()
	helper.Manifest.DPNWorkItem.Status = constants.StatusStarted
	helper.Manifest.DPNWorkItem.Stage = constants.StageValidate
	helper.SaveDPNWorkItem()

	if helper.WorkSummary.HasErrors() {
		checker.Context.MessageLog.Error("Error setting up manifest for WorkItem %s: %s",
			string(message.Body), helper.WorkSummary.AllErrorsAsString())
		// No use proceeding...
		checker.CleanupChannel <- helper
		return fmt.Errorf(helper.WorkSummary.AllErrorsAsString())
	}
	if helper.Manifest.DPNWorkItem.IsCompletedOrCancelled() {
		checker.Context.MessageLog.Info("Skipping WorkItem %d because status is %s",
			helper.Manifest.DPNWorkItem.Id, helper.Manifest.DPNWorkItem.Status)
		checker.CleanupChannel <- helper
		return nil
	}

	if helper.Manifest.ExpectedFixityValue == "" {
		helper.WorkSummary.AddError("ExpectedFixityValue for bag %s "+
			"is missing from manifest. Cannot validate.", helper.Manifest.DPNBag.UUID)
		helper.WorkSummary.ErrorIsFatal = true
		checker.CleanupChannel <- helper
		return nil
	}

	if checker.PreTestChannel != nil {
		checker.PreTestChannel <- helper
	} else {
		checker.ValidationChannel <- helper
	}
	return nil
}

func (checker *DPNFixityChecker) validate() {
	for helper := range checker.ValidationChannel {
		checker.Context.MessageLog.Info("Validating %s", helper.Manifest.LocalPath)
		// Validation can take long time.
		// Ping NSQ immediately before and after,
		// so we don't time out.
		helper.Manifest.NsqMessage.Touch()
		checker.ValidateBag(helper)
		helper.Manifest.NsqMessage.Touch()

		if helper.WorkSummary.HasErrors() {
			checker.CleanupChannel <- helper
		} else {
			checker.RecordChannel <- helper
		}
	}
}

// DPN fixity check requires us to validate the entire bag and
// extract the sha256 checksum of tagmanifest-sha256.txt file.
// If the bag is valid, and the fixity value of the tag manifest
// matches what's in the DPN registry, the fixity check passes.
// DPN currently has no way of recording that a fixity check has
// failed, other than a human looking at the records. We can record
// the DPNWorkItem as failed in Pharos.
func (checker *DPNFixityChecker) ValidateBag(helper *DPNRestoreHelper) {
	validator, err := validation.NewValidator(helper.Manifest.LocalPath,
		checker.BagValidationConfig, false)
	if err != nil {
		helper.WorkSummary.AddError(err.Error())
		helper.WorkSummary.ErrorIsFatal = true
	} else {
		// Validation can take a long time for large bags.
		summary, err := validator.Validate()
		if err != nil {
			helper.WorkSummary.AddError(err.Error())
		} else {
			checker.Context.MessageLog.Info("Finished validating %s",
				helper.Manifest.LocalPath)
			for _, validationError := range summary.Errors {
				helper.WorkSummary.AddError(validationError)
			}
			if summary.ErrorIsFatal {
				helper.WorkSummary.ErrorIsFatal = summary.ErrorIsFatal
			}
			checker.getTagManifestChecksum(helper, validator)
		}
	}
}

// The validator records the results of its work in a BoltDB
// file because we often get bags with over 100,000 files.
// We can extract the tagmanifest-sha256.txt checksum from the BoltDB.
func (checker *DPNFixityChecker) getTagManifestChecksum(helper *DPNRestoreHelper, validator *validation.Validator) {
	fileIdentifier := fmt.Sprintf("%s/tagmanifest-sha256.txt", helper.Manifest.DPNBag.UUID)
	checker.Context.MessageLog.Info("Looking up %s in BoltDB %s",
		fileIdentifier, validator.DBName())
	db, err := storage.NewBoltDB(validator.DBName())
	if err != nil {
		helper.WorkSummary.AddError("Error opening BoltDB: %v", err)
		return
	}
	defer db.Close()
	gf, err := db.GetGenericFile(fileIdentifier)
	if err != nil {
		helper.WorkSummary.AddError("Error finding file %s in BoltDB: %v", fileIdentifier, err)
		return
	}
	// Record on the manifest the actual sha256 digest that the validator
	// just calculated for the tagmanifest-sha256.txt file.
	helper.Manifest.ActualFixityValue = gf.IngestSha256
	checker.Context.MessageLog.Info("Validator calculated checksum %s for file %s",
		gf.IngestSha256, fileIdentifier)

	// TODO: Delete BoltDB?
	// TODO: Delete bag? (though not during unit tests)
}

// Record this fixity check in our local DPN REST server
func (checker *DPNFixityChecker) record() {
	for helper := range checker.RecordChannel {
		checker.SaveFixityRecord(helper)
		checker.CleanupChannel <- helper
	}
}

// Update NSQ and Pharos on the status of this task
func (checker *DPNFixityChecker) cleanup() {
	for helper := range checker.CleanupChannel {
		helper.WorkSummary.Finish()
		if helper.WorkSummary.HasErrors() {
			checker.FinishWithError(helper)
		} else {
			checker.FinishWithSuccess(helper)
		}
		// For testing only. The test code creates the PostTestChannel.
		// When running in demo & production, this channel is nil.
		if checker.PostTestChannel != nil {
			checker.PostTestChannel <- helper
		}
	}
}

// Save the fixity record to the local DPN REST server.
func (checker *DPNFixityChecker) SaveFixityRecord(helper *DPNRestoreHelper) {
	// Create a FixityCheck record and save it with
	// checker.LocalDPNRestClient.FixityCheckCreate()
	if helper.Manifest.ExpectedFixityValue == "" {
		helper.WorkSummary.AddError("Cannot create DPN FixityCheck record because " +
			"because ExpectedFixityValue is missing from manifest.")
		return
	}
	if helper.Manifest.ActualFixityValue == "" {
		helper.WorkSummary.AddError("Cannot create DPN FixityCheck record because " +
			"because ActualFixityValue is missing from manifest.")
		return
	}
	if helper.Manifest.FixityCheck == nil {
		helper.Manifest.FixityCheck = &models.FixityCheck{
			FixityCheckId: uuid.NewV4().String(),
			Bag:           helper.Manifest.DPNBag.UUID,
			Node:          checker.Context.Config.DPN.LocalNode,
			Success:       helper.Manifest.ExpectedFixityValue == helper.Manifest.ActualFixityValue,
			FixityAt:      time.Now().UTC(),
		}
	}
	resp := checker.LocalDPNRestClient.FixityCheckCreate(helper.Manifest.FixityCheck)
	if resp.Error != nil {
		helper.WorkSummary.AddError("Error saving FixityCheck to DPN REST server: %v",
			resp.Error)
		return
	}
	helper.Manifest.FixityCheck.CreatedAt = resp.FixityCheck().CreatedAt
	helper.Manifest.FixityCheckSavedAt = time.Now().UTC()
	if helper.Manifest.CheckCompletedAndFailed() {
		helper.WorkSummary.AddError(
			"Fixity check completed, and fixity record %s was saved to DPN. "+
				"Actual fixity value %s does not match expected fixity %s",
			helper.Manifest.FixityCheck.FixityCheckId,
			helper.Manifest.ActualFixityValue, helper.Manifest.ExpectedFixityValue)
		helper.WorkSummary.ErrorIsFatal = true
	}
}

func (checker *DPNFixityChecker) FinishWithSuccess(helper *DPNRestoreHelper) {
	helper.Manifest.DPNWorkItem.ClearNodeAndPid()
	note := fmt.Sprintf("Fixity check complete. Saved to DPN with FixityCheckId %s",
		helper.Manifest.FixityCheck.FixityCheckId)
	helper.Manifest.DPNWorkItem.Note = &note
	// TODO: To repurpose this code to support restoration,
	// branch here. If DPNWorkItem.Task is fixity check,
	// set Status to Success. If Task is restore, set Stage
	// to StageRestoring, status to StatusPending, and push the
	// bag into the restoration queue, or do whatever else is
	// necessary to complete the restore process.
	helper.WorkSummary.Finish()
	helper.Manifest.DPNWorkItem.Stage = constants.StageValidate
	helper.Manifest.DPNWorkItem.Status = constants.StatusSuccess
	helper.SaveDPNWorkItem()
	helper.Manifest.NsqMessage.Finish()
}

func (checker *DPNFixityChecker) FinishWithError(helper *DPNRestoreHelper) {
	helper.Manifest.DPNWorkItem.ClearNodeAndPid()
	// Copy errors into the DPNWorkItem note, so we can see them in
	// the Pharos UI.
	errors := helper.WorkSummary.AllErrorsAsString()
	helper.Manifest.DPNWorkItem.Note = &errors
	checker.Context.MessageLog.Error(errors)
	helper.WorkSummary.Finish()
	if helper.WorkSummary.ErrorIsFatal {
		// Mark the DPNWorkItem as failed
		checker.Context.MessageLog.Error("Error for %s is fatal. Not requeueing.",
			helper.Manifest.DPNWorkItem.Identifier)
		helper.Manifest.DPNWorkItem.Status = constants.StatusFailed
		helper.Manifest.DPNWorkItem.Retry = false
		helper.SaveDPNWorkItem()
		helper.Manifest.NsqMessage.Finish()
	} else {
		// Transient errors. Retry DPNWorkItem.
		// MINUTES_BETWEEN_RETRIES is defined in dpn_s3_retriever.go
		helper.Manifest.DPNWorkItem.Retry = true
		helper.SaveDPNWorkItem()
		helper.Manifest.NsqMessage.Requeue(MINUTES_BETWEEN_RETRIES * time.Minute)
	}
}
<|endoftext|>


<commit_before>package ekanite

import (
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/gorilla/csrf"
	"github.com/gorilla/securecookie"
)

// Server serves query client connections.
type HttpServer struct {
	iface    string
	Searcher Searcher

	addr     net.Addr
	template *template.Template

	Logger *log.Logger
}

// NewServer returns a new Server instance.
func NewHttpServer(iface string, searcher Searcher) *HttpServer {
	return &HttpServer{
		iface:    iface,
		Searcher: searcher,
		Logger:   log.New(os.Stderr, "[httpserver] ", log.LstdFlags),
	}
}

// Start instructs the Server to bind to the interface and accept connections.
func (s *HttpServer) Start() error {
	ln, err := net.Listen("tcp", s.iface)
	if err != nil {
		return err
	}

	s.addr = ln.Addr()

	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		ln.Close()
		return err
	}

	CSRF := csrf.Protect(securecookie.GenerateRandomKey(32),
		csrf.Secure(false),
		csrf.HttpOnly(true))

	go http.Serve(ln, CSRF(s))
	return nil
}

// Addr returns the address to which the Server is bound.
func (s *HttpServer) Addr() net.Addr {
	return s.addr
}

// ServeHTTP implements a http.Handler, serving the query interface for Ekanite
func (s *HttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	dontCache(w, r)

	if r.Method == "GET" || r.Method == "HEAD" {
		// HEAD is conveniently supported by net/http without further action
		err := serveIndex(s, w, r)

		if err != nil {
			s.Logger.Print("Error executing template: ", err)
			http.Error(w, "Error executing template: "+err.Error(), http.StatusInternalServerError)
			return
		}
	} else if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			s.Logger.Printf("Error parsing form '%s'", err)
			http.Error(w, "Error parsing form", http.StatusBadRequest)
			return
		}

		if len(r.FormValue("query")) == 0 {
			serveIndex(s, w, r)
			return
		}

		userQuery := r.FormValue("query")

		s.Logger.Printf("executing query '%s'", userQuery)

		resultSet, err := s.Searcher.Search(userQuery)
		var resultSlice []string

		if err != nil {
			s.Logger.Printf("Error executing query: '%s'", err)
			http.Error(w, "Error executing query: "+err.Error(), http.StatusInternalServerError)
			return
		}

		for s := range resultSet {
			resultSlice = append(resultSlice, s)
		}

		data := struct {
			Title         string
			Headline      string
			ReturnResults bool
			LogMessages   []string
			CsrfField     template.HTML
		}{
			"Ekanite query interface",
			"Ekanite - Listing " + strconv.Itoa(len(resultSlice)) + " results for '" + userQuery + "'",
			true,
			resultSlice,
			csrf.TemplateField(r),
		}

		if err := s.template.Execute(w, data); err != nil {
			s.Logger.Print("Error executing template: ", err)
		}

	} else {
		http.Error(w, "Unsupported method", http.StatusMethodNotAllowed)
	}
}

// serveIndex serves the plain index for the GET request and POST failovers
func serveIndex(s *HttpServer, w http.ResponseWriter, r *http.Request) error {
	data := struct {
		Title         string
		Headline      string
		ReturnResults bool
		LogMessages   []string
		CsrfField     template.HTML
	}{
		"Ekanite query interface",
		"Ekanite query interface",
		false,
		[]string{},
		csrf.TemplateField(r),
	}

	if err := s.template.Execute(w, data); err != nil {
		return err
	}

	return nil
}

// dontCache sets necessary headers to avoid client and intermediate caching of response
func dontCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Expires", time.Unix(0, 0).Format(time.RFC1123))
	w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123))
	w.Header().Set("Cache-Control", "private, no-store, max-age=0, no-cache, must-revalidate, post-check=0, pre-check=0")

	return
}

const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>

{{ $.Headline }}

Query language reference: bleve

	<form action="/" method="POST">
    <textarea name="query" cols="100" rows="2"></textarea>

    <input name="submit" type="submit" class="button" value="Query">
    {{ .CsrfField }}
	</form>

{{ if $.ReturnResults }}

	{{range $message := $.LogMessages }}

{{ $message }}

	{{ end }}

{{ end }}

`
<commit_msg>CSS fixes, indentation<commit_after>package ekanite

import (
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/gorilla/csrf"
	"github.com/gorilla/securecookie"
)

// Server serves query client connections.
type HttpServer struct {
	iface    string
	Searcher Searcher

	addr     net.Addr
	template *template.Template

	Logger *log.Logger
}

// NewServer returns a new Server instance.
func NewHttpServer(iface string, searcher Searcher) *HttpServer {
	return &HttpServer{
		iface:    iface,
		Searcher: searcher,
		Logger:   log.New(os.Stderr, "[httpserver] ", log.LstdFlags),
	}
}

// Start instructs the Server to bind to the interface and accept connections.
func (s *HttpServer) Start() error {
	ln, err := net.Listen("tcp", s.iface)
	if err != nil {
		return err
	}

	s.addr = ln.Addr()

	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		ln.Close()
		return err
	}

	CSRF := csrf.Protect(securecookie.GenerateRandomKey(32),
		csrf.Secure(false),
		csrf.HttpOnly(true))

	go http.Serve(ln, CSRF(s))
	return nil
}

// Addr returns the address to which the Server is bound.
func (s *HttpServer) Addr() net.Addr {
	return s.addr
}

// ServeHTTP implements a http.Handler, serving the query interface for Ekanite
func (s *HttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	dontCache(w, r)

	if r.Method == "GET" || r.Method == "HEAD" {
		// HEAD is conveniently supported by net/http without further action
		err := serveIndex(s, w, r)

		if err != nil {
			s.Logger.Print("Error executing template: ", err)
			http.Error(w, "Error executing template: "+err.Error(), http.StatusInternalServerError)
			return
		}
	} else if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			s.Logger.Printf("Error parsing form '%s'", err)
			http.Error(w, "Error parsing form", http.StatusBadRequest)
			return
		}

		if len(r.FormValue("query")) == 0 {
			serveIndex(s, w, r)
			return
		}

		userQuery := r.FormValue("query")

		s.Logger.Printf("executing query '%s'", userQuery)

		resultSet, err := s.Searcher.Search(userQuery)
		var resultSlice []string

		if err != nil {
			s.Logger.Printf("Error executing query: '%s'", err)
			http.Error(w, "Error executing query: "+err.Error(), http.StatusInternalServerError)
			return
		}

		for s := range resultSet {
			resultSlice = append(resultSlice, s)
		}

		data := struct {
			Title         string
			Headline      string
			ReturnResults bool
			LogMessages   []string
			CsrfField     template.HTML
		}{
			"Ekanite query interface",
			"Ekanite - Listing " + strconv.Itoa(len(resultSlice)) + " results for '" + userQuery + "'",
			true,
			resultSlice,
			csrf.TemplateField(r),
		}

		if err := s.template.Execute(w, data); err != nil {
			s.Logger.Print("Error executing template: ", err)
		}

	} else {
		http.Error(w, "Unsupported method", http.StatusMethodNotAllowed)
	}
}

// serveIndex serves the plain index for the GET request and POST failovers
func serveIndex(s *HttpServer, w http.ResponseWriter, r *http.Request) error {
	data := struct {
		Title         string
		Headline      string
		ReturnResults bool
		LogMessages   []string
		CsrfField     template.HTML
	}{
		"Ekanite query interface",
		"Ekanite query interface",
		false,
		[]string{},
		csrf.TemplateField(r),
	}

	if err := s.template.Execute(w, data); err != nil {
		return err
	}

	return nil
}

// dontCache sets necessary headers to avoid client and intermediate caching of response
func dontCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Expires", time.Unix(0, 0).Format(time.RFC1123))
	w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123))
	w.Header().Set("Cache-Control", "private, no-store, max-age=0, no-cache, must-revalidate, post-check=0, pre-check=0")

	return
}

const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>

{{ $.Headline }}

Query language reference: bleve

	<form action="/" method="POST">
    <textarea name="query" cols="100" rows="2"></textarea>

    <input name="submit" type="submit" class="button" value="Query">
    {{ .CsrfField }}
	</form>

{{ if $.ReturnResults }}

	{{range $message := $.LogMessages }}

{{ $message }}

	{{ end }}

{{ end }}

`
<|endoftext|>


<commit_before>package client

import (
	"testing"

	"github.com/open-policy-agent/opa/ast"
)

type regoTestCase struct {
	Name          string
	Rego          string
	Path          string
	ErrorExpected bool
	ExpectedRego  string
	ArityExpected int
	RequiredRules ruleArities
}

func runRegoTests(tt []regoTestCase, t *testing.T) {
	for _, tc := range tt {
		t.Run(tc.Name, func(t *testing.T) {
			path := tc.Path
			if path == "" {
				path = "default.test.path"
			}
			rego, err := ensureRegoConformance("test", path, tc.Rego)
			if (err == nil) && tc.ErrorExpected {
				t.Errorf("err = nil; want non-nil")
			}
			if (err != nil) && !tc.ErrorExpected {
				t.Errorf("err = \"%s\"; want nil", err)
			}
			if tc.ExpectedRego != "" && rego != tc.ExpectedRego {
				t.Errorf("ensureRegoConformance(%s) = %s; want %s", tc.Rego, rego, tc.ExpectedRego)
			}
		})
	}
}

func TestDataAccess(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "Empty String Fails",
			Rego:          "",
			ErrorExpected: true,
		},
		{
			Name:          "No Data Access",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access: Inventory",
			Rego:          "package hello v{data.inventory == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access Field",
			Rego:          `package hello v{data["inventory"] == 1}`,
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access Field Variable Assignment",
			Rego:          `package hello v{q := data["inventory"]; q.res == 7}`,
			ErrorExpected: false,
		},
		{
			Name:          "Invalid Data Access",
			Rego:          "package hello v{data.tribble == 1}",
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Param",
			Rego:          `package hello v[{"here": data.onering}]{1 == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access No Param",
			Rego:          `package hello v{data == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Variable",
			Rego:          `package hello v{q := "inventory"; data[q] == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Variable Assignment",
			Rego:          `package hello v{q := data; q.nonono == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Blank Iterator",
			Rego:          `package hello v{data[_] == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Object",
			Rego:          `package hello v{data[{"my": _}] == 1}`,
			ErrorExpected: true,
		},
	}, t)
}

func TestNoImportsAllowed(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "No Imports",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "One Import",
			Rego:          "package hello import data.foo v{1 == 1}",
			ErrorExpected: true,
		},
		{
			Name:          "Three Imports",
			Rego:          "package hello import data.foo import data.test import data.things v{1 == 1}",
			ErrorExpected: true,
		},
	}, t)
}

func TestPackageChange(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "Package Modified",
			Path:          "some.path",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
			ExpectedRego: `package some.path

v = true { equal(1, 1) }`,
		},
		{
			Name:          "Package Modified Other Path",
			Path:          "different.path",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
			ExpectedRego: `package different.path

v = true { equal(1, 1) }`,
		},
	}, t)
}

func TestGetRuleArity(t *testing.T) {
	tc := []regoTestCase{
		{
			Name:          "Nullary",
			Rego:          `package hello v{1 == 1}`,
			ArityExpected: 0,
		},
		{
			Name:          "Unary",
			Rego:          `package hello v[r]{r == 1}`,
			ArityExpected: 1,
		},
		{
			Name:          "Binary",
			Rego:          `package hello v[[r, d]]{r == 1; d == 2}`,
			ArityExpected: 2,
		},
		{
			Name:          "5-ary",
			Rego:          `package hello v[[a, b, c, d, e]]{a == 1; b == 2; c == 3; d == 4; e == 5 }`,
			ArityExpected: 5,
		},
		{
			Name:          "No Object",
			Rego:          `package hello v[{"arg": a}]{a == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "No Object in Array",
			Rego:          `package hello v[[{"arg": a}, b]]{a == 1; b == 2}`,
			ErrorExpected: true,
		},
		{
			Name:          "No Object in Array (reverse)",
			Rego:          `package hello v[[b, {"arg": a}]]{a == 1; b == 2}`,
			ErrorExpected: true,
		},
	}
	for _, tt := range tc {
		t.Run(tt.Name, func(t *testing.T) {
			module, err := ast.ParseModule("foo", tt.Rego)
			if err != nil {
				t.Fatalf("Error parsing rego: %s", err)
			}
			arity, err := getRuleArity(module.Rules[0])
			if (err == nil) && tt.ErrorExpected {
				t.Fatalf("err = nil; want non-nil")
			}
			if (err != nil) && !tt.ErrorExpected {
				t.Fatalf("err = \"%s\"; want nil", err)
			}
			if arity != tt.ArityExpected {
				t.Errorf("getRuleArity(%s) = %d, want %d", tt.Rego, arity, tt.ArityExpected)
			}
		})
	}
}

func TestRequireRules(t *testing.T) {
	tc := []regoTestCase{
		{
			Name:          "No Required Rules",
			Rego:          `package hello`,
			ErrorExpected: false,
		},
		{
			Name:          "Bad Rego",
			Rego:          `package hello {dangling bracket`,
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 0},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Unary",
			Rego:          `package hello r[v]{v == 1}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Binary",
			Rego:          `package hello r[[v, q]]{v == 1; q == 2}`,
			RequiredRules: ruleArities{"r": 2},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Extras",
			Rego:          `package hello r[v]{v == 1} q{3 == 3}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Multiple",
			Rego:          `package hello r[v]{v == 1} q{3 == 3}`,
			RequiredRules: ruleArities{"r": 1, "q": 0},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Missing",
			Rego:          `package hello`,
			RequiredRules: ruleArities{"r": 0},
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule Wrong Arity",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule Missing, Multiple",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 0, "q": 0},
			ErrorExpected: true,
		},
	}
	for _, tt := range tc {
		t.Run(tt.Name, func(t *testing.T) {
			err := requireRules(tt.Rego, tt.RequiredRules)
			if (err == nil) && tt.ErrorExpected {
				t.Fatalf("err = nil; want non-nil")
			}
			if (err != nil) && !tt.ErrorExpected {
				t.Fatalf("err = \"%s\"; want nil", err)
			}
		})
	}
}
<commit_msg>Test rego arity calculation against string args<commit_after>package client

import (
	"testing"

	"github.com/open-policy-agent/opa/ast"
)

type regoTestCase struct {
	Name          string
	Rego          string
	Path          string
	ErrorExpected bool
	ExpectedRego  string
	ArityExpected int
	RequiredRules ruleArities
}

func runRegoTests(tt []regoTestCase, t *testing.T) {
	for _, tc := range tt {
		t.Run(tc.Name, func(t *testing.T) {
			path := tc.Path
			if path == "" {
				path = "default.test.path"
			}
			rego, err := ensureRegoConformance("test", path, tc.Rego)
			if (err == nil) && tc.ErrorExpected {
				t.Errorf("err = nil; want non-nil")
			}
			if (err != nil) && !tc.ErrorExpected {
				t.Errorf("err = \"%s\"; want nil", err)
			}
			if tc.ExpectedRego != "" && rego != tc.ExpectedRego {
				t.Errorf("ensureRegoConformance(%s) = %s; want %s", tc.Rego, rego, tc.ExpectedRego)
			}
		})
	}
}

func TestDataAccess(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "Empty String Fails",
			Rego:          "",
			ErrorExpected: true,
		},
		{
			Name:          "No Data Access",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access: Inventory",
			Rego:          "package hello v{data.inventory == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access Field",
			Rego:          `package hello v{data["inventory"] == 1}`,
			ErrorExpected: false,
		},
		{
			Name:          "Valid Data Access Field Variable Assignment",
			Rego:          `package hello v{q := data["inventory"]; q.res == 7}`,
			ErrorExpected: false,
		},
		{
			Name:          "Invalid Data Access",
			Rego:          "package hello v{data.tribble == 1}",
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Param",
			Rego:          `package hello v[{"here": data.onering}]{1 == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access No Param",
			Rego:          `package hello v{data == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Variable",
			Rego:          `package hello v{q := "inventory"; data[q] == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Variable Assignment",
			Rego:          `package hello v{q := data; q.nonono == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Blank Iterator",
			Rego:          `package hello v{data[_] == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "Invalid Data Access Object",
			Rego:          `package hello v{data[{"my": _}] == 1}`,
			ErrorExpected: true,
		},
	}, t)
}

func TestNoImportsAllowed(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "No Imports",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
		},
		{
			Name:          "One Import",
			Rego:          "package hello import data.foo v{1 == 1}",
			ErrorExpected: true,
		},
		{
			Name:          "Three Imports",
			Rego:          "package hello import data.foo import data.test import data.things v{1 == 1}",
			ErrorExpected: true,
		},
	}, t)
}

func TestPackageChange(t *testing.T) {
	runRegoTests([]regoTestCase{
		{
			Name:          "Package Modified",
			Path:          "some.path",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
			ExpectedRego: `package some.path

v = true { equal(1, 1) }`,
		},
		{
			Name:          "Package Modified Other Path",
			Path:          "different.path",
			Rego:          "package hello v{1 == 1}",
			ErrorExpected: false,
			ExpectedRego: `package different.path

v = true { equal(1, 1) }`,
		},
	}, t)
}

func TestGetRuleArity(t *testing.T) {
	tc := []regoTestCase{
		{
			Name:          "Nullary",
			Rego:          `package hello v{1 == 1}`,
			ArityExpected: 0,
		},
		{
			Name:          "Unary",
			Rego:          `package hello v[r]{r == 1}`,
			ArityExpected: 1,
		},
		{
			Name:          "Binary",
			Rego:          `package hello v[[r, d]]{r == 1; d == 2}`,
			ArityExpected: 2,
		},
		{
			Name:          "5-ary",
			Rego:          `package hello v[[a, b, c, d, e]]{a == 1; b == 2; c == 3; d == 4; e == 5 }`,
			ArityExpected: 5,
		},
		{
			Name:          "No Object",
			Rego:          `package hello v[{"arg": a}]{a == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "No Object in Array",
			Rego:          `package hello v[[{"arg": a}, b]]{a == 1; b == 2}`,
			ErrorExpected: true,
		},
		{
			Name:          "No Object in Array (reverse)",
			Rego:          `package hello v[[b, {"arg": a}]]{a == 1; b == 2}`,
			ErrorExpected: true,
		},
		{
			Name:          "No String Key",
			Rego:          `package hello v["q"]{1 == 1}`,
			ErrorExpected: true,
		},
		{
			Name:          "No String Array Entry",
			Rego:          `package hello v[[b, "a"]]{b == 2}`,
			ErrorExpected: true,
		},
	}
	for _, tt := range tc {
		t.Run(tt.Name, func(t *testing.T) {
			module, err := ast.ParseModule("foo", tt.Rego)
			if err != nil {
				t.Fatalf("Error parsing rego: %s", err)
			}
			arity, err := getRuleArity(module.Rules[0])
			if (err == nil) && tt.ErrorExpected {
				t.Fatalf("err = nil; want non-nil")
			}
			if (err != nil) && !tt.ErrorExpected {
				t.Fatalf("err = \"%s\"; want nil", err)
			}
			if arity != tt.ArityExpected {
				t.Errorf("getRuleArity(%s) = %d, want %d", tt.Rego, arity, tt.ArityExpected)
			}
		})
	}
}

func TestRequireRules(t *testing.T) {
	tc := []regoTestCase{
		{
			Name:          "No Required Rules",
			Rego:          `package hello`,
			ErrorExpected: false,
		},
		{
			Name:          "Bad Rego",
			Rego:          `package hello {dangling bracket`,
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 0},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Unary",
			Rego:          `package hello r[v]{v == 1}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Binary",
			Rego:          `package hello r[[v, q]]{v == 1; q == 2}`,
			RequiredRules: ruleArities{"r": 2},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Extras",
			Rego:          `package hello r[v]{v == 1} q{3 == 3}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Multiple",
			Rego:          `package hello r[v]{v == 1} q{3 == 3}`,
			RequiredRules: ruleArities{"r": 1, "q": 0},
			ErrorExpected: false,
		},
		{
			Name:          "Required Rule Missing",
			Rego:          `package hello`,
			RequiredRules: ruleArities{"r": 0},
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule Wrong Arity",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 1},
			ErrorExpected: true,
		},
		{
			Name:          "Required Rule Missing, Multiple",
			Rego:          `package hello r{1 == 1}`,
			RequiredRules: ruleArities{"r": 0, "q": 0},
			ErrorExpected: true,
		},
	}
	for _, tt := range tc {
		t.Run(tt.Name, func(t *testing.T) {
			err := requireRules(tt.Rego, tt.RequiredRules)
			if (err == nil) && tt.ErrorExpected {
				t.Fatalf("err = nil; want non-nil")
			}
			if (err != nil) && !tt.ErrorExpected {
				t.Fatalf("err = \"%s\"; want nil", err)
			}
		})
	}
}
<|endoftext|>


<commit_before>package recycleme

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
)

func init() {
	canSendMail = false
}

func (f FetchableURL) Fetch(ean string) (Product, error) {
	return Product{}, nil
}

func (f FetchableURL) IsURLValidForEAN(url, ean string) bool {
	return fullURL(f.URL, ean) == url
}

func TestAddBlacklistHandler(t *testing.T) {
	nopFetcher := FetchableURL{URL: "http://www.example.com/%s/", WebsiteName: "Example.com"}
	data := url.Values{}
	data.Set("name", "test product")
	ean := "EAN_TEST"
	data.Set("ean", ean)
	url := fullURL(nopFetcher.URL, ean)
	data.Set("url", url)
	data.Set("website", nopFetcher.WebsiteName)

	req, err := http.NewRequest("POST", "/blacklist/add", strings.NewReader(data.Encode()))
	req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	if err != nil {
		t.Fatal(err)
	}

	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger := log.New(ioutil.Discard, "", 0)
		Blacklist.AddBlacklistHandler(w, r, logger, nopFetcher)
	})

	handler.ServeHTTP(rr, req)

	if status := rr.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	expected := "added"
	if rr.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rr.Body.String(), expected)
	}

	if !Blacklist.Contains(url) {
		t.Errorf("%v not added to blacklist", url)
	}
}
<commit_msg>test blacklist error on bad url<commit_after>package recycleme

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
)

func init() {
	canSendMail = false
}

func (f FetchableURL) Fetch(ean string) (Product, error) {
	return Product{}, nil
}

func (f FetchableURL) IsURLValidForEAN(url, ean string) bool {
	return fullURL(f.URL, ean) == url
}

func createPostRequest(uri string, data url.Values) (*http.Request, error) {
	req, err := http.NewRequest("POST", uri, strings.NewReader(data.Encode()))
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode())))
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	return req, nil
}

func TestAddBlacklistHandler(t *testing.T) {
	nopFetcher := FetchableURL{URL: "http://www.example.com/%s/", WebsiteName: "Example.com"}
	data := url.Values{}
	data.Set("name", "test product")
	ean := "EAN_TEST"
	data.Set("ean", ean)
	url := fullURL(nopFetcher.URL, ean)
	data.Set("url", url)
	data.Set("website", nopFetcher.WebsiteName)

	req, err := createPostRequest("/blacklist/add", data)
	if err != nil {
		t.Fatal(err)
	}

	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		logger := log.New(ioutil.Discard, "", 0)
		Blacklist.AddBlacklistHandler(w, r, logger, nopFetcher)
	})

	handler.ServeHTTP(rr, req)

	if status := rr.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	expected := "added"
	if rr.Body.String() != expected {
		t.Errorf("handler returned unexpected body: got %v want %v", rr.Body.String(), expected)
	}

	if !Blacklist.Contains(url) {
		t.Errorf("%v not added to blacklist", url)
	}

	data.Set("url", "invalid")
	req, err = createPostRequest("/blacklist/add", data)
	if err != nil {
		t.Fatal(err)
	}
	rr = httptest.NewRecorder()
	handler.ServeHTTP(rr, req)

	if status := rr.Code; status != http.StatusInternalServerError {
		t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusInternalServerError)
	}

}
<|endoftext|>


<commit_before>package goapi

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPOST(t *testing.T) {
	s := New()
	s.POST("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[POST])
}

func TestGET(t *testing.T) {
	s := New()
	s.GET("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[GET])
}

func TestPUT(t *testing.T) {
	s := New()
	s.PUT("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[PUT])
}

func TestDELETE(t *testing.T) {
	s := New()
	s.DELETE("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[DELETE])
}

func TestPATCH(t *testing.T) {
	s := New()
	s.PATCH("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[PATCH])
}

func TestOPTIONS(t *testing.T) {
	s := New()
	s.OPTIONS("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[OPTIONS])
}

func TestUseGlobal(t *testing.T) {
	s := New()
	s.Use("", 0, func(req *http.Request) Error { return nil })

	assert.NotEqual(t, 0, len(s.middleware))
}

func TestUseRoot(t *testing.T) {
	s := New()
	s.OPTIONS("/", func(w http.ResponseWriter, req *http.Request) {})
	s.PATCH("/", func(w http.ResponseWriter, req *http.Request) {})
	s.DELETE("/", func(w http.ResponseWriter, req *http.Request) {})
	s.PUT("/", func(w http.ResponseWriter, req *http.Request) {})
	s.GET("/", func(w http.ResponseWriter, req *http.Request) {})

	s.Use("/", 0, func(req *http.Request) Error { return nil })

	for _, r := range s.routes {
		assert.NotEqual(t, 0, len(r.middleware))
	}
}

func TestUseNodes(t *testing.T) {
	s := New()
	s.OPTIONS("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.PATCH("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.DELETE("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.PUT("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.GET("/x", func(w http.ResponseWriter, req *http.Request) {})

	s.Use("/x", 0, func(req *http.Request) Error { return nil })

	for _, root := range s.routes {
		assert.Equal(t, 0, len(root.middleware))
		node := root.nodes["x"]
		if assert.NotNil(t, node) {
			assert.NotEqual(t, 0, len(node.middleware))
		}
	}
}
<commit_msg>Add server test<commit_after>package goapi

import (
	"net/http"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPOST(t *testing.T) {
	s := New()
	s.POST("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[POST])
}

func TestGET(t *testing.T) {
	s := New()
	s.GET("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[GET])
}

func TestPUT(t *testing.T) {
	s := New()
	s.PUT("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[PUT])
}

func TestDELETE(t *testing.T) {
	s := New()
	s.DELETE("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[DELETE])
}

func TestPATCH(t *testing.T) {
	s := New()
	s.PATCH("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[PATCH])
}

func TestOPTIONS(t *testing.T) {
	s := New()
	s.OPTIONS("/", func(w http.ResponseWriter, req *http.Request) {})

	assert.NotNil(t, s.routes[OPTIONS])
}

func TestUseGlobal(t *testing.T) {
	s := New()
	s.Use("", 0, func(req *http.Request) Error { return nil })

	assert.NotEqual(t, 0, len(s.middleware))
}

func TestUseRoot(t *testing.T) {
	s := New()
	s.OPTIONS("/", func(w http.ResponseWriter, req *http.Request) {})
	s.PATCH("/", func(w http.ResponseWriter, req *http.Request) {})
	s.DELETE("/", func(w http.ResponseWriter, req *http.Request) {})
	s.PUT("/", func(w http.ResponseWriter, req *http.Request) {})
	s.GET("/", func(w http.ResponseWriter, req *http.Request) {})

	s.Use("/", 0, func(req *http.Request) Error { return nil })

	for _, r := range s.routes {
		assert.NotEqual(t, 0, len(r.middleware))
	}
}

func TestUseNodes(t *testing.T) {
	s := New()
	s.OPTIONS("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.PATCH("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.DELETE("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.PUT("/x", func(w http.ResponseWriter, req *http.Request) {})
	s.GET("/x", func(w http.ResponseWriter, req *http.Request) {})

	s.Use("/x", 0, func(req *http.Request) Error { return nil })

	for _, root := range s.routes {
		assert.Equal(t, 0, len(root.middleware))
		node := root.nodes["x"]
		if assert.NotNil(t, node) {
			assert.NotEqual(t, 0, len(node.middleware))
		}
	}
}

type mockResponseWriter struct{}

func (m *mockResponseWriter) Header() (h http.Header) {
	return http.Header{}
}
func (m *mockResponseWriter) Write(p []byte) (n int, err error) {
	return len(p), nil
}
func (m *mockResponseWriter) WriteString(s string) (n int, err error) {
	return len(s), nil
}
func (m *mockResponseWriter) WriteHeader(int) {}

func TestServer(t *testing.T) {
	s := New()

	serverd := false
	s.GET("/:param", func(w http.ResponseWriter, r *http.Request) {
		serverd = true
	})

	w := new(mockResponseWriter)
	req, _ := http.NewRequest("GET", "/x", nil)
	s.ServeHTTP(w, req)

	assert.Equal(t, true, serverd)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

// Process represents an operating system process.
type Process struct {
	Pid     int
	User    *user.User
	Command string

	// Alive is a flag used by ProcessMonitor to determine if it should remove
	// this process.
	Alive bool

	// Data from proc/<pid>/stat
	Pgrp  int
	Utime uint64
	Stime uint64
}

// NewProcess returns a new Process if a process is currently running on
// the system with the passed in Pid.
func NewProcess(pid int) *Process {
	p := &Process{
		Pid: pid,
	}

	if err := p.ParseCmdlineFile(); err != nil {
		return nil
	}

	if err := p.Update(); err != nil {
		return nil
	}

	return p
}

func (p *Process) Update() error {
	if err := p.StatProcDir(); err != nil {
		return err
	}

	if err := p.ParseStatFile(); err != nil {
		return err
	}

	return nil
}

// StatProcDir updates p with any information it needs from statting /proc/<pid>.
func (p *Process) StatProcDir() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid))

	var stat syscall.Stat_t
	err := syscall.Stat(path, &stat)
	if err != nil {
		return err
	}

	user, err := userByUid(strconv.FormatUint(uint64(stat.Uid), 10))
	if err != nil {
		panic(err)
	}
	p.User = user

	return nil
}

// ParseStatFile updates p with any information it needs from /proc/<pid>/stat.
func (p *Process) ParseStatFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid), "stat")

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	line := string(data)
	values := strings.Split(line, " ")

	pgrp, err := strconv.Atoi(values[4])
	if err != nil {
		panic(err)
	}
	p.Pgrp = pgrp

	utime, err := strconv.Atoi(values[13])
	if err != nil {
		panic(err)
	}
	p.Utime = uint64(utime)

	stime, err := strconv.Atoi(values[14])
	if err != nil {
		panic(err)
	}
	p.Stime = uint64(stime)

	return nil
}

// ParseCmdlineFile sets p's Command via /proc/<pid>/cmdline.
func (p *Process) ParseCmdlineFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid), "cmdline")

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	s := string(data)
	p.Command = strings.TrimSpace(strings.Replace(s, "\x00", " ", -1))
	return nil
}

func (p *Process) IsKernelThread() bool {
	return p.Pgrp == 0
}

type ByPid []*Process

func (p ByPid) Len() int           { return len(p) }
func (p ByPid) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p ByPid) Less(i, j int) bool { return p[i].Pid < p[j].Pid }
<commit_msg>Name the fields of /proc/<pid>/stat<commit_after>package main

import (
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
)

const (
	// The indices of the values in /proc/<pid>/stat
	statPidIdx = iota
	statCommIdx
	statStateIdx
	statPpidIdx
	statPgrpIdx
	statSessionIdx
	statTtyNrIdx
	statTpgidIdx
	statFlagsIdx
	statMinfltIdx
	statCminfltIdx
	statMajfltIdx
	statCmajfltIdx
	statUtimeIdx
	statStimeIdx
	statCutimeIdx
	statCstimeIdx
	statPriorityIdx
	statNiceIdx
	statNumThreadsIdx
	statItrealvalueIdx
	statStartTimeIdx
	statVsizeIdx
	statRssIdx
	statRsslimIdx
	statStartCodeIdx
	statEndCodeIdx
	statStartStackIdx
	statKstKespIdx
	statKstKeipIdx
	statSignalIdx
	statBlockedIdx
	statSigIgnoreIdx
	statSigCatchIdx
	statWchanIdx
	statNswapIdx
	statCnswapIdx
	statExitSignalIdx
	statProcessorIdx
	statRtPriorityIdx
	statPolicyIdx
	statDelayActBlkioTicksIdx
	statGuestTimeIdx
	statCguestTimeIdx
)

// Process represents an operating system process.
type Process struct {
	Pid     int
	User    *user.User
	Command string

	// Alive is a flag used by ProcessMonitor to determine if it should remove
	// this process.
	Alive bool

	// Data from /proc/<pid>/stat
	Pgrp  int
	Utime uint64
	Stime uint64
}

// NewProcess returns a new Process if a process is currently running on
// the system with the passed in Pid.
func NewProcess(pid int) *Process {
	p := &Process{
		Pid: pid,
	}

	if err := p.ParseCmdlineFile(); err != nil {
		return nil
	}

	if err := p.Update(); err != nil {
		return nil
	}

	return p
}

func (p *Process) Update() error {
	if err := p.StatProcDir(); err != nil {
		return err
	}

	if err := p.ParseStatFile(); err != nil {
		return err
	}

	return nil
}

// StatProcDir updates p with any information it needs from statting /proc/<pid>.
func (p *Process) StatProcDir() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid))

	var stat syscall.Stat_t
	err := syscall.Stat(path, &stat)
	if err != nil {
		return err
	}

	user, err := userByUid(strconv.FormatUint(uint64(stat.Uid), 10))
	if err != nil {
		panic(err)
	}
	p.User = user

	return nil
}

// ParseStatFile updates p with any information it needs from /proc/<pid>/stat.
func (p *Process) ParseStatFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid), "stat")

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	data, err := ioutil.ReadAll(file)
	if err != nil {
		return err
	}

	line := string(data)
	values := strings.Split(line, " ")

	pgrp, err := strconv.Atoi(values[statPgrpIdx])
	if err != nil {
		panic(err)
	}
	p.Pgrp = pgrp

	utime, err := strconv.Atoi(values[statUtimeIdx])
	if err != nil {
		panic(err)
	}
	p.Utime = uint64(utime)

	stime, err := strconv.Atoi(values[statStimeIdx])
	if err != nil {
		panic(err)
	}
	p.Stime = uint64(stime)

	return nil
}

// ParseCmdlineFile sets p's Command via /proc/<pid>/cmdline.
func (p *Process) ParseCmdlineFile() error {
	path := filepath.Join("/proc", strconv.Itoa(p.Pid), "cmdline")

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	s := string(data)
	p.Command = strings.TrimSpace(strings.Replace(s, "\x00", " ", -1))
	return nil
}

func (p *Process) IsKernelThread() bool {
	return p.Pgrp == 0
}

type ByPid []*Process

func (p ByPid) Len() int           { return len(p) }
func (p ByPid) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p ByPid) Less(i, j int) bool { return p[i].Pid < p[j].Pid }
<|endoftext|>


<commit_before>package deployment

import (
	"fmt"

	ct "github.com/flynn/flynn/controller/types"
)

func (d *DeployJob) deployOneByOne() error {
	log := d.logger.New("fn", "deployOneByOne")
	log.Info("starting one-by-one deployment")

	oldScale := make(map[string]int, len(d.oldReleaseState))
	for typ, count := range d.oldReleaseState {
		oldScale[typ] = count
		if d.isOmni(typ) {
			oldScale[typ] /= d.hostCount
		}
	}

	newScale := make(map[string]int, len(d.newReleaseState))
	for typ, count := range d.newReleaseState {
		newScale[typ] = count
		if d.isOmni(typ) {
			newScale[typ] /= d.hostCount
		}
	}

	olog := log.New("release_id", d.OldReleaseID)
	nlog := log.New("release_id", d.NewReleaseID)
	for typ, num := range d.Processes {
		diff := 1
		if d.isOmni(typ) {
			diff = d.hostCount
		}

		for i := newScale[typ]; i < num; i++ {
			nlog.Info("scaling new formation up by one", "type", typ)
			newScale[typ]++
			if err := d.client.PutFormation(&ct.Formation{
				AppID:     d.AppID,
				ReleaseID: d.NewReleaseID,
				Processes: newScale,
			}); err != nil {
				nlog.Error("error scaling new formation up by one", "type", typ, "err", err)
				return err
			}
			for i := 0; i < diff; i++ {
				d.deployEvents <- ct.DeploymentEvent{
					ReleaseID: d.NewReleaseID,
					JobState:  "starting",
					JobType:   typ,
				}
			}
			nlog.Info(fmt.Sprintf("waiting for %d job up event(s)", diff), "type", typ)
			if err := d.waitForJobEvents(d.NewReleaseID, jobEvents{typ: {"up": diff}}, nlog); err != nil {
				nlog.Error("error waiting for job up events", "err", err)
				return err
			}

			olog.Info("scaling old formation down by one", "type", typ)
			oldScale[typ]--
			if err := d.client.PutFormation(&ct.Formation{
				AppID:     d.AppID,
				ReleaseID: d.OldReleaseID,
				Processes: oldScale,
			}); err != nil {
				olog.Error("error scaling old formation down by one", "type", typ, "err", err)
				return err
			}
			for i := 0; i < diff; i++ {
				d.deployEvents <- ct.DeploymentEvent{
					ReleaseID: d.OldReleaseID,
					JobState:  "stopping",
					JobType:   typ,
				}
			}

			olog.Info(fmt.Sprintf("waiting for %d job down event(s)", diff), "type", typ)
			if err := d.waitForJobEvents(d.OldReleaseID, jobEvents{typ: {"down": diff}}, olog); err != nil {
				olog.Error("error waiting for job down events", "err", err)
				return err
			}
		}
	}

	// ensure any old leftover jobs are stopped (this can happen when new
	// workers continue deployments from old workers and still see the
	// old worker running even though it has been scaled down).
	log.Info("ensuring old formation is scaled down to zero")
	diff := make(jobEvents, len(oldScale))
	for typ, count := range oldScale {
		diff[typ] = map[string]int{"down": count}
	}
	if err := d.client.PutFormation(&ct.Formation{
		AppID:     d.AppID,
		ReleaseID: d.OldReleaseID,
	}); err != nil {
		log.Error("error scaling old formation down to zero", "err", err)
		return err
	}
	if diff.Count() > 0 {
		log.Info(fmt.Sprintf("waiting for %d job down event(s)", diff.Count()))
		if err := d.waitForJobEvents(d.OldReleaseID, diff, log); err != nil {
			log.Error("error waiting for job down events", "err", err)
			return err
		}
	}

	log.Info("finished one-by-one deployment")
	return nil
}
<commit_msg>deployer: Order one-by-one deployments<commit_after>package deployment

import (
	"fmt"
	"sort"

	ct "github.com/flynn/flynn/controller/types"
)

func (d *DeployJob) deployOneByOne() error {
	log := d.logger.New("fn", "deployOneByOne")
	log.Info("starting one-by-one deployment")

	oldScale := make(map[string]int, len(d.oldReleaseState))
	for typ, count := range d.oldReleaseState {
		oldScale[typ] = count
		if d.isOmni(typ) {
			oldScale[typ] /= d.hostCount
		}
	}

	newScale := make(map[string]int, len(d.newReleaseState))
	for typ, count := range d.newReleaseState {
		newScale[typ] = count
		if d.isOmni(typ) {
			newScale[typ] /= d.hostCount
		}
	}

	processTypes := make([]string, 0, len(d.Processes))
	for typ := range d.Processes {
		processTypes = append(processTypes, typ)
	}
	sort.Sort(sort.StringSlice(processTypes))

	olog := log.New("release_id", d.OldReleaseID)
	nlog := log.New("release_id", d.NewReleaseID)
	for _, typ := range processTypes {
		num := d.Processes[typ]
		diff := 1
		if d.isOmni(typ) {
			diff = d.hostCount
		}

		for i := newScale[typ]; i < num; i++ {
			nlog.Info("scaling new formation up by one", "type", typ)
			newScale[typ]++
			if err := d.client.PutFormation(&ct.Formation{
				AppID:     d.AppID,
				ReleaseID: d.NewReleaseID,
				Processes: newScale,
			}); err != nil {
				nlog.Error("error scaling new formation up by one", "type", typ, "err", err)
				return err
			}
			for i := 0; i < diff; i++ {
				d.deployEvents <- ct.DeploymentEvent{
					ReleaseID: d.NewReleaseID,
					JobState:  "starting",
					JobType:   typ,
				}
			}
			nlog.Info(fmt.Sprintf("waiting for %d job up event(s)", diff), "type", typ)
			if err := d.waitForJobEvents(d.NewReleaseID, jobEvents{typ: {"up": diff}}, nlog); err != nil {
				nlog.Error("error waiting for job up events", "err", err)
				return err
			}

			olog.Info("scaling old formation down by one", "type", typ)
			oldScale[typ]--
			if err := d.client.PutFormation(&ct.Formation{
				AppID:     d.AppID,
				ReleaseID: d.OldReleaseID,
				Processes: oldScale,
			}); err != nil {
				olog.Error("error scaling old formation down by one", "type", typ, "err", err)
				return err
			}
			for i := 0; i < diff; i++ {
				d.deployEvents <- ct.DeploymentEvent{
					ReleaseID: d.OldReleaseID,
					JobState:  "stopping",
					JobType:   typ,
				}
			}

			olog.Info(fmt.Sprintf("waiting for %d job down event(s)", diff), "type", typ)
			if err := d.waitForJobEvents(d.OldReleaseID, jobEvents{typ: {"down": diff}}, olog); err != nil {
				olog.Error("error waiting for job down events", "err", err)
				return err
			}
		}
	}

	// ensure any old leftover jobs are stopped (this can happen when new
	// workers continue deployments from old workers and still see the
	// old worker running even though it has been scaled down).
	log.Info("ensuring old formation is scaled down to zero")
	diff := make(jobEvents, len(oldScale))
	for typ, count := range oldScale {
		diff[typ] = map[string]int{"down": count}
	}
	if err := d.client.PutFormation(&ct.Formation{
		AppID:     d.AppID,
		ReleaseID: d.OldReleaseID,
	}); err != nil {
		log.Error("error scaling old formation down to zero", "err", err)
		return err
	}
	if diff.Count() > 0 {
		log.Info(fmt.Sprintf("waiting for %d job down event(s)", diff.Count()))
		if err := d.waitForJobEvents(d.OldReleaseID, diff, log); err != nil {
			log.Error("error waiting for job down events", "err", err)
			return err
		}
	}

	log.Info("finished one-by-one deployment")
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
    fmt.Printf("Apache header.\n")
}
<commit_msg>Delete bad-missing-license.go<commit_after><|endoftext|>


<commit_before>package elements

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
)

func getData(url string) (data []string) {
	resp, err := http.Get(url)
	if err != nil {
		return make([]string, 0)
	}

	defer resp.Body.Close()

	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		data = append(data, strings.TrimRight(scanner.Text(), "\n"))
		if err != nil {
			break
		}
	}
	return
}

func crawlData(url string) map[string]interface{} {
	data := make(map[string]interface{})
	urlData := getData(url)

	var key string
	for _, line := range urlData {

		// replace hyphens with underscores for JSON keys
		key = strings.Replace(line, "-", "_", -1)

		switch {
		default:
			data[key] = getData(url + line)[0]
		case line == "dynamic":
			break
		case line == "meta-data":
			data[key] = crawlData(url + line + "/")
		case line == "user-data":
			data[key] = strings.Join(getData(url+line+"/"), "")
		case strings.HasSuffix(line, "/"):
			data[key[:len(line)-1]] = crawlData(url + line)
		case strings.HasSuffix(url, "public-keys/"):
			keyId := strings.SplitN(line, "=", 2)[0]
			data[key] = crawlData(url + keyId + "/")
		}
	}
	return data
}

func jsonData(url string) ([]byte, error) {
	data, err := json.MarshalIndent(crawlData(url), "", "    ")
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (e *Elements) GetAWSElements() (map[string]interface{}, error) {
	url := "http://169.254.169.254/latest/"

	data, err := jsonData(url)
	if err != nil {
		return nil, fmt.Errorf("Unable to marshal data from ec2 crawl: %s", err)
	}

	ec2Elements := make(map[string]interface{})
	err = json.Unmarshal(data, &ec2Elements)
	if err != nil {
		return nil, fmt.Errorf("Unable to unmarshal data from ec2 JSON: %s", err)
	}

	return ec2Elements, nil
}

/*
Attribution: much of the code in this file was lifted from the ec2_metadata_dump
project: https://github.com/thbishop/ec2_metadata_dump, which is:

Copyright (c) 2013 Thomas Bishop
*/
<commit_msg>Conservative key assignment when crawling metadata<commit_after>package elements

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
)

func getData(url string) (data []string) {
	resp, err := http.Get(url)
	if err != nil {
		return make([]string, 0)
	}

	defer resp.Body.Close()

	scanner := bufio.NewScanner(resp.Body)
	for scanner.Scan() {
		data = append(data, strings.TrimRight(scanner.Text(), "\n"))
		if err != nil {
			break
		}
	}
	return
}

func crawlData(url string) map[string]interface{} {
	data := make(map[string]interface{})
	urlData := getData(url)

	var key string
	for _, line := range urlData {

		// replace hyphens with underscores for JSON keys
		key = strings.Replace(line, "-", "_", -1)

		switch {
		default:
			d := getData(url + line)
			if len(d) > 0 {
				data[key] = d[0]
			}
		case line == "dynamic":
			break
		case line == "meta-data":
			data[key] = crawlData(url + line + "/")
		case line == "user-data":
			data[key] = strings.Join(getData(url+line+"/"), "")
		case strings.HasSuffix(line, "/"):
			data[key[:len(line)-1]] = crawlData(url + line)
		case strings.HasSuffix(url, "public-keys/"):
			keyId := strings.SplitN(line, "=", 2)[0]
			data[key] = crawlData(url + keyId + "/")
		}
	}
	return data
}

func jsonData(url string) ([]byte, error) {
	data, err := json.MarshalIndent(crawlData(url), "", "    ")
	if err != nil {
		return nil, err
	}
	return data, nil
}

func (e *Elements) GetAWSElements() (map[string]interface{}, error) {
	url := "http://169.254.169.254/latest/"

	data, err := jsonData(url)
	if err != nil {
		return nil, fmt.Errorf("Unable to marshal data from ec2 crawl: %s", err)
	}

	ec2Elements := make(map[string]interface{})
	err = json.Unmarshal(data, &ec2Elements)
	if err != nil {
		return nil, fmt.Errorf("Unable to unmarshal data from ec2 JSON: %s", err)
	}

	return ec2Elements, nil
}

/*
Attribution: much of the code in this file was lifted from the ec2_metadata_dump
project: https://github.com/thbishop/ec2_metadata_dump, which is:

Copyright (c) 2013 Thomas Bishop
*/
<|endoftext|>


<commit_before>// Copyright 2014 Wei Shen (shenwei356@gmail.com). All rights reserved.
// Use of this source code is governed by a MIT-license
// that can be found in the LICENSE file.

// ParallelMap - A lock-free parallel map in go.
// ParallelMap uses a backend goroutine for the sequential excution of
// Get and Set or custom function, which was inspired by section 14.17
// in book << The Way to Go >>.
package ParallelMap

// The type of the map key is interface{}
type keyType interface{}

// The type of the map value is interface{}
type valueType interface{}

// Usage
//
//    import (
//        "fmt"
//        pmap "github.com/shenwei356/util/ParallelMap"
//        "runtime"
//    )
//
//    func main() {
//        runtime.GOMAXPROCS(runtime.NumCPU())
//
//        m := pmap.NewParallelMap()
//        m.Set("year", 2014)
//
//        if v, ok := m.Get("year"); ok {
//            fmt.Println(v)
//        }
//    }
type ParallelMap struct {
	// map
	Map map[keyType]valueType
	// backend goroutine for the sequential excution
	Op chan func()
}

func NewParallelMap() *ParallelMap {
	m := &ParallelMap{make(map[keyType]valueType), make(chan func())}
	go m.backend()
	return m
}

// Run operation channel as backend
func (this *ParallelMap) backend() {
	var f func()
	for {
		select {
		case f = <-this.Op:
			f()
		}
	}
}

// Getting element of the map is executed sequentially
func (this *ParallelMap) Get(key keyType) (valueType, bool) {
	/*v, ok := this.Map[key]
	return v, ok*/

	c1 := make(chan valueType)
	c2 := make(chan bool)
	this.Op <- func() {
		value, ok := this.Map[key]
		c1 <- value
		c2 <- ok
	}
	return <-c1, <-c2
}

// Setting operation is executed sequentially to ensure the
// operation is atomic.
func (this *ParallelMap) Set(key keyType, value valueType) {
	c := make(chan bool)
	this.Op <- func() {
		this.Map[key] = value
		c <- true
	}
	<-c
}

// Execute a custom function.
//
// Example: An element increasing function
//
//    map.ExecuteFunc(func() {
//        if v, ok := m.Map[i]; ok {
//            m.Map[i] = v.(int) + 1
//        } else {
//            m.Map[i] = int(1)
//        }
//    })
func (this *ParallelMap) ExecuteFunc(f func()) {
	c := make(chan bool)
	this.Op <- func() {
		f()
		c <- true
	}
	<-c
}
<commit_msg>improve the doc of ParallelMap<commit_after>// Copyright 2014 Wei Shen (shenwei356@gmail.com). All rights reserved.
// Use of this source code is governed by a MIT-license
// that can be found in the LICENSE file.

// ParallelMap - A lock-free parallel map in go.
// ParallelMap uses a backend goroutine for the sequential excution of
// Get and Set or custom function, which was inspired by section 14.17
// in book << The Way to Go >>.
package ParallelMap

// The type of the map key is interface{}
type keyType interface{}

// The type of the map value is interface{}
type valueType interface{}

// Usage
//
//    import (
//        "fmt"
//        pmap "github.com/shenwei356/util/ParallelMap"
//        "runtime"
//    )
//
//    func main() {
//        runtime.GOMAXPROCS(runtime.NumCPU())
//
//        m := pmap.NewParallelMap()
//        m.Set("year", 2014)
//
//        if v, ok := m.Get("year"); ok {
//            fmt.Println(v)
//        }
//    }
type ParallelMap struct {
	// map
	Map map[keyType]valueType
	// backend goroutine for the sequential excution
	Op chan func()
}

// NewParallelMap
func NewParallelMap() *ParallelMap {
	m := &ParallelMap{make(map[keyType]valueType), make(chan func())}
	go m.backend()
	return m
}

// Run operation channel as backend
func (this *ParallelMap) backend() {
	var f func()
	for {
		select {
		case f = <-this.Op:
			f()
		}
	}
}

// Getting element of the map is executed sequentially
func (this *ParallelMap) Get(key keyType) (valueType, bool) {
	/*v, ok := this.Map[key]
	return v, ok*/

	c1 := make(chan valueType)
	c2 := make(chan bool)
	this.Op <- func() {
		value, ok := this.Map[key]
		c1 <- value
		c2 <- ok
	}
	return <-c1, <-c2
}

// Setting operation is executed sequentially to ensure the
// operation is atomic.
func (this *ParallelMap) Set(key keyType, value valueType) {
	c := make(chan bool)
	this.Op <- func() {
		this.Map[key] = value
		c <- true
	}
	<-c
}

// Execute a custom function.
//
// Example: An element increasing function
//
//    m.ExecuteFunc(func() {
//        if v, ok := m.Map[i]; ok {
//            m.Map[i] = v.(int) + 1
//        } else {
//            m.Map[i] = int(1)
//        }
//    })
func (this *ParallelMap) ExecuteFunc(f func()) {
	c := make(chan bool)
	this.Op <- func() {
		f()
		c <- true
	}
	<-c
}
<|endoftext|>


<commit_before>package gojot

import (
	"os"
	"os/exec"
	"sort"
	"strings"
	"time"
)

func ListFiles(gitfolder string) []string {
	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("ListFiles Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", "master")
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := []string{}
	for _, document := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
		if len(document) < 1 {
			continue
		}
		if document[0] == '.' {
			document = DecryptOTP(document[1:])
			logger.Debug("Found document: %s", document)
			if document == ".deleted" || document == "gojot-key" || document == "gojot-hs" {
				continue
			}
			documents = append(documents, document)
		}
	}

	sort.Strings(documents)
	return documents
}

func ListFilesOfOne(gitfolder string, branch string) []string {
	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", branch)
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := []string{}
	for _, document := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
		if document[0] == '.' {
			continue
		}
		documents = append(documents, document)
	}

	return documents
}
<commit_msg>reverted key names<commit_after>package gojot

import (
	"os"
	"os/exec"
	"sort"
	"strings"
	"time"
)

func ListFiles(gitfolder string) []string {
	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("ListFiles Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", "master")
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := []string{}
	for _, document := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
		if len(document) < 1 {
			continue
		}
		if document[0] == '.' {
			document = DecryptOTP(document[1:])
			logger.Debug("Found document: %s", document)
			if document == ".deleted" || document == "jot-key" || document == "jot-hs" {
				continue
			}
			documents = append(documents, document)
		}
	}

	sort.Strings(documents)
	return documents
}

func ListFilesOfOne(gitfolder string, branch string) []string {
	defer timeTrack(time.Now(), "Listing files")
	cwd, _ := os.Getwd()
	defer os.Chdir(cwd)
	err := os.Chdir(gitfolder)
	if err != nil {
		logger.Error("Cannot chdir into " + gitfolder)
	}

	cmd := exec.Command("git", "ls-tree", "--name-only", branch)
	stdout, err := cmd.Output()
	if err != nil {
		logger.Error("Problem doing ls-tree")
	}
	documents := []string{}
	for _, document := range strings.Split(strings.TrimSpace(string(stdout)), "\n") {
		if document[0] == '.' {
			continue
		}
		documents = append(documents, document)
	}

	return documents
}
<|endoftext|>