<commit_before>package incremental

import (
	"context"
	"testing"
	"time"

	assert "github.com/stretchr/testify/require"
	"go.skia.org/infra/go/deepequal"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/git/repograph"
	git_testutils "go.skia.org/infra/go/git/testutils"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/task_scheduler/go/window"
)

func assertBranches(t *testing.T, gb *git_testutils.GitBuilder, actual map[string][]*gitinfo.GitBranch, expect map[string]string) {
	actualBranches := make(map[string]string, len(actual[gb.RepoUrl()]))
	for _, branch := range actual[gb.RepoUrl()] {
		actualBranches[branch.Name] = branch.Head
	}
	deepequal.AssertDeepEqual(t, expect, actualBranches)
}

func assertCommits(t *testing.T, gb *git_testutils.GitBuilder, actual map[string][]*vcsinfo.LongCommit, expect []string) {
	actualMap := util.NewStringSet(nil)
	for _, c := range actual[gb.RepoUrl()] {
		actualMap[c.Hash] = true
	}
	expectMap := util.NewStringSet(expect)
	deepequal.AssertDeepEqual(t, expectMap, actualMap)
}

func TestIncrementalCommits(t *testing.T) {
	testutils.MediumTest(t)

	// Setup.
	ctx := context.Background()
	gb := git_testutils.GitInit(t, ctx)
	defer gb.Cleanup()
	c0 := gb.CommitGen(ctx, "file1")
	wd, cleanupWd := testutils.TempDir(t)
	defer cleanupWd()
	repo, err := repograph.NewGraph(ctx, gb.Dir(), wd)
	assert.NoError(t, err)
	repos := repograph.Map{
		gb.RepoUrl(): repo,
	}
	N := 100
	w, err := window.New(24*time.Hour, N, repos)
	assert.NoError(t, err)
	cc := newCommitsCache(repos)

	// Initial update. Expect a single branch with one commit.
	branches, commits, err := cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c0,
	})
	assertCommits(t, gb, commits, []string{c0})

	// Update again, with no new commits. Expect empty response.
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{})
	assertCommits(t, gb, commits, []string{})

	// Passing in reset=true should give us ALL commits and branches,
	// regardless of whether they're new.
	branches, commits, err = cc.Update(ctx, w, true, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c0,
	})
	assertCommits(t, gb, commits, []string{c0})

	// Add some new commits.
	c1 := gb.CommitGen(ctx, "file1")
	c2 := gb.CommitGen(ctx, "file1")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c2,
	})
	assertCommits(t, gb, commits, []string{c1, c2})

	// Add a new branch, with no commits.
	gb.CreateBranchTrackBranch(ctx, "branch2", "origin/master")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c2,
		"branch2": c2,
	})
	assertCommits(t, gb, commits, []string{})

	// Add a commit on the new branch.
	c3 := gb.CommitGen(ctx, "file2")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c2,
		"branch2": c3,
	})
	assertCommits(t, gb, commits, []string{c3})

	// Merge branch2 back into master. Note that, since there are no new
	// commits on master, this does not create a merge commit but just
	// updates HEAD of master to point at c3.
	gb.CheckoutBranch(ctx, "master")
	mergeCommit := gb.MergeBranch(ctx, "branch2")
	assert.Equal(t, c3, mergeCommit)
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c3,
		"branch2": c3,
	})
	assertCommits(t, gb, commits, []string{})

	// Add a new branch. Add commits on both master and branch3.
	gb.CreateBranchTrackBranch(ctx, "branch3", "origin/master")
	c4 := gb.CommitGen(ctx, "file3")
	gb.CheckoutBranch(ctx, "master")
	c5 := gb.CommitGen(ctx, "file1")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c5,
		"branch2": c3,
		"branch3": c4,
	})
	assertCommits(t, gb, commits, []string{c4, c5})

	// Merge branch3 back into master. Because there are commits on both
	// branches, a merge commit will be created.
	c6 := gb.MergeBranch(ctx, "branch3")
	assert.NotEqual(t, c6, c4) // Ensure that we actually created a merge commit.
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c6,
		"branch2": c3,
		"branch3": c4,
	})
	assertCommits(t, gb, commits, []string{c6})
}
<commit_msg>Resize TestIncrementalCommits<commit_after>package incremental

import (
	"context"
	"testing"
	"time"

	assert "github.com/stretchr/testify/require"
	"go.skia.org/infra/go/deepequal"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/git/repograph"
	git_testutils "go.skia.org/infra/go/git/testutils"
	"go.skia.org/infra/go/testutils"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/go/vcsinfo"
	"go.skia.org/infra/task_scheduler/go/window"
)

func assertBranches(t *testing.T, gb *git_testutils.GitBuilder, actual map[string][]*gitinfo.GitBranch, expect map[string]string) {
	actualBranches := make(map[string]string, len(actual[gb.RepoUrl()]))
	for _, branch := range actual[gb.RepoUrl()] {
		actualBranches[branch.Name] = branch.Head
	}
	deepequal.AssertDeepEqual(t, expect, actualBranches)
}

func assertCommits(t *testing.T, gb *git_testutils.GitBuilder, actual map[string][]*vcsinfo.LongCommit, expect []string) {
	actualMap := util.NewStringSet(nil)
	for _, c := range actual[gb.RepoUrl()] {
		actualMap[c.Hash] = true
	}
	expectMap := util.NewStringSet(expect)
	deepequal.AssertDeepEqual(t, expectMap, actualMap)
}

func TestIncrementalCommits(t *testing.T) {
	testutils.LargeTest(t)

	// Setup.
	ctx := context.Background()
	gb := git_testutils.GitInit(t, ctx)
	defer gb.Cleanup()
	c0 := gb.CommitGen(ctx, "file1")
	wd, cleanupWd := testutils.TempDir(t)
	defer cleanupWd()
	repo, err := repograph.NewGraph(ctx, gb.Dir(), wd)
	assert.NoError(t, err)
	repos := repograph.Map{
		gb.RepoUrl(): repo,
	}
	N := 100
	w, err := window.New(24*time.Hour, N, repos)
	assert.NoError(t, err)
	cc := newCommitsCache(repos)

	// Initial update. Expect a single branch with one commit.
	branches, commits, err := cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c0,
	})
	assertCommits(t, gb, commits, []string{c0})

	// Update again, with no new commits. Expect empty response.
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{})
	assertCommits(t, gb, commits, []string{})

	// Passing in reset=true should give us ALL commits and branches,
	// regardless of whether they're new.
	branches, commits, err = cc.Update(ctx, w, true, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c0,
	})
	assertCommits(t, gb, commits, []string{c0})

	// Add some new commits.
	c1 := gb.CommitGen(ctx, "file1")
	c2 := gb.CommitGen(ctx, "file1")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master": c2,
	})
	assertCommits(t, gb, commits, []string{c1, c2})

	// Add a new branch, with no commits.
	gb.CreateBranchTrackBranch(ctx, "branch2", "origin/master")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c2,
		"branch2": c2,
	})
	assertCommits(t, gb, commits, []string{})

	// Add a commit on the new branch.
	c3 := gb.CommitGen(ctx, "file2")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c2,
		"branch2": c3,
	})
	assertCommits(t, gb, commits, []string{c3})

	// Merge branch2 back into master. Note that, since there are no new
	// commits on master, this does not create a merge commit but just
	// updates HEAD of master to point at c3.
	gb.CheckoutBranch(ctx, "master")
	mergeCommit := gb.MergeBranch(ctx, "branch2")
	assert.Equal(t, c3, mergeCommit)
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c3,
		"branch2": c3,
	})
	assertCommits(t, gb, commits, []string{})

	// Add a new branch. Add commits on both master and branch3.
	gb.CreateBranchTrackBranch(ctx, "branch3", "origin/master")
	c4 := gb.CommitGen(ctx, "file3")
	gb.CheckoutBranch(ctx, "master")
	c5 := gb.CommitGen(ctx, "file1")
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c5,
		"branch2": c3,
		"branch3": c4,
	})
	assertCommits(t, gb, commits, []string{c4, c5})

	// Merge branch3 back into master. Because there are commits on both
	// branches, a merge commit will be created.
	c6 := gb.MergeBranch(ctx, "branch3")
	assert.NotEqual(t, c6, c4) // Ensure that we actually created a merge commit.
	branches, commits, err = cc.Update(ctx, w, false, N)
	assert.NoError(t, err)
	assertBranches(t, gb, branches, map[string]string{
		"master":  c6,
		"branch2": c3,
		"branch3": c4,
	})
	assertCommits(t, gb, commits, []string{c6})
}
<|endoftext|>


<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.UUID,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "Requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.UUID,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {
		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.UUID))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link, localPath)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			SendToTroubleQueue(result, copier.ProcUtil)
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
}
<commit_msg>Temporarily turned off ssh for rsync for Rails testing<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
)

// copier.go copies tarred bags from other nodes via rsync.
// This is used when replicating content from other nodes.
// For putting together DPN bags from APTrust files, see fetcher.go.

type Copier struct {
	LookupChannel       chan *DPNResult
	CopyChannel         chan *DPNResult
	PostProcessChannel  chan *DPNResult
	DPNConfig           *DPNConfig
	ProcUtil            *bagman.ProcessUtil
	LocalClient         *DPNRestClient
	RemoteClients       map[string]*DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

type CopyResult struct {
	LocalPath       string
	ErrorMessage    string
	RsyncStdout     string
	RsyncStderr     string
	InfoMessage     string
	BagWasCopied    bool
}

func NewCopier(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Copier, error) {
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := GetRemoteClients(localClient, dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}
	copier := &Copier {
		DPNConfig: dpnConfig,
		ProcUtil: procUtil,
		LocalClient: localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	copier.LookupChannel = make(chan *DPNResult, workerBufferSize)
	copier.CopyChannel = make(chan *DPNResult, workerBufferSize)
	copier.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go copier.doLookup()
		go copier.doCopy()
		go copier.postProcess()
	}
	return copier, nil
}

func (copier *Copier) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	dpnResult := &DPNResult{}
	err := json.Unmarshal(message.Body, dpnResult)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq. " +
			"Error is: %v    JSON is: %s", err.Error(), string(message.Body))
		copier.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Set up the copy result
	dpnResult.CopyResult = &CopyResult{
		BagWasCopied: false,
	}
	dpnResult.NsqMessage = message

	// Start processing.
	dpnResult.Stage = STAGE_COPY
	copier.LookupChannel <- dpnResult
	copier.ProcUtil.MessageLog.Info("Put %s from %s into copy channel",
		dpnResult.DPNBag.UUID, dpnResult.DPNBag.AdminNode)
	return nil
}

// Look up the DPN bag on the admin node. Although we already have the
// bag object as bart of the DPNResult object, this request may have been
// sitting in the queue for many hours, and the replication request may
// have been fulfilled or cancelled in that time. So check the status on
// the authoritative node to avoid unnecessarily processing what might
// be hundreds of gigs of data.
func (copier *Copier) doLookup() {
	for result := range copier.LookupChannel {
		// Get a client to talk to the FromNode
		remoteClient := copier.RemoteClients[result.TransferRequest.FromNode]

		copier.ProcUtil.MessageLog.Debug(
			"Looking up ReplicationId %s, bag %s, on node %s ",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.UUID,
				result.TransferRequest.FromNode)

		// If we can find out for sure that this replication request should
		// not be processed, then don't process it...
		xfer, _ := remoteClient.ReplicationTransferGet(
			result.TransferRequest.ReplicationId)
		if xfer != nil && xfer.Status != "Requested" {
			message := fmt.Sprintf(
				"Cancelling copy of ReplicationId %s (bag %s) because " +
					"replication status on %s is %s",
				result.TransferRequest.ReplicationId,
				result.TransferRequest.UUID,
				result.TransferRequest.FromNode,
				xfer.Status)
			copier.ProcUtil.MessageLog.Info(message)
			result.CopyResult.InfoMessage = message
			result.Retry = false
			result.TransferRequest = xfer
			copier.PostProcessChannel <- result
			continue
		}
		// ...otherwise, proceed with processing.
		copier.CopyChannel <- result
	}
}

// Copy the file from the remote node to our local staging area.
// Calculate checksums.
func (copier *Copier) doCopy() {
	for result := range copier.CopyChannel {
		localPath := filepath.Join(
			copier.ProcUtil.Config.DPNStagingDirectory,
			fmt.Sprintf("%s.tar", result.TransferRequest.UUID))

		if !bagman.FileExists(copier.ProcUtil.Config.DPNStagingDirectory) {
			os.MkdirAll(copier.ProcUtil.Config.DPNStagingDirectory, 0755)
		}

		rsyncCommand := GetRsyncCommand(result.TransferRequest.Link, localPath)

		// Touch message on both sides of rsync, so NSQ doesn't time out.
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		output, err := rsyncCommand.CombinedOutput()
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		if err != nil {
			result.CopyResult.ErrorMessage = fmt.Sprintf("%s: %s",
				err.Error(), string(output))
		} else {
			result.LocalPath = localPath
			result.CopyResult.LocalPath = localPath
			result.CopyResult.BagWasCopied = true
			fileDigest, err := bagman.CalculateDigests(localPath)
			if result.NsqMessage != nil {
				result.NsqMessage.Touch()
			}
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Could not calculate checksums on '%s': %v",
					result.PackageResult.TarFilePath, err)
				copier.PostProcessChannel <- result
				continue
			}
			result.BagMd5Digest = fileDigest.Md5Digest
			result.BagSha256Digest = fileDigest.Sha256Digest
			result.BagSize = fileDigest.Size
		}
		copier.PostProcessChannel <- result
	}
}

func (copier *Copier) postProcess() {
	// On success, send to validation queue.
	// Otherwise, send to trouble queue.
	for result := range copier.PostProcessChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}
		result.ErrorMessage = result.CopyResult.ErrorMessage

		// On error, log and send to trouble queue
		if result.ErrorMessage != "" {
			copier.ProcUtil.MessageLog.Error(result.ErrorMessage)
			copier.ProcUtil.IncrementFailed()
			SendToTroubleQueue(result, copier.ProcUtil)
			if bagman.FileExists(result.CopyResult.LocalPath) {
				os.Remove(result.CopyResult.LocalPath)
				copier.ProcUtil.MessageLog.Debug(
					"Deleting bag file %s", result.CopyResult.LocalPath)
			}
		} else if result.CopyResult.BagWasCopied == false {
			// We didn't copy the bag, but there was no error.
			// This happens when the transfer request is marked
			// as completed or cancelled on the remote node.
			// Count this as success, because we did what we're
			// supposed to do in this case, which is nothing.
			copier.ProcUtil.IncrementSucceeded()
		} else {
			// We successfully copied the bag. Send it on to
			// the validation queue.
			copier.ProcUtil.IncrementSucceeded()
			SendToValidationQueue(result, copier.ProcUtil)
		}

		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			copier.WaitGroup.Done()
		} else {
			result.NsqMessage.Finish()
		}
		copier.ProcUtil.LogStats()

	}
}

func (copier *Copier) RunTest(dpnResult *DPNResult) {
	copier.WaitGroup.Add(1)
	copier.ProcUtil.MessageLog.Info("Putting %s into lookup channel",
		dpnResult.BagIdentifier)
	copier.CopyChannel <- dpnResult
	copier.WaitGroup.Wait()
}

// Returns a command object for copying from the remote location to
// the local filesystem. The copy is done via rsync over ssh, and
// the command will capture stdout and stderr. The copyFrom param
// should be a valid scp target in this format:
//
// remoteuser@remotehost:/remote/dir/bag.tar
//
// The copyTo param should be an absolute path on a locally-accessible
// file system, such as:
//
// /mnt/dpn/data/bag.tar
//
// Using this assumes a few things:
//
// 1. You have rsync installed.
// 2. You have an ssh client installed.
// 3. You have an entry in your ~/.ssh/config file specifying
//    connection and key information for the remote host.
//
// Usage:
//
// command := GetRsyncCommand("aptrust@tdr:bag.tar", "/mnt/dpn/bag.tar")
// err := command.Run()
// if err != nil {
//    ... do something ...
// }
//
// -- OR --
//
// output, err := command.CombinedOutput()
// if err != nil {
//    fmt.Println(err.Error())
//    fmt.Println(string(output))
// }

//
func GetRsyncCommand(copyFrom, copyTo string) (*exec.Cmd) {
//	rsync -avz -e ssh remoteuser@remotehost:/remote/dir /this/dir/
	// return exec.Command("rsync", "-avz", "-e",  "ssh", copyFrom, copyTo)
	// TODO: Undo this temp fix. Use proper config to specify whether to use SSH!
	return exec.Command("rsync", "-avz", copyFrom, copyTo)
}
<|endoftext|>


<commit_before>// Package note is the glue between the prosemirror models, the VFS, redis, the
// hub for realtime, etc.
package note

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/prosemirror-go/model"
	"github.com/cozy/prosemirror-go/transform"
)

// Document is the note document in memory. It is persisted to the VFS as a
// file, but with a debounce: the intermediate states are saved in Redis.
type Document struct {
	DocID    string          `json:"_id"`
	DocRev   string          `json:"_rev,omitempty"`
	Title    string          `json:"title"`
	DirID    string          `json:"dir_id,omitempty"`
	Revision int             `json:"revision"`
	Schema   json.RawMessage `json:"schema"`
	Content  interface{}     `json:"content,omitempty"`
}

// ID returns the directory qualified identifier
func (d *Document) ID() string { return d.DocID }

// Rev returns the directory revision
func (d *Document) Rev() string { return d.DocRev }

// DocType returns the document type
func (d *Document) DocType() string { return consts.NotesDocuments }

// Clone implements couchdb.Doc
func (d *Document) Clone() couchdb.Doc {
	cloned := *d
	// XXX The schema and the content are supposed to be immutable and, as
	// such, are not cloned.
	return &cloned
}

// SetID changes the directory qualified identifier
func (d *Document) SetID(id string) { d.DocID = id }

// SetRev changes the directory revision
func (d *Document) SetRev(rev string) { d.DocRev = rev }

// Create the file in the VFS for this note.
func (d *Document) Create(inst *instance.Instance) (*vfs.FileDoc, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	d.Revision = 0
	content, err := d.getInitialContent(inst)
	if err != nil {
		return nil, err
	}
	d.Content = content.ToJSON()

	// TODO markdown
	markdown := []byte(content.String())
	fileDoc, err := d.newFileDoc(inst, markdown)
	if err != nil {
		return nil, err
	}
	if err := writeFile(inst.VFS(), fileDoc, nil, markdown); err != nil {
		return nil, err
	}
	return fileDoc, nil
}

func (d *Document) getInitialContent(inst *instance.Instance) (*model.Node, error) {
	var spec model.SchemaSpec
	if err := json.Unmarshal(d.Schema, &spec); err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot read the schema: %s", err)
		return nil, ErrInvalidSchema
	}

	schema, err := model.NewSchema(&spec)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return nil, ErrInvalidSchema
	}

	// Create an empty document that matches the schema constraints.
	typ, err := schema.NodeType(schema.Spec.TopNode)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("The schema is invalid: %s", err)
		return nil, ErrInvalidSchema
	}
	node, err := typ.CreateAndFill()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("The topNode cannot be created: %s", err)
		return nil, ErrInvalidSchema
	}
	return node, nil
}

func (d *Document) getDirID(inst *instance.Instance) (string, error) {
	if d.DirID != "" {
		return d.DirID, nil
	}
	parent, err := ensureNotesDir(inst)
	if err != nil {
		return "", err
	}
	return parent.ID(), nil
}

func titleToFilename(title string) string {
	if title == "" {
		title = "New note"
	}
	name := strings.ReplaceAll(title, "/", "-")
	return name + ".cozy-note"
}

func (d *Document) newFileDoc(inst *instance.Instance, content []byte) (*vfs.FileDoc, error) {
	dirID, err := d.getDirID(inst)
	if err != nil {
		return nil, err
	}

	fileDoc, err := vfs.NewFileDoc(
		titleToFilename(d.Title),
		dirID,
		int64(len(content)),
		nil, // Let the VFS compute the md5sum
		"text/markdown",
		"text",
		time.Now(),
		false, // Not executable
		false, // Not trashed
		nil,   // No tags
	)
	if err != nil {
		return nil, err
	}

	fileDoc.Metadata = d.metadata()
	fileDoc.CozyMetadata = vfs.NewCozyMetadata(inst.PageURL("/", nil))
	return fileDoc, nil
}

func (d *Document) metadata() map[string]interface{} {
	return map[string]interface{}{
		"title":    d.Title,
		"content":  d.Content,
		"revision": d.Revision,
		"schema":   d.Schema,
	}
}

// TODO retry if another file with the same name already exists
func writeFile(fs vfs.VFS, fileDoc, oldDoc *vfs.FileDoc, content []byte) (err error) {
	var file vfs.File
	file, err = fs.CreateFile(fileDoc, oldDoc)
	if err != nil {
		return
	}
	defer func() {
		if cerr := file.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()
	_, err = file.Write(content)
	return
}

func ensureNotesDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	ref := couchdb.DocReference{
		Type: consts.Apps,
		ID:   consts.Apps + "/" + consts.NotesSlug,
	}
	key := []string{ref.Type, ref.ID}
	end := []string{ref.Type, ref.ID, couchdb.MaxString}
	req := &couchdb.ViewRequest{
		StartKey:    key,
		EndKey:      end,
		IncludeDocs: true,
	}
	var res couchdb.ViewResponse
	err := couchdb.ExecView(inst, couchdb.FilesReferencedByView, req, &res)
	if err != nil {
		return nil, err
	}

	fs := inst.VFS()
	if len(res.Rows) > 0 {
		return fs.DirByID(res.Rows[0].ID)
	}
	dirname := inst.Translate("Tree Notes")
	dir, err := vfs.NewDirDocWithPath(dirname, consts.RootDirID, "/", nil)
	if err != nil {
		return nil, err
	}
	dir.AddReferencedBy(ref)
	dir.CozyMetadata = vfs.NewCozyMetadata(inst.PageURL("/", nil))
	if err = fs.CreateDir(dir); err != nil {
		if !couchdb.IsConflictError(err) {
			return nil, err
		}
		dir, err = fs.DirByPath(dir.Fullpath)
		if err != nil {
			return nil, err
		}
		olddoc := dir.Clone().(*vfs.DirDoc)
		dir.AddReferencedBy(ref)
		_ = fs.UpdateDirDoc(olddoc, dir)
	}
	return dir, nil
}

// UpdateTitle changes the title of a note and renames the associated file.
// TODO add debounce
func UpdateTitle(inst *instance.Instance, file *vfs.FileDoc, title string) error {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return err
	}
	defer lock.Unlock()

	if len(file.Metadata) == 0 {
		return ErrInvalidFile
	}
	old, _ := file.Metadata["title"].(string)
	if old == title {
		return nil
	}

	olddoc := file.Clone().(*vfs.FileDoc)
	file.Metadata["title"] = title
	file.UpdatedAt = time.Now()
	file.CozyMetadata.UpdatedAt = file.UpdatedAt

	// If the file was renamed manually before, we will keep its name. Else, we
	// can rename with the new title.
	if rename := titleToFilename(old) == file.DocName; rename {
		file.DocName = titleToFilename(title)
		file.ResetFullpath()
	}

	return inst.VFS().UpdateFileDoc(olddoc, file)
}

// ApplySteps takes a note and some steps, and tries to apply them. It is an
// all or nothing change: if there is one error, the note won't be changed.
// TODO fetch last info for file (if debounce)
func ApplySteps(inst *instance.Instance, file *vfs.FileDoc, steps []couchdb.JSONDoc) error {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return err
	}
	defer lock.Unlock()

	if len(steps) == 0 {
		return ErrNoSteps
	}

	oldContent, ok := file.Metadata["content"].(map[string]interface{})
	if !ok {
		return ErrInvalidFile
	}
	schemaSpec, ok := file.Metadata["schema"].(map[string]interface{})
	if !ok {
		return ErrInvalidSchema
	}

	spec := model.SchemaSpecFromJSON(schemaSpec)
	schema, err := model.NewSchema(&spec)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return ErrInvalidSchema
	}

	doc, err := model.NodeFromJSON(schema, oldContent)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the document: %s", err)
		return ErrInvalidFile
	}

	for _, s := range steps {
		step, err := transform.StepFromJSON(schema, s.M)
		if err != nil {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot instantiate a step: %s", err)
			return ErrInvalidSteps
		}
		result := step.Apply(doc)
		if result.Failed != "" {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot apply a step: %s", err)
			return ErrCannotApply
		}
		doc = result.Doc
	}

	olddoc := file.Clone().(*vfs.FileDoc)
	file.Metadata["content"] = doc.ToJSON()
	// TODO markdown
	markdown := []byte(doc.String())

	// TODO add debounce
	return writeFile(inst.VFS(), file, olddoc, markdown)
}

var _ couchdb.Doc = &Document{}
<commit_msg>Increment the revision when a note is updated<commit_after>// Package note is the glue between the prosemirror models, the VFS, redis, the
// hub for realtime, etc.
package note

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/model/instance"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/prosemirror-go/model"
	"github.com/cozy/prosemirror-go/transform"
)

// Document is the note document in memory. It is persisted to the VFS as a
// file, but with a debounce: the intermediate states are saved in Redis.
type Document struct {
	DocID    string          `json:"_id"`
	DocRev   string          `json:"_rev,omitempty"`
	Title    string          `json:"title"`
	DirID    string          `json:"dir_id,omitempty"`
	Revision int             `json:"revision"`
	Schema   json.RawMessage `json:"schema"`
	Content  interface{}     `json:"content,omitempty"`
}

// ID returns the directory qualified identifier
func (d *Document) ID() string { return d.DocID }

// Rev returns the directory revision
func (d *Document) Rev() string { return d.DocRev }

// DocType returns the document type
func (d *Document) DocType() string { return consts.NotesDocuments }

// Clone implements couchdb.Doc
func (d *Document) Clone() couchdb.Doc {
	cloned := *d
	// XXX The schema and the content are supposed to be immutable and, as
	// such, are not cloned.
	return &cloned
}

// SetID changes the directory qualified identifier
func (d *Document) SetID(id string) { d.DocID = id }

// SetRev changes the directory revision
func (d *Document) SetRev(rev string) { d.DocRev = rev }

// Create the file in the VFS for this note.
func (d *Document) Create(inst *instance.Instance) (*vfs.FileDoc, error) {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return nil, err
	}
	defer lock.Unlock()

	d.Revision = 0
	content, err := d.getInitialContent(inst)
	if err != nil {
		return nil, err
	}
	d.Content = content.ToJSON()

	// TODO markdown
	markdown := []byte(content.String())
	fileDoc, err := d.newFileDoc(inst, markdown)
	if err != nil {
		return nil, err
	}
	if err := writeFile(inst.VFS(), fileDoc, nil, markdown); err != nil {
		return nil, err
	}
	return fileDoc, nil
}

func (d *Document) getInitialContent(inst *instance.Instance) (*model.Node, error) {
	var spec model.SchemaSpec
	if err := json.Unmarshal(d.Schema, &spec); err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot read the schema: %s", err)
		return nil, ErrInvalidSchema
	}

	schema, err := model.NewSchema(&spec)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return nil, ErrInvalidSchema
	}

	// Create an empty document that matches the schema constraints.
	typ, err := schema.NodeType(schema.Spec.TopNode)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("The schema is invalid: %s", err)
		return nil, ErrInvalidSchema
	}
	node, err := typ.CreateAndFill()
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("The topNode cannot be created: %s", err)
		return nil, ErrInvalidSchema
	}
	return node, nil
}

func (d *Document) getDirID(inst *instance.Instance) (string, error) {
	if d.DirID != "" {
		return d.DirID, nil
	}
	parent, err := ensureNotesDir(inst)
	if err != nil {
		return "", err
	}
	return parent.ID(), nil
}

func titleToFilename(title string) string {
	if title == "" {
		title = "New note"
	}
	name := strings.ReplaceAll(title, "/", "-")
	return name + ".cozy-note"
}

func (d *Document) newFileDoc(inst *instance.Instance, content []byte) (*vfs.FileDoc, error) {
	dirID, err := d.getDirID(inst)
	if err != nil {
		return nil, err
	}

	fileDoc, err := vfs.NewFileDoc(
		titleToFilename(d.Title),
		dirID,
		int64(len(content)),
		nil, // Let the VFS compute the md5sum
		"text/markdown",
		"text",
		time.Now(),
		false, // Not executable
		false, // Not trashed
		nil,   // No tags
	)
	if err != nil {
		return nil, err
	}

	fileDoc.Metadata = d.metadata()
	fileDoc.CozyMetadata = vfs.NewCozyMetadata(inst.PageURL("/", nil))
	return fileDoc, nil
}

func (d *Document) metadata() map[string]interface{} {
	return map[string]interface{}{
		"title":    d.Title,
		"content":  d.Content,
		"revision": d.Revision,
		"schema":   d.Schema,
	}
}

// TODO retry if another file with the same name already exists
func writeFile(fs vfs.VFS, fileDoc, oldDoc *vfs.FileDoc, content []byte) (err error) {
	var file vfs.File
	file, err = fs.CreateFile(fileDoc, oldDoc)
	if err != nil {
		return
	}
	defer func() {
		if cerr := file.Close(); cerr != nil && err == nil {
			err = cerr
		}
	}()
	_, err = file.Write(content)
	return
}

func ensureNotesDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	ref := couchdb.DocReference{
		Type: consts.Apps,
		ID:   consts.Apps + "/" + consts.NotesSlug,
	}
	key := []string{ref.Type, ref.ID}
	end := []string{ref.Type, ref.ID, couchdb.MaxString}
	req := &couchdb.ViewRequest{
		StartKey:    key,
		EndKey:      end,
		IncludeDocs: true,
	}
	var res couchdb.ViewResponse
	err := couchdb.ExecView(inst, couchdb.FilesReferencedByView, req, &res)
	if err != nil {
		return nil, err
	}

	fs := inst.VFS()
	if len(res.Rows) > 0 {
		return fs.DirByID(res.Rows[0].ID)
	}
	dirname := inst.Translate("Tree Notes")
	dir, err := vfs.NewDirDocWithPath(dirname, consts.RootDirID, "/", nil)
	if err != nil {
		return nil, err
	}
	dir.AddReferencedBy(ref)
	dir.CozyMetadata = vfs.NewCozyMetadata(inst.PageURL("/", nil))
	if err = fs.CreateDir(dir); err != nil {
		if !couchdb.IsConflictError(err) {
			return nil, err
		}
		dir, err = fs.DirByPath(dir.Fullpath)
		if err != nil {
			return nil, err
		}
		olddoc := dir.Clone().(*vfs.DirDoc)
		dir.AddReferencedBy(ref)
		_ = fs.UpdateDirDoc(olddoc, dir)
	}
	return dir, nil
}

// UpdateTitle changes the title of a note and renames the associated file.
// TODO add debounce
func UpdateTitle(inst *instance.Instance, file *vfs.FileDoc, title string) error {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return err
	}
	defer lock.Unlock()

	if len(file.Metadata) == 0 {
		return ErrInvalidFile
	}
	old, _ := file.Metadata["title"].(string)
	if old == title {
		return nil
	}

	olddoc := file.Clone().(*vfs.FileDoc)
	file.Metadata["title"] = title
	file.UpdatedAt = time.Now()
	file.CozyMetadata.UpdatedAt = file.UpdatedAt

	// If the file was renamed manually before, we will keep its name. Else, we
	// can rename with the new title.
	if rename := titleToFilename(old) == file.DocName; rename {
		file.DocName = titleToFilename(title)
		file.ResetFullpath()
	}

	return inst.VFS().UpdateFileDoc(olddoc, file)
}

// ApplySteps takes a note and some steps, and tries to apply them. It is an
// all or nothing change: if there is one error, the note won't be changed.
// TODO fetch last info for file (if debounce)
func ApplySteps(inst *instance.Instance, file *vfs.FileDoc, steps []couchdb.JSONDoc) error {
	lock := inst.NotesLock()
	if err := lock.Lock(); err != nil {
		return err
	}
	defer lock.Unlock()

	if len(steps) == 0 {
		return ErrNoSteps
	}

	oldContent, ok := file.Metadata["content"].(map[string]interface{})
	if !ok {
		return ErrInvalidFile
	}
	revision, ok := file.Metadata["revision"].(float64)
	if !ok {
		return ErrInvalidFile
	}
	rev := int(revision)
	schemaSpec, ok := file.Metadata["schema"].(map[string]interface{})
	if !ok {
		return ErrInvalidSchema
	}

	spec := model.SchemaSpecFromJSON(schemaSpec)
	schema, err := model.NewSchema(&spec)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the schema: %s", err)
		return ErrInvalidSchema
	}

	doc, err := model.NodeFromJSON(schema, oldContent)
	if err != nil {
		inst.Logger().WithField("nspace", "notes").
			Infof("Cannot instantiate the document: %s", err)
		return ErrInvalidFile
	}

	for _, s := range steps {
		rev++
		step, err := transform.StepFromJSON(schema, s.M)
		if err != nil {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot instantiate a step: %s", err)
			return ErrInvalidSteps
		}
		result := step.Apply(doc)
		if result.Failed != "" {
			inst.Logger().WithField("nspace", "notes").
				Infof("Cannot apply a step: %s", err)
			return ErrCannotApply
		}
		doc = result.Doc
	}

	// TODO persist the steps

	olddoc := file.Clone().(*vfs.FileDoc)
	file.Metadata["content"] = doc.ToJSON()
	file.Metadata["revision"] = rev
	// TODO markdown
	markdown := []byte(doc.String())

	// TODO add debounce
	return writeFile(inst.VFS(), file, olddoc, markdown)
}

var _ couchdb.Doc = &Document{}
<|endoftext|>


<commit_before>// +build windows

package uilive

import (
	"math"
	"syscall"
	"unsafe"
)

type consoleFontInfo struct {
	font     uint32
	fontSize coord
}

const (
	SmCxMin = 28
	SmCyMin = 29
)

var (
	tmpConsoleFontInfo        consoleFontInfo
	moduleUser32              = syscall.NewLazyDLL("user32.dll")
	procGetCurrentConsoleFont = kernel32.NewProc("GetCurrentConsoleFont")
	getSystemMetrics          = moduleUser32.NewProc("GetSystemMetrics")
)

func getCurrentConsoleFont(h syscall.Handle, info *consoleFontInfo) (err error) {
	r0, _, e1 := syscall.Syscall(
		procGetCurrentConsoleFont.Addr(), 3, uintptr(h), 0, uintptr(unsafe.Pointer(info)),
	)
	if int(r0) == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}

func getTermSize() (int, int) {
	out, err := syscall.Open("CONOUT$", syscall.O_RDWR, 0)
	if err != nil {
		return 0, 0
	}

	x, _, err := getSystemMetrics.Call(SmCxMin)
	y, _, err := getSystemMetrics.Call(SmCyMin)

	if x == 0 || y == 0 {
		if err != nil {
			panic(err)
		}
	}

	err = getCurrentConsoleFont(out, &tmpConsoleFontInfo)
	if err != nil {
		panic(err)
	}

	return int(math.Ceil(float64(x) / float64(tmpConsoleFontInfo.fontSize.x))), int(math.Ceil(float64(y) / float64(tmpConsoleFontInfo.fontSize.y)))
}
<commit_msg>Fix getTermSize() for Windows<commit_after>// +build windows

package uilive

import (
	"os"
	"unsafe"
)

func getTermSize() (int, int) {
	out, err := os.Open("CONOUT$")
	if err != nil {
		return 0, 0
	}

	var csbi consoleScreenBufferInfo
	ret, _, _ := procGetConsoleScreenBufferInfo.Call(out.Fd(), uintptr(unsafe.Pointer(&csbi)))
	if ret == 0 {
		return 0, 0
	}

	return int(csbi.window.right - csbi.window.left + 1), int(csbi.window.bottom - csbi.window.top + 1)
}
<|endoftext|>


<commit_before>package speed

// Registry defines a valid set of instance domains and metrics
type Registry interface {
	AddInstanceDomain(InstanceDomain) error                                                     // adds a InstanceDomain object to the writer
	AddInstanceDomainByName(name string, instances []string) (InstanceDomain, error)            // adds a InstanceDomain object after constructing it using passed name and instances
	AddMetric(Metric) error                                                                     // adds a Metric object to the writer
	AddMetricByName(name string, s MetricSemantics, t MetricType, u MetricUnit) (Metric, error) // adds a Metric object after parsing the name for Instances and InstanceDomains
	UpdateMetricByName(name string, val interface{}) error                                      // updates a Metric object by looking it up by name and updating its value
}
<commit_msg>registry: modify the Registry interface<commit_after>package speed

// Registry defines a valid set of instance domains and metrics
type Registry interface {
	HasInstanceDomain(name string) bool                                                           // checks if an instance domain of the passed name is already present or not
	HasMetric(name string) bool                                                                   // checks if an metric of the passed name is already present or not
	AddInstanceDomain(InstanceDomain) error                                                       // adds a InstanceDomain object to the writer
	AddInstanceDomainByName(name string, instances []string) (InstanceDomain, error)              // adds a InstanceDomain object after constructing it using passed name and instances
	AddMetric(Metric) error                                                                       // adds a Metric object to the writer
	AddMetricByString(name string, s MetricSemantics, t MetricType, u MetricUnit) (Metric, error) // adds a Metric object after parsing the passed string for Instances and InstanceDomains
	UpdateMetricByName(name string, val interface{}) error                                        // updates a Metric object by looking it up by name and updating its value
}
<|endoftext|>


<commit_before>package leaderboard

import (
	"testing"
)

func initLeaderboard() *Leaderboard {
	l := New()
	l.board = []*Player{
		{Username: "0", Planets: 10, Team: Color{13, 11, 92}},
		{Username: "1", Planets: 8, Team: Color{16, 5, 90}},
		{Username: "2", Planets: 7, Team: Color{16, 5, 90}},
		{Username: "3", Planets: 6, Team: Color{13, 11, 92}},
		{Username: "4", Planets: 5, Team: Color{13, 11, 92}},
		{Username: "5", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "6", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "7", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "8", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "9", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "10", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "11", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "12", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "13", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "14", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "15", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "16", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "17", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "18", Planets: 0, Team: Color{20, 6, 90}},
	}

	l.places = map[string]int{
		"0":  0,
		"1":  1,
		"2":  2,
		"3":  3,
		"4":  4,
		"5":  5,
		"6":  6,
		"7":  7,
		"8":  8,
		"9":  9,
		"10": 10,
		"11": 11,
		"12": 12,
		"13": 13,
		"14": 14,
		"15": 15,
		"16": 16,
		"17": 17,
		"18": 18,
	}

	l.teams = Teams{
		{Color: Color{13, 11, 92}},
		{Color: Color{20, 6, 90}},
		{Color: Color{16, 5, 90}},
	}

	l.RecountTeamsPlanets()

	return l
}

func xTestSimpleTransfer(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("1", "0")
	if l.board[0].Planets != 11 && l.board[1].Planets != 7 {
		t.Errorf(
			"0 has %d and 1 has %d planets, instead of 11 and 7",
			l.board[0].Planets,
			l.board[1].Planets,
		)
	}

	if l.board[0].Username != "0" {
		t.Errorf("0 is %s instead of 0", l.board[0].Username)
	}

	if l.board[1].Username != "1" {
		t.Errorf("1 is %s instead of 0", l.board[1].Username)
	}
}

func xTestMovingUp(t *testing.T) {
	l := initLeaderboard()
	l.board[4].Planets = 9

	l.moveUp("4")
	if l.board[1].Username != "4" {
		t.Errorf("4 is not in the 1 place, %s is there instead", l.board[1].Username)
	}

	if l.board[0].Username != "0" {
		t.Errorf("0 is not in the 0 place, %s is there instead", l.board[1].Username)
	}

	l.moveUp("0")
}

func xTestMovingDown(t *testing.T) {
	l := initLeaderboard()
	l.board[1].Planets = 2

	l.moveDown("1")
	if l.board[4].Username != "1" {
		t.Errorf("1 is not last, %s is there instead", l.board[4].Username)
	}
	l.moveUp(string(len(l.board) - 1))
}

func xTestPlace(t *testing.T) {
	l := initLeaderboard()
	results := []struct {
		in  string
		out int
	}{
		{"0", 0},
		{"4", 4},
		{"2", 2},
		{"42", 0},
	}

	for _, result := range results {
		place := l.Place(result.in)
		if place != result.out {
			t.Errorf("Player %#v is in place %d, expected %d", result.in, place, result.out)
		}
	}
}

func xTestPage(t *testing.T) {
	l := initLeaderboard()
	results := []struct {
		in  int64
		out int
		err bool
	}{
		{-1, 0, true},
		{0, 0, true},
		{1, 10, false},
		{2, 9, false},
		{3, 0, true},
	}

	for _, result := range results {
		page, err := l.Page(result.in)
		if len(page) != result.out || result.err != (err != nil) {
			t.Errorf(
				"Page %d returned %d (err: %t) players, expected %d (err: %t)",
				result.in, len(page), err != nil, result.out, result.err)
		}
	}
}

func xTestAdd(t *testing.T) {
	l := initLeaderboard()
	boardLengthBefore := l.Len()
	placesLengthBefore := len(l.places)
	teamsLengthBefore := len(l.teams)
	l.Add(&Player{Username: "panda", Planets: 42, Team: Color{13, 11, 92}})
	l.FindTeam(Color{21, 6, 90})
	l.Add(&Player{Username: "gophie", Planets: 42, Team: Color{21, 6, 90}})
	if boardLengthBefore+2 != l.Len() {
		t.Error("Board size did not changed after adding a player")
	}

	if placesLengthBefore+2 != len(l.places) {
		t.Error("Places map size did not changed after adding a player")
	}

	if teamsLengthBefore+1 != len(l.Teams()) {
		t.Error("Places map size did not changed after adding a player")
	}
}

func xTestSort(t *testing.T) {
	l := initLeaderboard()
	l.places["0"] = 20
	l.board[1].Planets = 128
	l.RecountTeamsPlanets()
	l.Sort()

	if l.board[0].Username != "1" {
		t.Error("Leaderboard.Sort() did not sorted the board")
	}

	if l.places["0"] != 1 {
		t.Error("Leaderboard.Sort() did not changed the places")
	}

	if l.teams[0] != l.FindTeam(Color{16, 5, 90}) {
		t.Error("Leaderboard.Sort() did not sort teams")
	}
}

func xTestChangingPlacesAndPlanets(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("3", "4")
	l.Transfer("0", "4")

	if l.places["4"] > l.places["3"] {
		t.Errorf(
			"Player 4 is on place %d, Player 3 - %d",
			l.places["4"],
			l.places["3"],
		)
	}

	planets := l.board[l.places["4"]].Planets
	if planets != 7 {
		t.Errorf("Player 4 is has %d planets, expected 7", planets)
	}
}

func xTestTakePlanetsWithoutOwner(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("", "4")
	l.Transfer("", "4")

	if l.places["4"] > l.places["3"] {
		t.Errorf(
			"Player 4 is on place %d, Player 3 - %d",
			l.places["4"],
			l.places["3"],
		)
	}
}

func TestTeamPlanetsTransfer(t *testing.T) {
	d := New()
	d.board = []*Player{
		{Username: "0", Planets: 10, Team: Color{13, 11, 92}},
		{Username: "1", Planets: 10, Team: Color{16, 5, 90}},
		{Username: "2", Planets: 9, Team: Color{16, 5, 90}},
		{Username: "3", Planets: 8, Team: Color{13, 11, 92}},
		{Username: "4", Planets: 7, Team: Color{6, 9, 90}},
	}

	d.places = map[string]int{
		"0": 0,
		"1": 1,
		"2": 2,
		"3": 3,
		"4": 4,
	}

	d.teams = Teams{
		{Color: Color{13, 11, 92}},
		{Color: Color{6, 9, 90}},
		{Color: Color{16, 5, 90}},
	}

	d.RecountTeamsPlanets()

	d.Transfer("", "3")
	d.Transfer("", "3")
	d.Transfer("", "2")

	planets := d.FindTeam(Color{13, 11, 92}).Planets
	if planets != 20 {
		t.Errorf("This team has %d", planets)
	}
}
<commit_msg>Change xTest to Test<commit_after>package leaderboard

import (
	"testing"
)

func initLeaderboard() *Leaderboard {
	l := New()
	l.board = []*Player{
		{Username: "0", Planets: 10, Team: Color{13, 11, 92}},
		{Username: "1", Planets: 8, Team: Color{16, 5, 90}},
		{Username: "2", Planets: 7, Team: Color{16, 5, 90}},
		{Username: "3", Planets: 6, Team: Color{13, 11, 92}},
		{Username: "4", Planets: 5, Team: Color{13, 11, 92}},
		{Username: "5", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "6", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "7", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "8", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "9", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "10", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "11", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "12", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "13", Planets: 0, Team: Color{13, 11, 92}},
		{Username: "14", Planets: 0, Team: Color{20, 6, 90}},
		{Username: "15", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "16", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "17", Planets: 0, Team: Color{16, 5, 90}},
		{Username: "18", Planets: 0, Team: Color{20, 6, 90}},
	}

	l.places = map[string]int{
		"0":  0,
		"1":  1,
		"2":  2,
		"3":  3,
		"4":  4,
		"5":  5,
		"6":  6,
		"7":  7,
		"8":  8,
		"9":  9,
		"10": 10,
		"11": 11,
		"12": 12,
		"13": 13,
		"14": 14,
		"15": 15,
		"16": 16,
		"17": 17,
		"18": 18,
	}

	l.teams = Teams{
		{Color: Color{13, 11, 92}},
		{Color: Color{20, 6, 90}},
		{Color: Color{16, 5, 90}},
	}

	l.RecountTeamsPlanets()

	return l
}

func TestSimpleTransfer(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("1", "0")
	if l.board[0].Planets != 11 && l.board[1].Planets != 7 {
		t.Errorf(
			"0 has %d and 1 has %d planets, instead of 11 and 7",
			l.board[0].Planets,
			l.board[1].Planets,
		)
	}

	if l.board[0].Username != "0" {
		t.Errorf("0 is %s instead of 0", l.board[0].Username)
	}

	if l.board[1].Username != "1" {
		t.Errorf("1 is %s instead of 0", l.board[1].Username)
	}
}

func TestMovingUp(t *testing.T) {
	l := initLeaderboard()
	l.board[4].Planets = 9

	l.moveUp("4")
	if l.board[1].Username != "4" {
		t.Errorf("4 is not in the 1 place, %s is there instead", l.board[1].Username)
	}

	if l.board[0].Username != "0" {
		t.Errorf("0 is not in the 0 place, %s is there instead", l.board[1].Username)
	}

	l.moveUp("0")
}

func TestMovingDown(t *testing.T) {
	l := initLeaderboard()
	l.board[1].Planets = 2

	l.moveDown("1")
	if l.board[4].Username != "1" {
		t.Errorf("1 is not last, %s is there instead", l.board[4].Username)
	}
	l.moveUp(string(len(l.board) - 1))
}

func TestPlace(t *testing.T) {
	l := initLeaderboard()
	results := []struct {
		in  string
		out int
	}{
		{"0", 0},
		{"4", 4},
		{"2", 2},
		{"42", 0},
	}

	for _, result := range results {
		place := l.Place(result.in)
		if place != result.out {
			t.Errorf("Player %#v is in place %d, expected %d", result.in, place, result.out)
		}
	}
}

func TestPage(t *testing.T) {
	l := initLeaderboard()
	results := []struct {
		in  int64
		out int
		err bool
	}{
		{-1, 0, true},
		{0, 0, true},
		{1, 10, false},
		{2, 9, false},
		{3, 0, true},
	}

	for _, result := range results {
		page, err := l.Page(result.in)
		if len(page) != result.out || result.err != (err != nil) {
			t.Errorf(
				"Page %d returned %d (err: %t) players, expected %d (err: %t)",
				result.in, len(page), err != nil, result.out, result.err)
		}
	}
}

func TestAdd(t *testing.T) {
	l := initLeaderboard()
	boardLengthBefore := l.Len()
	placesLengthBefore := len(l.places)
	teamsLengthBefore := len(l.teams)
	l.Add(&Player{Username: "panda", Planets: 42, Team: Color{13, 11, 92}})
	l.FindTeam(Color{21, 6, 90})
	l.Add(&Player{Username: "gophie", Planets: 42, Team: Color{21, 6, 90}})
	if boardLengthBefore+2 != l.Len() {
		t.Error("Board size did not changed after adding a player")
	}

	if placesLengthBefore+2 != len(l.places) {
		t.Error("Places map size did not changed after adding a player")
	}

	if teamsLengthBefore+1 != len(l.Teams()) {
		t.Error("Places map size did not changed after adding a player")
	}
}

func TestSort(t *testing.T) {
	l := initLeaderboard()
	l.places["0"] = 20
	l.board[1].Planets = 128
	l.RecountTeamsPlanets()
	l.Sort()

	if l.board[0].Username != "1" {
		t.Error("Leaderboard.Sort() did not sorted the board")
	}

	if l.places["0"] != 1 {
		t.Error("Leaderboard.Sort() did not changed the places")
	}

	if l.teams[0] != l.FindTeam(Color{16, 5, 90}) {
		t.Error("Leaderboard.Sort() did not sort teams")
	}
}

func TestChangingPlacesAndPlanets(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("3", "4")
	l.Transfer("0", "4")

	if l.places["4"] > l.places["3"] {
		t.Errorf(
			"Player 4 is on place %d, Player 3 - %d",
			l.places["4"],
			l.places["3"],
		)
	}

	planets := l.board[l.places["4"]].Planets
	if planets != 7 {
		t.Errorf("Player 4 is has %d planets, expected 7", planets)
	}
}

func TestTakePlanetsWithoutOwner(t *testing.T) {
	l := initLeaderboard()
	l.Transfer("", "4")
	l.Transfer("", "4")

	if l.places["4"] > l.places["3"] {
		t.Errorf(
			"Player 4 is on place %d, Player 3 - %d",
			l.places["4"],
			l.places["3"],
		)
	}
}

func TestTeamPlanetsTransfer(t *testing.T) {
	d := New()
	d.board = []*Player{
		{Username: "0", Planets: 10, Team: Color{13, 11, 92}},
		{Username: "1", Planets: 10, Team: Color{16, 5, 90}},
		{Username: "2", Planets: 9, Team: Color{16, 5, 90}},
		{Username: "3", Planets: 8, Team: Color{13, 11, 92}},
		{Username: "4", Planets: 7, Team: Color{6, 9, 90}},
	}

	d.places = map[string]int{
		"0": 0,
		"1": 1,
		"2": 2,
		"3": 3,
		"4": 4,
	}

	d.teams = Teams{
		{Color: Color{13, 11, 92}},
		{Color: Color{6, 9, 90}},
		{Color: Color{16, 5, 90}},
	}

	d.RecountTeamsPlanets()

	d.Transfer("", "3")
	d.Transfer("", "3")
	d.Transfer("", "2")

	planets := d.FindTeam(Color{13, 11, 92}).Planets
	if planets != 20 {
		t.Errorf("This team has %d", planets)
	}
}
<|endoftext|>


<commit_before>package tardeploy

import (
	"fmt"

	"github.com/prometheus/log"

	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

type (
	// Configuration contains the daemon config
	Configuration struct {
		Directories DirectoryConfiguration
		Application ApplicationHandling
	}

	// ApplicationHandling  provides information about handling older versions
	ApplicationHandling struct {
		NumberOfBackups int
	}

	FileSecurity struct {
		User  string // User or UID for file/directory owner
		Group string // Group or UID for file/directory owner
	}

	// DirectoryConfiguration contains all data required to handle deployments
	DirectoryConfiguration struct {
		TarballDirectory     string       // TarballDirectory denotes the place to put the tarballs
		WebRootDirectory     string       // WebRootDirectory denotes the root for the web
		ApplicationDirectory string       // ApplicationDirectory - where to store the applications
		Security             FileSecurity // Chown information
	}
)

func LoadConfiguration() (*Configuration, error) {
	// Searches for config file in given paths and read it
	if err := viper.ReadInConfig(); err != nil {
		return nil, errors.Wrap(err, "Error reading config file")
	}

	// Confirm which config file is used
	log.Println(fmt.Sprintf("Using config:         [%s]", viper.ConfigFileUsed()))

	var config Configuration
	err := viper.Unmarshal(&config)

	return &config, err
}

func init() {
	viper.AddConfigPath("/etc/tardeploy")   // look in system config driectory
	viper.AddConfigPath("$HOME/.tardeploy") // maybe user space
	viper.AddConfigPath(".")                // local config
	viper.SetConfigName("tardeploy")        // file is named tardeploy.[yaml|json|toml]

	viper.AutomaticEnv() // read in environment variables that match
}
<commit_msg>Comment for public struct<commit_after>package tardeploy

import (
	"fmt"

	"github.com/prometheus/log"

	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

type (
	// Configuration contains the daemon config
	Configuration struct {
		Directories DirectoryConfiguration
		Application ApplicationHandling
	}

	// ApplicationHandling  provides information about handling older versions
	ApplicationHandling struct {
		NumberOfBackups int
	}

	// FileSecurity defines the ownership of files
	FileSecurity struct {
		User  string // User or UID for file/directory owner
		Group string // Group or UID for file/directory owner
	}

	// DirectoryConfiguration contains all data required to handle deployments
	DirectoryConfiguration struct {
		TarballDirectory     string       // TarballDirectory denotes the place to put the tarballs
		WebRootDirectory     string       // WebRootDirectory denotes the root for the web
		ApplicationDirectory string       // ApplicationDirectory - where to store the applications
		Security             FileSecurity // Chown information
	}
)

func LoadConfiguration() (*Configuration, error) {
	// Searches for config file in given paths and read it
	if err := viper.ReadInConfig(); err != nil {
		return nil, errors.Wrap(err, "Error reading config file")
	}

	// Confirm which config file is used
	log.Println(fmt.Sprintf("Using config:         [%s]", viper.ConfigFileUsed()))

	var config Configuration
	err := viper.Unmarshal(&config)

	return &config, err
}

func init() {
	viper.AddConfigPath("/etc/tardeploy")   // look in system config driectory
	viper.AddConfigPath("$HOME/.tardeploy") // maybe user space
	viper.AddConfigPath(".")                // local config
	viper.SetConfigName("tardeploy")        // file is named tardeploy.[yaml|json|toml]

	viper.AutomaticEnv() // read in environment variables that match
}
<|endoftext|>


<commit_before>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package navigator

import (
	"github.com/nelsam/gxui"
	"github.com/nelsam/vidar/commander/bind"
	"github.com/nelsam/vidar/setting"
)

type ProjectSetter interface {
	Name() string
	Menu() string
	SetProject(settings.Project)
	Exec(interface{}) bind.Status
}

type Projects struct {
	theme gxui.Theme
	cmdr  Commander

	button          gxui.Button
	projects        gxui.List
	projectsAdapter *gxui.DefaultAdapter

	projectFrame gxui.Control
}

func NewProjectsPane(cmdr Commander, driver gxui.Driver, theme gxui.Theme, projFrame gxui.Control) *Projects {
	pane := &Projects{
		cmdr:            cmdr,
		theme:           theme,
		projectFrame:    projFrame,
		button:          createIconButton(driver, theme, "projects.png"),
		projects:        theme.CreateList(),
		projectsAdapter: gxui.CreateDefaultAdapter(),
	}
	pane.projectsAdapter.SetItems(settings.Projects())
	pane.projects.SetAdapter(pane.projectsAdapter)
	pane.projects.OnSelectionChanged(func(selected gxui.AdapterItem) {
		opener := pane.cmdr.Bindable("open-project").(ProjectSetter)
		opener.SetProject(selected.(settings.Project))
		pane.cmdr.Execute(opener)
	})
	return pane
}

func (p *Projects) Add(project settings.Project) {
	projects := append(p.projectsAdapter.Items().([]settings.Project), project)
	p.projectsAdapter.SetItems(projects)
}

func (p *Projects) Button() gxui.Button {
	return p.button
}

func (p *Projects) Frame() gxui.Control {
	return p.projects
}

func (p *Projects) Projects() []settings.Project {
	return p.projectsAdapter.Items().([]settings.Project)
}

func (p *Projects) OnComplete(onComplete func(bind.Bindable)) {
	opener := p.cmdr.Bindable("open-project").(ProjectSetter)
	p.projects.OnSelectionChanged(func(selected gxui.AdapterItem) {
		opener.SetProject(selected.(settings.Project))
		onComplete(opener)
	})
}
<commit_msg>Remove excess methods from navigator.ProjectSetter<commit_after>// This is free and unencumbered software released into the public
// domain.  For more information, see <http://unlicense.org> or the
// accompanying UNLICENSE file.

package navigator

import (
	"github.com/nelsam/gxui"
	"github.com/nelsam/vidar/commander/bind"
	"github.com/nelsam/vidar/setting"
)

type ProjectSetter interface {
	bind.Bindable

	SetProject(settings.Project)
}

type Projects struct {
	theme gxui.Theme
	cmdr  Commander

	button          gxui.Button
	projects        gxui.List
	projectsAdapter *gxui.DefaultAdapter

	projectFrame gxui.Control
}

func NewProjectsPane(cmdr Commander, driver gxui.Driver, theme gxui.Theme, projFrame gxui.Control) *Projects {
	pane := &Projects{
		cmdr:            cmdr,
		theme:           theme,
		projectFrame:    projFrame,
		button:          createIconButton(driver, theme, "projects.png"),
		projects:        theme.CreateList(),
		projectsAdapter: gxui.CreateDefaultAdapter(),
	}
	pane.projectsAdapter.SetItems(settings.Projects())
	pane.projects.SetAdapter(pane.projectsAdapter)
	pane.projects.OnSelectionChanged(func(selected gxui.AdapterItem) {
		opener := pane.cmdr.Bindable("open-project").(ProjectSetter)
		opener.SetProject(selected.(settings.Project))
		pane.cmdr.Execute(opener)
	})
	return pane
}

func (p *Projects) Add(project settings.Project) {
	projects := append(p.projectsAdapter.Items().([]settings.Project), project)
	p.projectsAdapter.SetItems(projects)
}

func (p *Projects) Button() gxui.Button {
	return p.button
}

func (p *Projects) Frame() gxui.Control {
	return p.projects
}

func (p *Projects) Projects() []settings.Project {
	return p.projectsAdapter.Items().([]settings.Project)
}

func (p *Projects) OnComplete(onComplete func(bind.Bindable)) {
	opener := p.cmdr.Bindable("open-project").(ProjectSetter)
	p.projects.OnSelectionChanged(func(selected gxui.AdapterItem) {
		opener.SetProject(selected.(settings.Project))
		onComplete(opener)
	})
}
<|endoftext|>


<commit_before>package dpn

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"io"
	"os"
	"sync"
	"time"
)

// StorageResult maintains information about the state of
// an attempt to store a DPN bag in AWS Glacier.
type StorageResult struct {

	// The NSQ message. This will be always have a value
	// in production, and will be nil when running local
	// developer tests.
	NsqMessage      *nsq.Message  `json:"-"`

	// BagIdentifier is the APTrust bag identifier. If this is
	// a non-empty value, it means this bag came from APTrust,
	// and we will need to record a PREMIS event noting that
	// it was ingested into DPN. If the bag identifier is empty,
	// this bag came from somewhere else. We're just replicating
	// and we don't need to store a PREMIS event in Fluctus.
	BagIdentifier   string

	// UUID is the DPN identifier for this bag.
	UUID            string

	// The path to the bag, which is stored on disk as a tar file.
	TarFilePath     string

	// The URL of this file in Glacier. This will be empty until
	// we actually manage to store the file.
	StorageURL      string

	// A message describing what went wrong in the storage process.
	// If we have a StorageURL and ErrorMessage is empty,
	// storage succeeded.
	ErrorMessage    string

	// The file's md5 digest. We need this to copy to Amazon S3/Glacier.
	Md5Digest       string

	// Should we try again to store this object? Usually, this is
	// true if we encounter network errors, false if there's some
	// fatal error, like TarFilePath cannot be found.
	Retry           bool
}

type Storer struct {
	DigestChannel       chan *StorageResult
	StorageChannel      chan *StorageResult
	CleanupChannel      chan *StorageResult
	PostProcessChannel  chan *StorageResult
	ProcUtil            *bagman.ProcessUtil
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil) (*Storer) {
	storer := &Storer{
		ProcUtil: procUtil,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.DigestChannel = make(chan *StorageResult, workerBufferSize)
	storer.StorageChannel = make(chan *StorageResult, workerBufferSize)
	storer.CleanupChannel = make(chan *StorageResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *StorageResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.calculateDigest()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var result *StorageResult
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	result.NsqMessage = message
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath, bagIdentifier)
	storer.DigestChannel <- result
	return nil
}

func (storer *Storer) calculateDigest() {
	for result := range storer.DigestChannel {
		if result.Md5Digest != "" {
			storer.StorageChannel <- result
		}
		md5Hash := md5.New()
		reader, err := os.Open(result.TarFilePath)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		bytesWritten, err := io.Copy(md5Hash, reader)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error running md5 checksum on file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		if bytesWritten != fileInfo.Size() {
			result.ErrorMessage = fmt.Sprintf("Error running md5 checksum on file '%s': " +
				"read only %d of %d bytes.",
				result.TarFilePath, bytesWritten, fileInfo.Size())
			storer.PostProcessChannel <- result
			continue
		}
		reader.Close()
		result.Md5Digest = fmt.Sprintf("%x", md5Hash.Sum(nil))
		storer.StorageChannel <- result
	}
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		reader, err := os.Open(result.TarFilePath)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		fileName := fmt.Sprintf("%s.tar", result.UUID)
		url, err := storer.ProcUtil.S3Client.SaveToS3(
			storer.ProcUtil.Config.DPNPreservationBucket,
			fileName,
			"application/x-tar",
			reader,
			fileInfo.Size(),
			options)
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		result.StorageURL = url

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		if result.ErrorMessage == "" && result.StorageURL != "" {
			err := os.Remove(result.TarFilePath)
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath, err)
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				storer.SendToRecordQueue(result)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					storer.SendToTroubleQueue(result)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath)
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *StorageResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.Md5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) SendToRecordQueue(result *StorageResult) {
	// Record has to record PREMIS event in Fluctus if
	// BagIdentifier is present. It will definitely have
	// to record information in the DPN REST API.
	err := bagman.Enqueue(storer.ProcUtil.Config.NsqdHttpAddress,
		storer.ProcUtil.Config.DPNRecordWorker.NsqTopic, result)
	if err != nil {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.UUID
		}
		message := fmt.Sprintf("Could not send '%s' (at %s) to record queue: %v",
			bagIdentifier, result.TarFilePath, err)
		result.ErrorMessage += message
		storer.ProcUtil.MessageLog.Error(message)
		storer.SendToTroubleQueue(result)
	}
}

func (storer *Storer) SendToTroubleQueue(result *StorageResult) {
	result.ErrorMessage += " This item has been queued for administrative review."
	err := bagman.Enqueue(storer.ProcUtil.Config.NsqdHttpAddress,
		storer.ProcUtil.Config.DPNTroubleWorker.NsqTopic, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v",
			result.BagIdentifier, err)
		storer.ProcUtil.MessageLog.Error("Original error on '%s' was %s",
			result.BagIdentifier, result.ErrorMessage)
	}
}

func (storer *Storer) RunTest(result *StorageResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.DigestChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<commit_msg>Added logging statement describing cleanup<commit_after>package dpn

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"io"
	"os"
	"sync"
	"time"
)

// StorageResult maintains information about the state of
// an attempt to store a DPN bag in AWS Glacier.
type StorageResult struct {

	// The NSQ message. This will be always have a value
	// in production, and will be nil when running local
	// developer tests.
	NsqMessage      *nsq.Message  `json:"-"`

	// BagIdentifier is the APTrust bag identifier. If this is
	// a non-empty value, it means this bag came from APTrust,
	// and we will need to record a PREMIS event noting that
	// it was ingested into DPN. If the bag identifier is empty,
	// this bag came from somewhere else. We're just replicating
	// and we don't need to store a PREMIS event in Fluctus.
	BagIdentifier   string

	// UUID is the DPN identifier for this bag.
	UUID            string

	// The path to the bag, which is stored on disk as a tar file.
	TarFilePath     string

	// The URL of this file in Glacier. This will be empty until
	// we actually manage to store the file.
	StorageURL      string

	// A message describing what went wrong in the storage process.
	// If we have a StorageURL and ErrorMessage is empty,
	// storage succeeded.
	ErrorMessage    string

	// The file's md5 digest. We need this to copy to Amazon S3/Glacier.
	Md5Digest       string

	// Should we try again to store this object? Usually, this is
	// true if we encounter network errors, false if there's some
	// fatal error, like TarFilePath cannot be found.
	Retry           bool
}

type Storer struct {
	DigestChannel       chan *StorageResult
	StorageChannel      chan *StorageResult
	CleanupChannel      chan *StorageResult
	PostProcessChannel  chan *StorageResult
	ProcUtil            *bagman.ProcessUtil
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil) (*Storer) {
	storer := &Storer{
		ProcUtil: procUtil,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.DigestChannel = make(chan *StorageResult, workerBufferSize)
	storer.StorageChannel = make(chan *StorageResult, workerBufferSize)
	storer.CleanupChannel = make(chan *StorageResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *StorageResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.calculateDigest()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var result *StorageResult
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	result.NsqMessage = message
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath, bagIdentifier)
	storer.DigestChannel <- result
	return nil
}

func (storer *Storer) calculateDigest() {
	for result := range storer.DigestChannel {
		if result.Md5Digest != "" {
			storer.StorageChannel <- result
		}
		md5Hash := md5.New()
		reader, err := os.Open(result.TarFilePath)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		bytesWritten, err := io.Copy(md5Hash, reader)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error running md5 checksum on file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		if bytesWritten != fileInfo.Size() {
			result.ErrorMessage = fmt.Sprintf("Error running md5 checksum on file '%s': " +
				"read only %d of %d bytes.",
				result.TarFilePath, bytesWritten, fileInfo.Size())
			storer.PostProcessChannel <- result
			continue
		}
		reader.Close()
		result.Md5Digest = fmt.Sprintf("%x", md5Hash.Sum(nil))
		storer.StorageChannel <- result
	}
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		reader, err := os.Open(result.TarFilePath)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath, err)
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		fileName := fmt.Sprintf("%s.tar", result.UUID)
		url, err := storer.ProcUtil.S3Client.SaveToS3(
			storer.ProcUtil.Config.DPNPreservationBucket,
			fileName,
			"application/x-tar",
			reader,
			fileInfo.Size(),
			options)
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		result.StorageURL = url

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		if result.ErrorMessage == "" && result.StorageURL != "" {
			err := os.Remove(result.TarFilePath)
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath, err)
			} else {
				storer.ProcUtil.MessageLog.Info(
					"After successful upload, deleted local DPN bag at %s",
					result.TarFilePath)
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				storer.SendToRecordQueue(result)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					storer.SendToTroubleQueue(result)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath)
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *StorageResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.Md5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) SendToRecordQueue(result *StorageResult) {
	// Record has to record PREMIS event in Fluctus if
	// BagIdentifier is present. It will definitely have
	// to record information in the DPN REST API.
	err := bagman.Enqueue(storer.ProcUtil.Config.NsqdHttpAddress,
		storer.ProcUtil.Config.DPNRecordWorker.NsqTopic, result)
	if err != nil {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.UUID
		}
		message := fmt.Sprintf("Could not send '%s' (at %s) to record queue: %v",
			bagIdentifier, result.TarFilePath, err)
		result.ErrorMessage += message
		storer.ProcUtil.MessageLog.Error(message)
		storer.SendToTroubleQueue(result)
	}
}

func (storer *Storer) SendToTroubleQueue(result *StorageResult) {
	result.ErrorMessage += " This item has been queued for administrative review."
	err := bagman.Enqueue(storer.ProcUtil.Config.NsqdHttpAddress,
		storer.ProcUtil.Config.DPNTroubleWorker.NsqTopic, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v",
			result.BagIdentifier, err)
		storer.ProcUtil.MessageLog.Error("Original error on '%s' was %s",
			result.BagIdentifier, result.ErrorMessage)
	}
}

func (storer *Storer) RunTest(result *StorageResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.DigestChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<|endoftext|>


<commit_before>// Copyright 2016 ePoxy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//////////////////////////////////////////////////////////////////////////////

// Package storage includes the Host record definition. Host records represent
// a managed machine and store the next stage configuration. Host records are
// saved to persistent storage.
package storage

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"time"
)

// These variables provide indirection for the default function implementations.
// Each can be reassigned with an alternate implementation for unit tests.
var (
	randRead = rand.Read
	timeNow  = time.Now
)

// CollectedInformation stores information received directly from iPXE clients.
type CollectedInformation struct {
	Platform         string
	BuildArch        string
	Serial           string
	Asset            string
	UUID             string
	Manufacturer     string
	Product          string
	Chip             string
	MAC              string
	IP               string
	Version          string
	PublicSSHHostKey string
}

// TODO: SessionIDs and Sequence structs should be map[string]string, that
// store target stage names as keys. This prevents hard-coding the target names,
// the SessionID names and the Sequence stage names.

// SessionIDs contains the three session IDs generated when requesting a stage1 target.
type SessionIDs struct {
	Stage2ID string // Needed for requesting the stage2.json target.
	Stage3ID string // Needed for requesting the stage3.json target.
	ReportID string // Needed for requesting the report target.
}

// TODO: Sequences could be a separate type stored in datastore. These could be
// named and referenced by Host objects by name.

// Sequence represents a set of operator-provided iPXE scripts or JSON nextboot Configs.
type Sequence struct {
	// Stage1ChainURL is the absolute URL to an iPXE script for booting from stage1 to stage2.
	Stage1ChainURL string
	// Stage2ChainURL is the absolute URL to a JSON config for booting from stage2 to stage3.
	Stage2ChainURL string
	// Stage3ChainURL is the absolute URL to a JSON config for running commands in stage3. For
	// example, "flashrom", or "join global k8s cluster".
	Stage3ChainURL string
}

// NextURL returns the Chain URL corresponding to the given stage name.
func (s Sequence) NextURL(stage string) string {
	switch stage {
	case "stage1":
		return s.Stage1ChainURL
	case "stage2":
		return s.Stage2ChainURL
	case "stage3":
		return s.Stage3ChainURL
	default:
		// TODO: support a default error url.
		return ""
	}
}

// A Host represents the configuration of a server managed by ePoxy.
type Host struct {
	// Name is the FQDN of the host.
	Name string
	// IPv4Addr is the IPv4 address the booting machine will use to connect to the API.
	IPv4Addr string

	// Boot is the typical boot sequence for this Host.
	Boot Sequence
	// Update is an alternate boot sequence, typically used to update the system, e.g. reinstall, reflash.
	Update Sequence

	// UpdateEnabled controls whether ePoxy returns the Update Sequence (true)
	// or Boot Sequence (false) Chain URLs.
	UpdateEnabled bool

	// CurrentSessionIDs are the most recently generated session ids for a booting machine.
	CurrentSessionIDs SessionIDs
	// LastSessionCreation is the time when CurrentSessionIDs was generated.
	LastSessionCreation time.Time
	// LastReport is the time of the most recent report for this host.
	LastReport time.Time
	// LastSuccess is the time of the most recent successful report from this host.
	LastSuccess time.Time
	// CollectedInformation reported by the host.
	CollectedInformation CollectedInformation
}

// String serializes a Host record. All string type Host fields should be UTF8.
func (h *Host) String() string {
	// Errors only occur for non-UTF8 characters in strings or unmarshalable types (which we don't have).
	b, _ := json.MarshalIndent(h, "", "    ")
	return string(b)
}

// GenerateSessionIDs creates new random session IDs for the host's CurrentSessionIDs.
// On success, the host LastSessionCreation is updated to the current time.
func (h *Host) GenerateSessionIDs() {
	h.CurrentSessionIDs.Stage2ID = generateSessionID()
	h.CurrentSessionIDs.Stage3ID = generateSessionID()
	h.CurrentSessionIDs.ReportID = generateSessionID()
	h.LastSessionCreation = timeNow()
}

// CurrentSequence returns the currently enabled boot sequence.
func (h *Host) CurrentSequence() Sequence {
	if h.UpdateEnabled {
		return h.Update
	}
	return h.Boot
}

// randomSessionByteCount is the number of bytes used to generate random session IDs.
const randomSessionByteCount = 20

// generateSessionId creates a random session ID.
func generateSessionID() string {
	b := make([]byte, randomSessionByteCount)
	_, err := randRead(b)
	if err != nil {
		// Only possible if randRead fails to read len(b) bytes.
		panic(err)
	}
	// RawURLEncoding does not pad encoded string with "=".
	return base64.RawURLEncoding.EncodeToString(b)
}
<commit_msg>Add logic to generate an Extension session ID<commit_after>// Copyright 2016 ePoxy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//////////////////////////////////////////////////////////////////////////////

// Package storage includes the Host record definition. Host records represent
// a managed machine and store the next stage configuration. Host records are
// saved to persistent storage.
package storage

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"time"
)

// These variables provide indirection for the default function implementations.
// Each can be reassigned with an alternate implementation for unit tests.
var (
	randRead = rand.Read
	timeNow  = time.Now
)

// CollectedInformation stores information received directly from iPXE clients.
// Field names correspond to iPXE variable names.
type CollectedInformation struct {
	Platform         string
	BuildArch        string
	Serial           string
	Asset            string
	UUID             string
	Manufacturer     string
	Product          string
	Chip             string
	MAC              string
	IP               string
	Version          string
	PublicSSHHostKey string
}

// TODO: SessionIDs and Sequence structs should be map[string]string, that
// store target stage names as keys. This prevents hard-coding the target names,
// the SessionID names and the Sequence stage names.

// SessionIDs contains the three session IDs generated when requesting a stage1 target.
type SessionIDs struct {
	Stage2ID string // Needed for requesting the stage2.json target.
	Stage3ID string // Needed for requesting the stage3.json target.
	ReportID string // Needed for requesting the report target.
	// TODO: support multiple extensions.
	ExtensionID string // Needed for requesting the extension target.
}

// TODO: Sequences could be a separate type stored in datastore. These could be
// named and referenced by Host objects by name.

// Sequence represents a set of operator-provided iPXE scripts or JSON nextboot Configs.
type Sequence struct {
	// Stage1ChainURL is the absolute URL to an iPXE script for booting from stage1 to stage2.
	Stage1ChainURL string
	// Stage2ChainURL is the absolute URL to a JSON config for booting from stage2 to stage3.
	Stage2ChainURL string
	// Stage3ChainURL is the absolute URL to a JSON config for running commands in stage3. For
	// example, "flashrom", or "join global k8s cluster".
	Stage3ChainURL string
}

// NextURL returns the Chain URL corresponding to the given stage name.
func (s Sequence) NextURL(stage string) string {
	switch stage {
	case "stage1":
		return s.Stage1ChainURL
	case "stage2":
		return s.Stage2ChainURL
	case "stage3":
		return s.Stage3ChainURL
	default:
		// TODO: support a default error url.
		return ""
	}
}

// A Host represents the configuration of a server managed by ePoxy.
type Host struct {
	// Name is the FQDN of the host.
	Name string
	// IPv4Addr is the IPv4 address the booting machine will use to connect to the API.
	IPv4Addr string

	// TODO: add IPv6Addr.

	// Boot is the typical boot sequence for this Host.
	Boot Sequence
	// Update is an alternate boot sequence, typically used to update the system, e.g. reinstall, reflash.
	Update Sequence

	// UpdateEnabled controls whether ePoxy returns the Update Sequence (true)
	// or Boot Sequence (false) Chain URLs.
	UpdateEnabled bool

	// Extensions is an arry of extensions enabled for this host.
	Extensions []string

	// CurrentSessionIDs are the most recently generated session ids for a booting machine.
	CurrentSessionIDs SessionIDs
	// LastSessionCreation is the time when CurrentSessionIDs was generated.
	LastSessionCreation time.Time
	// LastReport is the time of the most recent report for this host.
	LastReport time.Time
	// LastSuccess is the time of the most recent successful report from this host.
	LastSuccess time.Time
	// CollectedInformation reported by the host.
	CollectedInformation CollectedInformation
}

// String serializes a Host record. All string type Host fields should be UTF8.
func (h *Host) String() string {
	// Errors only occur for non-UTF8 characters in strings or unmarshalable types (which we don't have).
	b, _ := json.MarshalIndent(h, "", "    ")
	return string(b)
}

// GenerateSessionIDs creates new random session IDs for the host's CurrentSessionIDs.
// On success, the host LastSessionCreation is updated to the current time.
func (h *Host) GenerateSessionIDs() {
	h.CurrentSessionIDs.Stage2ID = generateSessionID()
	h.CurrentSessionIDs.Stage3ID = generateSessionID()
	h.CurrentSessionIDs.ReportID = generateSessionID()
	h.CurrentSessionIDs.ExtensionID = generateSessionID()
	h.LastSessionCreation = timeNow()
}

// CurrentSequence returns the currently enabled boot sequence.
func (h *Host) CurrentSequence() Sequence {
	if h.UpdateEnabled {
		return h.Update
	}
	return h.Boot
}

// randomSessionByteCount is the number of bytes used to generate random session IDs.
const randomSessionByteCount = 20

// generateSessionId creates a random session ID.
func generateSessionID() string {
	b := make([]byte, randomSessionByteCount)
	_, err := randRead(b)
	if err != nil {
		// Only possible if randRead fails to read len(b) bytes.
		panic(err)
	}
	// RawURLEncoding does not pad encoded string with "=".
	return base64.RawURLEncoding.EncodeToString(b)
}
<|endoftext|>