<commit_before>package stringset_test

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/jamesjoshuahill/stringset"
)

var _ = Describe("StringSet", func() {
	Context("when empty", func() {
		It("does not contain a member", func() {
			emptySet := stringset.New()
			Expect(emptySet.Contains("monkeys")).To(BeFalse())
		})

		It("lists no members", func() {
			emptySet := stringset.New()
			Expect(emptySet.Members()).To(BeEmpty())
		})

		It("prints neatly", func() {
			emptySet := stringset.New()
			Expect(fmt.Sprintf("%v", emptySet)).To(Equal(`{}`))
		})

		It("is empty", func() {
			emptySet := stringset.New()
			Expect(emptySet.Empty()).To(BeTrue())
		})

		It("has an order of 0", func() {
			emptySet := stringset.New()
			Expect(emptySet.Order()).To(Equal(0))
		})

		It("is a subset of itself", func() {
			emptySet := stringset.New()
			Expect(emptySet.IsSubset(emptySet)).To(BeTrue())
		})
	})

	Context("when it has one member", func() {
		It("contains the member", func() {
			set := stringset.New("monkeys")
			Expect(set.Contains("monkeys")).To(BeTrue())
		})

		It("does not contain another member", func() {
			set := stringset.New("bananas")
			Expect(set.Contains("monkeys")).To(BeFalse())
		})

		It("prints neatly", func() {
			set := stringset.New("monkeys")
			Expect(fmt.Sprintf("%v", set)).To(Equal(`{monkeys}`))
		})

		It("is not empty", func() {
			set := stringset.New("monkeys")
			Expect(set.Empty()).To(BeFalse())
		})

		It("has an order of one", func() {
			set := stringset.New("monkeys")
			Expect(set.Order()).To(Equal(1))
		})

		It("is a subset of itself", func() {
			set := stringset.New("monkeys")
			Expect(set.IsSubset(set)).To(BeTrue())
		})
	})

	Context("when it has some members", func() {
		It("lists all members", func() {
			set := stringset.New("monkeys", "bananas", "trees")
			Expect(set.Members()).To(ContainElement("monkeys"))
			Expect(set.Members()).To(ContainElement("bananas"))
			Expect(set.Members()).To(ContainElement("trees"))
		})

		It("prints neatly", func() {
			set := stringset.New("monkeys", "bananas", "trees")
			Expect(fmt.Sprintf("%v", set)).To(Or(
				Equal(`{bananas monkeys trees}`),
				Equal(`{bananas trees monkeys}`),
				Equal(`{monkeys bananas trees}`),
				Equal(`{monkeys trees bananas}`),
				Equal(`{trees bananas monkeys}`),
				Equal(`{trees monkeys bananas}`),
			))
		})

		It("is not empty", func() {
			set := stringset.New("monkeys", "bananas", "trees")
			Expect(set.Empty()).To(BeFalse())
		})

		It("has an order of three", func() {
			set := stringset.New("monkeys", "bananas", "trees")
			Expect(set.Order()).To(Equal(3))
		})

		It("is a subset of itself", func() {
			set := stringset.New("monkeys", "bananas", "trees")
			Expect(set.IsSubset(set)).To(BeTrue())
		})
	})

	Context("when empty and the other set has a member", func() {
		It("subtracts nothing", func() {
			emptySet := stringset.New()
			other := stringset.New("trees")
			Expect(emptySet.Subtract(other)).To(Equal(emptySet))
		})

		It("has no members in common", func() {
			emptySet := stringset.New()
			other := stringset.New("trees")
			Expect(emptySet.Intersection(other)).To(Equal(emptySet))
		})

		It("includes all members in the union", func() {
			emptySet := stringset.New()
			other := stringset.New("trees")
			Expect(emptySet.Union(other)).To(Equal(other))

			By("not changing the set")
			Expect(emptySet).To(Equal(stringset.New()))
		})

		It("includes the member in the symmetric difference", func() {
			emptySet := stringset.New()
			other := stringset.New("trees")
			Expect(emptySet.SymmetricDifference(other)).To(Equal(other))

			By("not changing the set")
			Expect(emptySet).To(Equal(stringset.New()))
		})

		It("is a subset of the other", func() {
			emptySet := stringset.New()
			other := stringset.New("trees")
			Expect(emptySet.IsSubset(other)).To(BeTrue())
		})
	})

	Context("when it has members and the other set is a subset", func() {
		It("subtracts the member in common", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas")
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys")))
		})

		It("intersects", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas")
			Expect(set.Intersection(other)).To(Equal(stringset.New("bananas")))
		})

		It("includes all members in the union", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas")
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("subtracts the member in common from the symmetric difference", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas")
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas")
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})

	Context("when it has members and the other set intersects", func() {
		It("subtracts the member in common", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas", "trees")
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys")))
		})

		It("intersects", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas", "trees")
			Expect(set.Intersection(other)).To(Equal(stringset.New("bananas")))
		})

		It("includes all members in the union", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas", "trees")
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("subtracts the member in common from the symmetric difference", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas", "trees")
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("bananas", "trees")
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})

	Context("when it has members and the other set does not intersect", func() {
		It("does not subtract any members", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("trees", "sunshine")
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("does not intersect", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("trees")
			Expect(set.Intersection(other)).To(Equal(stringset.New()))
		})

		It("includes all members in the union", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("trees")
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("includes all members in the symmetric difference", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("trees")
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys", "bananas", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			set := stringset.New("monkeys", "bananas")
			other := stringset.New("trees")
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})
})
<commit_msg>Refactor tests<commit_after>package stringset_test

import (
	"fmt"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/jamesjoshuahill/stringset"
)

var _ = Describe("StringSet", func() {
	var set stringset.StringSet
	var other stringset.StringSet

	Context("when is has no members", func() {
		BeforeEach(func() {
			set = stringset.New()
		})

		It("does not contain a member", func() {
			Expect(set.Contains("monkeys")).To(BeFalse())
		})

		It("lists no members", func() {
			Expect(set.Members()).To(BeEmpty())
		})

		It("prints neatly", func() {
			Expect(fmt.Sprintf("%v", set)).To(Equal(`{}`))
		})

		It("is empty", func() {
			Expect(set.Empty()).To(BeTrue())
		})

		It("has an order of 0", func() {
			Expect(set.Order()).To(Equal(0))
		})

		It("is a subset of itself", func() {
			Expect(set.IsSubset(set)).To(BeTrue())
		})
	})

	Context("when it has one member", func() {
		BeforeEach(func() {
			set = stringset.New("monkeys")
		})

		It("contains the member", func() {
			Expect(set.Contains("monkeys")).To(BeTrue())
		})

		It("does not contain another member", func() {
			Expect(set.Contains("bananas")).To(BeFalse())
		})

		It("prints neatly", func() {
			Expect(fmt.Sprintf("%v", set)).To(Equal(`{monkeys}`))
		})

		It("is not empty", func() {
			Expect(set.Empty()).To(BeFalse())
		})

		It("has an order of one", func() {
			Expect(set.Order()).To(Equal(1))
		})

		It("is a subset of itself", func() {
			Expect(set.IsSubset(set)).To(BeTrue())
		})
	})

	Context("when it has some members", func() {
		BeforeEach(func() {
			set = stringset.New("monkeys", "bananas", "trees")
		})

		It("lists all members", func() {
			Expect(set.Members()).To(ContainElement("monkeys"))
			Expect(set.Members()).To(ContainElement("bananas"))
			Expect(set.Members()).To(ContainElement("trees"))
		})

		It("prints neatly", func() {
			Expect(fmt.Sprintf("%v", set)).To(Or(
				Equal(`{bananas monkeys trees}`),
				Equal(`{bananas trees monkeys}`),
				Equal(`{monkeys bananas trees}`),
				Equal(`{monkeys trees bananas}`),
				Equal(`{trees bananas monkeys}`),
				Equal(`{trees monkeys bananas}`),
			))
		})

		It("is not empty", func() {
			Expect(set.Empty()).To(BeFalse())
		})

		It("has an order of three", func() {
			Expect(set.Order()).To(Equal(3))
		})

		It("is a subset of itself", func() {
			Expect(set.IsSubset(set)).To(BeTrue())
		})
	})

	Context("when empty and the other set has a member", func() {
		BeforeEach(func() {
			set = stringset.New()
			other = stringset.New("trees")
		})

		It("subtracts nothing", func() {
			Expect(set.Subtract(other)).To(Equal(stringset.New()))
		})

		It("has no members in common", func() {
			Expect(set.Intersection(other)).To(Equal(stringset.New()))
		})

		It("includes all members in the union", func() {
			Expect(set.Union(other)).To(Equal(other))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New()))
		})

		It("includes the member in the symmetric difference", func() {
			Expect(set.SymmetricDifference(other)).To(Equal(other))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New()))
		})

		It("is a subset of the other", func() {
			Expect(set.IsSubset(other)).To(BeTrue())
		})
	})

	Context("when it has members and the other set is a subset", func() {
		BeforeEach(func() {
			set = stringset.New("monkeys", "bananas")
			other = stringset.New("bananas")
		})

		It("subtracts the member in common", func() {
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys")))
		})

		It("intersects", func() {
			Expect(set.Intersection(other)).To(Equal(stringset.New("bananas")))
		})

		It("includes all members in the union", func() {
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("subtracts the member in common from the symmetric difference", func() {
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})

	Context("when it has members and the other set intersects", func() {
		BeforeEach(func() {
			set = stringset.New("monkeys", "bananas")
			other = stringset.New("bananas", "trees")
		})

		It("subtracts the member in common", func() {
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys")))
		})

		It("intersects", func() {
			Expect(set.Intersection(other)).To(Equal(stringset.New("bananas")))
		})

		It("includes all members in the union", func() {
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("subtracts the member in common from the symmetric difference", func() {
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys", "trees")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})

	Context("when it has members and the other set does not intersect", func() {
		BeforeEach(func() {
			set = stringset.New("monkeys", "bananas")
			other = stringset.New("trees", "sunshine")
		})

		It("does not subtract any members", func() {
			Expect(set.Subtract(other)).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("does not intersect", func() {
			Expect(set.Intersection(other)).To(Equal(stringset.New()))
		})

		It("includes all members in the union", func() {
			Expect(set.Union(other)).To(Equal(stringset.New("monkeys", "bananas", "trees", "sunshine")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("includes all members in the symmetric difference", func() {
			Expect(set.SymmetricDifference(other)).To(Equal(stringset.New("monkeys", "bananas", "trees", "sunshine")))

			By("not changing the set")
			Expect(set).To(Equal(stringset.New("monkeys", "bananas")))
		})

		It("is not a subset of the other", func() {
			Expect(set.IsSubset(other)).To(BeFalse())
		})
	})
})
<|endoftext|>


<commit_before>package test

import (
	"fmt"

	"github.com/DATA-DOG/godog/gherkin"
	"github.com/Originate/git-town/test/helpers"
	"github.com/sergi/go-diff/diffmatchpatch"
)

// DataTable allows comparing user-generated data with Gherkin tables.
type DataTable struct {
	// cells contains table data organized as rows and columns
	cells [][]string
}

// FromGherkin provides a DataTable instance populated with data from the given Gherkin table.
func FromGherkin(table *gherkin.DataTable) (result DataTable) {
	for _, tableRow := range table.Rows {
		resultRow := make([]string, len(tableRow.Cells))
		for i, tableCell := range tableRow.Cells {
			resultRow[i] = tableCell.Value
		}
		result.AddRow(resultRow...)
	}
	return result
}

// AddRow adds the given row of table data to this table.
func (table *DataTable) AddRow(elements ...string) {
	table.cells = append(table.cells, elements)
}

// columns provides the table data organized into columns.
func (table *DataTable) columns() (result [][]string) {
	for column := range table.cells[0] {
		colData := []string{}
		for row := range table.cells {
			colData = append(colData, table.cells[row][column])
		}
		result = append(result, colData)
	}
	return result
}

// Equal indicates whether this DataTable instance is equal to the given Gherkin table.
// If both are equal it returns an empty string,
// otherwise a diff printable on the console.
func (table *DataTable) Equal(other *gherkin.DataTable) (diff string, errorCount int) {
	if len(table.cells) == 0 {
		return "your data is empty", 1
	}
	gherkinTable := FromGherkin(other)
	dmp := diffmatchpatch.New()
	diffs := dmp.DiffMain(gherkinTable.String(), table.String(), false)
	if len(diffs) == 1 && diffs[0].Type == 0 {
		return "", 0
	}
	return dmp.DiffPrettyText(diffs), len(diffs)
}

// String provides the data in this DataTable instance formatted in Gherkin table format.
func (table *DataTable) String() (result string) {
	// determine how to format each column
	formatStrings := []string{}
	for _, width := range table.widths() {
		formatStrings = append(formatStrings, fmt.Sprintf("| %%-%dv ", width))
	}

	// render the table using this format
	for row := range table.cells {
		for col := range table.cells[row] {
			result += fmt.Sprintf(formatStrings[col], table.cells[row][col])
		}
		result += "|\n"
	}
	return result
}

// widths provides the widths of all columns.
func (table *DataTable) widths() (result []int) {
	for _, column := range table.columns() {
		result = append(result, helpers.LongestStringLength(column))
	}
	return result
}
<commit_msg>Document the zero value (#1289)<commit_after>package test

import (
	"fmt"

	"github.com/DATA-DOG/godog/gherkin"
	"github.com/Originate/git-town/test/helpers"
	"github.com/sergi/go-diff/diffmatchpatch"
)

// DataTable allows comparing user-generated data with Gherkin tables.
// The zero value is an empty DataTable.
type DataTable struct {
	// cells contains table data organized as rows and columns
	cells [][]string
}

// FromGherkin provides a DataTable instance populated with data from the given Gherkin table.
func FromGherkin(table *gherkin.DataTable) (result DataTable) {
	for _, tableRow := range table.Rows {
		resultRow := make([]string, len(tableRow.Cells))
		for i, tableCell := range tableRow.Cells {
			resultRow[i] = tableCell.Value
		}
		result.AddRow(resultRow...)
	}
	return result
}

// AddRow adds the given row of table data to this table.
func (table *DataTable) AddRow(elements ...string) {
	table.cells = append(table.cells, elements)
}

// columns provides the table data organized into columns.
func (table *DataTable) columns() (result [][]string) {
	for column := range table.cells[0] {
		colData := []string{}
		for row := range table.cells {
			colData = append(colData, table.cells[row][column])
		}
		result = append(result, colData)
	}
	return result
}

// Equal indicates whether this DataTable instance is equal to the given Gherkin table.
// If both are equal it returns an empty string,
// otherwise a diff printable on the console.
func (table *DataTable) Equal(other *gherkin.DataTable) (diff string, errorCount int) {
	if len(table.cells) == 0 {
		return "your data is empty", 1
	}
	gherkinTable := FromGherkin(other)
	dmp := diffmatchpatch.New()
	diffs := dmp.DiffMain(gherkinTable.String(), table.String(), false)
	if len(diffs) == 1 && diffs[0].Type == 0 {
		return "", 0
	}
	return dmp.DiffPrettyText(diffs), len(diffs)
}

// String provides the data in this DataTable instance formatted in Gherkin table format.
func (table *DataTable) String() (result string) {
	// determine how to format each column
	formatStrings := []string{}
	for _, width := range table.widths() {
		formatStrings = append(formatStrings, fmt.Sprintf("| %%-%dv ", width))
	}

	// render the table using this format
	for row := range table.cells {
		for col := range table.cells[row] {
			result += fmt.Sprintf(formatStrings[col], table.cells[row][col])
		}
		result += "|\n"
	}
	return result
}

// widths provides the widths of all columns.
func (table *DataTable) widths() (result []int) {
	for _, column := range table.columns() {
		result = append(result, helpers.LongestStringLength(column))
	}
	return result
}
<|endoftext|>


<commit_before>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	processFilesToCopyToCache(request.FilesToCopyToCache, rootDirectoryName)
	if len(oldTriggers.Triggers) > 0 {
		processMakeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		processDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers,
			false)
		processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
			&oldTriggers, false)
		processChangeInodes(request.InodesToChange, rootDirectoryName,
			&oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processMakeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	processDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers,
		true)
	processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
		request.Triggers, true)
	processChangeInodes(request.InodesToChange, rootDirectoryName,
		request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Post-Update() debugging sleep complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processFilesToCopyToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		logger.Printf("Copy: %s to cache\n", fileToCopy.Name)
		// TODO(rgooch): Implement.
	}
}

func processMakeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			logger.Printf("Make inode: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processHardlinksToMake(hardlinksToMake []sub.Hardlink,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		triggers.Match(hardlink.NewLink)
		if takeAction {
			logger.Printf("Link: %s => %s\n", hardlink.NewLink, hardlink.Target)
			// TODO(rgooch): Implement.
			// err := os.Link(path.Join(rootDirectoryName, hardlink.Target),
			//	path.Join(rootDirectoryName, hardlink.NewLink))
		}
	}
}

func processDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			logger.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(directoriesToMake []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if skipPath(newdir.Name) {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			logger.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeDirectories(directoriesToChange []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, directory := range directoriesToChange {
		if takeAction {
			logger.Printf("Change directory: %s\n", directory.Name)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeInodes(inodesToChange []sub.Inode,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		if takeAction {
			logger.Printf("Change inode: %s\n", inode.Name)
			// TODO(rgooch): Implement.
		}
	}
}

func skipPath(pathname string) bool {
	if scannerConfiguration.ScanFilter.Match(pathname) {
		return true
	}
	if pathname == "/.subd" {
		return true
	}
	if strings.HasPrefix(pathname, "/.subd/") {
		return true
	}
	return false
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print("Rebooting")
				// TODO(rgooch): Remove debugging output.
				cmd := exec.Command("echo", "reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					logger.Print(err)
				}
				return
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Remove debugging output.
		cmd := exec.Command("run-in-mntns", ppid, "echo", "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			logger.Print(err)
		}
		// TODO(rgooch): Implement.
	}
}
<commit_msg>Implement processFilesToCopyToCache().<commit_after>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"os/exec"
	"path"
	"strings"
	"syscall"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	startTime := time.Now()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	processFilesToCopyToCache(request.FilesToCopyToCache, rootDirectoryName)
	if len(oldTriggers.Triggers) > 0 {
		processMakeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		processDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers,
			false)
		processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
			&oldTriggers, false)
		processChangeInodes(request.InodesToChange, rootDirectoryName,
			&oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processMakeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	processDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers,
		true)
	processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
		request.Triggers, true)
	processChangeInodes(request.InodesToChange, rootDirectoryName,
		request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	timeTaken := time.Since(startTime)
	logger.Printf("Update() completed in %s\n", timeTaken)
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Post-Update() debugging sleep complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processFilesToCopyToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		sourcePathname := path.Join(rootDirectoryName, fileToCopy.Name)
		destPathname := path.Join(objectsDir,
			objectcache.HashToFilename(fileToCopy.Hash))
		if copyFile(destPathname, sourcePathname) {
			logger.Printf("Copied: %s to cache\n", sourcePathname)
		}
	}
}

func copyFile(destPathname, sourcePathname string) bool {
	sourceFile, err := os.Open(sourcePathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer sourceFile.Close()
	dirname := path.Dir(destPathname)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return false
	}
	destFile, err := os.Create(destPathname)
	if err != nil {
		logger.Println(err)
		return false
	}
	defer destFile.Close()
	_, err = io.Copy(destFile, sourceFile)
	if err != nil {
		return false
	}
	return true
}

func processMakeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		if takeAction {
			logger.Printf("Make inode: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processHardlinksToMake(hardlinksToMake []sub.Hardlink,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		triggers.Match(hardlink.NewLink)
		if takeAction {
			logger.Printf("Link: %s => %s\n", hardlink.NewLink, hardlink.Target)
			// TODO(rgooch): Implement.
			// err := os.Link(path.Join(rootDirectoryName, hardlink.Target),
			//	path.Join(rootDirectoryName, hardlink.NewLink))
		}
	}
}

func processDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			logger.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(directoriesToMake []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if skipPath(newdir.Name) {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			logger.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeDirectories(directoriesToChange []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, directory := range directoriesToChange {
		if takeAction {
			logger.Printf("Change directory: %s\n", directory.Name)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeInodes(inodesToChange []sub.Inode,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		if takeAction {
			logger.Printf("Change inode: %s\n", inode.Name)
			// TODO(rgooch): Implement.
		}
	}
}

func skipPath(pathname string) bool {
	if scannerConfiguration.ScanFilter.Match(pathname) {
		return true
	}
	if pathname == "/.subd" {
		return true
	}
	if strings.HasPrefix(pathname, "/.subd/") {
		return true
	}
	return false
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print("Rebooting")
				// TODO(rgooch): Remove debugging output.
				cmd := exec.Command("echo", "reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					logger.Print(err)
				}
				return
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Remove debugging output.
		cmd := exec.Command("run-in-mntns", ppid, "echo", "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			logger.Print(err)
		}
		// TODO(rgooch): Implement.
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"github.com/hashicorp/terraform/helper/resource"
)

// resourceMap is the mapping of resources we support to their basic
// operations. This makes it easy to implement new resource types.
var resourceMap *resource.Map

func init() {
	resourceMap = &resource.Map{
		Mapping: map[string]resource.Resource{
			"aws_instance": resource.Resource{
				Create:  resource_aws_instance_create,
				Destroy: resource_aws_instance_destroy,
				Diff:    resource_aws_instance_diff,
				Refresh: resource_aws_instance_refresh,
			},
			"aws_elb": resource.Resource{
				Create:  resource_aws_elb_create,
				Update:  resource_aws_elb_update,
				Destroy: resource_aws_elb_destroy,
				Diff:    resource_aws_elb_diff,
				Refresh: resource_aws_elb_refresh,
			},
		},
	}
}
<commit_msg>providers/aws: ABC, Its as Easy as 123<commit_after>package aws

import (
	"github.com/hashicorp/terraform/helper/resource"
)

// resourceMap is the mapping of resources we support to their basic
// operations. This makes it easy to implement new resource types.
var resourceMap *resource.Map

func init() {
	resourceMap = &resource.Map{
		Mapping: map[string]resource.Resource{
			"aws_elb": resource.Resource{
				Create:  resource_aws_elb_create,
				Update:  resource_aws_elb_update,
				Destroy: resource_aws_elb_destroy,
				Diff:    resource_aws_elb_diff,
				Refresh: resource_aws_elb_refresh,
			},

			"aws_instance": resource.Resource{
				Create:  resource_aws_instance_create,
				Destroy: resource_aws_instance_destroy,
				Diff:    resource_aws_instance_diff,
				Refresh: resource_aws_instance_refresh,
			},
		},
	}
}
<|endoftext|>


<commit_before>// Package exec is the entry point for security automation Cloud Functions.
package exec

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"fmt"

	"cloud.google.com/go/pubsub"
	"github.com/googlecloudplatform/threat-automation/clients"
	"github.com/googlecloudplatform/threat-automation/cloudfunctions"
	"github.com/googlecloudplatform/threat-automation/entities"
)

var (
	// TODO(tomfitzgerald): Handle these configuration options elsewhere & better.
	// folderID specifies which folder RevokeExternalGrantsFolders should remove members from.
	folderIDs = []string{"111185550749"}
	// disallowed contains a list of external domains RevokeExternalGrantsFolders should remove.
	disallowed = []string{"test.com", "gmail.com"}
)

const (
	authFile = "credentials/auth.json"
)

// RevokeExternalGrantsFolders is the entry point for IAM revoker Cloud Function.
//
// This Cloud Function will be triggered when Event Threat Detection
// detects an anomalous IAM grant. Once triggered this function will
// attempt to revoke the external members added to the policy if they match the provided
// list of disallowed domains. Additionally this method will only remove members if the
// project they were added to is within the specified folders. This configuration allows
// you to take a remediation action only on specific members and folders. For example,
// you may have a folder "development" where users can experiment without strict policies.
// However in your "production" folder you may want to revoke any grants that ETD finds as
// long as they match the domains you specify.
//
// Permissions required
//
// By default the service account used can only revoke projects that are found within the
// folder ID specified within `action-revoke-member-folders.tf`.
func RevokeExternalGrantsFolders(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)

	return cloudfunctions.RevokeExternalGrantsFolders(ctx, m, r, folderIDs, disallowed)
}

// SnapshotDisk is the entry point for the auto creation of GCE snapshots Cloud Function.
//
// This Cloud Function will respond to Event Threat Detection **bad IP** findings. Once a bad IP
// finding is received this Cloud Function will look for any existing disk snapshots for the
// affected instance. If there are recent snapshots then no action is taken. If we have not
// taken a snapshot recently, take a new snapshot for each disk within the instance.
//
// Permissions required
//
// By default the service account can only be used to create snapshots for the projects
// specified in `action-snaphot-disk.tf`
//
// TODO: Support assigning roles at the folder and organization level.
func SnapshotDisk(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)

	cs, err := clients.NewCompute(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize compute client: %q", err)
	}
	h := entities.NewHost(cs)
	return cloudfunctions.CreateSnapshot(ctx, m, r, h)
}

// CloseBucket will remove any public users from buckets found within the provided folders.
func CloseBucket(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)
	return cloudfunctions.CloseBucket(ctx, m, r, folderIDs)
}

//StopInstance stops instance on gce
func StopInstance(ctx context.Context, m pubsub.Message) error {
	cs, err := clients.NewCompute(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize compute client: %q", err)
	}
	h := entities.NewHost(cs)
	return cloudfunctions.StopInstance(ctx, m, h)
}
<commit_msg>Remove stop instance cloud function for wile<commit_after>// Package exec is the entry point for security automation Cloud Functions.
package exec

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"fmt"

	"cloud.google.com/go/pubsub"
	"github.com/googlecloudplatform/threat-automation/clients"
	"github.com/googlecloudplatform/threat-automation/cloudfunctions"
	"github.com/googlecloudplatform/threat-automation/entities"
)

var (
	// TODO(tomfitzgerald): Handle these configuration options elsewhere & better.
	// folderID specifies which folder RevokeExternalGrantsFolders should remove members from.
	folderIDs = []string{"111185550749"}
	// disallowed contains a list of external domains RevokeExternalGrantsFolders should remove.
	disallowed = []string{"test.com", "gmail.com"}
)

const (
	authFile = "credentials/auth.json"
)

// RevokeExternalGrantsFolders is the entry point for IAM revoker Cloud Function.
//
// This Cloud Function will be triggered when Event Threat Detection
// detects an anomalous IAM grant. Once triggered this function will
// attempt to revoke the external members added to the policy if they match the provided
// list of disallowed domains. Additionally this method will only remove members if the
// project they were added to is within the specified folders. This configuration allows
// you to take a remediation action only on specific members and folders. For example,
// you may have a folder "development" where users can experiment without strict policies.
// However in your "production" folder you may want to revoke any grants that ETD finds as
// long as they match the domains you specify.
//
// Permissions required
//
// By default the service account used can only revoke projects that are found within the
// folder ID specified within `action-revoke-member-folders.tf`.
func RevokeExternalGrantsFolders(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)

	return cloudfunctions.RevokeExternalGrantsFolders(ctx, m, r, folderIDs, disallowed)
}

// SnapshotDisk is the entry point for the auto creation of GCE snapshots Cloud Function.
//
// This Cloud Function will respond to Event Threat Detection **bad IP** findings. Once a bad IP
// finding is received this Cloud Function will look for any existing disk snapshots for the
// affected instance. If there are recent snapshots then no action is taken. If we have not
// taken a snapshot recently, take a new snapshot for each disk within the instance.
//
// Permissions required
//
// By default the service account can only be used to create snapshots for the projects
// specified in `action-snaphot-disk.tf`
//
// TODO: Support assigning roles at the folder and organization level.
func SnapshotDisk(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)

	cs, err := clients.NewCompute(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize compute client: %q", err)
	}
	h := entities.NewHost(cs)
	return cloudfunctions.CreateSnapshot(ctx, m, r, h)
}

// CloseBucket will remove any public users from buckets found within the provided folders.
func CloseBucket(ctx context.Context, m pubsub.Message) error {
	crm, err := clients.NewCloudResourceManager(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize cloud resource manager client: %q", err)
	}

	stg, err := clients.NewStorage(ctx, authFile)
	if err != nil {
		return fmt.Errorf("failed to initialize storage client: %q", err)
	}
	r := entities.NewResource(crm, stg)
	return cloudfunctions.CloseBucket(ctx, m, r, folderIDs)
}
<|endoftext|>


<commit_before>//  Copyright 2017 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package test

import (
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"testing/quick"
	"time"
)

var r = rand.New(rand.NewSource(time.Now().UnixNano()))

func equal(this, that interface{}) bool {
	eqMethod := reflect.ValueOf(this).MethodByName("Equal")
	res := eqMethod.Call([]reflect.Value{reflect.ValueOf(that)})
	return res[0].Interface().(bool)
}

func random(this interface{}) interface{} {
	v, ok := quick.Value(reflect.TypeOf(this), r)
	if !ok {
		panic(fmt.Sprintf("unable to generate value for type: %T", this))
	}
	return v.Interface()
}

func TestEqual(t *testing.T) {
	structs := []interface{}{
		&BuiltInTypes{},
		&PtrToBuiltInTypes{},
		&SliceOfBuiltInTypes{},
		&SliceOfPtrToBuiltInTypes{},
		&ArrayOfBuiltInTypes{},
		&ArrayOfPtrToBuiltInTypes{},

		&SliceToSlice{},
		&SomeComplexTypes{},
		&RecursiveType{},
	}
	for _, this := range structs {
		desc := reflect.TypeOf(this).Elem().Name()
		t.Run(desc, func(t *testing.T) {
			for i := 0; i < 100; i++ {
				if !equal(this, this) {
					t.Fatal("empty not equal to itself")
				}
				this = random(this)
				if !equal(this, this) {
					t.Fatal("random not equal to itself")
				}
				that := random(this)
				for reflect.ValueOf(that).IsNil() {
					that = random(this)
				}
				if equal(this, that) {
					t.Fatalf("random %#v equal to another random %#v", this, that)
				}
			}
		})
	}
}
<commit_msg>better equal testing<commit_after>//  Copyright 2017 Walter Schulze
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package test

import (
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"testing/quick"
	"time"
)

var r = rand.New(rand.NewSource(time.Now().UnixNano()))

func equal(this, that interface{}) bool {
	eqMethod := reflect.ValueOf(this).MethodByName("Equal")
	res := eqMethod.Call([]reflect.Value{reflect.ValueOf(that)})
	return res[0].Interface().(bool)
}

func random(this interface{}) interface{} {
	v, ok := quick.Value(reflect.TypeOf(this), r)
	if !ok {
		panic(fmt.Sprintf("unable to generate value for type: %T", this))
	}
	return v.Interface()
}

func TestEqual(t *testing.T) {
	structs := []interface{}{
		&BuiltInTypes{},
		&PtrToBuiltInTypes{},
		&SliceOfBuiltInTypes{},
		&SliceOfPtrToBuiltInTypes{},
		&ArrayOfBuiltInTypes{},
		&ArrayOfPtrToBuiltInTypes{},

		&SliceToSlice{},
		&SomeComplexTypes{},
		&RecursiveType{},
	}
	for _, this := range structs {
		desc := reflect.TypeOf(this).Elem().Name()
		t.Run(desc, func(t *testing.T) {
			for i := 0; i < 100; i++ {
				this = random(this)
				that := random(this)
				if want, got := reflect.DeepEqual(this, that), equal(this, that); want != got {
					t.Fatalf("want %v got %v\n this = %#v\n that = %#v", want, got, this, that)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/boltdb/bolt"
)

var (
	QuotaError = errors.New("payment required")

	bucketName = []byte("cache")
)

type Cache struct {
	db *bolt.DB
}

func NewCache(dir string) (*Cache, error) {
	db, err := bolt.Open(dir, 0666, nil)
	if err != nil {
		return nil, err
	}
	tx, err := db.Begin(true)
	if err != nil {
		return nil, err
	}
	bucket := tx.Bucket(bucketName)
	if bucket == nil {
		_, err = tx.CreateBucket(bucketName)
		if err != nil {
			return nil, err
		}
	}
	err = tx.Commit()
	if err != nil {
		return nil, err
	}
	return &Cache{
		db: db,
	}, nil
}

func (c *Cache) Close() error {
	return c.db.Close()
}

func (c *Cache) Put(key string, data []byte) error {
	tx, err := c.db.Begin(true)
	if err != nil {
		return err
	}
	defer func() {
		if tx != nil {
			tx.Rollback()
		}
	}()
	bucket := tx.Bucket(bucketName)
	err = bucket.Put([]byte(key), data)
	if err != nil {
		return err
	}
	err = tx.Commit()
	tx = nil
	return err
}

func (c *Cache) Get(key string) ([]byte, error) {
	tx, err := c.db.Begin(false)
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()
	bucket := tx.Bucket(bucketName)
	temp := bucket.Get([]byte(key))
	data := make([]byte, len(temp))
	copy(data, temp)
	return data, nil
}

type Geocoder struct {
	key   string
	cache *Cache
}

func NewGeocoder(key, cacheDir string) (*Geocoder, error) {
	cache, err := NewCache(cacheDir)
	if err != nil {
		return nil, err
	}
	return &Geocoder{
		key:   key,
		cache: cache,
	}, nil
}

func (g *Geocoder) Close() error {
	return g.cache.Close()
}

type LocRate struct {
	Limit     int `json:"limit"`
	Remaining int `json:"remaining"`
}

type LocComponent struct {
	City        string `json:"city"`
	PostCode    string `json:"postcode"`
	County      string `json:"county"`
	State       string `json:"state"`
	Country     string `json:"country"`
	CountryCode string `json:"country_code"`
}

func (c *LocComponent) String() string {
	values := []struct {
		Field string
		Value string
	}{
		{"city", c.City},
		{"postcode", c.PostCode},
		{"county", c.County},
		{"state", c.State},
		{"country", c.Country},
	}
	s := ""
	written := false
	for _, v := range values {
		if v.Value == "" {
			continue
		}
		if written {
			s += ", "
		}
		s += fmt.Sprintf("%s: %s", v.Field, v.Value)
		written = true
	}
	return s
}

type LocResult struct {
	Component LocComponent `json:"components"`
}

type Location struct {
	Cached  bool
	Rate    LocRate     `json:"rate"`
	Results []LocResult `json:"results"`
}

func makeKeyAndCountryCode(q, code string) (string, string) {
	code = strings.ToLower(code)
	if code == "" {
		code = "unk"
	}
	return q + "-" + code, code
}

func (g *Geocoder) GeocodeFromCache(q, countryCode string) (*Location, error) {
	key, countryCode := makeKeyAndCountryCode(q, countryCode)
	data, err := g.cache.Get(key)
	if err != nil {
		return nil, err
	}
	if len(data) == 0 {
		return nil, nil
	}
	res := &Location{}
	err = json.Unmarshal(data, res)
	res.Cached = true
	return res, err
}

func (g *Geocoder) Geocode(q, countryCode string, offline bool) (*Location, error) {
	res, err := g.GeocodeFromCache(q, countryCode)
	if err != nil || res != nil || offline {
		return res, err
	}
	r, err := g.rawGeocode(q, countryCode)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	data, err := ioutil.ReadAll(&io.LimitedReader{
		R: r,
		N: 4 * 1024 * 1024,
	})
	if err != nil {
		return nil, err
	}
	res = &Location{}
	err = json.Unmarshal(data, res)
	if err != nil {
		return nil, err
	}
	key, _ := makeKeyAndCountryCode(q, countryCode)
	err = g.cache.Put(key, data)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, res)
	return res, err
}

func (g *Geocoder) rawGeocode(q, countryCode string) (io.ReadCloser, error) {
	u := fmt.Sprintf("http://api.opencagedata.com/geocode/v1/json?q=%s&key=%s",
		url.QueryEscape(q), url.QueryEscape(g.key))
	if countryCode != "" {
		u += "&countrycode=" + url.QueryEscape(countryCode)
	}
	rsp, err := http.Get(u)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != 200 {
		rsp.Body.Close()
		if rsp.StatusCode == 402 {
			return nil, QuotaError
		}
		return nil, fmt.Errorf("geocoding failed with %s", rsp.Status)
	}
	return rsp.Body, nil
}

var (
	geocodeCmd   = app.Command("geocode", "geocode location with OpenCage")
	geocodeQuery = geocodeCmd.Arg("query", "geocoding query").Required().String()
)

func geocode(cfg *Config) error {
	key := cfg.GeocodingKey()
	if key == "" {
		return fmt.Errorf("geocoding key is not set, please configure APEC_GEOCODING_KEY")
	}
	geocoder, err := NewGeocoder(key, cfg.Geocoder())
	if err != nil {
		return err
	}
	defer geocoder.Close()
	loc, err := geocoder.Geocode(*geocodeQuery, "fr", false)
	if err != nil {
		return err
	}
	if loc.Cached {
		fmt.Printf("cached: true\n")
	}
	fmt.Printf("remaining: %d\n", loc.Rate.Remaining)
	for _, res := range loc.Results {
		comp := res.Component
		fmt.Printf("%s\n", comp.String())
	}
	return nil
}
<commit_msg>geocoder: simplify bucket initialization<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/boltdb/bolt"
)

var (
	QuotaError = errors.New("payment required")

	bucketName = []byte("cache")
)

type Cache struct {
	db *bolt.DB
}

func NewCache(dir string) (*Cache, error) {
	db, err := bolt.Open(dir, 0666, nil)
	if err != nil {
		return nil, err
	}
	tx, err := db.Begin(true)
	if err != nil {
		return nil, err
	}
	_, err = tx.CreateBucketIfNotExists(bucketName)
	if err != nil {
		return nil, err
	}
	err = tx.Commit()
	if err != nil {
		return nil, err
	}
	return &Cache{
		db: db,
	}, nil
}

func (c *Cache) Close() error {
	return c.db.Close()
}

func (c *Cache) Put(key string, data []byte) error {
	tx, err := c.db.Begin(true)
	if err != nil {
		return err
	}
	defer func() {
		if tx != nil {
			tx.Rollback()
		}
	}()
	bucket := tx.Bucket(bucketName)
	err = bucket.Put([]byte(key), data)
	if err != nil {
		return err
	}
	err = tx.Commit()
	tx = nil
	return err
}

func (c *Cache) Get(key string) ([]byte, error) {
	tx, err := c.db.Begin(false)
	if err != nil {
		return nil, err
	}
	defer tx.Rollback()
	bucket := tx.Bucket(bucketName)
	temp := bucket.Get([]byte(key))
	data := make([]byte, len(temp))
	copy(data, temp)
	return data, nil
}

type Geocoder struct {
	key   string
	cache *Cache
}

func NewGeocoder(key, cacheDir string) (*Geocoder, error) {
	cache, err := NewCache(cacheDir)
	if err != nil {
		return nil, err
	}
	return &Geocoder{
		key:   key,
		cache: cache,
	}, nil
}

func (g *Geocoder) Close() error {
	return g.cache.Close()
}

type LocRate struct {
	Limit     int `json:"limit"`
	Remaining int `json:"remaining"`
}

type LocComponent struct {
	City        string `json:"city"`
	PostCode    string `json:"postcode"`
	County      string `json:"county"`
	State       string `json:"state"`
	Country     string `json:"country"`
	CountryCode string `json:"country_code"`
}

func (c *LocComponent) String() string {
	values := []struct {
		Field string
		Value string
	}{
		{"city", c.City},
		{"postcode", c.PostCode},
		{"county", c.County},
		{"state", c.State},
		{"country", c.Country},
	}
	s := ""
	written := false
	for _, v := range values {
		if v.Value == "" {
			continue
		}
		if written {
			s += ", "
		}
		s += fmt.Sprintf("%s: %s", v.Field, v.Value)
		written = true
	}
	return s
}

type LocResult struct {
	Component LocComponent `json:"components"`
}

type Location struct {
	Cached  bool
	Rate    LocRate     `json:"rate"`
	Results []LocResult `json:"results"`
}

func makeKeyAndCountryCode(q, code string) (string, string) {
	code = strings.ToLower(code)
	if code == "" {
		code = "unk"
	}
	return q + "-" + code, code
}

func (g *Geocoder) GeocodeFromCache(q, countryCode string) (*Location, error) {
	key, countryCode := makeKeyAndCountryCode(q, countryCode)
	data, err := g.cache.Get(key)
	if err != nil {
		return nil, err
	}
	if len(data) == 0 {
		return nil, nil
	}
	res := &Location{}
	err = json.Unmarshal(data, res)
	res.Cached = true
	return res, err
}

func (g *Geocoder) Geocode(q, countryCode string, offline bool) (*Location, error) {
	res, err := g.GeocodeFromCache(q, countryCode)
	if err != nil || res != nil || offline {
		return res, err
	}
	r, err := g.rawGeocode(q, countryCode)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	data, err := ioutil.ReadAll(&io.LimitedReader{
		R: r,
		N: 4 * 1024 * 1024,
	})
	if err != nil {
		return nil, err
	}
	res = &Location{}
	err = json.Unmarshal(data, res)
	if err != nil {
		return nil, err
	}
	key, _ := makeKeyAndCountryCode(q, countryCode)
	err = g.cache.Put(key, data)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, res)
	return res, err
}

func (g *Geocoder) rawGeocode(q, countryCode string) (io.ReadCloser, error) {
	u := fmt.Sprintf("http://api.opencagedata.com/geocode/v1/json?q=%s&key=%s",
		url.QueryEscape(q), url.QueryEscape(g.key))
	if countryCode != "" {
		u += "&countrycode=" + url.QueryEscape(countryCode)
	}
	rsp, err := http.Get(u)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != 200 {
		rsp.Body.Close()
		if rsp.StatusCode == 402 {
			return nil, QuotaError
		}
		return nil, fmt.Errorf("geocoding failed with %s", rsp.Status)
	}
	return rsp.Body, nil
}

var (
	geocodeCmd   = app.Command("geocode", "geocode location with OpenCage")
	geocodeQuery = geocodeCmd.Arg("query", "geocoding query").Required().String()
)

func geocode(cfg *Config) error {
	key := cfg.GeocodingKey()
	if key == "" {
		return fmt.Errorf("geocoding key is not set, please configure APEC_GEOCODING_KEY")
	}
	geocoder, err := NewGeocoder(key, cfg.Geocoder())
	if err != nil {
		return err
	}
	defer geocoder.Close()
	loc, err := geocoder.Geocode(*geocodeQuery, "fr", false)
	if err != nil {
		return err
	}
	if loc.Cached {
		fmt.Printf("cached: true\n")
	}
	fmt.Printf("remaining: %d\n", loc.Rate.Remaining)
	for _, res := range loc.Results {
		comp := res.Component
		fmt.Printf("%s\n", comp.String())
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/intervention-engine/hdsfhir"
	"io/ioutil"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "upload"
	app.Usage = "Convert health-data-standards JSON to FHIR JSON and upload it to a FHIR Server"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "fhir, f",
			Usage: "URL for the FHIR server",
		},
		cli.StringFlag{
			Name:  "json, j",
			Usage: "Path to the directory of JSON files",
		},
		cli.StringFlag{
			Name:  "single, s",
			Usage: "Path to the a single JSON file",
		},
	}
	app.Action = func(c *cli.Context) {
		fhirUrl := c.String("fhir")
		path := c.String("json")
		singlePath := c.String("single")
		if fhirUrl == "" || (path == "" && singlePath == "") {
			fmt.Println("You must provide a FHIR URL and path to JSON files")
		} else {
			var fileNames []string
			if path != "" {
				files, err := ioutil.ReadDir(path)
				if err != nil {
					panic("Couldn't read the directory" + err.Error())
				}
				for _, file := range files {
					fileNames = append(fileNames, path+"/"+file.Name())
				}
			} else {
				fileNames = []string{singlePath}
			}
			for _, file := range fileNames {
				patient := &hdsfhir.Patient{}
				jsonBlob, err := ioutil.ReadFile(file)
				if err != nil {
					panic("Couldn't read the JSON file" + err.Error())
				}
				json.Unmarshal(jsonBlob, patient)
				patient.PostToFHIRServer(fhirUrl)
			}

		}
	}

	app.Run(os.Args)
}
<commit_msg>Added option to shift times of conditions, encounters, vitals, birthdate, medications, and procedures when uploading. <commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/intervention-engine/hdsfhir"
	"io/ioutil"
	"time"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "upload"
	app.Usage = "Convert health-data-standards JSON to FHIR JSON and upload it to a FHIR Server"
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "offset, o",
			Usage: "How many years to offset dates by",
		},
		cli.StringFlag{
			Name:  "fhir, f",
			Usage: "URL for the FHIR server",
		},
		cli.StringFlag{
			Name:  "json, j",
			Usage: "Path to the directory of JSON files",
		},
		cli.StringFlag{
			Name:  "single, s",
			Usage: "Path to the a single JSON file",
		},
	}
	app.Action = func(c *cli.Context) {
		offset := c.Int("offset")
		fhirUrl := c.String("fhir")
		path := c.String("json")
		singlePath := c.String("single")
		if fhirUrl == "" || (path == "" && singlePath == "") {
			fmt.Println("You must provide a FHIR URL and path to JSON files")
		} else {
			var fileNames []string
			if path != "" {
				files, err := ioutil.ReadDir(path)
				if err != nil {
					panic("Couldn't read the directory" + err.Error())
				}
				for _, file := range files {
					fileNames = append(fileNames, path+"/"+file.Name())
				}
			} else {
				fileNames = []string{singlePath}
			}
			for _, file := range fileNames {
				patient := &hdsfhir.Patient{}
				jsonBlob, err := ioutil.ReadFile(file)
				if err != nil {
					panic("Couldn't read the JSON file" + err.Error())
				}
				json.Unmarshal(jsonBlob, patient)
				patient.UnixBirthTime = time.Unix(patient.UnixBirthTime,0).AddDate(offset,0,0).Unix()

        for _,cond := range patient.Conditions {
          cond.StartTime = time.Unix(cond.StartTime, 0).AddDate(offset,0,0).Unix()
        }
        for _,enc := range patient.Encounters {
          enc.StartTime = time.Unix(enc.StartTime, 0).AddDate(offset,0,0).Unix()
        }
        for _,med := range patient.Medications {
          med.StartTime = time.Unix(med.StartTime, 0).AddDate(offset,0,0).Unix()
        }
        for _,vit := range patient.VitalSigns {
          vit.StartTime = time.Unix(vit.StartTime, 0).AddDate(offset,0,0).Unix()
        }
        for _,proc := range patient.Procedures {
          proc.StartTime = time.Unix(proc.StartTime, 0).AddDate(offset,0,0).Unix()
        }
				patient.PostToFHIRServer(fhirUrl)
			}

		}
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package torrent

import "github.com/anacrolix/torrent/metainfo"

// Provides access to regions of torrent data that correspond to its files.
type File struct {
	t      Torrent
	path   string
	offset int64
	length int64
	fi     metainfo.FileInfo
}

// Data for this file begins this far into the torrent.
func (f *File) Offset() int64 {
	return f.offset
}

func (f File) FileInfo() metainfo.FileInfo {
	return f.fi
}

func (f File) Path() string {
	return f.path
}

func (f *File) Length() int64 {
	return f.length
}

type FilePieceState struct {
	Bytes int64 // Bytes within the piece that are part of this File.
	PieceState
}

// Returns the state of pieces in this file.
func (f *File) State() (ret []FilePieceState) {
	pieceSize := int64(f.t.usualPieceSize())
	off := f.offset % pieceSize
	remaining := f.length
	for i := int(f.offset / pieceSize); ; i++ {
		if remaining == 0 {
			break
		}
		len1 := pieceSize - off
		if len1 > remaining {
			len1 = remaining
		}
		f.t.cl.mu.RLock()
		ps := f.t.pieceState(i)
		f.t.cl.mu.RUnlock()
		ret = append(ret, FilePieceState{len1, ps})
		off = 0
		remaining -= len1
	}
	return
}

func (f *File) PrioritizeRegion(off, len int64) {
	if off < 0 || off >= f.length {
		return
	}
	if off+len > f.length {
		len = f.length - off
	}
	off += f.offset
	f.t.SetRegionPriority(off, len)
}
<commit_msg>Add File.DisplayPath<commit_after>package torrent

import (
	"strings"

	"github.com/anacrolix/torrent/metainfo"
)

// Provides access to regions of torrent data that correspond to its files.
type File struct {
	t      Torrent
	path   string
	offset int64
	length int64
	fi     metainfo.FileInfo
}

// Data for this file begins this far into the torrent.
func (f *File) Offset() int64 {
	return f.offset
}

func (f File) FileInfo() metainfo.FileInfo {
	return f.fi
}

func (f File) Path() string {
	return f.path
}

func (f *File) Length() int64 {
	return f.length
}

// The relative file path for a multi-file torrent, and the torrent name for a
// single-file torrent.
func (f *File) DisplayPath() string {
	fip := f.FileInfo().Path
	if len(fip) == 0 {
		return f.t.Info().Name
	}
	return strings.Join(fip, "/")

}

type FilePieceState struct {
	Bytes int64 // Bytes within the piece that are part of this File.
	PieceState
}

// Returns the state of pieces in this file.
func (f *File) State() (ret []FilePieceState) {
	pieceSize := int64(f.t.usualPieceSize())
	off := f.offset % pieceSize
	remaining := f.length
	for i := int(f.offset / pieceSize); ; i++ {
		if remaining == 0 {
			break
		}
		len1 := pieceSize - off
		if len1 > remaining {
			len1 = remaining
		}
		f.t.cl.mu.RLock()
		ps := f.t.pieceState(i)
		f.t.cl.mu.RUnlock()
		ret = append(ret, FilePieceState{len1, ps})
		off = 0
		remaining -= len1
	}
	return
}

func (f *File) PrioritizeRegion(off, len int64) {
	if off < 0 || off >= f.length {
		return
	}
	if off+len > f.length {
		len = f.length - off
	}
	off += f.offset
	f.t.SetRegionPriority(off, len)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package nitro

import "os"
import "bufio"
import "errors"
import "github.com/couchbase/goforestdb"
import "bytes"

const DiskBlockSize = 512 * 1024

var (
	ErrNotEnoughSpace = errors.New("Not enough space in the buffer")
	forestdbConfig    *forestdb.Config
)

func init() {
	forestdbConfig = forestdb.DefaultConfig()
	forestdbConfig.SetSeqTreeOpt(forestdb.SEQTREE_NOT_USE)
	forestdbConfig.SetBufferCacheSize(1024 * 1024)

}

type FileWriter interface {
	Open(path string) error
	WriteItem(*Item) error
	Close() error
}

type FileReader interface {
	Open(path string) error
	ReadItem() (*Item, error)
	Close() error
}

func (m *Nitro) newFileWriter(t FileType) FileWriter {
	var w FileWriter
	if t == RawdbFile {
		w = &rawFileWriter{db: m}
	} else if t == ForestdbFile {
		w = &forestdbFileWriter{db: m}
	}
	return w
}

func (m *Nitro) newFileReader(t FileType) FileReader {
	var r FileReader
	if t == RawdbFile {
		r = &rawFileReader{db: m}
	} else if t == ForestdbFile {
		r = &forestdbFileReader{db: m}
	}
	return r
}

type rawFileWriter struct {
	db   *Nitro
	fd   *os.File
	w    *bufio.Writer
	buf  []byte
	path string
}

func (f *rawFileWriter) Open(path string) error {
	var err error
	f.fd, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0755)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.w = bufio.NewWriterSize(f.fd, DiskBlockSize)
	}
	return err
}

func (f *rawFileWriter) WriteItem(itm *Item) error {
	return f.db.EncodeItem(itm, f.buf, f.w)
}

func (f *rawFileWriter) Close() error {
	terminator := &Item{}

	if err := f.WriteItem(terminator); err != nil {
		return err
	}

	f.w.Flush()
	return f.fd.Close()
}

type rawFileReader struct {
	db   *Nitro
	fd   *os.File
	r    *bufio.Reader
	buf  []byte
	path string
}

func (f *rawFileReader) Open(path string) error {
	var err error
	f.fd, err = os.Open(path)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.r = bufio.NewReaderSize(f.fd, DiskBlockSize)
	}
	return err
}

func (f *rawFileReader) ReadItem() (*Item, error) {
	return f.db.DecodeItem(f.buf, f.r)
}

func (f *rawFileReader) Close() error {
	return f.fd.Close()
}

type forestdbFileWriter struct {
	db    *Nitro
	file  *forestdb.File
	store *forestdb.KVStore
	buf   []byte
	wbuf  bytes.Buffer
}

func (f *forestdbFileWriter) Open(path string) error {
	var err error
	f.file, err = forestdb.Open(path, forestdbConfig)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.store, err = f.file.OpenKVStoreDefault(nil)
	}

	return err
}

func (f *forestdbFileWriter) WriteItem(itm *Item) error {
	f.wbuf.Reset()
	err := f.db.EncodeItem(itm, f.buf, &f.wbuf)
	if err == nil {
		err = f.store.SetKV(f.wbuf.Bytes(), nil)
	}

	return err
}

func (f *forestdbFileWriter) Close() error {
	err := f.file.Commit(forestdb.COMMIT_NORMAL)
	if err == nil {
		err = f.store.Close()
		if err == nil {
			err = f.file.Close()
		}
	}

	return err
}

type forestdbFileReader struct {
	db    *Nitro
	file  *forestdb.File
	store *forestdb.KVStore
	iter  *forestdb.Iterator
	buf   []byte
}

func (f *forestdbFileReader) Open(path string) error {
	var err error

	f.file, err = forestdb.Open(path, forestdbConfig)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.store, err = f.file.OpenKVStoreDefault(nil)
		if err == nil {
			f.iter, err = f.store.IteratorInit(nil, nil, forestdb.ITR_NONE)
		}
	}

	return err
}

func (f *forestdbFileReader) ReadItem() (*Item, error) {
	itm := &Item{}
	doc, err := f.iter.Get()
	if err == forestdb.RESULT_ITERATOR_FAIL {
		return nil, nil
	}

	f.iter.Next()
	if err == nil {
		rbuf := bytes.NewBuffer(doc.Key())
		itm, err = f.db.DecodeItem(f.buf, rbuf)
	}

	return itm, err
}

func (f *forestdbFileReader) Close() error {
	f.iter.Close()
	f.store.Close()
	return f.file.Close()
}
<commit_msg>Remove support for forestdb files<commit_after>// Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package nitro

import "os"
import "bufio"
import "errors"

const DiskBlockSize = 512 * 1024

var (
	ErrNotEnoughSpace = errors.New("Not enough space in the buffer")
)

type FileWriter interface {
	Open(path string) error
	WriteItem(*Item) error
	Close() error
}

type FileReader interface {
	Open(path string) error
	ReadItem() (*Item, error)
	Close() error
}

func (m *Nitro) newFileWriter(t FileType) FileWriter {
	var w FileWriter
	if t == RawdbFile {
		w = &rawFileWriter{db: m}
	}
	return w
}

func (m *Nitro) newFileReader(t FileType) FileReader {
	var r FileReader
	if t == RawdbFile {
		r = &rawFileReader{db: m}
	}
	return r
}

type rawFileWriter struct {
	db   *Nitro
	fd   *os.File
	w    *bufio.Writer
	buf  []byte
	path string
}

func (f *rawFileWriter) Open(path string) error {
	var err error
	f.fd, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0755)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.w = bufio.NewWriterSize(f.fd, DiskBlockSize)
	}
	return err
}

func (f *rawFileWriter) WriteItem(itm *Item) error {
	return f.db.EncodeItem(itm, f.buf, f.w)
}

func (f *rawFileWriter) Close() error {
	terminator := &Item{}

	if err := f.WriteItem(terminator); err != nil {
		return err
	}

	f.w.Flush()
	return f.fd.Close()
}

type rawFileReader struct {
	db   *Nitro
	fd   *os.File
	r    *bufio.Reader
	buf  []byte
	path string
}

func (f *rawFileReader) Open(path string) error {
	var err error
	f.fd, err = os.Open(path)
	if err == nil {
		f.buf = make([]byte, encodeBufSize)
		f.r = bufio.NewReaderSize(f.fd, DiskBlockSize)
	}
	return err
}

func (f *rawFileReader) ReadItem() (*Item, error) {
	return f.db.DecodeItem(f.buf, f.r)
}

func (f *rawFileReader) Close() error {
	return f.fd.Close()
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package main

import (
	"github.com/jacobsa/gcloud/gcs"

	"bazil.org/fuse"
)

type file struct {
	bucket     gcs.Bucket
	objectName string
	size       uint64
}

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: 0400,
		Size: f.size,
	}
}
<commit_msg>Added the skeleton for a read-only file implementation.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package main

import (
	"os"
	"sync"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
)

// A remote object's name and metadata, along with a local temporary file that
// contains its contents (when initialized).
type file struct {
	bucket     gcs.Bucket
	objectName string
	size       uint64

	mu       sync.RWMutex
	tempFile *os.File // GUARDED_BY(mu)
}

// Make sure file implements the interfaces we think it does.
var (
	_ fs.Node           = &file{}
	_ fs.Handle         = &file{}
	_ fs.HandleReader   = &file{}
	_ fs.HandleReleaser = &file{}
)

func (f *file) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: 0400,
		Size: f.size,
	}
}

// If the file contents have not yet been fetched to a temporary file, fetch
// them.
func (f *file) ensureTempFile(ctx context.Context) error

// Throw away the local temporary file, if any.
func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error

// Ensure that the local temporary file is initialized, then read from it.
func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error
<|endoftext|>


<commit_before>package scipipe

import (
	"io/ioutil"
	"os"
	"time"
)

// ======= FileTarget ========

type FileTarget struct {
	path string
}

func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	return ft
}

func (ft *FileTarget) GetPath() string {
	return ft.path
}

func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

func (ft *FileTarget) Open() *os.File {
	f, err := os.Open(ft.GetPath())
	Check(err)
	return f
}

func (ft *FileTarget) Read() []byte {
	dat, err := ioutil.ReadFile(ft.GetPath())
	Check(err)
	return dat
}

func (ft *FileTarget) Write(dat []byte) {
	err := ioutil.WriteFile(ft.GetTempPath(), dat, 0644)
	ft.Atomize()
	Check(err)
}

func (ft *FileTarget) Atomize() {
	time.Sleep(0 * time.Millisecond) // TODO: Remove in production. Just for demo purposes!
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
}

func (ft *FileTarget) Exists() bool {
	if _, err := os.Stat(ft.GetPath()); err == nil {
		return true
	}
	return false
}

// ======= FileQueue =======

type FileQueue struct {
	process
	Out       chan *FileTarget
	FilePaths []string
}

func FQ(fps ...string) (fq *FileQueue) {
	return NewFileQueue(fps...)
}

func NewFileQueue(fps ...string) (fq *FileQueue) {
	filePaths := []string{}
	for _, fp := range fps {
		filePaths = append(filePaths, fp)
	}
	fq = &FileQueue{
		Out:       make(chan *FileTarget, BUFSIZE),
		FilePaths: filePaths,
	}
	return
}

func (proc *FileQueue) Run() {
	defer close(proc.Out)
	for _, fp := range proc.FilePaths {
		proc.Out <- NewFileTarget(fp)
	}
}

// ======= Sink =======

type Sink struct {
	process
	In chan *FileTarget
}

func NewSink() (s *Sink) {
	return &Sink{}
}

func (proc *Sink) Run() {
	for ft := range proc.In {
		Debug.Println("Received file in sink: ", ft.GetPath())
	}
}
<commit_msg>Add a buffer field to the FileTarget<commit_after>package scipipe

import (
	"bytes"
	"io/ioutil"
	"os"
	"time"
)

// ======= FileTarget ========

type FileTarget struct {
	path   string
	buffer *bytes.Buffer
	stream bool
}

func NewFileTarget(path string) *FileTarget {
	ft := new(FileTarget)
	ft.path = path
	//Don't init buffer if not needed?
	//buf := make([]byte, 0, 128)
	//ft.buffer = bytes.NewBuffer(buf)
	return ft
}

func (ft *FileTarget) GetPath() string {
	return ft.path
}

func (ft *FileTarget) GetTempPath() string {
	return ft.path + ".tmp"
}

func (ft *FileTarget) Open() *os.File {
	f, err := os.Open(ft.GetPath())
	Check(err)
	return f
}

func (ft *FileTarget) Read() []byte {
	dat, err := ioutil.ReadFile(ft.GetPath())
	Check(err)
	return dat
}

func (ft *FileTarget) Write(dat []byte) {
	err := ioutil.WriteFile(ft.GetTempPath(), dat, 0644)
	ft.Atomize()
	Check(err)
}

func (ft *FileTarget) Atomize() {
	time.Sleep(0 * time.Millisecond) // TODO: Remove in production. Just for demo purposes!
	err := os.Rename(ft.GetTempPath(), ft.path)
	Check(err)
}

func (ft *FileTarget) Exists() bool {
	if _, err := os.Stat(ft.GetPath()); err == nil {
		return true
	}
	return false
}

// ======= FileQueue =======

type FileQueue struct {
	process
	Out       chan *FileTarget
	FilePaths []string
}

func FQ(fps ...string) (fq *FileQueue) {
	return NewFileQueue(fps...)
}

func NewFileQueue(fps ...string) (fq *FileQueue) {
	filePaths := []string{}
	for _, fp := range fps {
		filePaths = append(filePaths, fp)
	}
	fq = &FileQueue{
		Out:       make(chan *FileTarget, BUFSIZE),
		FilePaths: filePaths,
	}
	return
}

func (proc *FileQueue) Run() {
	defer close(proc.Out)
	for _, fp := range proc.FilePaths {
		proc.Out <- NewFileTarget(fp)
	}
}

// ======= Sink =======

type Sink struct {
	process
	In chan *FileTarget
}

func NewSink() (s *Sink) {
	return &Sink{}
}

func (proc *Sink) Run() {
	for ft := range proc.In {
		Debug.Println("Received file in sink: ", ft.GetPath())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"strconv"

	"github.com/andrewstuart/limio"
)

var serveAPI = flag.Bool("serve", false, "serve the api")
var searchType = flag.String("t", "movie", "the type of search to perform")
var rateLimit = flag.String("r", "", "the rate limit")
var nc = flag.Bool("nocache", false, "skip cache")
var clr = flag.Bool("clear", false, "clear cache")

var downRate int

func init() {
	flag.Parse()

	if *searchType == "tv" {
		*searchType = "tvsearch"
	}

	if *rateLimit == "" {
		*rateLimit = os.Getenv("SAB_RATE")
	}

	if len(*rateLimit) > 0 {
		rl := []byte(*rateLimit)
		unit := rl[len(rl)-1]
		rl = rl[:len(rl)-1]

		qty, err := strconv.ParseFloat(string(rl), 64)

		if err != nil {
			log.Printf("Bad quantity: %s\n", *rateLimit)
		}

		switch unit {
		case 'm':
			downRate = int(qty * float64(limio.MB))
		case 'k':
			downRate = int(qty * float64(limio.KB))
		}
	}
}
<commit_msg>Update limio directory<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"strconv"

	"astuart.co/limio"
)

var serveAPI = flag.Bool("serve", false, "serve the api")
var searchType = flag.String("t", "movie", "the type of search to perform")
var rateLimit = flag.String("r", "", "the rate limit")
var nc = flag.Bool("nocache", false, "skip cache")
var clr = flag.Bool("clear", false, "clear cache")

var downRate int

func init() {
	flag.Parse()

	if *searchType == "tv" {
		*searchType = "tvsearch"
	}

	if *rateLimit == "" {
		*rateLimit = os.Getenv("SAB_RATE")
	}

	if len(*rateLimit) > 0 {
		rl := []byte(*rateLimit)
		unit := rl[len(rl)-1]
		rl = rl[:len(rl)-1]

		qty, err := strconv.ParseFloat(string(rl), 64)

		if err != nil {
			log.Printf("Bad quantity: %s\n", *rateLimit)
		}

		switch unit {
		case 'm':
			downRate = int(qty * float64(limio.MB))
		case 'k':
			downRate = int(qty * float64(limio.KB))
		}
	}
}
<|endoftext|>


<commit_before>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything expect remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<commit_msg>Fix wording for dry-run flag in useage message for garbage collector.<commit_after>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<|endoftext|>


<commit_before>package regressiontests

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
)

type Issue struct {
	Linter   string `json:"linter"`
	Severity string `json:"severity"`
	Path     string `json:"path"`
	Line     int    `json:"line"`
	Col      int    `json:"col"`
	Message  string `json:"message"`
}

func (i *Issue) String() string {
	col := ""
	if i.Col != 0 {
		col = fmt.Sprintf("%d", i.Col)
	}
	return fmt.Sprintf("%s:%d:%s:%s: %s (%s)", strings.TrimSpace(i.Path), i.Line, col, i.Severity, strings.TrimSpace(i.Message), i.Linter)
}

type Issues []Issue

func (e Issues) Len() int           { return len(e) }
func (e Issues) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
func (e Issues) Less(i, j int) bool { return e[i].String() < e[j].String() }

// ExpectIssues runs gometalinter and expects it to generate exactly the
// issues provided.
func ExpectIssues(t *testing.T, linter string, source string, expected Issues, extraFlags ...string) {
	// Write source to temporary directory.
	dir, err := ioutil.TempDir(".", "gometalinter-")
	if !assert.NoError(t, err) {
		return
	}
	defer os.RemoveAll(dir)
	w, err := os.Create(filepath.Join(dir, "test.go"))
	if !assert.NoError(t, err) {
		return
	}
	defer os.Remove(w.Name())
	_, err = w.WriteString(source)
	_ = w.Close()
	if !assert.NoError(t, err) {
		return
	}

	// Run gometalinter.
	args := []string{"go", "run", "../main.go", "../directives.go", "../config.go", "../checkstyle.go", "../aggregate.go", "--disable-all", "--enable", linter, "--json", dir}
	args = append(args, extraFlags...)
	cmd := exec.Command(args[0], args[1:]...)
	if !assert.NoError(t, err) {
		return
	}
	output, _ := cmd.Output()
	var actual Issues
	err = json.Unmarshal(output, &actual)
	if !assert.NoError(t, err) {
		fmt.Printf("Output: %s\n", output)
		return
	}

	// Remove output from other linters.
	actualForLinter := Issues{}
	for _, issue := range actual {
		if issue.Linter == linter || linter == "" {
			// Normalise path.
			issue.Path = "test.go"
			issue.Message = strings.Replace(issue.Message, w.Name(), "test.go", -1)
			issue.Message = strings.Replace(issue.Message, dir, "", -1)
			actualForLinter = append(actualForLinter, issue)
		}
	}
	sort.Sort(expected)
	sort.Sort(actualForLinter)

	assert.Equal(t, expected, actualForLinter)
}
<commit_msg>Update regressiontest support.go to allow for new files.<commit_after>package regressiontests

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

type Issue struct {
	Linter   string `json:"linter"`
	Severity string `json:"severity"`
	Path     string `json:"path"`
	Line     int    `json:"line"`
	Col      int    `json:"col"`
	Message  string `json:"message"`
}

func (i *Issue) String() string {
	col := ""
	if i.Col != 0 {
		col = fmt.Sprintf("%d", i.Col)
	}
	return fmt.Sprintf("%s:%d:%s:%s: %s (%s)", strings.TrimSpace(i.Path), i.Line, col, i.Severity, strings.TrimSpace(i.Message), i.Linter)
}

type Issues []Issue

func (e Issues) Len() int           { return len(e) }
func (e Issues) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
func (e Issues) Less(i, j int) bool { return e[i].String() < e[j].String() }

// ExpectIssues runs gometalinter and expects it to generate exactly the
// issues provided.
func ExpectIssues(t *testing.T, linter string, source string, expected Issues, extraFlags ...string) {
	// Write source to temporary directory.
	dir, err := ioutil.TempDir(".", "gometalinter-")
	require.NoError(t, err)
	defer os.RemoveAll(dir)

	testFile := filepath.Join(dir, "test.go")
	err = ioutil.WriteFile(testFile, []byte(source), 0644)
	require.NoError(t, err)

	// Run gometalinter.
	binary, cleanup := buildBinary(t)
	defer cleanup()
	args := []string{"-d", "--disable-all", "--enable", linter, "--json", dir}
	args = append(args, extraFlags...)
	cmd := exec.Command(binary, args...)
	errBuffer := new(bytes.Buffer)
	cmd.Stderr = errBuffer
	require.NoError(t, err)

	output, _ := cmd.Output()
	var actual Issues
	err = json.Unmarshal(output, &actual)
	if !assert.NoError(t, err) {
		fmt.Printf("Stderr: %s\n", errBuffer)
		fmt.Printf("Output: %s\n", output)
		return
	}

	// Remove output from other linters.
	actualForLinter := Issues{}
	for _, issue := range actual {
		if issue.Linter == linter || linter == "" {
			// Normalise path.
			issue.Path = "test.go"
			issue.Message = strings.Replace(issue.Message, testFile, "test.go", -1)
			issue.Message = strings.Replace(issue.Message, dir, "", -1)
			actualForLinter = append(actualForLinter, issue)
		}
	}
	sort.Sort(expected)
	sort.Sort(actualForLinter)

	if !assert.Equal(t, expected, actualForLinter) {
		fmt.Printf("Stderr: %s\n", errBuffer)
		fmt.Printf("Output: %s\n", output)
	}
}

func buildBinary(t *testing.T) (string, func()) {
	tmpdir, err := ioutil.TempDir("", "regression-test")
	require.NoError(t, err)
	path := filepath.Join(tmpdir, "binary")
	cmd := exec.Command("go", "build", "-o", path, "..")
	require.NoError(t, cmd.Run())
	return path, func() { os.RemoveAll(tmpdir) }
}
<|endoftext|>