<commit_before>// Package parse implements handling parse tasks for BUILD files.
//
// The actual work to interpret them is done by the //src/parse/asp package; this
// package handles requests for parsing build targets and triggering them to
// start building when ready.
package parse

import (
	"fmt"
	"path"
	"strings"

	"github.com/thought-machine/please/src/cli"
	"github.com/thought-machine/please/src/cli/logging"
	"github.com/thought-machine/please/src/core"
	"github.com/thought-machine/please/src/fs"
)

var log = logging.Log

// Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package.
// It is not an error if the package has already been parsed.
//
// By default, after the package is parsed, any targets that are now needed for the build and ready
// to be built are queued, and any new packages are queued for parsing. When a specific label is requested
// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it:
// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only
// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added.
// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed
// even when we're not otherwise building targets.
func Parse(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) {
	if err := parse(tid, state, label, dependent, forSubinclude); err != nil {
		state.LogBuildError(tid, label, core.ParseFailed, err, "Failed to parse package")
	}
}

func parse(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) error {
	if !forSubinclude {
		state.Parser.WaitForInit()
	}
	subrepo, err := checkSubrepo(tid, state, label, dependent, forSubinclude)
	if err != nil {
		return err
	}

	// See if something else has parsed this package first.
	pkg := state.SyncParsePackage(label)
	if pkg != nil {
		// Does exist, all we need to do is toggle on this target
		return activateTarget(state, pkg, label, dependent, forSubinclude)
	}
	// If we get here then it falls to us to parse this package.
	state.LogParseResult(tid, label, core.PackageParsing, "Parsing...")

	if subrepo != nil && subrepo.Target != nil {
		// We have got the definition of the subrepo but it depends on something, make sure that has been built.
		state.WaitForTargetAndEnsureDownload(subrepo.Target.Label, label)
		if err := subrepo.LoadSubrepoConfig(); err != nil {
			return err
		}
	}

	if subrepo != nil {
		state = subrepo.State
		if !forSubinclude {
			state.Parser.WaitForInit()
		}
	}

	// Subrepo & nothing else means we just want to ensure that subrepo is present.
	if label.Subrepo != "" && label.PackageName == "" && label.Name == "" {
		return nil
	}
	pkg, err = parsePackage(state, label, dependent, subrepo)

	if err != nil {
		return err
	}
	state.LogParseResult(tid, label, core.PackageParsed, "Parsed package")
	return activateTarget(state, pkg, label, dependent, forSubinclude)
}

// checkSubrepo checks whether this guy exists within a subrepo. If so we will need to make sure that's available first.
func checkSubrepo(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) (*core.Subrepo, error) {
	if label.Subrepo == "" {
		return nil, nil
	} else if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	}

	sl := label.SubrepoLabel(state, dependent.Subrepo)

	// Local subincludes are when we subinclude from a subrepo defined in the current package
	localSubinclude := sl.PackageName == dependent.PackageName && forSubinclude

	// If we're including from the same package, we don't want to parse the subrepo package
	if !localSubinclude {
		if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, "", label); err != nil {
			return nil, err
		} else if !handled {
			// They may have meant a subrepo that was defined in the dependent label's subrepo rather than the host
			// repo
			if _, err := parseSubrepoPackage(tid, state, sl.PackageName, dependent.Subrepo, label); err != nil {
				return nil, err
			}
		}
	}
	if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	} else if subrepo := checkArchSubrepo(state, label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	if !localSubinclude {
		// Fix for #577; fallback like above, it might be defined within the subrepo.
		if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, dependent.Subrepo, label); handled && err == nil {
			return state.Graph.Subrepo(label.Subrepo), nil
		}
		return nil, fmt.Errorf("Subrepo %s is not defined (referenced by %s)", label.Subrepo, dependent)
	}
	// For local subincludes, the subrepo has to already be defined at this point in the BUILD file
	return nil, fmt.Errorf("Subrepo %v is not defined yet. It must appear before it is used by subinclude()", sl)
}

// parseSubrepoPackage parses a package to make sure subrepos are available.
func parseSubrepoPackage(tid int, state *core.BuildState, pkg, subrepo string, dependent core.BuildLabel) (bool, error) {
	if state.Graph.Package(pkg, subrepo) == nil {
		// Don't have it already, must parse.
		label := core.BuildLabel{Subrepo: subrepo, PackageName: pkg, Name: "all"}
		return true, parse(tid, state, label, dependent, true)
	}
	return false, nil
}

// checkArchSubrepo checks if a target refers to a cross-compiling subrepo.
// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to.
func checkArchSubrepo(state *core.BuildState, name string) *core.Subrepo {
	var arch cli.Arch
	if err := arch.UnmarshalFlag(name); err == nil {
		return state.Graph.MaybeAddSubrepo(core.SubrepoForArch(state, arch))
	}
	return nil
}

// activateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses.
func activateTarget(state *core.BuildState, pkg *core.Package, label, dependent core.BuildLabel, forSubinclude bool) error {
	if !label.IsAllTargets() && state.Graph.Target(label) == nil {
		if label.Subrepo == "" && label.PackageName == "" && label.Name == dependent.Subrepo {
			if subrepo := checkArchSubrepo(state, label.Name); subrepo != nil {
				state.ArchSubrepoInitialised(label)
				return nil
			}
		}
		if state.Config.Bazel.Compatibility && forSubinclude {
			// Bazel allows some things that look like build targets but aren't - notably the syntax
			// to load(). It suits us to treat that as though it is one, but we now have to
			// implicitly make it available.
			exportFile(state, pkg, label)
		} else {
			msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name)
			if dependent != core.OriginalTarget {
				msg += fmt.Sprintf(" (depended on by %s)", dependent)
			}
			return fmt.Errorf(msg + suggestTargets(pkg, label, dependent))
		}
	}
	if state.ParsePackageOnly && !forSubinclude {
		return nil // Some kinds of query don't need a full recursive parse.
	} else if label.IsAllTargets() {
		if dependent == core.OriginalTarget {
			for _, target := range pkg.AllTargets() {
				// Don't activate targets that were added in a post-build function; that causes a race condition
				// between the post-build functions running and other things trying to activate them too early.
				if state.ShouldInclude(target) && !target.AddedPostBuild {
					// Must always do this for coverage because we need to calculate sources of
					// non-test targets later on.
					if !state.NeedTests || target.IsTest() || state.NeedCoverage {
						if err := state.QueueTarget(target.Label, dependent, dependent.IsAllTargets()); err != nil {
							return err
						}
					}
				}
			}
		}
	} else {
		for _, l := range state.Graph.DependentTargets(dependent, label) {
			// We use :all to indicate a dependency needed for parse.
			if err := state.QueueTarget(l, dependent, forSubinclude || dependent.IsAllTargets()); err != nil {
				return err
			}
		}
	}
	return nil
}

// parsePackage parses a BUILD file and adds the package to the build graph
func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) {
	packageName := label.PackageName
	pkg := core.NewPackage(packageName)
	pkg.Subrepo = subrepo
	if subrepo != nil {
		pkg.SubrepoName = subrepo.Name
	}
	// Only load the internal package for the host repo's state
	if state.ParentState == nil && packageName == InternalPackageName {
		pkgStr, err := GetInternalPackage(state.Config)
		if err != nil {
			return nil, fmt.Errorf("failed to generate internal package: %w", err)
		}
		if err := state.Parser.ParseReader(pkg, strings.NewReader(pkgStr)); err != nil {
			return nil, fmt.Errorf("failed to parse internal package: %w", err)
		}
	} else {
		filename, dir := buildFileName(state, label.PackageName, subrepo)
		if filename != "" {
			pkg.Filename = filename
			if err := state.Parser.ParseFile(pkg, pkg.Filename); err != nil {
				return nil, err
			}
		} else {
			exists := core.PathExists(dir)
			// Handle quite a few cases to provide more obvious error messages.
			if dependent != core.OriginalTarget && exists {
				return nil, fmt.Errorf("%s depends on %s, but there's no %s file in %s/", dependent, label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			} else if dependent != core.OriginalTarget {
				return nil, fmt.Errorf("%s depends on %s, but the directory %s doesn't exist: %s", dependent, label, dir, packageName)
			} else if exists {
				return nil, fmt.Errorf("Can't build %s; there's no %s file in %s/", label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			}
			return nil, fmt.Errorf("Can't build %s; the directory %s doesn't exist", label, dir)
		}
	}

	// Verifies some details of the output files. This can only be perfomed after the whole package has been parsed as
	// it guarantees that all necessary information between targets has been retrieved.
	if state.Config.FeatureFlags.PackageOutputsStrictness {
		go pkg.MustVerifyOutputs()
	} else {
		go pkg.VerifyOutputs()
	}

	state.Graph.AddPackage(pkg) // Calling this means nobody else will add entries to pendingTargets for this package.
	return pkg, nil
}

// buildFileName returns the name of the BUILD file for a package, or the empty string if one
// doesn't exist. It also returns the directory that it looked in.
func buildFileName(state *core.BuildState, pkgName string, subrepo *core.Subrepo) (string, string) {
	config := state.Config
	if subrepo != nil {
		pkgName = subrepo.Dir(pkgName)
		config = subrepo.State.Config
	}
	// Bazel defines targets in its "external" package from its WORKSPACE file.
	// We will fake this by treating that as an actual package file...
	// TODO(peterebden): They may be moving away from their "external" nomenclature?
	if state.Config.Bazel.Compatibility && pkgName == "external" || pkgName == "workspace" {
		return "WORKSPACE", ""
	}
	for _, buildFileName := range config.Parse.BuildFileName {
		if filename := path.Join(pkgName, buildFileName); fs.FileExists(filename) {
			return filename, pkgName
		}
	}
	return "", pkgName
}

// exportFile adds a single-file export target. This is primarily used for Bazel compat.
func exportFile(state *core.BuildState, pkg *core.Package, label core.BuildLabel) {
	t := core.NewBuildTarget(label)
	t.Subrepo = pkg.Subrepo
	t.IsFilegroup = true
	t.AddSource(core.NewFileLabel(label.Name, pkg))
	state.AddTarget(pkg, t)
}
<commit_msg>Fix plugin_repo erroring out on top level BUILD (#2432)<commit_after>// Package parse implements handling parse tasks for BUILD files.
//
// The actual work to interpret them is done by the //src/parse/asp package; this
// package handles requests for parsing build targets and triggering them to
// start building when ready.
package parse

import (
	"fmt"
	"path"
	"strings"

	"github.com/thought-machine/please/src/cli"
	"github.com/thought-machine/please/src/cli/logging"
	"github.com/thought-machine/please/src/core"
	"github.com/thought-machine/please/src/fs"
)

var log = logging.Log

// Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package.
// It is not an error if the package has already been parsed.
//
// By default, after the package is parsed, any targets that are now needed for the build and ready
// to be built are queued, and any new packages are queued for parsing. When a specific label is requested
// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it:
// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only
// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added.
// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed
// even when we're not otherwise building targets.
func Parse(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) {
	if err := parse(tid, state, label, dependent, forSubinclude); err != nil {
		state.LogBuildError(tid, label, core.ParseFailed, err, "Failed to parse package")
	}
}

func parse(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) error {
	if !forSubinclude {
		state.Parser.WaitForInit()
	}
	subrepo, err := checkSubrepo(tid, state, label, dependent, forSubinclude)
	if err != nil {
		return err
	}

	// See if something else has parsed this package first.
	pkg := state.SyncParsePackage(label)
	if pkg != nil {
		// Does exist, all we need to do is toggle on this target
		return activateTarget(state, pkg, label, dependent, forSubinclude)
	}
	// If we get here then it falls to us to parse this package.
	state.LogParseResult(tid, label, core.PackageParsing, "Parsing...")

	if subrepo != nil && subrepo.Target != nil {
		// We have got the definition of the subrepo but it depends on something, make sure that has been built.
		state.WaitForTargetAndEnsureDownload(subrepo.Target.Label, label)
		if err := subrepo.LoadSubrepoConfig(); err != nil {
			return err
		}
	}

	if subrepo != nil {
		state = subrepo.State
		if !forSubinclude {
			state.Parser.WaitForInit()
		}
	}

	// Subrepo & nothing else means we just want to ensure that subrepo is present.
	if label.Subrepo != "" && label.PackageName == "" && label.Name == "" {
		return nil
	}
	pkg, err = parsePackage(state, label, dependent, subrepo)

	if err != nil {
		return err
	}
	state.LogParseResult(tid, label, core.PackageParsed, "Parsed package")
	return activateTarget(state, pkg, label, dependent, forSubinclude)
}

// checkSubrepo checks whether this guy exists within a subrepo. If so we will need to make sure that's available first.
func checkSubrepo(tid int, state *core.BuildState, label, dependent core.BuildLabel, forSubinclude bool) (*core.Subrepo, error) {
	if label.Subrepo == "" {
		return nil, nil
	} else if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	}

	sl := label.SubrepoLabel(state, dependent.Subrepo)

	// Local subincludes are when we subinclude from a subrepo defined in the current package
	localSubinclude := label.Subrepo == dependent.Subrepo && label.PackageName == dependent.PackageName && forSubinclude

	// If we're including from the same package, we don't want to parse the subrepo package
	if !localSubinclude {
		if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, "", label); err != nil {
			return nil, err
		} else if !handled {
			// They may have meant a subrepo that was defined in the dependent label's subrepo rather than the host
			// repo
			if _, err := parseSubrepoPackage(tid, state, sl.PackageName, dependent.Subrepo, label); err != nil {
				return nil, err
			}
		}
	}
	if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	} else if subrepo := checkArchSubrepo(state, label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	if !localSubinclude {
		// Fix for #577; fallback like above, it might be defined within the subrepo.
		if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, dependent.Subrepo, label); handled && err == nil {
			return state.Graph.Subrepo(label.Subrepo), nil
		}
		return nil, fmt.Errorf("Subrepo %s is not defined (referenced by %s)", label.Subrepo, dependent)
	}
	// For local subincludes, the subrepo has to already be defined at this point in the BUILD file
	return nil, fmt.Errorf("Subrepo %v is not defined yet. It must appear before it is used by subinclude()", sl)
}

// parseSubrepoPackage parses a package to make sure subrepos are available.
func parseSubrepoPackage(tid int, state *core.BuildState, pkg, subrepo string, dependent core.BuildLabel) (bool, error) {
	if state.Graph.Package(pkg, subrepo) == nil {
		// Don't have it already, must parse.
		label := core.BuildLabel{Subrepo: subrepo, PackageName: pkg, Name: "all"}
		return true, parse(tid, state, label, dependent, true)
	}
	return false, nil
}

// checkArchSubrepo checks if a target refers to a cross-compiling subrepo.
// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to.
func checkArchSubrepo(state *core.BuildState, name string) *core.Subrepo {
	var arch cli.Arch
	if err := arch.UnmarshalFlag(name); err == nil {
		return state.Graph.MaybeAddSubrepo(core.SubrepoForArch(state, arch))
	}
	return nil
}

// activateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses.
func activateTarget(state *core.BuildState, pkg *core.Package, label, dependent core.BuildLabel, forSubinclude bool) error {
	if !label.IsAllTargets() && state.Graph.Target(label) == nil {
		if label.Subrepo == "" && label.PackageName == "" && label.Name == dependent.Subrepo {
			if subrepo := checkArchSubrepo(state, label.Name); subrepo != nil {
				state.ArchSubrepoInitialised(label)
				return nil
			}
		}
		if state.Config.Bazel.Compatibility && forSubinclude {
			// Bazel allows some things that look like build targets but aren't - notably the syntax
			// to load(). It suits us to treat that as though it is one, but we now have to
			// implicitly make it available.
			exportFile(state, pkg, label)
		} else {
			msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name)
			if dependent != core.OriginalTarget {
				msg += fmt.Sprintf(" (depended on by %s)", dependent)
			}
			return fmt.Errorf(msg + suggestTargets(pkg, label, dependent))
		}
	}
	if state.ParsePackageOnly && !forSubinclude {
		return nil // Some kinds of query don't need a full recursive parse.
	} else if label.IsAllTargets() {
		if dependent == core.OriginalTarget {
			for _, target := range pkg.AllTargets() {
				// Don't activate targets that were added in a post-build function; that causes a race condition
				// between the post-build functions running and other things trying to activate them too early.
				if state.ShouldInclude(target) && !target.AddedPostBuild {
					// Must always do this for coverage because we need to calculate sources of
					// non-test targets later on.
					if !state.NeedTests || target.IsTest() || state.NeedCoverage {
						if err := state.QueueTarget(target.Label, dependent, dependent.IsAllTargets()); err != nil {
							return err
						}
					}
				}
			}
		}
	} else {
		for _, l := range state.Graph.DependentTargets(dependent, label) {
			// We use :all to indicate a dependency needed for parse.
			if err := state.QueueTarget(l, dependent, forSubinclude || dependent.IsAllTargets()); err != nil {
				return err
			}
		}
	}
	return nil
}

// parsePackage parses a BUILD file and adds the package to the build graph
func parsePackage(state *core.BuildState, label, dependent core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) {
	packageName := label.PackageName
	pkg := core.NewPackage(packageName)
	pkg.Subrepo = subrepo
	if subrepo != nil {
		pkg.SubrepoName = subrepo.Name
	}
	// Only load the internal package for the host repo's state
	if state.ParentState == nil && packageName == InternalPackageName {
		pkgStr, err := GetInternalPackage(state.Config)
		if err != nil {
			return nil, fmt.Errorf("failed to generate internal package: %w", err)
		}
		if err := state.Parser.ParseReader(pkg, strings.NewReader(pkgStr)); err != nil {
			return nil, fmt.Errorf("failed to parse internal package: %w", err)
		}
	} else {
		filename, dir := buildFileName(state, label.PackageName, subrepo)
		if filename != "" {
			pkg.Filename = filename
			if err := state.Parser.ParseFile(pkg, pkg.Filename); err != nil {
				return nil, err
			}
		} else {
			exists := core.PathExists(dir)
			// Handle quite a few cases to provide more obvious error messages.
			if dependent != core.OriginalTarget && exists {
				return nil, fmt.Errorf("%s depends on %s, but there's no %s file in %s/", dependent, label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			} else if dependent != core.OriginalTarget {
				return nil, fmt.Errorf("%s depends on %s, but the directory %s doesn't exist: %s", dependent, label, dir, packageName)
			} else if exists {
				return nil, fmt.Errorf("Can't build %s; there's no %s file in %s/", label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			}
			return nil, fmt.Errorf("Can't build %s; the directory %s doesn't exist", label, dir)
		}
	}

	// Verifies some details of the output files. This can only be perfomed after the whole package has been parsed as
	// it guarantees that all necessary information between targets has been retrieved.
	if state.Config.FeatureFlags.PackageOutputsStrictness {
		go pkg.MustVerifyOutputs()
	} else {
		go pkg.VerifyOutputs()
	}

	state.Graph.AddPackage(pkg) // Calling this means nobody else will add entries to pendingTargets for this package.
	return pkg, nil
}

// buildFileName returns the name of the BUILD file for a package, or the empty string if one
// doesn't exist. It also returns the directory that it looked in.
func buildFileName(state *core.BuildState, pkgName string, subrepo *core.Subrepo) (string, string) {
	config := state.Config
	if subrepo != nil {
		pkgName = subrepo.Dir(pkgName)
		config = subrepo.State.Config
	}
	// Bazel defines targets in its "external" package from its WORKSPACE file.
	// We will fake this by treating that as an actual package file...
	// TODO(peterebden): They may be moving away from their "external" nomenclature?
	if state.Config.Bazel.Compatibility && pkgName == "external" || pkgName == "workspace" {
		return "WORKSPACE", ""
	}
	for _, buildFileName := range config.Parse.BuildFileName {
		if filename := path.Join(pkgName, buildFileName); fs.FileExists(filename) {
			return filename, pkgName
		}
	}
	return "", pkgName
}

// exportFile adds a single-file export target. This is primarily used for Bazel compat.
func exportFile(state *core.BuildState, pkg *core.Package, label core.BuildLabel) {
	t := core.NewBuildTarget(label)
	t.Subrepo = pkg.Subrepo
	t.IsFilegroup = true
	t.AddSource(core.NewFileLabel(label.Name, pkg))
	state.AddTarget(pkg, t)
}
<|endoftext|>


<commit_before>package testflight_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("When a resource type depends on another resource type", func() {
	BeforeEach(func() {
		setAndUnpausePipeline("fixtures/recursive-resource-checking.yml")
	})

	It("can be checked shallowly and errors when parent has no version", func() {
		check := spawnFly("check-resource", "-r", inPipeline("recursive-custom-resource"), "--shallow")
		<-check.Exited
		Expect(check).To(gexec.Exit(1))
		Expect(check.Err).To(gbytes.Say("resource's parent type '.*' has no version"))
	})

	It("can be checked recursively", func() {
		check := fly("check-resource", "-r", inPipeline("recursive-custom-resource"))
		Expect(check).To(gbytes.Say("mock-resource-parent.*succeeded"))
		Expect(check).To(gbytes.Say("mock-resource-child.*succeeded"))
		Expect(check).To(gbytes.Say("mock-resource-grandchild.*succeeded"))
		Expect(check).To(gbytes.Say("recursive-custom-resource.*succeeded"))
	})
})
<commit_msg>Update testflight/custom_resource_check_test.go<commit_after>package testflight_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("When a resource type depends on another resource type", func() {
	BeforeEach(func() {
		setAndUnpausePipeline("fixtures/recursive-resource-checking.yml")
	})

	It("can be checked shallowly and errors when parent has no version", func() {
		check := spawnFly("check-resource", "-r", inPipeline("recursive-custom-resource"), "--shallow")
		<-check.Exited
		Expect(check).To(gexec.Exit(1))
		Expect(check.Err).To(gbytes.Say("resource type '.*' has no version"))
	})

	It("can be checked recursively", func() {
		check := fly("check-resource", "-r", inPipeline("recursive-custom-resource"))
		Expect(check).To(gbytes.Say("mock-resource-parent.*succeeded"))
		Expect(check).To(gbytes.Say("mock-resource-child.*succeeded"))
		Expect(check).To(gbytes.Say("mock-resource-grandchild.*succeeded"))
		Expect(check).To(gbytes.Say("recursive-custom-resource.*succeeded"))
	})
})
<|endoftext|>


<commit_before>package signalfx

import (
	"math"
	"sync"
	"sync/atomic"

	"github.com/zvelo/go-signalfx/sfxproto"
)

// A Bucket trakcs groups of values, reporting the min/max as gauges, and
// count/sum/sum of squares as a cumulative counter. All operations on Buckets
// are goroutine safe.
type Bucket struct {
	metric       string
	dimensions   sfxproto.Dimensions
	count        uint64
	min          int64
	max          int64
	sum          int64
	sumOfSquares int64
	mu           sync.Mutex
}

func (b *Bucket) lock() {
	b.mu.Lock()
}

func (b *Bucket) unlock() {
	b.mu.Unlock()
}

// Clone makes a deep copy of a Bucket
func (b *Bucket) Clone() *Bucket {
	// use the mutex to keep all operations part of the same transaction
	b.lock()
	defer b.unlock()

	return &Bucket{
		metric:       b.metric,             // can't use Metric() since we already have a lock
		dimensions:   b.dimensions.Clone(), // can't use Dimensions() since we already have a lock
		count:        b.Count(),
		min:          b.Min(),
		max:          b.Max(),
		sum:          b.Sum(),
		sumOfSquares: b.SumOfSquares(),
	}
}

// Equal returns whether two buckets are exactly equal
func (b *Bucket) Equal(r *Bucket) bool {
	// lock the state of both buckets as this operation is effectively a
	// transaction on an exact state of a whole bucket

	b.lock()
	defer b.unlock()

	r.lock()
	defer r.unlock()

	if b.metric != r.metric {
		return false
	}

	if !b.dimensions.Equal(r.dimensions) {
		return false
	}

	if b.Count() != r.Count() {
		return false
	}

	if b.Min() != r.Min() {
		return false
	}

	if b.Max() != r.Max() {
		return false
	}

	if b.Sum() != r.Sum() {
		return false
	}

	if b.SumOfSquares() != r.SumOfSquares() {
		return false
	}

	return true
}

// Metric returns the metric name of the Bucket
func (b *Bucket) Metric() string {
	b.lock()
	defer b.unlock()

	return b.metric
}

// SetMetric sets the metric name of the Bucket
func (b *Bucket) SetMetric(name string) {
	b.lock()
	defer b.unlock()

	b.metric = name
}

// Dimensions returns a copy of the dimensions of the Bucket. Changes are not
// reflected inside the Bucket itself.
func (b *Bucket) Dimensions() sfxproto.Dimensions {
	b.lock()
	defer b.unlock()

	return b.dimensions.Clone()
}

// SetDimension adds or overwrites the dimension at key with value. If the key
// or value is empty, no changes are made
func (b *Bucket) SetDimension(key, value string) {
	if key == "" || value == "" {
		return
	}

	b.lock()
	defer b.unlock()

	b.dimensions[key] = value
}

// SetDimensions adds or overwrites multiple dimensions. Because the passed in
// dimensions can not be locked by this method, it is important that the caller
// ensures its state does not change for the duration of the operation.
func (b *Bucket) SetDimensions(dims sfxproto.Dimensions) {
	for key, value := range dims {
		b.SetDimension(key, value)
	}
}

// RemoveDimension removes one or more dimensions with the given keys
func (b *Bucket) RemoveDimension(keys ...string) {
	b.lock()
	defer b.unlock()

	for _, key := range keys {
		delete(b.dimensions, key)
	}
}

// Count returns the number of items added to the Bucket
func (b *Bucket) Count() uint64 {
	return atomic.LoadUint64(&b.count)
}

// Min returns the lowest item added to the Bucket
func (b *Bucket) Min() int64 {
	return atomic.LoadInt64(&b.min)
}

// Max returns the highest item added to the Bucket
func (b *Bucket) Max() int64 {
	return atomic.LoadInt64(&b.max)
}

// Sum returns the sum of all items added to the Bucket
func (b *Bucket) Sum() int64 {
	return atomic.LoadInt64(&b.sum)
}

// SumOfSquares returns the sum of the square of all items added to the Bucket
func (b *Bucket) SumOfSquares() int64 {
	return atomic.LoadInt64(&b.sumOfSquares)
}

// NewBucket creates a new Bucket. Because the passed in dimensions can not be
// locked by this method, it is important that the caller ensures its state does
// not change for the duration of the operation.
func NewBucket(metric string, dimensions sfxproto.Dimensions) *Bucket {
	ret := &Bucket{
		metric:     metric,
		dimensions: dimensions.Clone(),
	}
	return ret
}

// Add an item to the Bucket, later reporting the result in the next report
// cycle.
func (b *Bucket) Add(val int64) {
	// still use atomic though, so that the atomic "getters" will never read
	// from an inconsistent state
	count := atomic.AddUint64(&b.count, 1)
	atomic.AddInt64(&b.sum, val)
	atomic.AddInt64(&b.sumOfSquares, val*val)

	if count == 1 {
		if cur := b.Min(); cur == 0 {
			if !atomic.CompareAndSwapInt64(&b.min, cur, val) {
				b.setMin(val)
			}
		} else {
			b.setMin(val)
		}

		if cur := b.Max(); cur == 0 {
			if !atomic.CompareAndSwapInt64(&b.max, cur, val) {
				b.setMax(val)
			}
		} else {
			b.setMax(val)
		}

		return
	}

	b.setMin(val)
	b.setMax(val)
}

func (b *Bucket) setMin(val int64) {
	for {
		if cur := b.Min(); cur > val || cur == math.MaxInt64 {
			if atomic.CompareAndSwapInt64(&b.min, cur, val) {
				break
			}
		} else {
			break
		}
	}
}

func (b *Bucket) setMax(val int64) {
	for {
		if cur := b.Max(); cur < val || cur == math.MinInt64 {
			if atomic.CompareAndSwapInt64(&b.max, cur, val) {
				break
			}
		} else {
			break
		}
	}
}

func (b *Bucket) dimFor(defaultDims sfxproto.Dimensions, rollup string) sfxproto.Dimensions {
	b.lock()
	defer b.unlock()

	dims := defaultDims.Append(b.dimensions)
	dims["rollup"] = rollup

	return dims
}

// CountDataPoint returns a DataPoint representing the Bucket's Count. Because
// the passed in dimensions can not be locked by this method, it is important
// that the caller ensures its state does not change for the duration of the
// operation.
func (b *Bucket) CountDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.Count(), b.dimFor(defaultDims, "count"))
	return dp
}

// SumDataPoint returns a DataPoint representing the Bucket's Sum. Because the
// passed in dimensions can not be locked by this method, it is important that
// the caller ensures its state does not change for the duration of the
// operation.
func (b *Bucket) SumDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.Sum(), b.dimFor(defaultDims, "sum"))
	return dp
}

// SumOfSquaresDataPoint returns a DataPoint representing the Bucket's
// SumOfSquares. Because the passed in dimensions can not be locked by this
// method, it is important that the caller ensures its state does not change for
// the duration of the operation.
func (b *Bucket) SumOfSquaresDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.SumOfSquares(), b.dimFor(defaultDims, "sumsquare"))
	return dp
}

// MinDataPoint returns a DataPoint representing the Bucket's Min. Note that
// this resets the Min value. nil is returned if no items have been added to the
// bucket since it was created or last reset. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) MinDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	min := atomic.SwapInt64(&b.min, math.MaxInt64)

	if b.Count() != 0 && min != math.MaxInt64 {
		dp, _ := NewGauge(b.Metric()+".min", min, b.dimFor(defaultDims, "min"))
		return dp
	}

	return nil
}

// MaxDataPoint returns a DataPoint representing the Bucket's Max. Note that
// this resets the Max value. nil is returned if no items have been added to the
// bucket since it was created or last reset. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) MaxDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	max := atomic.SwapInt64(&b.max, math.MinInt64)

	if b.Count() != 0 && max != math.MinInt64 {
		dp, _ := NewGauge(b.Metric()+".max", max, b.dimFor(defaultDims, "max"))
		return dp
	}

	return nil
}

// DataPoints returns a DataPoints object with DataPoint values for Count, Sum,
// SumOfSquares, Min and Max (if set). Note that this resets both the Min and
// Max values. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) DataPoints(defaultDims sfxproto.Dimensions) *DataPoints {
	return NewDataPoints(5).
		Add(b.CountDataPoint(defaultDims)).
		Add(b.SumDataPoint(defaultDims)).
		Add(b.SumOfSquaresDataPoint(defaultDims)).
		Add(b.MinDataPoint(defaultDims)).
		Add(b.MaxDataPoint(defaultDims))
}
<commit_msg>revert unnecessary change<commit_after>package signalfx

import (
	"math"
	"sync"
	"sync/atomic"

	"github.com/zvelo/go-signalfx/sfxproto"
)

// A Bucket trakcs groups of values, reporting the min/max as gauges, and
// count/sum/sum of squares as a cumulative counter. All operations on Buckets
// are goroutine safe.
type Bucket struct {
	metric       string
	dimensions   sfxproto.Dimensions
	count        uint64
	min          int64
	max          int64
	sum          int64
	sumOfSquares int64
	mu           sync.Mutex
}

func (b *Bucket) lock() {
	b.mu.Lock()
}

func (b *Bucket) unlock() {
	b.mu.Unlock()
}

// Clone makes a deep copy of a Bucket
func (b *Bucket) Clone() *Bucket {
	// use the mutex to keep all operations part of the same transaction
	b.lock()
	defer b.unlock()

	return &Bucket{
		metric:       b.metric,             // can't use Metric() since we already have a lock
		dimensions:   b.dimensions.Clone(), // can't use Dimensions() since we already have a lock
		count:        b.Count(),
		min:          b.Min(),
		max:          b.Max(),
		sum:          b.Sum(),
		sumOfSquares: b.SumOfSquares(),
	}
}

// Equal returns whether two buckets are exactly equal
func (b *Bucket) Equal(r *Bucket) bool {
	// lock the state of both buckets as this operation is effectively a
	// transaction on an exact state of a whole bucket

	b.lock()
	defer b.unlock()

	r.lock()
	defer r.unlock()

	if b.metric != r.metric {
		return false
	}

	if !b.dimensions.Equal(r.dimensions) {
		return false
	}

	if b.Count() != r.Count() {
		return false
	}

	if b.Min() != r.Min() {
		return false
	}

	if b.Max() != r.Max() {
		return false
	}

	if b.Sum() != r.Sum() {
		return false
	}

	if b.SumOfSquares() != r.SumOfSquares() {
		return false
	}

	return true
}

// Metric returns the metric name of the Bucket
func (b *Bucket) Metric() string {
	b.lock()
	defer b.unlock()

	return b.metric
}

// SetMetric sets the metric name of the Bucket
func (b *Bucket) SetMetric(name string) {
	b.lock()
	defer b.unlock()

	b.metric = name
}

// Dimensions returns a copy of the dimensions of the Bucket. Changes are not
// reflected inside the Bucket itself.
func (b *Bucket) Dimensions() sfxproto.Dimensions {
	b.lock()
	defer b.unlock()

	return b.dimensions.Clone()
}

// SetDimension adds or overwrites the dimension at key with value. If the key
// or value is empty, no changes are made
func (b *Bucket) SetDimension(key, value string) {
	if key == "" || value == "" {
		return
	}

	b.lock()
	defer b.unlock()

	b.dimensions[key] = value
}

// SetDimensions adds or overwrites multiple dimensions. Because the passed in
// dimensions can not be locked by this method, it is important that the caller
// ensures its state does not change for the duration of the operation.
func (b *Bucket) SetDimensions(dims sfxproto.Dimensions) {
	for key, value := range dims {
		b.SetDimension(key, value)
	}
}

// RemoveDimension removes one or more dimensions with the given keys
func (b *Bucket) RemoveDimension(keys ...string) {
	b.lock()
	defer b.unlock()

	for _, key := range keys {
		delete(b.dimensions, key)
	}
}

// Count returns the number of items added to the Bucket
func (b *Bucket) Count() uint64 {
	return atomic.LoadUint64(&b.count)
}

// Min returns the lowest item added to the Bucket
func (b *Bucket) Min() int64 {
	return atomic.LoadInt64(&b.min)
}

// Max returns the highest item added to the Bucket
func (b *Bucket) Max() int64 {
	return atomic.LoadInt64(&b.max)
}

// Sum returns the sum of all items added to the Bucket
func (b *Bucket) Sum() int64 {
	return atomic.LoadInt64(&b.sum)
}

// SumOfSquares returns the sum of the square of all items added to the Bucket
func (b *Bucket) SumOfSquares() int64 {
	return atomic.LoadInt64(&b.sumOfSquares)
}

// NewBucket creates a new Bucket. Because the passed in dimensions can not be
// locked by this method, it is important that the caller ensures its state does
// not change for the duration of the operation.
func NewBucket(metric string, dimensions sfxproto.Dimensions) *Bucket {
	return &Bucket{
		metric:     metric,
		dimensions: dimensions.Clone(),
	}
}

// Add an item to the Bucket, later reporting the result in the next report
// cycle.
func (b *Bucket) Add(val int64) {
	// still use atomic though, so that the atomic "getters" will never read
	// from an inconsistent state
	count := atomic.AddUint64(&b.count, 1)
	atomic.AddInt64(&b.sum, val)
	atomic.AddInt64(&b.sumOfSquares, val*val)

	if count == 1 {
		if cur := b.Min(); cur == 0 {
			if !atomic.CompareAndSwapInt64(&b.min, cur, val) {
				b.setMin(val)
			}
		} else {
			b.setMin(val)
		}

		if cur := b.Max(); cur == 0 {
			if !atomic.CompareAndSwapInt64(&b.max, cur, val) {
				b.setMax(val)
			}
		} else {
			b.setMax(val)
		}

		return
	}

	b.setMin(val)
	b.setMax(val)
}

func (b *Bucket) setMin(val int64) {
	for {
		if cur := b.Min(); cur > val || cur == math.MaxInt64 {
			if atomic.CompareAndSwapInt64(&b.min, cur, val) {
				break
			}
		} else {
			break
		}
	}
}

func (b *Bucket) setMax(val int64) {
	for {
		if cur := b.Max(); cur < val || cur == math.MinInt64 {
			if atomic.CompareAndSwapInt64(&b.max, cur, val) {
				break
			}
		} else {
			break
		}
	}
}

func (b *Bucket) dimFor(defaultDims sfxproto.Dimensions, rollup string) sfxproto.Dimensions {
	b.lock()
	defer b.unlock()

	dims := defaultDims.Append(b.dimensions)
	dims["rollup"] = rollup

	return dims
}

// CountDataPoint returns a DataPoint representing the Bucket's Count. Because
// the passed in dimensions can not be locked by this method, it is important
// that the caller ensures its state does not change for the duration of the
// operation.
func (b *Bucket) CountDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.Count(), b.dimFor(defaultDims, "count"))
	return dp
}

// SumDataPoint returns a DataPoint representing the Bucket's Sum. Because the
// passed in dimensions can not be locked by this method, it is important that
// the caller ensures its state does not change for the duration of the
// operation.
func (b *Bucket) SumDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.Sum(), b.dimFor(defaultDims, "sum"))
	return dp
}

// SumOfSquaresDataPoint returns a DataPoint representing the Bucket's
// SumOfSquares. Because the passed in dimensions can not be locked by this
// method, it is important that the caller ensures its state does not change for
// the duration of the operation.
func (b *Bucket) SumOfSquaresDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	dp, _ := NewCounter(b.Metric(), b.SumOfSquares(), b.dimFor(defaultDims, "sumsquare"))
	return dp
}

// MinDataPoint returns a DataPoint representing the Bucket's Min. Note that
// this resets the Min value. nil is returned if no items have been added to the
// bucket since it was created or last reset. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) MinDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	min := atomic.SwapInt64(&b.min, math.MaxInt64)

	if b.Count() != 0 && min != math.MaxInt64 {
		dp, _ := NewGauge(b.Metric()+".min", min, b.dimFor(defaultDims, "min"))
		return dp
	}

	return nil
}

// MaxDataPoint returns a DataPoint representing the Bucket's Max. Note that
// this resets the Max value. nil is returned if no items have been added to the
// bucket since it was created or last reset. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) MaxDataPoint(defaultDims sfxproto.Dimensions) *DataPoint {
	max := atomic.SwapInt64(&b.max, math.MinInt64)

	if b.Count() != 0 && max != math.MinInt64 {
		dp, _ := NewGauge(b.Metric()+".max", max, b.dimFor(defaultDims, "max"))
		return dp
	}

	return nil
}

// DataPoints returns a DataPoints object with DataPoint values for Count, Sum,
// SumOfSquares, Min and Max (if set). Note that this resets both the Min and
// Max values. Because the passed in dimensions
// can not be locked by this method, it is important that the caller ensures its
// state does not change for the duration of the operation.
func (b *Bucket) DataPoints(defaultDims sfxproto.Dimensions) *DataPoints {
	return NewDataPoints(5).
		Add(b.CountDataPoint(defaultDims)).
		Add(b.SumDataPoint(defaultDims)).
		Add(b.SumOfSquaresDataPoint(defaultDims)).
		Add(b.MinDataPoint(defaultDims)).
		Add(b.MaxDataPoint(defaultDims))
}
<|endoftext|>


<commit_before>package cypress

import "sync"

var pbBufPool sync.Pool

func init() {
	pbBufPool.New = newBuf
}

func newBuf() interface{} {
	return make([]byte, 128)
}
<commit_msg>Use a closure rather than a standalone func<commit_after>package cypress

import "sync"

var pbBufPool sync.Pool

func init() {
	pbBufPool.New = func() interface{} {
		return make([]byte, 128)
	}
}
<|endoftext|>


<commit_before>package buffer

import (
	"errors"
	"fmt"
	"os"
	"sync"
	"syscall"

	"github.com/ashishgandhi/buffer/binary"
)

// buffer file format related sizes
const (
	// head
	size     = 4 // bytes in frame header used to store size of frame
	sequence = 8 // bytes in frame header used to store sequence of record
	total    = size + sequence

	// tail
	metadata = 4 << 10 // bytes in file at the end reserved for metadata
)

var maxBytes uint32 = (1 << (uint(size) * 8)) - total // the biggest size of data a frame can store

type frame []byte

var (
	errFrameTooBig = fmt.Errorf("buffer: data more than max %d bytes", maxBytes)
	errDataTooBig  = errors.New("buffer: data overflows buffer")
)

func newFrame(seq uint64, data []byte) (frame, error) {
	if len(data) > int(maxBytes) {
		return nil, errFrameTooBig
	}
	var f frame = make([]byte, len(data)+total)
	binary.PutLittleEndianUint32(f, 0, uint32(len(f)))
	binary.PutLittleEndianUint64(f, size, seq)
	copy(f[total:], data)
	return f, nil
}

// size returns the size of frame encoded in the frame.
// If the complete frame was read it'd match the length.
// It returns 0 if not enough of the frame has been read
// to determine the encoded size.
func (f frame) size() uint32 {
	if len(f) < size {
		return 0
	}
	return binary.GetLittleEndianUint32(f, 0)
}

// seq returns the record sequence encoded in the frame.
// It returns 0 if not enough of the frame has been read
// to determine the encoded sequence.
func (f frame) seq() uint64 {
	if len(f) < total {
		return 0
	}
	return binary.GetLittleEndianUint64(f, size)
}

// data returns the record data stored in the frame.
// It returns nil if not enough of the frame has been
// read to read at least one byte of data.
//
// The underlying array of the slice returned may be
// recycled.
func (f frame) data() []byte {
	if len(f) < total {
		return nil
	}
	return f[total:]
}

type Buffer struct {
	capacity uint64
	nextSeq  uint64
	biggest  uint32 // largest frame seen, doesn't reset but helps frameOfLen
	length   uint64 // total data stored

	// These are start offsets for first and
	// last records maintained by the buffer.
	// They are abosulte and not wrapped.
	first uint64
	last  uint64

	filename string
	data     []byte

	mu sync.RWMutex // protects whole Buffer
}

func New(capacity int, filename string) (*Buffer, error) {
	if _, err := os.Stat(filename); !os.IsNotExist(err) {
		return nil, ErrFileExists
	}

	f, err := os.Create(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fsize := capacity + metadata
	if err := syscall.Truncate(filename, int64(fsize)); err != nil {
		return nil, err
	}

	data, err := syscall.Mmap(
		int(f.Fd()), 0, fsize,
		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED,
	)
	// TODO(ashish): Call msync periodically.
	if err != nil {
		return nil, err
	}

	b := &Buffer{
		capacity: uint64(capacity), // since it's used with uint64s a lot more
		filename: f.Name(),
		data:     data,
	}
	b.updateMeta()

	return b, nil
}

// Len returns the total data bytes stored in the buffer.
// This does not include the space used for framing.
func (b *Buffer) Len() uint64 {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.length
}

// Remaining returns the total bytes remaining to be read
// before reading everything in the buffer.
func (b *Buffer) Remaining(from Cursor) uint64 {
	b.mu.RLock()
	defer b.mu.RUnlock()

	if from.offset > b.last {
		return 0
	}

	off := from.offset
	if off < b.first {
		off = b.first
	}
	remaining := b.last - off
	remaining += uint64(b.frameSize(b.last))
	return remaining
}

// NOTE: No memlock because do not want to deal with RLIMIT_MEMLOCK
// on the edge boxes yet.

func (b *Buffer) updateMeta() {
	// First 8 bytes have the first frame offset,
	// next 8 bytes have the last frame offset,
	// next 8 bytes are the next sequence number,
	// next 4 bytes are the biggest data record we've seen,
	// next 8 bytes are the total data in the buffer.
	off := int(b.capacity)
	binary.PutLittleEndianUint64(b.data, off, b.first)
	binary.PutLittleEndianUint64(b.data, off+8, b.last)
	binary.PutLittleEndianUint64(b.data, off+16, b.nextSeq)
	binary.PutLittleEndianUint32(b.data, off+24, b.biggest)
	binary.PutLittleEndianUint64(b.data, off+28, b.length)
}

func Load(filename string) (*Buffer, error) {
	b := &Buffer{filename: filename}
	err := b.reload()
	return b, err
}

// reload reads back info about b from b.filename.
// b.filename should be set.
func (b *Buffer) reload() error {
	stat, err := os.Stat(b.filename)
	if err != nil {
		return err
	}
	fsize := uint64(stat.Size())
	b.capacity = fsize - metadata

	f, err := os.OpenFile(b.filename, os.O_RDWR, 0600)
	if err != nil {
		return err
	}

	data, err := syscall.Mmap(
		int(f.Fd()), 0, int(fsize),
		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED,
	)
	if err != nil {
		return err
	}
	b.data = data

	off := int(b.capacity)
	b.first = binary.GetLittleEndianUint64(b.data, off)
	b.last = binary.GetLittleEndianUint64(b.data, off+8)
	b.nextSeq = binary.GetLittleEndianUint64(b.data, off+16)
	b.biggest = binary.GetLittleEndianUint32(b.data, off+24)
	b.length = binary.GetLittleEndianUint64(b.data, off+28)

	return nil
}

// Insert adds data into b. Caller is free to recycle the underlying
// array of the slice.
func (b *Buffer) Insert(data []byte) error {
	if len(data) == 0 {
		return nil
	}

	b.mu.Lock()
	defer b.mu.Unlock()

	frame, err := newFrame(b.nextSeq, data) // can be avoided
	if err != nil {
		return err
	}

	fsize := frame.size()
	fsize64 := uint64(fsize)
	if fsize64 > b.capacity {
		return errDataTooBig
	}

	b.last = b.nextFrameOffset(b.last) // new last where we'd start writing frame
	b.updateFirst(fsize64)

	start := b.last % b.capacity
	end := start + fsize64

	var wrap bool
	if end > b.capacity {
		wrap = true
		end = b.capacity
	}
	copyoff := end - start
	copy(b.data[start:end], frame[:copyoff])
	if wrap {
		copy(b.data, frame[copyoff:])
	}

	if fsize > b.biggest {
		b.biggest = fsize
	}
	b.nextSeq++
	b.length += uint64(len(data))
	b.updateMeta()
	return nil
}

// updateFirst updates the offset of the first record in b.
//
// This should be called before updating b.biggest or writing any data but
// after updating b.last to the value where the new record would be written.
func (b *Buffer) updateFirst(fsize uint64) {
	if b.biggest == 0 {
		// Just starting out, no need to update.
		return
	}

	var (
		start    = b.last % b.capacity
		end      = (start + fsize) % b.capacity
		wrapping = end <= start
	)

	if start == end {
		b.length = 0
		b.first = b.last
		return
	}

	for {
		if b.first == b.last {
			// b can fit only the new incoming record.
			return
		}

		firstWrapped := b.first % b.capacity

		if wrapping {
			if end <= firstWrapped && firstWrapped < start {
				return
			}
		} else {
			if end <= firstWrapped {
				return
			}
			if start > firstWrapped {
				return
			}
		}

		second := b.nextFrameOffset(b.first)
		b.length -= (second - b.first - total)
		b.first = second

		// May need to discard multiple records at the begining.
	}
}

// nextFrameOffset returns the offset from where the next frame
// should start. It is an absolute offset and not wrapped around.
func (b *Buffer) nextFrameOffset(offset uint64) uint64 {
	return offset + uint64(b.frameSize(offset))
}

// frame returns the frame encoded at offset.
func (b *Buffer) frame(offset uint64) frame {
	return b.frameOfLen(offset, b.frameSize(offset))
}

// frameSize returns the size of the frame encoded at offset.
func (b *Buffer) frameSize(offset uint64) uint32 {
	f := b.frameOfLen(offset, size)
	return f.size()
}

// frameOfLen returns a frame of length starting at offset. The size
// of frame may differ from the length.
func (b *Buffer) frameOfLen(offset uint64, length uint32) frame {
	if length > b.biggest {
		// This can happen, say, when a non-header region is
		// interpreted as a header region and may have junk
		// frame length.
		return nil
	}

	var f frame

	start := offset % b.capacity
	end := start + uint64(length)

	if end > b.capacity {
		// The record wraps around. Until we mmap the file back-to-back
		// we cannot slice b.data and need to copy the wrapped contents.
		f = make([]byte, length)
		copy(f, b.data[start:b.capacity])
		copy(f[b.capacity-start:], b.data[:end%b.capacity])
	} else {
		f = b.data[start:end]
	}
	return f
}

var (
	ErrFileExists  = errors.New("buffer: log file already exists")
	ErrPartialData = errors.New("buffer: partial data returned")
	ErrNotArrived  = errors.New("buffer: data has not yet arrived")
)

// Read copies record that c points to into data. It returns number of bytes copied,
// a cursor to the next record, and error if there was any.
//
// If c points to a record that has been ejected from b the call is analogous to
// b.ReadFirst.
//
// If c points to a record that has not yet arrived an ErrNotArrived error is returned.
//
// If data does not have enough room for record an ErrPartialData error is returned.
func (b *Buffer) Read(data []byte, c Cursor) (n int, next Cursor, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	seq, offset := c.seq, c.offset

	if seq >= b.nextSeq || offset > b.last {
		return 0, next, ErrNotArrived
	}

	f := b.frame(offset)
	if f.size() == 0 || f.seq() != seq {
		return b.readFirst(data)
	}

	return b.readOffset(data, offset)
}

// ReadFirst reads the first (oldest) record in b. It's return values are analogous
// to Read.
func (b *Buffer) ReadFirst(data []byte) (n int, next Cursor, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.readOffset(data, b.first)
}

func (b *Buffer) readFirst(data []byte) (n int, next Cursor, err error) {
	return b.readOffset(data, b.first)
}

func (b *Buffer) readOffset(data []byte, offset uint64) (n int, next Cursor, err error) {
	if b.biggest == 0 {
		return 0, next, ErrNotArrived
	}

	f := b.frame(offset)
	d := f.data()
	n = copy(data, d)

	if n != len(d) {
		err = ErrPartialData
	}

	next.offset = b.nextFrameOffset(offset)

	if next.offset-offset == b.capacity {
		// There is only one frame in b.
		next.seq = b.nextSeq
		return
	}

	if offset == b.last {
		next.seq = b.nextSeq
	} else {
		nextf := b.frame(next.offset)
		next.seq = nextf.seq()
	}

	return
}

var pageSize uint64

func init() {
	pageSize = uint64(os.Getpagesize())
}

// Unmap unmaps the buffer file from memory.
func (b *Buffer) Unmap() {
	if err := syscall.Munmap(b.data); err != nil {
		// Munmap should only fail if we pass it a bad pointer
		// which should not happen. If it does something has
		// gone terribly wrong and should not proceed further.
		panic(err)
	}
}

// Close calls b.Unmap and deletes the buffer file.
func (b *Buffer) Close() error {
	b.Unmap()
	return os.Remove(b.filename)
}

type Cursor struct {
	offset uint64
	seq    uint64
}

// Seq returns the sequence number of the record c points
// to in Buffer. Sequence numbers are sequential.
func (c Cursor) Seq() uint64 {
	return c.seq
}
<commit_msg>bug fix: unable to read from buffer when offset is less than offset of first record<commit_after>package buffer

import (
	"errors"
	"fmt"
	"os"
	"sync"
	"syscall"

	"github.com/ashishgandhi/buffer/binary"
)

// buffer file format related sizes
const (
	// head
	size     = 4 // bytes in frame header used to store size of frame
	sequence = 8 // bytes in frame header used to store sequence of record
	total    = size + sequence

	// tail
	metadata = 4 << 10 // bytes in file at the end reserved for metadata
)

var maxBytes uint32 = (1 << (uint(size) * 8)) - total // the biggest size of data a frame can store

type frame []byte

var (
	errFrameTooBig = fmt.Errorf("buffer: data more than max %d bytes", maxBytes)
	errDataTooBig  = errors.New("buffer: data overflows buffer")
)

func newFrame(seq uint64, data []byte) (frame, error) {
	if len(data) > int(maxBytes) {
		return nil, errFrameTooBig
	}
	var f frame = make([]byte, len(data)+total)
	binary.PutLittleEndianUint32(f, 0, uint32(len(f)))
	binary.PutLittleEndianUint64(f, size, seq)
	copy(f[total:], data)
	return f, nil
}

// size returns the size of frame encoded in the frame.
// If the complete frame was read it'd match the length.
// It returns 0 if not enough of the frame has been read
// to determine the encoded size.
func (f frame) size() uint32 {
	if len(f) < size {
		return 0
	}
	return binary.GetLittleEndianUint32(f, 0)
}

// seq returns the record sequence encoded in the frame.
// It returns 0 if not enough of the frame has been read
// to determine the encoded sequence.
func (f frame) seq() uint64 {
	if len(f) < total {
		return 0
	}
	return binary.GetLittleEndianUint64(f, size)
}

// data returns the record data stored in the frame.
// It returns nil if not enough of the frame has been
// read to read at least one byte of data.
//
// The underlying array of the slice returned may be
// recycled.
func (f frame) data() []byte {
	if len(f) < total {
		return nil
	}
	return f[total:]
}

type Buffer struct {
	capacity uint64
	nextSeq  uint64
	biggest  uint32 // largest frame seen, doesn't reset but helps frameOfLen
	length   uint64 // total data stored

	// These are start offsets for first and
	// last records maintained by the buffer.
	// They are abosulte and not wrapped.
	first uint64
	last  uint64

	filename string
	data     []byte

	mu sync.RWMutex // protects whole Buffer
}

func New(capacity int, filename string) (*Buffer, error) {
	if _, err := os.Stat(filename); !os.IsNotExist(err) {
		return nil, ErrFileExists
	}

	f, err := os.Create(filename)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	fsize := capacity + metadata
	if err := syscall.Truncate(filename, int64(fsize)); err != nil {
		return nil, err
	}

	data, err := syscall.Mmap(
		int(f.Fd()), 0, fsize,
		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED,
	)
	// TODO(ashish): Call msync periodically.
	if err != nil {
		return nil, err
	}

	b := &Buffer{
		capacity: uint64(capacity), // since it's used with uint64s a lot more
		filename: f.Name(),
		data:     data,
	}
	b.updateMeta()

	return b, nil
}

// Len returns the total data bytes stored in the buffer.
// This does not include the space used for framing.
func (b *Buffer) Len() uint64 {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.length
}

// Remaining returns the total bytes remaining to be read
// before reading everything in the buffer.
func (b *Buffer) Remaining(from Cursor) uint64 {
	b.mu.RLock()
	defer b.mu.RUnlock()

	if from.offset > b.last {
		return 0
	}

	off := from.offset
	if off < b.first {
		off = b.first
	}
	remaining := b.last - off
	remaining += uint64(b.frameSize(b.last))
	return remaining
}

// NOTE: No memlock because do not want to deal with RLIMIT_MEMLOCK
// on the edge boxes yet.

func (b *Buffer) updateMeta() {
	// First 8 bytes have the first frame offset,
	// next 8 bytes have the last frame offset,
	// next 8 bytes are the next sequence number,
	// next 4 bytes are the biggest data record we've seen,
	// next 8 bytes are the total data in the buffer.
	off := int(b.capacity)
	binary.PutLittleEndianUint64(b.data, off, b.first)
	binary.PutLittleEndianUint64(b.data, off+8, b.last)
	binary.PutLittleEndianUint64(b.data, off+16, b.nextSeq)
	binary.PutLittleEndianUint32(b.data, off+24, b.biggest)
	binary.PutLittleEndianUint64(b.data, off+28, b.length)
}

func Load(filename string) (*Buffer, error) {
	b := &Buffer{filename: filename}
	err := b.reload()
	return b, err
}

// reload reads back info about b from b.filename.
// b.filename should be set.
func (b *Buffer) reload() error {
	stat, err := os.Stat(b.filename)
	if err != nil {
		return err
	}
	fsize := uint64(stat.Size())
	b.capacity = fsize - metadata

	f, err := os.OpenFile(b.filename, os.O_RDWR, 0600)
	if err != nil {
		return err
	}

	data, err := syscall.Mmap(
		int(f.Fd()), 0, int(fsize),
		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED,
	)
	if err != nil {
		return err
	}
	b.data = data

	off := int(b.capacity)
	b.first = binary.GetLittleEndianUint64(b.data, off)
	b.last = binary.GetLittleEndianUint64(b.data, off+8)
	b.nextSeq = binary.GetLittleEndianUint64(b.data, off+16)
	b.biggest = binary.GetLittleEndianUint32(b.data, off+24)
	b.length = binary.GetLittleEndianUint64(b.data, off+28)

	return nil
}

// Insert adds data into b. Caller is free to recycle the underlying
// array of the slice.
func (b *Buffer) Insert(data []byte) error {
	if len(data) == 0 {
		return nil
	}

	b.mu.Lock()
	defer b.mu.Unlock()

	frame, err := newFrame(b.nextSeq, data) // can be avoided
	if err != nil {
		return err
	}

	fsize := frame.size()
	fsize64 := uint64(fsize)
	if fsize64 > b.capacity {
		return errDataTooBig
	}

	b.last = b.nextFrameOffset(b.last) // new last where we'd start writing frame
	b.updateFirst(fsize64)

	start := b.last % b.capacity
	end := start + fsize64

	var wrap bool
	if end > b.capacity {
		wrap = true
		end = b.capacity
	}
	copyoff := end - start
	copy(b.data[start:end], frame[:copyoff])
	if wrap {
		copy(b.data, frame[copyoff:])
	}

	if fsize > b.biggest {
		b.biggest = fsize
	}
	b.nextSeq++
	b.length += uint64(len(data))
	b.updateMeta()
	return nil
}

// updateFirst updates the offset of the first record in b.
//
// This should be called before updating b.biggest or writing any data but
// after updating b.last to the value where the new record would be written.
func (b *Buffer) updateFirst(fsize uint64) {
	if b.biggest == 0 {
		// Just starting out, no need to update.
		return
	}

	var (
		start    = b.last % b.capacity
		end      = (start + fsize) % b.capacity
		wrapping = end <= start
	)

	if start == end {
		b.length = 0
		b.first = b.last
		return
	}

	for {
		if b.first == b.last {
			// b can fit only the new incoming record.
			return
		}

		firstWrapped := b.first % b.capacity

		if wrapping {
			if end <= firstWrapped && firstWrapped < start {
				return
			}
		} else {
			if end <= firstWrapped {
				return
			}
			if start > firstWrapped {
				return
			}
		}

		second := b.nextFrameOffset(b.first)
		b.length -= (second - b.first - total)
		b.first = second

		// May need to discard multiple records at the begining.
	}
}

// nextFrameOffset returns the offset from where the next frame
// should start. It is an absolute offset and not wrapped around.
func (b *Buffer) nextFrameOffset(offset uint64) uint64 {
	return offset + uint64(b.frameSize(offset))
}

// frame returns the frame encoded at offset.
func (b *Buffer) frame(offset uint64) frame {
	return b.frameOfLen(offset, b.frameSize(offset))
}

// frameSize returns the size of the frame encoded at offset.
func (b *Buffer) frameSize(offset uint64) uint32 {
	f := b.frameOfLen(offset, size)
	return f.size()
}

// frameOfLen returns a frame of length starting at offset. The size
// of frame may differ from the length.
func (b *Buffer) frameOfLen(offset uint64, length uint32) frame {
	if length > b.biggest {
		// This can happen, say, when a non-header region is
		// interpreted as a header region and may have junk
		// frame length.
		return nil
	}

	var f frame

	start := offset % b.capacity
	end := start + uint64(length)

	if end > b.capacity {
		// The record wraps around. Until we mmap the file back-to-back
		// we cannot slice b.data and need to copy the wrapped contents.
		f = make([]byte, length)
		copy(f, b.data[start:b.capacity])
		copy(f[b.capacity-start:], b.data[:end%b.capacity])
	} else {
		f = b.data[start:end]
	}
	return f
}

var (
	ErrFileExists  = errors.New("buffer: log file already exists")
	ErrPartialData = errors.New("buffer: partial data returned")
	ErrNotArrived  = errors.New("buffer: data has not yet arrived")
)

// Read copies record that c points to into data. It returns number of bytes copied,
// a cursor to the next record, and error if there was any.
//
// If c points to a record that has been ejected from b the call is analogous to
// b.ReadFirst.
//
// If c points to a record that has not yet arrived an ErrNotArrived error is returned.
//
// If data does not have enough room for record an ErrPartialData error is returned.
func (b *Buffer) Read(data []byte, c Cursor) (n int, next Cursor, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()

	seq, offset := c.seq, c.offset

	if seq >= b.nextSeq || offset > b.last {
		return 0, next, ErrNotArrived
	}

	f := b.frame(offset)
	if f.size() == 0 || f.seq() != seq || offset < b.first {
		return b.readFirst(data)
	}

	return b.readOffset(data, offset)
}

// ReadFirst reads the first (oldest) record in b. It's return values are analogous
// to Read.
func (b *Buffer) ReadFirst(data []byte) (n int, next Cursor, err error) {
	b.mu.RLock()
	defer b.mu.RUnlock()
	return b.readOffset(data, b.first)
}

func (b *Buffer) readFirst(data []byte) (n int, next Cursor, err error) {
	return b.readOffset(data, b.first)
}

func (b *Buffer) readOffset(data []byte, offset uint64) (n int, next Cursor, err error) {
	if b.biggest == 0 {
		return 0, next, ErrNotArrived
	}

	f := b.frame(offset)
	d := f.data()
	n = copy(data, d)

	if n != len(d) {
		err = ErrPartialData
	}

	next.offset = b.nextFrameOffset(offset)

	if next.offset-offset == b.capacity {
		// There is only one frame in b.
		next.seq = b.nextSeq
		return
	}

	if offset == b.last {
		next.seq = b.nextSeq
	} else {
		nextf := b.frame(next.offset)
		next.seq = nextf.seq()
	}

	return
}

var pageSize uint64

func init() {
	pageSize = uint64(os.Getpagesize())
}

// Unmap unmaps the buffer file from memory.
func (b *Buffer) Unmap() {
	if err := syscall.Munmap(b.data); err != nil {
		// Munmap should only fail if we pass it a bad pointer
		// which should not happen. If it does something has
		// gone terribly wrong and should not proceed further.
		panic(err)
	}
}

// Close calls b.Unmap and deletes the buffer file.
func (b *Buffer) Close() error {
	b.Unmap()
	return os.Remove(b.filename)
}

type Cursor struct {
	offset uint64
	seq    uint64
}

// Seq returns the sequence number of the record c points
// to in Buffer. Sequence numbers are sequential.
func (c Cursor) Seq() uint64 {
	return c.seq
}
<|endoftext|>


<commit_before>package cl11

import (
	"fmt"
	"reflect"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A buffer object stores a one-dimensional collection of elements (though there
// are "rectangle" operations). Elements of a buffer object can be a scalar data
// type (such as an int, float), vector data type, or a user-defined structure.
type Buffer struct {
	id clw.Mem

	// The context the buffer was created on.
	Context *Context

	// The size of the buffer in bytes.
	Size int64

	// Usage information for the buffer from the device's point of view.
	Flags MemFlags

	// The host backed memory for the buffer, if applicable.
	Host interface{}

	// The parent buffer, if applicable.
	Buffer *Buffer

	// The offset in bytes within the parent buffer.
	Origin int64
}

// Creates a buffer object on the device.
//
// Creates an uninitialized buffer on the device. The size is in bytes.
func (c *Context) CreateDeviceBuffer(size int64, mf MemFlags) (*Buffer, error) {

	memory, err := clw.CreateBuffer(c.id, clw.MemFlags(mf), clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

// Create a buffer object on the device.
//
// Create a buffer object initialized with the passed value. The size and
// contents are determined by the passed value.
func (c *Context) CreateDeviceBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

// Create a buffer object for the device backed by host memory.
//
// Create a device accessible buffer object on the host. The host value must be
// addressable. The OpenCL implementation is allowed to cache the data on the
// device.
func (c *Context) CreateDeviceBufferFromHostMem(mf MemFlags, host interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemUseHostPointer

	value := reflect.ValueOf(host)
	if kind := value.Kind(); kind != reflect.Ptr && kind != reflect.Slice {
		return nil, wrapError(fmt.Errorf("host value not addressable"))
	} else if kind == reflect.Ptr {
		for {
			value = value.Elem()
			if value.Kind() != reflect.Ptr {
				break
			}
		}
	}
	pointer, size := addressablePointerAndSize(value)

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Host: host, Flags: mf}, nil
}

// Creates a buffer object that is host accessible.
//
// Creates an uninitialized buffer in pinned memory. The size is in bytes. This
// memory is not pageable and allows for DMA copies (which are faster).
func (c *Context) CreateHostBuffer(size int64, mf MemFlags) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

// Creates a buffer object that is host accessible.
//
// Creates an initialized buffer in pinned memory. The size and contents are
// determined by value. This memory is not pageable and allows for DMA copies
// (which are faster).
func (c *Context) CreateHostBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

// Creates a buffer object from an existing object with the passed offset and
// size in bytes.
//
// If the size and offset are the same as another sub buffer the implementation
// may return the same sub buffer and increment the reference count.
func (b *Buffer) CreateSubBuffer(mf MemFlags, origin, size int64) (*Buffer, error) {

	region := clw.BufferRegion{clw.Size(origin), clw.Size(size)}

	memory, err := clw.CreateSubBuffer(b.id, clw.MemFlags(mf), region)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: b.Context, Size: size, Flags: mf, Buffer: b, Origin: origin}, nil
}

// Increments the buffer object reference count.
//
// The OpenCL commands that return a buffer perform an implicit retain.
func (b *Buffer) Retain() error {
	return clw.RetainMemObject(b.id)
}

// Decrements the buffer object reference count.
//
// After the buffers reference count becomes zero and commands queued for
// execution that use the buffer have finished the buffer is deleted.
func (b *Buffer) Release() error {
	return clw.ReleaseMemObject(b.id)
}

// Return the buffer's reference count.
//
// The reference count returned should be considered immediately stale. It is
// unsuitable for general use in applications. This feature is provided for
// identifying memory leaks.
func (b *Buffer) ReferenceCount() (int, error) {

	var count clw.Uint
	err := clw.GetMemObjectInfo(b.id, clw.MemReferenceCount, clw.Size(unsafe.Sizeof(count)), unsafe.Pointer(&count),
		nil)
	if err != nil {
		return 0, err
	}

	return int(count), nil
}

// Enqueues a command to copy from one buffer object to another.
//
// Source offset, destination offset, and size are in bytes.
func (cq *CommandQueue) CopyBuffer(src, dst *Buffer, srcOffset, dstOffset, size int64, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBuffer(cq.id, src.id, dst.id, clw.Size(srcOffset), clw.Size(dstOffset), clw.Size(size),
		cq.toEvents(waitList), event)
}

// Enqueues a command to copy a rectangular region from the buffer object to
// another buffer object.
//
// See Rect definition for how source and destination are defined.
func (cq *CommandQueue) CopyBufferRect(src, dst *Buffer, r *Rect, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBufferRect(cq.id, src.id, dst.id, r.srcOrigin(), r.dstOrigin(), r.region(), r.srcRowPitch(),
		r.srcSlicePitch(), r.dstRowPitch(), r.dstSlicePitch(), cq.toEvents(waitList), event)
}

// Enqueues a command to map a region of the buffer object given by buffer into
// the host address space.
func (cq *CommandQueue) MapBuffer(b *Buffer, bc BlockingCall, flags MapFlags, offset, size int64, waitList []*Event,
	e *Event) (*MappedBuffer, error) {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandMapBuffer
		e.CommandQueue = cq
	}

	pointer, err := clw.EnqueueMapBuffer(cq.id, b.id, clw.Bool(bc), clw.MapFlags(flags), clw.Size(offset),
		clw.Size(size), cq.toEvents(waitList), event)
	if err != nil {
		return nil, err
	}

	return &MappedBuffer{pointer, size, b.id}, nil
}

// Enqueue commands to read from a buffer object to host memory.
//
// Offset is in bytes. The destination must be addressable. If the buffer object
// is backed by host memory then all commands that use it and sub buffers must
// have finished execution and it must not be mapped otherwise the results are
// undefined.
func (cq *CommandQueue) ReadBuffer(b *Buffer, bc BlockingCall, offset int64, dst interface{}, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	err = clw.EnqueueReadBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	return nil
}

// Enqueue commands to write to a buffer object from host memory.
//
// Offset is in bytes. If the buffer object is backed by host memory then all
// commands that use it and sub buffers must have finished execution and it must
// not be mapped otherwise the results are undefined. If the source is
// addressable a reference will be held to prevent it being garbage collected,
// it is the user's responsibility to ensure that the source data is valid at
// the time the write is actually performed. If the source isn't addressable a
// copy will be created.
func (cq *CommandQueue) WriteBuffer(b *Buffer, bc BlockingCall, offset int64, src interface{}, waitList []*Event,
	e *Event) error {

	// Ensure we always have an event if not blocking. The event will be used to
	// register a callback. Thus the source data is guaranteed to be referenced
	// somewhere to preventing it from being garbage collected. Once the event
	// has completed and the callback is triggered (doing nothing) the reference
	// to the source data will be removed allowing it to be garbage collected.
	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandWriteBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(src)

	if err != nil {
		// The source value is not addressable so create a local copy of it.
		var scratch [scratchSize]byte
		pointer, size = getPointerAndSize(src, unsafe.Pointer(&scratch[0]))
		src = &scratch
	}

	err = clw.EnqueueWriteBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	// Set a no-op callback, just need hold a reference to the source data.
	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, src)
		if err != nil {
			return err
		}
	}

	return nil
}

// Enqueue commands to read from a rectangular region from a buffer object to
// host memory.
//
// See Rect definition for offset is defined. The destination must be
// addressable. If the buffer object is backed by host memory then all commands
// that use it and sub buffers must have finished execution and it must not be
// mapped otherwise the results are undefined.
func (cq *CommandQueue) ReadBufferRect(b *Buffer, bc BlockingCall, offset *Rect, dst interface{}, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, _, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	err = clw.EnqueueReadBufferRect(cq.id, b.id, clw.Bool(bc), offset.dstOrigin(), offset.srcOrigin(), offset.region(),
		offset.dstRowPitch(), offset.dstSlicePitch(), offset.srcRowPitch(), offset.dstRowPitch(), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	return nil
}

// Enqueue commands to write a rectangular region to a buffer object from host
// memory.
//
// See Rect definition for offset is defined. If the buffer object is backed by
// host memory then all commands that use it and sub buffers must have finished
// execution and it must not be mapped otherwise the results are undefined. If
// the source is addressable a reference will be held to prevent it being
// garbage collected, it is the user's responsibility to ensure that the source
// data is valid at the time the write is actually performed. If the source
// isn't addressable a copy will be created.
func (cq *CommandQueue) WriteBufferRect(b *Buffer, bc BlockingCall, offset *Rect, src interface{}, waitList []*Event,
	e *Event) error {

	// Ensure we always have an event if not blocking. The event will be used to
	// register a callback. Thus the source data is guaranteed to be referenced
	// somewhere to preventing it from being garbage collected. Once the event
	// has completed and the callback is triggered (doing nothing) the reference
	// to the source data will be removed allowing it to be garbage collected.
	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandWriteBuffer
		e.CommandQueue = cq
	}

	pointer, _, err := tryPointerAndSize(src)

	if err != nil {
		// The source value is not addressable so create a local copy of it.
		var scratch [scratchSize]byte
		pointer, _ = getPointerAndSize(src, unsafe.Pointer(&scratch[0]))
		src = &scratch
	}

	err = clw.EnqueueWriteBufferRect(cq.id, b.id, clw.Bool(bc), offset.dstOrigin(), offset.srcOrigin(), offset.region(),
		offset.dstRowPitch(), offset.dstSlicePitch(), offset.srcRowPitch(), offset.srcSlicePitch(), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	// Set a no-op callback, just need hold a reference to the source data.
	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, src)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Shortened code a bit.<commit_after>package cl11

import (
	"fmt"
	"reflect"
	"unsafe"

	clw "github.com/rdwilliamson/clw11"
)

// A buffer object stores a one-dimensional collection of elements (though there
// are "rectangle" operations). Elements of a buffer object can be a scalar data
// type (such as an int, float), vector data type, or a user-defined structure.
type Buffer struct {
	id clw.Mem

	// The context the buffer was created on.
	Context *Context

	// The size of the buffer in bytes.
	Size int64

	// Usage information for the buffer from the device's point of view.
	Flags MemFlags

	// The host backed memory for the buffer, if applicable.
	Host interface{}

	// The parent buffer, if applicable.
	Buffer *Buffer

	// The offset in bytes within the parent buffer.
	Origin int64
}

// Creates a buffer object on the device.
//
// Creates an uninitialized buffer on the device. The size is in bytes.
func (c *Context) CreateDeviceBuffer(size int64, mf MemFlags) (*Buffer, error) {

	memory, err := clw.CreateBuffer(c.id, clw.MemFlags(mf), clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

// Create a buffer object on the device.
//
// Create a buffer object initialized with the passed value. The size and
// contents are determined by the passed value.
func (c *Context) CreateDeviceBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

// Create a buffer object for the device backed by host memory.
//
// Create a device accessible buffer object on the host. The host value must be
// addressable. The OpenCL implementation is allowed to cache the data on the
// device.
func (c *Context) CreateDeviceBufferFromHostMem(mf MemFlags, host interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemUseHostPointer

	value := reflect.ValueOf(host)
	if kind := value.Kind(); kind != reflect.Ptr && kind != reflect.Slice {
		return nil, wrapError(fmt.Errorf("host value not addressable"))
	} else if kind == reflect.Ptr {
		for {
			value = value.Elem()
			if value.Kind() != reflect.Ptr {
				break
			}
		}
	}
	pointer, size := addressablePointerAndSize(value)

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Host: host, Flags: mf}, nil
}

// Creates a buffer object that is host accessible.
//
// Creates an uninitialized buffer in pinned memory. The size is in bytes. This
// memory is not pageable and allows for DMA copies (which are faster).
func (c *Context) CreateHostBuffer(size int64, mf MemFlags) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), nil)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: size, Flags: mf}, nil
}

// Creates a buffer object that is host accessible.
//
// Creates an initialized buffer in pinned memory. The size and contents are
// determined by value. This memory is not pageable and allows for DMA copies
// (which are faster).
func (c *Context) CreateHostBufferInitializedBy(mf MemFlags, value interface{}) (*Buffer, error) {

	flags := clw.MemFlags(mf) | clw.MemAllocHostPointer | clw.MemCopyHostPointer

	var scratch [scratchSize]byte
	pointer, size := getPointerAndSize(value, unsafe.Pointer(&scratch[0]))

	memory, err := clw.CreateBuffer(c.id, flags, clw.Size(size), pointer)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: c, Size: int64(size), Flags: mf}, nil
}

// Creates a buffer object from an existing object with the passed offset and
// size in bytes.
//
// If the size and offset are the same as another sub buffer the implementation
// may return the same sub buffer and increment the reference count.
func (b *Buffer) CreateSubBuffer(mf MemFlags, origin, size int64) (*Buffer, error) {

	region := clw.BufferRegion{clw.Size(origin), clw.Size(size)}

	memory, err := clw.CreateSubBuffer(b.id, clw.MemFlags(mf), region)
	if err != nil {
		return nil, err
	}

	return &Buffer{id: memory, Context: b.Context, Size: size, Flags: mf, Buffer: b, Origin: origin}, nil
}

// Increments the buffer object reference count.
//
// The OpenCL commands that return a buffer perform an implicit retain.
func (b *Buffer) Retain() error {
	return clw.RetainMemObject(b.id)
}

// Decrements the buffer object reference count.
//
// After the buffers reference count becomes zero and commands queued for
// execution that use the buffer have finished the buffer is deleted.
func (b *Buffer) Release() error {
	return clw.ReleaseMemObject(b.id)
}

// Return the buffer's reference count.
//
// The reference count returned should be considered immediately stale. It is
// unsuitable for general use in applications. This feature is provided for
// identifying memory leaks.
func (b *Buffer) ReferenceCount() (int, error) {

	var count clw.Uint
	err := clw.GetMemObjectInfo(b.id, clw.MemReferenceCount, clw.Size(unsafe.Sizeof(count)), unsafe.Pointer(&count),
		nil)
	if err != nil {
		return 0, err
	}

	return int(count), nil
}

// Enqueues a command to copy from one buffer object to another.
//
// Source offset, destination offset, and size are in bytes.
func (cq *CommandQueue) CopyBuffer(src, dst *Buffer, srcOffset, dstOffset, size int64, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBuffer(cq.id, src.id, dst.id, clw.Size(srcOffset), clw.Size(dstOffset), clw.Size(size),
		cq.toEvents(waitList), event)
}

// Enqueues a command to copy a rectangular region from the buffer object to
// another buffer object.
//
// See Rect definition for how source and destination are defined.
func (cq *CommandQueue) CopyBufferRect(src, dst *Buffer, r *Rect, waitList []*Event, e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandCopyBuffer
		e.CommandQueue = cq
	}

	return clw.EnqueueCopyBufferRect(cq.id, src.id, dst.id, r.srcOrigin(), r.dstOrigin(), r.region(), r.srcRowPitch(),
		r.srcSlicePitch(), r.dstRowPitch(), r.dstSlicePitch(), cq.toEvents(waitList), event)
}

// Enqueues a command to map a region of the buffer object given by buffer into
// the host address space.
func (cq *CommandQueue) MapBuffer(b *Buffer, bc BlockingCall, flags MapFlags, offset, size int64, waitList []*Event,
	e *Event) (*MappedBuffer, error) {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandMapBuffer
		e.CommandQueue = cq
	}

	pointer, err := clw.EnqueueMapBuffer(cq.id, b.id, clw.Bool(bc), clw.MapFlags(flags), clw.Size(offset),
		clw.Size(size), cq.toEvents(waitList), event)
	if err != nil {
		return nil, err
	}

	return &MappedBuffer{pointer, size, b.id}, nil
}

// Enqueue commands to read from a buffer object to host memory.
//
// Offset is in bytes. The destination must be addressable. If the buffer object
// is backed by host memory then all commands that use it and sub buffers must
// have finished execution and it must not be mapped otherwise the results are
// undefined.
func (cq *CommandQueue) ReadBuffer(b *Buffer, bc BlockingCall, offset int64, dst interface{}, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	return clw.EnqueueReadBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
}

// Enqueue commands to write to a buffer object from host memory.
//
// Offset is in bytes. If the buffer object is backed by host memory then all
// commands that use it and sub buffers must have finished execution and it must
// not be mapped otherwise the results are undefined. If the source is
// addressable a reference will be held to prevent it being garbage collected,
// it is the user's responsibility to ensure that the source data is valid at
// the time the write is actually performed. If the source isn't addressable a
// copy will be created.
func (cq *CommandQueue) WriteBuffer(b *Buffer, bc BlockingCall, offset int64, src interface{}, waitList []*Event,
	e *Event) error {

	// Ensure we always have an event if not blocking. The event will be used to
	// register a callback. Thus the source data is guaranteed to be referenced
	// somewhere to preventing it from being garbage collected. Once the event
	// has completed and the callback is triggered (doing nothing) the reference
	// to the source data will be removed allowing it to be garbage collected.
	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandWriteBuffer
		e.CommandQueue = cq
	}

	pointer, size, err := tryPointerAndSize(src)

	if err != nil {
		// The source value is not addressable so create a local copy of it.
		var scratch [scratchSize]byte
		pointer, size = getPointerAndSize(src, unsafe.Pointer(&scratch[0]))
		src = &scratch
	}

	err = clw.EnqueueWriteBuffer(cq.id, b.id, clw.Bool(bc), clw.Size(offset), clw.Size(size), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	// Set a no-op callback, just need hold a reference to the source data.
	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, src)
		if err != nil {
			return err
		}
	}

	return nil
}

// Enqueue commands to read from a rectangular region from a buffer object to
// host memory.
//
// See Rect definition for offset is defined. The destination must be
// addressable. If the buffer object is backed by host memory then all commands
// that use it and sub buffers must have finished execution and it must not be
// mapped otherwise the results are undefined.
func (cq *CommandQueue) ReadBufferRect(b *Buffer, bc BlockingCall, offset *Rect, dst interface{}, waitList []*Event,
	e *Event) error {

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandReadBuffer
		e.CommandQueue = cq
	}

	pointer, _, err := tryPointerAndSize(dst)
	if err != nil {
		return wrapError(err)
	}

	return clw.EnqueueReadBufferRect(cq.id, b.id, clw.Bool(bc), offset.dstOrigin(), offset.srcOrigin(), offset.region(),
		offset.dstRowPitch(), offset.dstSlicePitch(), offset.srcRowPitch(), offset.dstRowPitch(), pointer,
		cq.toEvents(waitList), event)
}

// Enqueue commands to write a rectangular region to a buffer object from host
// memory.
//
// See Rect definition for offset is defined. If the buffer object is backed by
// host memory then all commands that use it and sub buffers must have finished
// execution and it must not be mapped otherwise the results are undefined. If
// the source is addressable a reference will be held to prevent it being
// garbage collected, it is the user's responsibility to ensure that the source
// data is valid at the time the write is actually performed. If the source
// isn't addressable a copy will be created.
func (cq *CommandQueue) WriteBufferRect(b *Buffer, bc BlockingCall, offset *Rect, src interface{}, waitList []*Event,
	e *Event) error {

	// Ensure we always have an event if not blocking. The event will be used to
	// register a callback. Thus the source data is guaranteed to be referenced
	// somewhere to preventing it from being garbage collected. Once the event
	// has completed and the callback is triggered (doing nothing) the reference
	// to the source data will be removed allowing it to be garbage collected.
	if e == nil && bc == NonBlocking {
		e = &Event{}
	}

	var event *clw.Event
	if e != nil {
		event = &e.id
		e.Context = cq.Context
		e.CommandType = CommandWriteBuffer
		e.CommandQueue = cq
	}

	pointer, _, err := tryPointerAndSize(src)

	if err != nil {
		// The source value is not addressable so create a local copy of it.
		var scratch [scratchSize]byte
		pointer, _ = getPointerAndSize(src, unsafe.Pointer(&scratch[0]))
		src = &scratch
	}

	err = clw.EnqueueWriteBufferRect(cq.id, b.id, clw.Bool(bc), offset.dstOrigin(), offset.srcOrigin(), offset.region(),
		offset.dstRowPitch(), offset.dstSlicePitch(), offset.srcRowPitch(), offset.srcSlicePitch(), pointer,
		cq.toEvents(waitList), event)
	if err != nil {
		return err
	}

	// Set a no-op callback, just need hold a reference to the source data.
	if bc == NonBlocking {
		err = e.SetCallback(noOpEventCallback, src)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>