<commit_before>package main

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/pkg/errors"

	"github.com/spf13/pflag"

	"github.com/golang/protobuf/proto"

	"github.com/havoc-io/mutagen/cmd"
	"github.com/havoc-io/mutagen/cmd/profile"
	"github.com/havoc-io/mutagen/pkg/sync"
)

const (
	snapshotFile = "snapshot_test"
	cacheFile    = "cache_test"
)

var usage = `scan_bench [-h|--help] [-p|--profile] [-i|--ignore=<pattern>] <path>
`

func main() {
	// Parse command line arguments.
	flagSet := pflag.NewFlagSet("scan_bench", pflag.ContinueOnError)
	flagSet.SetOutput(ioutil.Discard)
	var ignores []string
	var enableProfile bool
	flagSet.StringSliceVarP(&ignores, "ignore", "i", nil, "specify ignore paths")
	flagSet.BoolVarP(&enableProfile, "profile", "p", false, "enable profiling")
	if err := flagSet.Parse(os.Args[1:]); err != nil {
		if err == pflag.ErrHelp {
			fmt.Fprint(os.Stdout, usage)
			return
		} else {
			cmd.Fatal(errors.Wrap(err, "unable to parse command line"))
		}
	}
	arguments := flagSet.Args()
	if len(arguments) != 1 {
		cmd.Fatal(errors.New("invalid number of paths specified"))
	}
	path := arguments[0]

	// Print information.
	fmt.Println("Analyzing", path)

	// Create a snapshot without any cache. If requested, enable CPU and memory
	// profiling.
	var profiler *profile.Profile
	var err error
	if enableProfile {
		if profiler, err = profile.New("scan_cold"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start := time.Now()
	snapshot, preservesExecutability, recomposeUnicode, cache, err := sync.Scan(path, sha1.New(), nil, ignores, sync.SymlinkMode_SymlinkPortable)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to create snapshot"))
	} else if snapshot == nil {
		cmd.Fatal(errors.New("target doesn't exist"))
	}
	stop := time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Cold scan took", stop.Sub(start))
	fmt.Println("Root preserves executability:", preservesExecutability)
	fmt.Println("Root requires Unicode recomposition:", recomposeUnicode)

	// Create a snapshot with a cache. If requested, enable CPU and memory
	// profiling.
	if enableProfile {
		if profiler, err = profile.New("scan_warm"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	snapshot, preservesExecutability, recomposeUnicode, _, err = sync.Scan(path, sha1.New(), cache, ignores, sync.SymlinkMode_SymlinkPortable)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to create snapshot"))
	} else if snapshot == nil {
		cmd.Fatal(errors.New("target has been deleted since original snapshot"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Warm scan took", stop.Sub(start))
	fmt.Println("Root preserves executability:", preservesExecutability)
	fmt.Println("Root requires Unicode recomposition:", recomposeUnicode)

	// Serialize it.
	start = time.Now()
	serializedSnapshot, err := proto.Marshal(snapshot)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to serialize snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot serialization took", stop.Sub(start))

	// Deserialize it.
	start = time.Now()
	deserializedSnapshot := &sync.Entry{}
	if err = proto.Unmarshal(serializedSnapshot, deserializedSnapshot); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to deserialize snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot deserialization took", stop.Sub(start))

	// Validate the deserialized snapshot.
	start = time.Now()
	if err = deserializedSnapshot.EnsureValid(); err != nil {
		cmd.Fatal(errors.Wrap(err, "deserialized snapshot invalid"))
	}
	stop = time.Now()
	fmt.Println("Snapshot validation took", stop.Sub(start))

	// Write the serialized snapshot to disk.
	start = time.Now()
	if err = ioutil.WriteFile(snapshotFile, serializedSnapshot, 0600); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to write snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot write took", stop.Sub(start))

	// Read the serialized snapshot from disk.
	start = time.Now()
	if _, err = ioutil.ReadFile(snapshotFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to read snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot read took", stop.Sub(start))

	// Wipe the temporary file.
	if err = os.Remove(snapshotFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to remove snapshot"))
	}

	// TODO: I'd like to add a stable serialization benchmark since that's what
	// we really care about (especially since it has to copy the entire entry
	// tree), but I also don't want to expose that machinery publicly.

	// Print other information.
	fmt.Println("Serialized snapshot size is", len(serializedSnapshot), "bytes")
	fmt.Println(
		"Original/deserialized snapshots equivalent?",
		deserializedSnapshot.Equal(snapshot),
	)

	// Checksum it.
	start = time.Now()
	sha1.Sum(serializedSnapshot)
	stop = time.Now()
	fmt.Println("SHA-1 snapshot digest took", stop.Sub(start))

	// TODO: I'd like to add a copy benchmark since copying is used in a lot of
	// our transformation functions, but I also don't want to expose this
	// function publicly.

	// Serialize the cache.
	start = time.Now()
	serializedCache, err := proto.Marshal(cache)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to serialize cache"))
	}
	stop = time.Now()
	fmt.Println("Cache serialization took", stop.Sub(start))

	// Deserialize the cache.
	start = time.Now()
	deserializedCache := &sync.Cache{}
	if err = proto.Unmarshal(serializedCache, deserializedCache); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to deserialize cache"))
	}
	stop = time.Now()
	fmt.Println("Cache deserialization took", stop.Sub(start))

	// Write the serialized cache to disk.
	start = time.Now()
	if err = ioutil.WriteFile(cacheFile, serializedCache, 0600); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to write cache"))
	}
	stop = time.Now()
	fmt.Println("Cache write took", stop.Sub(start))

	// Read the serialized cache from disk.
	start = time.Now()
	if _, err = ioutil.ReadFile(cacheFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to read cache"))
	}
	stop = time.Now()
	fmt.Println("Cache read took", stop.Sub(start))

	// Wipe the temporary file.
	if err = os.Remove(cacheFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to remove cache"))
	}

	// Print other information.
	fmt.Println("Serialized cache size is", len(serializedCache), "bytes")
}
<commit_msg>Added profiling to snapshot/cache (de)serialization in scan_bench.<commit_after>package main

import (
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"os"
	"time"

	"github.com/pkg/errors"

	"github.com/spf13/pflag"

	"github.com/golang/protobuf/proto"

	"github.com/havoc-io/mutagen/cmd"
	"github.com/havoc-io/mutagen/cmd/profile"
	"github.com/havoc-io/mutagen/pkg/sync"
)

const (
	snapshotFile = "snapshot_test"
	cacheFile    = "cache_test"
)

var usage = `scan_bench [-h|--help] [-p|--profile] [-i|--ignore=<pattern>] <path>
`

func main() {
	// Parse command line arguments.
	flagSet := pflag.NewFlagSet("scan_bench", pflag.ContinueOnError)
	flagSet.SetOutput(ioutil.Discard)
	var ignores []string
	var enableProfile bool
	flagSet.StringSliceVarP(&ignores, "ignore", "i", nil, "specify ignore paths")
	flagSet.BoolVarP(&enableProfile, "profile", "p", false, "enable profiling")
	if err := flagSet.Parse(os.Args[1:]); err != nil {
		if err == pflag.ErrHelp {
			fmt.Fprint(os.Stdout, usage)
			return
		} else {
			cmd.Fatal(errors.Wrap(err, "unable to parse command line"))
		}
	}
	arguments := flagSet.Args()
	if len(arguments) != 1 {
		cmd.Fatal(errors.New("invalid number of paths specified"))
	}
	path := arguments[0]

	// Print information.
	fmt.Println("Analyzing", path)

	// Create a snapshot without any cache. If requested, enable CPU and memory
	// profiling.
	var profiler *profile.Profile
	var err error
	if enableProfile {
		if profiler, err = profile.New("scan_cold"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start := time.Now()
	snapshot, preservesExecutability, recomposeUnicode, cache, err := sync.Scan(path, sha1.New(), nil, ignores, sync.SymlinkMode_SymlinkPortable)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to create snapshot"))
	} else if snapshot == nil {
		cmd.Fatal(errors.New("target doesn't exist"))
	}
	stop := time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Cold scan took", stop.Sub(start))
	fmt.Println("Root preserves executability:", preservesExecutability)
	fmt.Println("Root requires Unicode recomposition:", recomposeUnicode)

	// Create a snapshot with a cache. If requested, enable CPU and memory
	// profiling.
	if enableProfile {
		if profiler, err = profile.New("scan_warm"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	snapshot, preservesExecutability, recomposeUnicode, _, err = sync.Scan(path, sha1.New(), cache, ignores, sync.SymlinkMode_SymlinkPortable)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to create snapshot"))
	} else if snapshot == nil {
		cmd.Fatal(errors.New("target has been deleted since original snapshot"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Warm scan took", stop.Sub(start))
	fmt.Println("Root preserves executability:", preservesExecutability)
	fmt.Println("Root requires Unicode recomposition:", recomposeUnicode)

	// Serialize it.
	if enableProfile {
		if profiler, err = profile.New("serialize_snapshot"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	serializedSnapshot, err := proto.Marshal(snapshot)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to serialize snapshot"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Snapshot serialization took", stop.Sub(start))

	// Deserialize it.
	if enableProfile {
		if profiler, err = profile.New("deserialize_snapshot"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	deserializedSnapshot := &sync.Entry{}
	if err = proto.Unmarshal(serializedSnapshot, deserializedSnapshot); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to deserialize snapshot"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Snapshot deserialization took", stop.Sub(start))

	// Validate the deserialized snapshot.
	start = time.Now()
	if err = deserializedSnapshot.EnsureValid(); err != nil {
		cmd.Fatal(errors.Wrap(err, "deserialized snapshot invalid"))
	}
	stop = time.Now()
	fmt.Println("Snapshot validation took", stop.Sub(start))

	// Write the serialized snapshot to disk.
	start = time.Now()
	if err = ioutil.WriteFile(snapshotFile, serializedSnapshot, 0600); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to write snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot write took", stop.Sub(start))

	// Read the serialized snapshot from disk.
	start = time.Now()
	if _, err = ioutil.ReadFile(snapshotFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to read snapshot"))
	}
	stop = time.Now()
	fmt.Println("Snapshot read took", stop.Sub(start))

	// Wipe the temporary file.
	if err = os.Remove(snapshotFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to remove snapshot"))
	}

	// TODO: I'd like to add a stable serialization benchmark since that's what
	// we really care about (especially since it has to copy the entire entry
	// tree), but I also don't want to expose that machinery publicly.

	// Print other information.
	fmt.Println("Serialized snapshot size is", len(serializedSnapshot), "bytes")
	fmt.Println(
		"Original/deserialized snapshots equivalent?",
		deserializedSnapshot.Equal(snapshot),
	)

	// Checksum it.
	start = time.Now()
	sha1.Sum(serializedSnapshot)
	stop = time.Now()
	fmt.Println("SHA-1 snapshot digest took", stop.Sub(start))

	// TODO: I'd like to add a copy benchmark since copying is used in a lot of
	// our transformation functions, but I also don't want to expose this
	// function publicly.

	// Serialize the cache.
	if enableProfile {
		if profiler, err = profile.New("serialize_cache"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	serializedCache, err := proto.Marshal(cache)
	if err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to serialize cache"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Cache serialization took", stop.Sub(start))

	// Deserialize the cache.
	if enableProfile {
		if profiler, err = profile.New("deserialize_cache"); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to create profiler"))
		}
	}
	start = time.Now()
	deserializedCache := &sync.Cache{}
	if err = proto.Unmarshal(serializedCache, deserializedCache); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to deserialize cache"))
	}
	stop = time.Now()
	if enableProfile {
		if err = profiler.Finalize(); err != nil {
			cmd.Fatal(errors.Wrap(err, "unable to finalize profiler"))
		}
		profiler = nil
	}
	fmt.Println("Cache deserialization took", stop.Sub(start))

	// Write the serialized cache to disk.
	start = time.Now()
	if err = ioutil.WriteFile(cacheFile, serializedCache, 0600); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to write cache"))
	}
	stop = time.Now()
	fmt.Println("Cache write took", stop.Sub(start))

	// Read the serialized cache from disk.
	start = time.Now()
	if _, err = ioutil.ReadFile(cacheFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to read cache"))
	}
	stop = time.Now()
	fmt.Println("Cache read took", stop.Sub(start))

	// Wipe the temporary file.
	if err = os.Remove(cacheFile); err != nil {
		cmd.Fatal(errors.Wrap(err, "unable to remove cache"))
	}

	// Print other information.
	fmt.Println("Serialized cache size is", len(serializedCache), "bytes")
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cfg holds configuration shared by multiple parts
// of the go command.
package cfg

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"runtime"

	"cmd/internal/objabi"
)

// These are general "build flags" used by build and other commands.
var (
	BuildA                 bool   // -a flag
	BuildBuildmode         string // -buildmode flag
	BuildContext           = defaultContext()
	BuildMod               string             // -mod flag
	BuildI                 bool               // -i flag
	BuildLinkshared        bool               // -linkshared flag
	BuildMSan              bool               // -msan flag
	BuildN                 bool               // -n flag
	BuildO                 string             // -o flag
	BuildP                 = runtime.NumCPU() // -p flag
	BuildPkgdir            string             // -pkgdir flag
	BuildRace              bool               // -race flag
	BuildToolexec          []string           // -toolexec flag
	BuildToolchainName     string
	BuildToolchainCompiler func() string
	BuildToolchainLinker   func() string
	BuildV                 bool // -v flag
	BuildWork              bool // -work flag
	BuildX                 bool // -x flag

	CmdName string // "build", "install", "list", "mod tidy", etc.

	DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
)

func defaultContext() build.Context {
	ctxt := build.Default
	ctxt.JoinPath = filepath.Join // back door to say "do not use go command"
	return ctxt
}

func init() {
	BuildToolchainCompiler = func() string { return "missing-compiler" }
	BuildToolchainLinker = func() string { return "missing-linker" }
}

// An EnvVar is an environment variable Name=Value.
type EnvVar struct {
	Name  string
	Value string
}

// OrigEnv is the original environment of the program at startup.
var OrigEnv []string

// CmdEnv is the new environment for running go tool commands.
// User binaries (during go test or go run) are run with OrigEnv,
// not CmdEnv.
var CmdEnv []EnvVar

// Global build parameters (used during package load)
var (
	Goarch    = BuildContext.GOARCH
	Goos      = BuildContext.GOOS
	ExeSuffix string
	Gopath    = filepath.SplitList(BuildContext.GOPATH)

	// ModulesEnabled specifies whether the go command is running
	// in module-aware mode (as opposed to GOPATH mode).
	// It is equal to modload.Enabled, but not all packages can import modload.
	ModulesEnabled bool

	// GoModInGOPATH records whether we've found a go.mod in GOPATH/src
	// in GO111MODULE=auto mode. In that case, we don't use modules
	// but people might expect us to, so 'go get' warns.
	GoModInGOPATH string
)

func init() {
	if Goos == "windows" {
		ExeSuffix = ".exe"
	}
}

var (
	GOROOT       = findGOROOT()
	GOBIN        = os.Getenv("GOBIN")
	GOROOTbin    = filepath.Join(GOROOT, "bin")
	GOROOTpkg    = filepath.Join(GOROOT, "pkg")
	GOROOTsrc    = filepath.Join(GOROOT, "src")
	GOROOT_FINAL = findGOROOT_FINAL()

	// Used in envcmd.MkEnv and build ID computations.
	GOARM    = fmt.Sprint(objabi.GOARM)
	GO386    = objabi.GO386
	GOMIPS   = objabi.GOMIPS
	GOMIPS64 = objabi.GOMIPS64
)

// Update build context to use our computed GOROOT.
func init() {
	BuildContext.GOROOT = GOROOT
	if runtime.Compiler != "gccgo" {
		// Note that we must use runtime.GOOS and runtime.GOARCH here,
		// as the tool directory does not move based on environment
		// variables. This matches the initialization of ToolDir in
		// go/build, except for using GOROOT rather than
		// runtime.GOROOT.
		build.ToolDir = filepath.Join(GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH)
	}
}

// There is a copy of findGOROOT, isSameDir, and isGOROOT in
// x/tools/cmd/godoc/goroot.go.
// Try to keep them in sync for now.

// findGOROOT returns the GOROOT value, using either an explicitly
// provided environment variable, a GOROOT that contains the current
// os.Executable value, or else the GOROOT that the binary was built
// with from runtime.GOROOT().
//
// There is a copy of this code in x/tools/cmd/godoc/goroot.go.
func findGOROOT() string {
	if env := os.Getenv("GOROOT"); env != "" {
		return filepath.Clean(env)
	}
	def := filepath.Clean(runtime.GOROOT())
	if runtime.Compiler == "gccgo" {
		// gccgo has no real GOROOT, and it certainly doesn't
		// depend on the executable's location.
		return def
	}
	exe, err := os.Executable()
	if err == nil {
		exe, err = filepath.Abs(exe)
		if err == nil {
			if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
				// If def (runtime.GOROOT()) and dir are the same
				// directory, prefer the spelling used in def.
				if isSameDir(def, dir) {
					return def
				}
				return dir
			}
			exe, err = filepath.EvalSymlinks(exe)
			if err == nil {
				if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
					if isSameDir(def, dir) {
						return def
					}
					return dir
				}
			}
		}
	}
	return def
}

func findGOROOT_FINAL() string {
	def := GOROOT
	if env := os.Getenv("GOROOT_FINAL"); env != "" {
		def = filepath.Clean(env)
	}
	return def
}

// isSameDir reports whether dir1 and dir2 are the same directory.
func isSameDir(dir1, dir2 string) bool {
	if dir1 == dir2 {
		return true
	}
	info1, err1 := os.Stat(dir1)
	info2, err2 := os.Stat(dir2)
	return err1 == nil && err2 == nil && os.SameFile(info1, info2)
}

// isGOROOT reports whether path looks like a GOROOT.
//
// It does this by looking for the path/pkg/tool directory,
// which is necessary for useful operation of the cmd/go tool,
// and is not typically present in a GOPATH.
//
// There is a copy of this code in x/tools/cmd/godoc/goroot.go.
func isGOROOT(path string) bool {
	stat, err := os.Stat(filepath.Join(path, "pkg", "tool"))
	if err != nil {
		return false
	}
	return stat.IsDir()
}
<commit_msg>cmd/go/internal/cfg: remove unused Gopath variable<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cfg holds configuration shared by multiple parts
// of the go command.
package cfg

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"runtime"

	"cmd/internal/objabi"
)

// These are general "build flags" used by build and other commands.
var (
	BuildA                 bool   // -a flag
	BuildBuildmode         string // -buildmode flag
	BuildContext           = defaultContext()
	BuildMod               string             // -mod flag
	BuildI                 bool               // -i flag
	BuildLinkshared        bool               // -linkshared flag
	BuildMSan              bool               // -msan flag
	BuildN                 bool               // -n flag
	BuildO                 string             // -o flag
	BuildP                 = runtime.NumCPU() // -p flag
	BuildPkgdir            string             // -pkgdir flag
	BuildRace              bool               // -race flag
	BuildToolexec          []string           // -toolexec flag
	BuildToolchainName     string
	BuildToolchainCompiler func() string
	BuildToolchainLinker   func() string
	BuildV                 bool // -v flag
	BuildWork              bool // -work flag
	BuildX                 bool // -x flag

	CmdName string // "build", "install", "list", "mod tidy", etc.

	DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
)

func defaultContext() build.Context {
	ctxt := build.Default
	ctxt.JoinPath = filepath.Join // back door to say "do not use go command"
	return ctxt
}

func init() {
	BuildToolchainCompiler = func() string { return "missing-compiler" }
	BuildToolchainLinker = func() string { return "missing-linker" }
}

// An EnvVar is an environment variable Name=Value.
type EnvVar struct {
	Name  string
	Value string
}

// OrigEnv is the original environment of the program at startup.
var OrigEnv []string

// CmdEnv is the new environment for running go tool commands.
// User binaries (during go test or go run) are run with OrigEnv,
// not CmdEnv.
var CmdEnv []EnvVar

// Global build parameters (used during package load)
var (
	Goarch    = BuildContext.GOARCH
	Goos      = BuildContext.GOOS
	ExeSuffix string

	// ModulesEnabled specifies whether the go command is running
	// in module-aware mode (as opposed to GOPATH mode).
	// It is equal to modload.Enabled, but not all packages can import modload.
	ModulesEnabled bool

	// GoModInGOPATH records whether we've found a go.mod in GOPATH/src
	// in GO111MODULE=auto mode. In that case, we don't use modules
	// but people might expect us to, so 'go get' warns.
	GoModInGOPATH string
)

func init() {
	if Goos == "windows" {
		ExeSuffix = ".exe"
	}
}

var (
	GOROOT       = findGOROOT()
	GOBIN        = os.Getenv("GOBIN")
	GOROOTbin    = filepath.Join(GOROOT, "bin")
	GOROOTpkg    = filepath.Join(GOROOT, "pkg")
	GOROOTsrc    = filepath.Join(GOROOT, "src")
	GOROOT_FINAL = findGOROOT_FINAL()

	// Used in envcmd.MkEnv and build ID computations.
	GOARM    = fmt.Sprint(objabi.GOARM)
	GO386    = objabi.GO386
	GOMIPS   = objabi.GOMIPS
	GOMIPS64 = objabi.GOMIPS64
)

// Update build context to use our computed GOROOT.
func init() {
	BuildContext.GOROOT = GOROOT
	if runtime.Compiler != "gccgo" {
		// Note that we must use runtime.GOOS and runtime.GOARCH here,
		// as the tool directory does not move based on environment
		// variables. This matches the initialization of ToolDir in
		// go/build, except for using GOROOT rather than
		// runtime.GOROOT.
		build.ToolDir = filepath.Join(GOROOT, "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH)
	}
}

// There is a copy of findGOROOT, isSameDir, and isGOROOT in
// x/tools/cmd/godoc/goroot.go.
// Try to keep them in sync for now.

// findGOROOT returns the GOROOT value, using either an explicitly
// provided environment variable, a GOROOT that contains the current
// os.Executable value, or else the GOROOT that the binary was built
// with from runtime.GOROOT().
//
// There is a copy of this code in x/tools/cmd/godoc/goroot.go.
func findGOROOT() string {
	if env := os.Getenv("GOROOT"); env != "" {
		return filepath.Clean(env)
	}
	def := filepath.Clean(runtime.GOROOT())
	if runtime.Compiler == "gccgo" {
		// gccgo has no real GOROOT, and it certainly doesn't
		// depend on the executable's location.
		return def
	}
	exe, err := os.Executable()
	if err == nil {
		exe, err = filepath.Abs(exe)
		if err == nil {
			if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
				// If def (runtime.GOROOT()) and dir are the same
				// directory, prefer the spelling used in def.
				if isSameDir(def, dir) {
					return def
				}
				return dir
			}
			exe, err = filepath.EvalSymlinks(exe)
			if err == nil {
				if dir := filepath.Join(exe, "../.."); isGOROOT(dir) {
					if isSameDir(def, dir) {
						return def
					}
					return dir
				}
			}
		}
	}
	return def
}

func findGOROOT_FINAL() string {
	def := GOROOT
	if env := os.Getenv("GOROOT_FINAL"); env != "" {
		def = filepath.Clean(env)
	}
	return def
}

// isSameDir reports whether dir1 and dir2 are the same directory.
func isSameDir(dir1, dir2 string) bool {
	if dir1 == dir2 {
		return true
	}
	info1, err1 := os.Stat(dir1)
	info2, err2 := os.Stat(dir2)
	return err1 == nil && err2 == nil && os.SameFile(info1, info2)
}

// isGOROOT reports whether path looks like a GOROOT.
//
// It does this by looking for the path/pkg/tool directory,
// which is necessary for useful operation of the cmd/go tool,
// and is not typically present in a GOPATH.
//
// There is a copy of this code in x/tools/cmd/godoc/goroot.go.
func isGOROOT(path string) bool {
	stat, err := os.Stat(filepath.Join(path, "pkg", "tool"))
	if err != nil {
		return false
	}
	return stat.IsDir()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Only build this file if libffi is supported.

// +build libffi

package runtime

import "unsafe"

// This file contains the code that converts a Go type to an FFI type.
// This has to be written in Go because it allocates memory in the Go heap.

// C functions to return pointers to libffi variables.

func ffi_type_pointer() *__ffi_type
func ffi_type_sint8() *__ffi_type
func ffi_type_sint16() *__ffi_type
func ffi_type_sint32() *__ffi_type
func ffi_type_sint64() *__ffi_type
func ffi_type_uint8() *__ffi_type
func ffi_type_uint16() *__ffi_type
func ffi_type_uint32() *__ffi_type
func ffi_type_uint64() *__ffi_type
func ffi_type_float() *__ffi_type
func ffi_type_double() *__ffi_type
func ffi_supports_complex() bool
func ffi_type_complex_float() *__ffi_type
func ffi_type_complex_double() *__ffi_type
func ffi_type_void() *__ffi_type

// C functions defined in libffi.

//extern ffi_prep_cif
func ffi_prep_cif(*_ffi_cif, _ffi_abi, uint32, *__ffi_type, **__ffi_type) _ffi_status

// ffiFuncToCIF is called from C code.
//go:linkname ffiFuncToCIF runtime.ffiFuncToCIF

// ffiFuncToCIF builds an _ffi_cif struct for function described by ft.
func ffiFuncToCIF(ft *functype, isInterface bool, isMethod bool, cif *_ffi_cif) {
	nparams := len(ft.in)
	nargs := nparams
	if isInterface {
		nargs++
	}
	args := make([]*__ffi_type, nargs)
	i := 0
	off := 0
	if isInterface {
		args[0] = ffi_type_pointer()
		off = 1
	} else if isMethod {
		args[0] = ffi_type_pointer()
		i = 1
	}
	for ; i < nparams; i++ {
		args[i+off] = typeToFFI(ft.in[i])
	}

	rettype := funcReturnFFI(ft)

	var pargs **__ffi_type
	if len(args) > 0 {
		pargs = &args[0]
	}
	status := ffi_prep_cif(cif, _FFI_DEFAULT_ABI, uint32(nargs), rettype, pargs)
	if status != _FFI_OK {
		throw("ffi_prep_cif failed")
	}
}

// funcReturnFFI returns the FFI definition of the return type of ft.
func funcReturnFFI(ft *functype) *__ffi_type {
	c := len(ft.out)
	if c == 0 {
		return ffi_type_void()
	}

	// Compile a function that returns a zero-sized value as
	// though it returns void. This works around a problem in
	// libffi: it can't represent a zero-sized value.
	var size uintptr
	for _, v := range ft.out {
		size += v.size
	}
	if size == 0 {
		return ffi_type_void()
	}

	if c == 1 {
		return typeToFFI(ft.out[0])
	}

	elements := make([]*__ffi_type, c+1)
	for i, v := range ft.out {
		elements[i] = typeToFFI(v)
	}
	elements[c] = nil

	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// typeToFFI returns the __ffi_type for a Go type.
func typeToFFI(typ *_type) *__ffi_type {
	switch typ.kind & kindMask {
	case kindBool:
		switch unsafe.Sizeof(false) {
		case 1:
			return ffi_type_uint8()
		case 4:
			return ffi_type_uint32()
		default:
			throw("bad bool size")
			return nil
		}
	case kindInt:
		return intToFFI()
	case kindInt8:
		return ffi_type_sint8()
	case kindInt16:
		return ffi_type_sint16()
	case kindInt32:
		return ffi_type_sint32()
	case kindInt64:
		return ffi_type_sint64()
	case kindUint:
		switch unsafe.Sizeof(uint(0)) {
		case 4:
			return ffi_type_uint32()
		case 8:
			return ffi_type_uint64()
		default:
			throw("bad uint size")
			return nil
		}
	case kindUint8:
		return ffi_type_uint8()
	case kindUint16:
		return ffi_type_uint16()
	case kindUint32:
		return ffi_type_uint32()
	case kindUint64:
		return ffi_type_uint64()
	case kindUintptr:
		switch unsafe.Sizeof(uintptr(0)) {
		case 4:
			return ffi_type_uint32()
		case 8:
			return ffi_type_uint64()
		default:
			throw("bad uinptr size")
			return nil
		}
	case kindFloat32:
		return ffi_type_float()
	case kindFloat64:
		return ffi_type_double()
	case kindComplex64:
		if ffi_supports_complex() {
			return ffi_type_complex_float()
		} else {
			return complexToFFI(ffi_type_float())
		}
	case kindComplex128:
		if ffi_supports_complex() {
			return ffi_type_complex_double()
		} else {
			return complexToFFI(ffi_type_double())
		}
	case kindArray:
		return arrayToFFI((*arraytype)(unsafe.Pointer(typ)))
	case kindChan, kindFunc, kindMap, kindPtr, kindUnsafePointer:
		// These types are always simple pointers, and for FFI
		// purposes nothing else matters.
		return ffi_type_pointer()
	case kindInterface:
		return interfaceToFFI()
	case kindSlice:
		return sliceToFFI((*slicetype)(unsafe.Pointer(typ)))
	case kindString:
		return stringToFFI()
	case kindStruct:
		return structToFFI((*structtype)(unsafe.Pointer(typ)))
	default:
		throw("unknown type kind")
		return nil
	}
}

// interfaceToFFI returns an ffi_type for a Go interface type.
// This is used for both empty and non-empty interface types.
func interfaceToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffi_type_pointer()
	elements[1] = elements[0]
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// stringToFFI returns an ffi_type for a Go string type.
func stringToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffi_type_pointer()
	elements[1] = intToFFI()
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// structToFFI returns an ffi_type for a Go struct type.
func structToFFI(typ *structtype) *__ffi_type {
	c := len(typ.fields)
	if c == 0 {
		return emptyStructToFFI()
	}

	fields := make([]*__ffi_type, 0, c+1)
	checkPad := false
	for i, v := range typ.fields {
		// Skip zero-sized fields; they confuse libffi,
		// and there is no value to pass in any case.
		// We do have to check whether the alignment of the
		// zero-sized field introduces any padding for the
		// next field.
		if v.typ.size == 0 {
			checkPad = true
			continue
		}

		if checkPad {
			off := uintptr(0)
			for j := i - 1; j >= 0; j-- {
				if typ.fields[j].typ.size > 0 {
					off = typ.fields[j].offset() + typ.fields[j].typ.size
					break
				}
			}
			off += uintptr(v.typ.align) - 1
			off &^= uintptr(v.typ.align) - 1
			if off != v.offset() {
				fields = append(fields, padFFI(v.offset()-off))
			}
			checkPad = false
		}

		fields = append(fields, typeToFFI(v.typ))
	}

	fields = append(fields, nil)

	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &fields[0],
	}
}

// sliceToFFI returns an ffi_type for a Go slice type.
func sliceToFFI(typ *slicetype) *__ffi_type {
	elements := make([]*__ffi_type, 4)
	elements[0] = ffi_type_pointer()
	elements[1] = intToFFI()
	elements[2] = elements[1]
	elements[3] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// complexToFFI returns an ffi_type for a Go complex type.
// This is only used if libffi does not support complex types internally
// for this target.
func complexToFFI(ffiFloatType *__ffi_type) *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffiFloatType
	elements[1] = ffiFloatType
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// arrayToFFI returns an ffi_type for a Go array type.
func arrayToFFI(typ *arraytype) *__ffi_type {
	if typ.len == 0 {
		return emptyStructToFFI()
	}
	elements := make([]*__ffi_type, typ.len+1)
	et := typeToFFI(typ.elem)
	for i := uintptr(0); i < typ.len; i++ {
		elements[i] = et
	}
	elements[typ.len] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// intToFFI returns an ffi_type for the Go int type.
func intToFFI() *__ffi_type {
	switch unsafe.Sizeof(0) {
	case 4:
		return ffi_type_sint32()
	case 8:
		return ffi_type_sint64()
	default:
		throw("bad int size")
		return nil
	}
}

// emptyStructToFFI returns an ffi_type for an empty struct.
// The libffi library won't accept a struct with no fields.
func emptyStructToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 2)
	elements[0] = ffi_type_void()
	elements[1] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// padFFI returns a padding field of the given size
func padFFI(size uintptr) *__ffi_type {
	elements := make([]*__ffi_type, size+1)
	for i := uintptr(0); i < size; i++ {
		elements[i] = ffi_type_uint8()
	}
	elements[size] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

//go:linkname makeCIF reflect.makeCIF

// makeCIF is used by the reflect package to allocate a CIF.
func makeCIF(ft *functype) *_ffi_cif {
	cif := new(_ffi_cif)
	ffiFuncToCIF(ft, false, false, cif)
	return cif
}
<commit_msg>runtime: add padding to FFI type of struct ending with zero-sized field<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Only build this file if libffi is supported.

// +build libffi

package runtime

import "unsafe"

// This file contains the code that converts a Go type to an FFI type.
// This has to be written in Go because it allocates memory in the Go heap.

// C functions to return pointers to libffi variables.

func ffi_type_pointer() *__ffi_type
func ffi_type_sint8() *__ffi_type
func ffi_type_sint16() *__ffi_type
func ffi_type_sint32() *__ffi_type
func ffi_type_sint64() *__ffi_type
func ffi_type_uint8() *__ffi_type
func ffi_type_uint16() *__ffi_type
func ffi_type_uint32() *__ffi_type
func ffi_type_uint64() *__ffi_type
func ffi_type_float() *__ffi_type
func ffi_type_double() *__ffi_type
func ffi_supports_complex() bool
func ffi_type_complex_float() *__ffi_type
func ffi_type_complex_double() *__ffi_type
func ffi_type_void() *__ffi_type

// C functions defined in libffi.

//extern ffi_prep_cif
func ffi_prep_cif(*_ffi_cif, _ffi_abi, uint32, *__ffi_type, **__ffi_type) _ffi_status

// ffiFuncToCIF is called from C code.
//go:linkname ffiFuncToCIF runtime.ffiFuncToCIF

// ffiFuncToCIF builds an _ffi_cif struct for function described by ft.
func ffiFuncToCIF(ft *functype, isInterface bool, isMethod bool, cif *_ffi_cif) {
	nparams := len(ft.in)
	nargs := nparams
	if isInterface {
		nargs++
	}
	args := make([]*__ffi_type, nargs)
	i := 0
	off := 0
	if isInterface {
		args[0] = ffi_type_pointer()
		off = 1
	} else if isMethod {
		args[0] = ffi_type_pointer()
		i = 1
	}
	for ; i < nparams; i++ {
		args[i+off] = typeToFFI(ft.in[i])
	}

	rettype := funcReturnFFI(ft)

	var pargs **__ffi_type
	if len(args) > 0 {
		pargs = &args[0]
	}
	status := ffi_prep_cif(cif, _FFI_DEFAULT_ABI, uint32(nargs), rettype, pargs)
	if status != _FFI_OK {
		throw("ffi_prep_cif failed")
	}
}

// funcReturnFFI returns the FFI definition of the return type of ft.
func funcReturnFFI(ft *functype) *__ffi_type {
	c := len(ft.out)
	if c == 0 {
		return ffi_type_void()
	}

	// Compile a function that returns a zero-sized value as
	// though it returns void. This works around a problem in
	// libffi: it can't represent a zero-sized value.
	var size uintptr
	for _, v := range ft.out {
		size += v.size
	}
	if size == 0 {
		return ffi_type_void()
	}

	if c == 1 {
		return typeToFFI(ft.out[0])
	}

	elements := make([]*__ffi_type, c+1)
	for i, v := range ft.out {
		elements[i] = typeToFFI(v)
	}
	elements[c] = nil

	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// typeToFFI returns the __ffi_type for a Go type.
func typeToFFI(typ *_type) *__ffi_type {
	switch typ.kind & kindMask {
	case kindBool:
		switch unsafe.Sizeof(false) {
		case 1:
			return ffi_type_uint8()
		case 4:
			return ffi_type_uint32()
		default:
			throw("bad bool size")
			return nil
		}
	case kindInt:
		return intToFFI()
	case kindInt8:
		return ffi_type_sint8()
	case kindInt16:
		return ffi_type_sint16()
	case kindInt32:
		return ffi_type_sint32()
	case kindInt64:
		return ffi_type_sint64()
	case kindUint:
		switch unsafe.Sizeof(uint(0)) {
		case 4:
			return ffi_type_uint32()
		case 8:
			return ffi_type_uint64()
		default:
			throw("bad uint size")
			return nil
		}
	case kindUint8:
		return ffi_type_uint8()
	case kindUint16:
		return ffi_type_uint16()
	case kindUint32:
		return ffi_type_uint32()
	case kindUint64:
		return ffi_type_uint64()
	case kindUintptr:
		switch unsafe.Sizeof(uintptr(0)) {
		case 4:
			return ffi_type_uint32()
		case 8:
			return ffi_type_uint64()
		default:
			throw("bad uinptr size")
			return nil
		}
	case kindFloat32:
		return ffi_type_float()
	case kindFloat64:
		return ffi_type_double()
	case kindComplex64:
		if ffi_supports_complex() {
			return ffi_type_complex_float()
		} else {
			return complexToFFI(ffi_type_float())
		}
	case kindComplex128:
		if ffi_supports_complex() {
			return ffi_type_complex_double()
		} else {
			return complexToFFI(ffi_type_double())
		}
	case kindArray:
		return arrayToFFI((*arraytype)(unsafe.Pointer(typ)))
	case kindChan, kindFunc, kindMap, kindPtr, kindUnsafePointer:
		// These types are always simple pointers, and for FFI
		// purposes nothing else matters.
		return ffi_type_pointer()
	case kindInterface:
		return interfaceToFFI()
	case kindSlice:
		return sliceToFFI((*slicetype)(unsafe.Pointer(typ)))
	case kindString:
		return stringToFFI()
	case kindStruct:
		return structToFFI((*structtype)(unsafe.Pointer(typ)))
	default:
		throw("unknown type kind")
		return nil
	}
}

// interfaceToFFI returns an ffi_type for a Go interface type.
// This is used for both empty and non-empty interface types.
func interfaceToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffi_type_pointer()
	elements[1] = elements[0]
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// stringToFFI returns an ffi_type for a Go string type.
func stringToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffi_type_pointer()
	elements[1] = intToFFI()
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// structToFFI returns an ffi_type for a Go struct type.
func structToFFI(typ *structtype) *__ffi_type {
	c := len(typ.fields)
	if c == 0 {
		return emptyStructToFFI()
	}

	fields := make([]*__ffi_type, 0, c+1)
	checkPad := false
	lastzero := false
	for i, v := range typ.fields {
		// Skip zero-sized fields; they confuse libffi,
		// and there is no value to pass in any case.
		// We do have to check whether the alignment of the
		// zero-sized field introduces any padding for the
		// next field.
		if v.typ.size == 0 {
			checkPad = true
			lastzero = true
			continue
		}
		lastzero = false

		if checkPad {
			off := uintptr(0)
			for j := i - 1; j >= 0; j-- {
				if typ.fields[j].typ.size > 0 {
					off = typ.fields[j].offset() + typ.fields[j].typ.size
					break
				}
			}
			off += uintptr(v.typ.align) - 1
			off &^= uintptr(v.typ.align) - 1
			if off != v.offset() {
				fields = append(fields, padFFI(v.offset()-off))
			}
			checkPad = false
		}

		fields = append(fields, typeToFFI(v.typ))
	}

	if lastzero {
		// The compiler adds one byte padding to non-empty struct ending
		// with a zero-sized field (types.cc:get_backend_struct_fields).
		// Add this padding to the FFI type.
		fields = append(fields, ffi_type_uint8())
	}

	fields = append(fields, nil)

	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &fields[0],
	}
}

// sliceToFFI returns an ffi_type for a Go slice type.
func sliceToFFI(typ *slicetype) *__ffi_type {
	elements := make([]*__ffi_type, 4)
	elements[0] = ffi_type_pointer()
	elements[1] = intToFFI()
	elements[2] = elements[1]
	elements[3] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// complexToFFI returns an ffi_type for a Go complex type.
// This is only used if libffi does not support complex types internally
// for this target.
func complexToFFI(ffiFloatType *__ffi_type) *__ffi_type {
	elements := make([]*__ffi_type, 3)
	elements[0] = ffiFloatType
	elements[1] = ffiFloatType
	elements[2] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// arrayToFFI returns an ffi_type for a Go array type.
func arrayToFFI(typ *arraytype) *__ffi_type {
	if typ.len == 0 {
		return emptyStructToFFI()
	}
	elements := make([]*__ffi_type, typ.len+1)
	et := typeToFFI(typ.elem)
	for i := uintptr(0); i < typ.len; i++ {
		elements[i] = et
	}
	elements[typ.len] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// intToFFI returns an ffi_type for the Go int type.
func intToFFI() *__ffi_type {
	switch unsafe.Sizeof(0) {
	case 4:
		return ffi_type_sint32()
	case 8:
		return ffi_type_sint64()
	default:
		throw("bad int size")
		return nil
	}
}

// emptyStructToFFI returns an ffi_type for an empty struct.
// The libffi library won't accept a struct with no fields.
func emptyStructToFFI() *__ffi_type {
	elements := make([]*__ffi_type, 2)
	elements[0] = ffi_type_void()
	elements[1] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

// padFFI returns a padding field of the given size
func padFFI(size uintptr) *__ffi_type {
	elements := make([]*__ffi_type, size+1)
	for i := uintptr(0); i < size; i++ {
		elements[i] = ffi_type_uint8()
	}
	elements[size] = nil
	return &__ffi_type{
		_type:    _FFI_TYPE_STRUCT,
		elements: &elements[0],
	}
}

//go:linkname makeCIF reflect.makeCIF

// makeCIF is used by the reflect package to allocate a CIF.
func makeCIF(ft *functype) *_ffi_cif {
	cif := new(_ffi_cif)
	ffiFuncToCIF(ft, false, false, cif)
	return cif
}
<|endoftext|>


<commit_before>package moby

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

type tarWriter interface {
	Close() error
	Flush() error
	Write(b []byte) (n int, err error)
	WriteHeader(hdr *tar.Header) error
}

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

// Unfortunately there are some files that Docker always makes appear in a running image and
// export shows them. In particular we have no way for a user to specify their own resolv.conf.
// Even if we were not using docker export to get the image, users of docker build cannot override
// the resolv.conf either, as it is not writeable and bind mounted in.

var exclude = map[string]bool{
	".dockerenv":   true,
	"Dockerfile":   true,
	"dev/console":  true,
	"dev/pts":      true,
	"dev/shm":      true,
	"etc/hostname": true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`,
	"etc/resolv.conf": `
# no resolv.conf configured
`,
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw tarWriter) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != '/' {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == '/' {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			ModTime:  defaultModTime,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

// ImageTar takes a Docker image and outputs it to a tar stream
func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) {
	log.Debugf("image tar: %s %s", ref, prefix)
	if prefix != "" && prefix[len(prefix)-1] != '/' {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	err := tarPrefix(prefix, tw)
	if err != nil {
		return err
	}

	if pull || trust {
		err := dockerPull(ref, pull, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", ref, err)
		}
	}
	container, err := dockerCreate(ref.String())
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if strings.Contains(err.Error(), "No such image") {
			err := dockerPull(ref, true, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", ref, err)
			}
			container, err = dockerCreate(ref.String())
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", ref, err)
			}
		} else {
			return fmt.Errorf("Failed to create docker image %s: %v", ref, err)
		}
	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	defer func() {
		contents.Close()

		if err := dockerRm(container); e == nil && err != nil {
			e = fmt.Errorf("Failed to docker rm container %s: %v", container, err)
		}
	}()

	// now we need to filter out some files from the resulting tar archive

	tr := tar.NewReader(contents)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		// force PAX format, since it allows for unlimited Name/Linkname
		// and we move all files below prefix.
		hdr.Format = tar.FormatPAX
		if exclude[hdr.Name] {
			log.Debugf("image tar: %s %s exclude %s", ref, prefix, hdr.Name)
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else if replace[hdr.Name] != "" {
			if hdr.Name != "etc/resolv.conf" || resolv == "" {
				contents := replace[hdr.Name]
				hdr.Size = int64(len(contents))
				hdr.Name = prefix + hdr.Name
				hdr.ModTime = defaultModTime
				log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
				buf := bytes.NewBufferString(contents)
				_, err = io.Copy(tw, buf)
				if err != nil {
					return err
				}
			} else {
				// replace resolv.conf with specified symlink
				hdr.Name = prefix + hdr.Name
				hdr.Size = 0
				hdr.Typeflag = tar.TypeSymlink
				hdr.Linkname = resolv
				hdr.ModTime = defaultModTime
				log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
			}
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else {
			log.Debugf("image tar: %s %s add %s", ref, prefix, hdr.Name)
			hdr.Name = prefix + hdr.Name
			if hdr.Typeflag == tar.TypeLink {
				// hard links are referenced by full path so need to be adjusted
				hdr.Linkname = prefix + hdr.Linkname
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			_, err = io.Copy(tw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error { // nolint: lll
	// if read only, just unpack in rootfs/ but otherwise set up for overlay
	rootExtract := "rootfs"
	if !readonly {
		rootExtract = "lower"
	}

	// See if we have extracted this image previously
	root := path.Join(prefix, rootExtract)
	var foundElsewhere = dupMap[ref.String()] != ""
	if !foundElsewhere {
		if err := ImageTar(ref, root+"/", tw, trust, pull, ""); err != nil {
			return err
		}
		dupMap[ref.String()] = root
	} else {
		if err := tarPrefix(prefix+"/", tw); err != nil {
			return err
		}
		root = dupMap[ref.String()]
	}

	hdr := &tar.Header{
		Name:    path.Join(prefix, "config.json"),
		Mode:    0644,
		Size:    int64(len(config)),
		ModTime: defaultModTime,
		Format:  tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf := bytes.NewBuffer(config)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	var rootfsMounts []specs.Mount
	if !readonly {
		// add a tmp directory to be used as a mount point for tmpfs for upper, work
		tmp := path.Join(prefix, "tmp")
		hdr = &tar.Header{
			Name:     tmp,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			ModTime:  defaultModTime,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		// add rootfs as merged mount point
		hdr = &tar.Header{
			Name:     path.Join(prefix, "rootfs"),
			Mode:     0755,
			Typeflag: tar.TypeDir,
			ModTime:  defaultModTime,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		overlayOptions := []string{"lowerdir=/" + root, "upperdir=/" + path.Join(tmp, "upper"), "workdir=/" + path.Join(tmp, "work")}
		rootfsMounts = []specs.Mount{
			{Source: "tmpfs", Type: "tmpfs", Destination: "/" + tmp},
			// remount private as nothing else should see the temporary layers
			{Destination: "/" + tmp, Options: []string{"remount", "private"}},
			{Source: "overlay", Type: "overlay", Destination: "/" + path.Join(prefix, "rootfs"), Options: overlayOptions},
		}
	} else {
		if foundElsewhere {
			// we need to make the mountpoint at rootfs
			hdr = &tar.Header{
				Name:     path.Join(prefix, "rootfs"),
				Mode:     0755,
				Typeflag: tar.TypeDir,
				ModTime:  defaultModTime,
				Format:   tar.FormatPAX,
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
		}
		// either bind from another location, or bind from self to make sure it is a mountpoint as runc prefers this
		rootfsMounts = []specs.Mount{
			{Source: "/" + root, Destination: "/" + path.Join(prefix, "rootfs"), Options: []string{"bind"}},
		}
	}

	// Prepend the rootfs onto the user specified mounts.
	runtimeMounts := append(rootfsMounts, *runtime.Mounts...)
	runtime.Mounts = &runtimeMounts

	// write the runtime config
	runtimeConfig, err := json.MarshalIndent(runtime, "", "    ")
	if err != nil {
		return fmt.Errorf("Failed to create runtime config for %s: %v", ref, err)
	}

	hdr = &tar.Header{
		Name:    path.Join(prefix, "runtime.json"),
		Mode:    0644,
		Size:    int64(len(runtimeConfig)),
		ModTime: defaultModTime,
		Format:  tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf = bytes.NewBuffer(runtimeConfig)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, ref, string(config), string(runtimeConfig))

	return nil
}
<commit_msg>build: Improve debug output<commit_after>package moby

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"path"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/opencontainers/runtime-spec/specs-go"
	log "github.com/sirupsen/logrus"
)

type tarWriter interface {
	Close() error
	Flush() error
	Write(b []byte) (n int, err error)
	WriteHeader(hdr *tar.Header) error
}

// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
// used the containerd libraries to do this instead locally direct from a local image
// cache as it would be much simpler.

// Unfortunately there are some files that Docker always makes appear in a running image and
// export shows them. In particular we have no way for a user to specify their own resolv.conf.
// Even if we were not using docker export to get the image, users of docker build cannot override
// the resolv.conf either, as it is not writeable and bind mounted in.

var exclude = map[string]bool{
	".dockerenv":   true,
	"Dockerfile":   true,
	"dev/console":  true,
	"dev/pts":      true,
	"dev/shm":      true,
	"etc/hostname": true,
}

var replace = map[string]string{
	"etc/hosts": `127.0.0.1       localhost
::1     localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`,
	"etc/resolv.conf": `
# no resolv.conf configured
`,
}

// tarPrefix creates the leading directories for a path
func tarPrefix(path string, tw tarWriter) error {
	if path == "" {
		return nil
	}
	if path[len(path)-1] != '/' {
		return fmt.Errorf("path does not end with /: %s", path)
	}
	path = path[:len(path)-1]
	if path[0] == '/' {
		return fmt.Errorf("path should be relative: %s", path)
	}
	mkdir := ""
	for _, dir := range strings.Split(path, "/") {
		mkdir = mkdir + dir
		hdr := &tar.Header{
			Name:     mkdir,
			Mode:     0755,
			ModTime:  defaultModTime,
			Typeflag: tar.TypeDir,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		mkdir = mkdir + "/"
	}
	return nil
}

// ImageTar takes a Docker image and outputs it to a tar stream
func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) {
	log.Debugf("image tar: %s %s", ref, prefix)
	if prefix != "" && prefix[len(prefix)-1] != '/' {
		return fmt.Errorf("prefix does not end with /: %s", prefix)
	}

	err := tarPrefix(prefix, tw)
	if err != nil {
		return err
	}

	if pull || trust {
		err := dockerPull(ref, pull, trust)
		if err != nil {
			return fmt.Errorf("Could not pull image %s: %v", ref, err)
		}
	}
	container, err := dockerCreate(ref.String())
	if err != nil {
		// if the image wasn't found, pull it down.  Bail on other errors.
		if strings.Contains(err.Error(), "No such image") {
			err := dockerPull(ref, true, trust)
			if err != nil {
				return fmt.Errorf("Could not pull image %s: %v", ref, err)
			}
			container, err = dockerCreate(ref.String())
			if err != nil {
				return fmt.Errorf("Failed to docker create image %s: %v", ref, err)
			}
		} else {
			return fmt.Errorf("Failed to create docker image %s: %v", ref, err)
		}
	}
	contents, err := dockerExport(container)
	if err != nil {
		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
	}
	defer func() {
		contents.Close()

		if err := dockerRm(container); e == nil && err != nil {
			e = fmt.Errorf("Failed to docker rm container %s: %v", container, err)
		}
	}()

	// now we need to filter out some files from the resulting tar archive

	tr := tar.NewReader(contents)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		// force PAX format, since it allows for unlimited Name/Linkname
		// and we move all files below prefix.
		hdr.Format = tar.FormatPAX
		if exclude[hdr.Name] {
			log.Debugf("image tar: %s %s exclude %s", ref, prefix, hdr.Name)
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else if replace[hdr.Name] != "" {
			if hdr.Name != "etc/resolv.conf" || resolv == "" {
				contents := replace[hdr.Name]
				hdr.Size = int64(len(contents))
				hdr.Name = prefix + hdr.Name
				hdr.ModTime = defaultModTime
				log.Debugf("image tar: %s %s add %s (replaced)", ref, prefix, hdr.Name)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
				buf := bytes.NewBufferString(contents)
				_, err = io.Copy(tw, buf)
				if err != nil {
					return err
				}
			} else {
				// replace resolv.conf with specified symlink
				hdr.Name = prefix + hdr.Name
				hdr.Size = 0
				hdr.Typeflag = tar.TypeSymlink
				hdr.Linkname = resolv
				hdr.ModTime = defaultModTime
				log.Debugf("image tar: %s %s add resolv symlink /etc/resolv.conf -> %s", ref, prefix, resolv)
				if err := tw.WriteHeader(hdr); err != nil {
					return err
				}
			}
			_, err = io.Copy(ioutil.Discard, tr)
			if err != nil {
				return err
			}
		} else {
			log.Debugf("image tar: %s %s add %s (original)", ref, prefix, hdr.Name)
			hdr.Name = prefix + hdr.Name
			if hdr.Typeflag == tar.TypeLink {
				// hard links are referenced by full path so need to be adjusted
				hdr.Linkname = prefix + hdr.Linkname
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			_, err = io.Copy(tw, tr)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
func ImageBundle(prefix string, ref *reference.Spec, config []byte, runtime Runtime, tw tarWriter, trust bool, pull bool, readonly bool, dupMap map[string]string) error { // nolint: lll
	// if read only, just unpack in rootfs/ but otherwise set up for overlay
	rootExtract := "rootfs"
	if !readonly {
		rootExtract = "lower"
	}

	// See if we have extracted this image previously
	root := path.Join(prefix, rootExtract)
	var foundElsewhere = dupMap[ref.String()] != ""
	if !foundElsewhere {
		if err := ImageTar(ref, root+"/", tw, trust, pull, ""); err != nil {
			return err
		}
		dupMap[ref.String()] = root
	} else {
		if err := tarPrefix(prefix+"/", tw); err != nil {
			return err
		}
		root = dupMap[ref.String()]
	}

	hdr := &tar.Header{
		Name:    path.Join(prefix, "config.json"),
		Mode:    0644,
		Size:    int64(len(config)),
		ModTime: defaultModTime,
		Format:  tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf := bytes.NewBuffer(config)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	var rootfsMounts []specs.Mount
	if !readonly {
		// add a tmp directory to be used as a mount point for tmpfs for upper, work
		tmp := path.Join(prefix, "tmp")
		hdr = &tar.Header{
			Name:     tmp,
			Mode:     0755,
			Typeflag: tar.TypeDir,
			ModTime:  defaultModTime,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		// add rootfs as merged mount point
		hdr = &tar.Header{
			Name:     path.Join(prefix, "rootfs"),
			Mode:     0755,
			Typeflag: tar.TypeDir,
			ModTime:  defaultModTime,
			Format:   tar.FormatPAX,
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		overlayOptions := []string{"lowerdir=/" + root, "upperdir=/" + path.Join(tmp, "upper"), "workdir=/" + path.Join(tmp, "work")}
		rootfsMounts = []specs.Mount{
			{Source: "tmpfs", Type: "tmpfs", Destination: "/" + tmp},
			// remount private as nothing else should see the temporary layers
			{Destination: "/" + tmp, Options: []string{"remount", "private"}},
			{Source: "overlay", Type: "overlay", Destination: "/" + path.Join(prefix, "rootfs"), Options: overlayOptions},
		}
	} else {
		if foundElsewhere {
			// we need to make the mountpoint at rootfs
			hdr = &tar.Header{
				Name:     path.Join(prefix, "rootfs"),
				Mode:     0755,
				Typeflag: tar.TypeDir,
				ModTime:  defaultModTime,
				Format:   tar.FormatPAX,
			}
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
		}
		// either bind from another location, or bind from self to make sure it is a mountpoint as runc prefers this
		rootfsMounts = []specs.Mount{
			{Source: "/" + root, Destination: "/" + path.Join(prefix, "rootfs"), Options: []string{"bind"}},
		}
	}

	// Prepend the rootfs onto the user specified mounts.
	runtimeMounts := append(rootfsMounts, *runtime.Mounts...)
	runtime.Mounts = &runtimeMounts

	// write the runtime config
	runtimeConfig, err := json.MarshalIndent(runtime, "", "    ")
	if err != nil {
		return fmt.Errorf("Failed to create runtime config for %s: %v", ref, err)
	}

	hdr = &tar.Header{
		Name:    path.Join(prefix, "runtime.json"),
		Mode:    0644,
		Size:    int64(len(runtimeConfig)),
		ModTime: defaultModTime,
		Format:  tar.FormatPAX,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	buf = bytes.NewBuffer(runtimeConfig)
	if _, err := io.Copy(tw, buf); err != nil {
		return err
	}

	log.Debugf("image bundle: %s %s cfg: %s runtime: %s", prefix, ref, string(config), string(runtimeConfig))

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013-2014, Jeremy Bingham (<jbingham@gmail.com>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package search

import (
	"encoding/gob"
	"fmt"
	"github.com/ctdk/goiardi/client"
	"github.com/ctdk/goiardi/databag"
	"github.com/ctdk/goiardi/environment"
	"github.com/ctdk/goiardi/node"
	"github.com/ctdk/goiardi/role"
	"testing"
	"time"
)

// Most search testing can be handled fine with chef-pedant, but that's no
// reason to not have some go tests for it too.

var node1 *node.Node
var node2 *node.Node
var node3 *node.Node
var node4 *node.Node
var role1 *role.Role
var role2 *role.Role
var role3 *role.Role
var role4 *role.Role
var env1 *environment.ChefEnvironment
var env2 *environment.ChefEnvironment
var env3 *environment.ChefEnvironment
var env4 *environment.ChefEnvironment
var client1 *client.Client
var client2 *client.Client
var client3 *client.Client
var client4 *client.Client
var dbag1 *databag.DataBag
var dbag2 *databag.DataBag
var dbag3 *databag.DataBag
var dbag4 *databag.DataBag

func makeSearchItems() int {
	/* Gotta populate the search index */
	nodes := make([]*node.Node, 4)
	roles := make([]*role.Role, 4)
	envs := make([]*environment.ChefEnvironment, 4)
	clients := make([]*client.Client, 4)
	dbags := make([]*databag.DataBag, 4)
	gob.Register(new(node.Node))
	gob.Register(new(role.Role))
	gob.Register(new(environment.ChefEnvironment))
	gob.Register(new(client.Client))
	gob.Register(new(databag.DataBag))

	for i := 0; i < 4; i++ {
		nodes[i], _ = node.New(fmt.Sprintf("node%d", i))
		nodes[i].Save()
		roles[i], _ = role.New(fmt.Sprintf("role%d", i))
		roles[i].Save()
		envs[i], _ = environment.New(fmt.Sprintf("env%d", i))
		envs[i].Save()
		clients[i], _ = client.New(fmt.Sprintf("client%d", i))
		clients[i].Save()
		dbags[i], _ = databag.New(fmt.Sprintf("databag%d", i))
		dbags[i].Save()
		dbi := make(map[string]interface{})
		dbi["id"] = fmt.Sprintf("dbi%d", i)
		dbi["foo"] = fmt.Sprintf("dbag_item_%d", i)
		dbags[i].NewDBItem(dbi)
	}
	node1 = nodes[0]
	node2 = nodes[1]
	node3 = nodes[2]
	node4 = nodes[3]
	role1 = roles[0]
	role2 = roles[1]
	role3 = roles[2]
	role4 = roles[3]
	env1 = envs[0]
	env2 = envs[1]
	env3 = envs[2]
	env4 = envs[3]
	client1 = clients[0]
	client2 = clients[1]
	client3 = clients[2]
	client4 = clients[3]
	dbag1 = dbags[0]
	dbag2 = dbags[1]
	dbag3 = dbags[2]
	dbag4 = dbags[3]

	/* Make this function return something so the compiler's happy building
	 * the tests. */
	return 1
}

var v = makeSearchItems()

func TestFoo(t *testing.T) {
	return
}

/* Only basic search tests are here. The stronger tests are handled in
 * chef-pedant, but these tests are meant to check basic search functionality.
 */

func TestSearchNode(t *testing.T) {
	n, _ := Search("node", "name:node1")
	if n[0].(*node.Node).Name != "node1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchNodeAll(t *testing.T) {
	n, _ := Search("node", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchRole(t *testing.T) {
	r, _ := Search("role", "name:role1")
	if r[0].(*role.Role).Name != "role1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchRoleAll(t *testing.T) {
	n, _ := Search("role", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchEnv(t *testing.T) {
	e, _ := Search("environment", "name:env1")
	if e[0].(*environment.ChefEnvironment).Name != "env1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchEnvAll(t *testing.T) {
	n, _ := Search("environment", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchClient(t *testing.T) {
	c, _ := Search("client", "name:client1")
	if c[0].(*client.Client).Name != "client1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchClientAll(t *testing.T) {
	n, _ := Search("client", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchDbag(t *testing.T) {
	d, _ := Search("databag1", "foo:dbag_item_1")
	if len(d) == 0 {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchDbagAll(t *testing.T) {
	d, _ := Search("databag1", "*:*")
	if len(d) != 1 {
		t.Errorf("Incorrect number of items returned, expected 1, got %d", len(d))
	}
}

// Probably don't want this as an always test, but it's handy to have available.
/*
func TestEmbiggenSearch(t *testing.T) {
	for i := 4; i < 35000; i++ {
		n, _ := node.New(fmt.Sprintf("node%d", i))
		n.Save()
		r, _ := role.New(fmt.Sprintf("role%d", i))
		r.Save()
		e, _ := environment.New(fmt.Sprintf("env%d", i))
		e.Save()
		c, _ := client.New(fmt.Sprintf("client%d", i))
		c.Save()
		d, _ := databag.New(fmt.Sprintf("databag%d", i))
		d.Save()
		dbi := make(map[string]interface{})
		dbi["id"] = fmt.Sprintf("dbi%d", i)
		dbi["foo"] = fmt.Sprintf("dbag_item_%d", i)
		d.NewDBItem(dbi)
	}
	time.Sleep(1 * time.Second)
	n, _ := Search("client", "*:*")
	if len(n) != 35000 {
		t.Errorf("Incorrect number of items returned, expected 500, got %d", len(n))
	}
	c, _ := Search("node", "*:*")
	if len(c) != 35000 {
		t.Errorf("Incorrect number of nodes returned, expected 500, got %d", len(n))
	}
	e, _ := Search("environment", "name:env11666")
	if e[0].(*environment.ChefEnvironment).Name != "env11666" {
		t.Errorf("nothing returned from search")
	}
}
*/
<commit_msg>Added another simple FALSE AND TRUE test<commit_after>/*
 * Copyright (c) 2013-2014, Jeremy Bingham (<jbingham@gmail.com>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package search

import (
	"encoding/gob"
	"fmt"
	"github.com/ctdk/goiardi/client"
	"github.com/ctdk/goiardi/databag"
	"github.com/ctdk/goiardi/environment"
	"github.com/ctdk/goiardi/node"
	"github.com/ctdk/goiardi/role"
	"testing"
	"time"
)

// Most search testing can be handled fine with chef-pedant, but that's no
// reason to not have some go tests for it too.

var node1 *node.Node
var node2 *node.Node
var node3 *node.Node
var node4 *node.Node
var role1 *role.Role
var role2 *role.Role
var role3 *role.Role
var role4 *role.Role
var env1 *environment.ChefEnvironment
var env2 *environment.ChefEnvironment
var env3 *environment.ChefEnvironment
var env4 *environment.ChefEnvironment
var client1 *client.Client
var client2 *client.Client
var client3 *client.Client
var client4 *client.Client
var dbag1 *databag.DataBag
var dbag2 *databag.DataBag
var dbag3 *databag.DataBag
var dbag4 *databag.DataBag

func makeSearchItems() int {
	/* Gotta populate the search index */
	nodes := make([]*node.Node, 4)
	roles := make([]*role.Role, 4)
	envs := make([]*environment.ChefEnvironment, 4)
	clients := make([]*client.Client, 4)
	dbags := make([]*databag.DataBag, 4)
	gob.Register(new(node.Node))
	gob.Register(new(role.Role))
	gob.Register(new(environment.ChefEnvironment))
	gob.Register(new(client.Client))
	gob.Register(new(databag.DataBag))

	for i := 0; i < 4; i++ {
		nodes[i], _ = node.New(fmt.Sprintf("node%d", i))
		nodes[i].Save()
		roles[i], _ = role.New(fmt.Sprintf("role%d", i))
		roles[i].Save()
		envs[i], _ = environment.New(fmt.Sprintf("env%d", i))
		envs[i].Save()
		clients[i], _ = client.New(fmt.Sprintf("client%d", i))
		clients[i].Save()
		dbags[i], _ = databag.New(fmt.Sprintf("databag%d", i))
		dbags[i].Save()
		dbi := make(map[string]interface{})
		dbi["id"] = fmt.Sprintf("dbi%d", i)
		dbi["foo"] = fmt.Sprintf("dbag_item_%d", i)
		dbags[i].NewDBItem(dbi)
	}
	node1 = nodes[0]
	node2 = nodes[1]
	node3 = nodes[2]
	node4 = nodes[3]
	role1 = roles[0]
	role2 = roles[1]
	role3 = roles[2]
	role4 = roles[3]
	env1 = envs[0]
	env2 = envs[1]
	env3 = envs[2]
	env4 = envs[3]
	client1 = clients[0]
	client2 = clients[1]
	client3 = clients[2]
	client4 = clients[3]
	dbag1 = dbags[0]
	dbag2 = dbags[1]
	dbag3 = dbags[2]
	dbag4 = dbags[3]

	/* Make this function return something so the compiler's happy building
	 * the tests. */
	return 1
}

var v = makeSearchItems()

func TestFoo(t *testing.T) {
	return
}

/* Only basic search tests are here. The stronger tests are handled in
 * chef-pedant, but these tests are meant to check basic search functionality.
 */

func TestSearchNode(t *testing.T) {
	n, _ := Search("node", "name:node1")
	if n[0].(*node.Node).Name != "node1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchNodeAll(t *testing.T) {
	n, _ := Search("node", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchNodeFalse(t *testing.T) {
	n, _ := Search("node", "foo:bar AND NOT foo:bar")
	if len(n) != 0 {
		t.Errorf("Incorrect number of items returned, expected 0, got %d", len(n))
	}
}

func TestSearchRole(t *testing.T) {
	r, _ := Search("role", "name:role1")
	if r[0].(*role.Role).Name != "role1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchRoleAll(t *testing.T) {
	n, _ := Search("role", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchEnv(t *testing.T) {
	e, _ := Search("environment", "name:env1")
	if e[0].(*environment.ChefEnvironment).Name != "env1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchEnvAll(t *testing.T) {
	n, _ := Search("environment", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchClient(t *testing.T) {
	c, _ := Search("client", "name:client1")
	if c[0].(*client.Client).Name != "client1" {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchClientAll(t *testing.T) {
	n, _ := Search("client", "*:*")
	if len(n) != 4 {
		t.Errorf("Incorrect number of items returned, expected 4, got %d", len(n))
	}
}

func TestSearchDbag(t *testing.T) {
	d, _ := Search("databag1", "foo:dbag_item_1")
	if len(d) == 0 {
		t.Errorf("nothing returned from search")
	}
}

func TestSearchDbagAll(t *testing.T) {
	d, _ := Search("databag1", "*:*")
	if len(d) != 1 {
		t.Errorf("Incorrect number of items returned, expected 1, got %d", len(d))
	}
}

// Probably don't want this as an always test, but it's handy to have available.
/*
func TestEmbiggenSearch(t *testing.T) {
	for i := 4; i < 35000; i++ {
		n, _ := node.New(fmt.Sprintf("node%d", i))
		n.Save()
		r, _ := role.New(fmt.Sprintf("role%d", i))
		r.Save()
		e, _ := environment.New(fmt.Sprintf("env%d", i))
		e.Save()
		c, _ := client.New(fmt.Sprintf("client%d", i))
		c.Save()
		d, _ := databag.New(fmt.Sprintf("databag%d", i))
		d.Save()
		dbi := make(map[string]interface{})
		dbi["id"] = fmt.Sprintf("dbi%d", i)
		dbi["foo"] = fmt.Sprintf("dbag_item_%d", i)
		d.NewDBItem(dbi)
	}
	time.Sleep(1 * time.Second)
	n, _ := Search("client", "*:*")
	if len(n) != 35000 {
		t.Errorf("Incorrect number of items returned, expected 500, got %d", len(n))
	}
	c, _ := Search("node", "*:*")
	if len(c) != 35000 {
		t.Errorf("Incorrect number of nodes returned, expected 500, got %d", len(n))
	}
	e, _ := Search("environment", "name:env11666")
	if e[0].(*environment.ChefEnvironment).Name != "env11666" {
		t.Errorf("nothing returned from search")
	}
}
*/
<|endoftext|>


<commit_before>// Copyright © 2016 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"io"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/shenwei356/xopen"
	"github.com/spf13/cobra"
)

// replaceCmd represents the replace command
var replaceCmd = &cobra.Command{
	Use:   "replace",
	Short: "replace name/sequence by regular expression",
	Long: `replace name/sequence by regular expression.

Note that the replacement supports capture variables.
e.g. $1 represents the text of the first submatch.
ATTENTION: use SINGLE quote NOT double quotes in *nix OS.

Examples: Adding space to all bases.

    seqkit replace -p "(.)" -r '$1 ' -s

Or use the \ escape character.

    seqkit replace -p "(.)" -r "\$1 " -s

more on: http://shenwei356.github.io/seqkit/usage/#replace

Special repalcement symbols:

	{nr}	Record number, starting from 1
	{kv}	Corresponding value of the key ($1) by key-value file

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		alphabet := config.Alphabet
		idRegexp := config.IDRegexp
		lineWidth := config.LineWidth
		outFile := config.OutFile
		seq.AlphabetGuessSeqLenghtThreshold = config.AlphabetGuessSeqLength
		seq.ValidateSeq = false
		runtime.GOMAXPROCS(config.Threads)

		pattern := getFlagString(cmd, "pattern")
		replacement := []byte(getFlagString(cmd, "replacement"))
		kvFile := getFlagString(cmd, "kv-file")

		bySeq := getFlagBool(cmd, "by-seq")
		// byName := getFlagBool(cmd, "by-name")
		ignoreCase := getFlagBool(cmd, "ignore-case")

		if pattern == "" {
			checkError(fmt.Errorf("flags -p (--pattern) needed"))
		}
		p := pattern
		if ignoreCase {
			p = "(?i)" + p
		}
		patternRegexp, err := regexp.Compile(p)
		checkError(err)

		var replaceWithNR bool
		if reNR.Match(replacement) {
			replaceWithNR = true
		}

		var replaceWithKV bool
		var kvs map[string]string
		if reKV.Match(replacement) {
			replaceWithKV = true
			if !regexp.MustCompile(`\(.+\)`).MatchString(pattern) {
				checkError(fmt.Errorf(`value of -p (--pattern) must contains "(" and ")" to capture data which is used specify the KEY`))
			}
			if kvFile == "" {
				checkError(fmt.Errorf(`since repalcement symbol "{kv}"/"{KV}" found in value of flag -r (--replacement), tab-delimited key-value file should be given by flag -k (--kv-file)`))
			}
			log.Infof("read key-value file: %s", kvFile)
			kvs, err = readKVs(kvFile)
			if err != nil {
				checkError(fmt.Errorf("read key-value file: %s", err))
			}
			if len(kvs) == 0 {
				checkError(fmt.Errorf("no valid data in key-value file: %s", kvFile))
			}

			if ignoreCase {
				kvs2 := make(map[string]string, len(kvs))
				for k, v := range kvs {
					kvs2[strings.ToLower(k)] = v
				}
				kvs = kvs2
			}

			log.Infof("%d pairs of key-value loaded", len(kvs))
		}

		files := getFileList(args)

		outfh, err := xopen.Wopen(outFile)
		checkError(err)
		defer outfh.Close()

		var r []byte
		var found [][]byte
		var k string
		var ok bool
		for _, file := range files {

			fastxReader, err := fastx.NewReader(alphabet, file, idRegexp)
			checkError(err)
			nr := 0
			for {
				record, err := fastxReader.Read()
				if err != nil {
					if err == io.EOF {
						break
					}
					checkError(err)
					break
				}

				nr++
				if bySeq {
					record.Seq.Seq = patternRegexp.ReplaceAll(record.Seq.Seq, replacement)
				} else {
					r = replacement

					if replaceWithNR {
						r = reNR.ReplaceAll(r, []byte(strconv.Itoa(nr)))
					}

					if replaceWithKV {
						found = patternRegexp.FindSubmatch(record.Name)
						if len(found) > 0 {
							k = string(found[1])
							if ignoreCase {
								k = strings.ToLower(k)
							}
							if _, ok = kvs[k]; ok {
								r = reKV.ReplaceAll(r, []byte(kvs[k]))
							} else {
								r = reKV.ReplaceAll(r, found[1])
							}
						}
					}

				}

				record.Name = patternRegexp.ReplaceAll(record.Name, r)

				record.FormatToWriter(outfh, lineWidth)
			}

		}
	},
}

func init() {
	RootCmd.AddCommand(replaceCmd)
	replaceCmd.Flags().StringP("pattern", "p", "", "search regular expression")
	replaceCmd.Flags().StringP("replacement", "r", "",
		"replacement. supporting capture variables. "+
			" e.g. $1 represents the text of the first submatch. "+
			"ATTENTION: use SINGLE quote NOT double quotes in *nix OS or "+
			`use the \ escape character. Record number is also supported by "{nr}"`)
	// replaceCmd.Flags().BoolP("by-name", "n", false, "replace full name instead of just id")
	replaceCmd.Flags().BoolP("by-seq", "s", false, "replace seq")
	replaceCmd.Flags().BoolP("ignore-case", "i", false, "ignore case")
	replaceCmd.Flags().StringP("kv-file", "k", "",
		`tab-delimited key-value file for replacing key with value when using "{kv}" in -r (--replacement)`)
}

var reNR = regexp.MustCompile(`\{(NR|nr)\}`)
var reKV = regexp.MustCompile(`\{(KV|kv)\}`)
<commit_msg>fix replace.go<commit_after>// Copyright © 2016 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"io"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"github.com/shenwei356/bio/seq"
	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/shenwei356/xopen"
	"github.com/spf13/cobra"
)

// replaceCmd represents the replace command
var replaceCmd = &cobra.Command{
	Use:   "replace",
	Short: "replace name/sequence by regular expression",
	Long: `replace name/sequence by regular expression.

Note that the replacement supports capture variables.
e.g. $1 represents the text of the first submatch.
ATTENTION: use SINGLE quote NOT double quotes in *nix OS.

Examples: Adding space to all bases.

    seqkit replace -p "(.)" -r '$1 ' -s

Or use the \ escape character.

    seqkit replace -p "(.)" -r "\$1 " -s

more on: http://shenwei356.github.io/seqkit/usage/#replace

Special repalcement symbols:

	{nr}	Record number, starting from 1
	{kv}	Corresponding value of the key ($1) by key-value file

`,
	Run: func(cmd *cobra.Command, args []string) {
		config := getConfigs(cmd)
		alphabet := config.Alphabet
		idRegexp := config.IDRegexp
		lineWidth := config.LineWidth
		outFile := config.OutFile
		seq.AlphabetGuessSeqLenghtThreshold = config.AlphabetGuessSeqLength
		seq.ValidateSeq = false
		runtime.GOMAXPROCS(config.Threads)

		pattern := getFlagString(cmd, "pattern")
		replacement := []byte(getFlagString(cmd, "replacement"))
		kvFile := getFlagString(cmd, "kv-file")

		bySeq := getFlagBool(cmd, "by-seq")
		// byName := getFlagBool(cmd, "by-name")
		ignoreCase := getFlagBool(cmd, "ignore-case")

		if pattern == "" {
			checkError(fmt.Errorf("flags -p (--pattern) needed"))
		}
		p := pattern
		if ignoreCase {
			p = "(?i)" + p
		}
		patternRegexp, err := regexp.Compile(p)
		checkError(err)

		var replaceWithNR bool
		if reNR.Match(replacement) {
			replaceWithNR = true
		}

		var replaceWithKV bool
		var kvs map[string]string
		if reKV.Match(replacement) {
			replaceWithKV = true
			if !regexp.MustCompile(`\(.+\)`).MatchString(pattern) {
				checkError(fmt.Errorf(`value of -p (--pattern) must contains "(" and ")" to capture data which is used specify the KEY`))
			}
			if kvFile == "" {
				checkError(fmt.Errorf(`since repalcement symbol "{kv}"/"{KV}" found in value of flag -r (--replacement), tab-delimited key-value file should be given by flag -k (--kv-file)`))
			}
			log.Infof("read key-value file: %s", kvFile)
			kvs, err = readKVs(kvFile)
			if err != nil {
				checkError(fmt.Errorf("read key-value file: %s", err))
			}
			if len(kvs) == 0 {
				checkError(fmt.Errorf("no valid data in key-value file: %s", kvFile))
			}

			if ignoreCase {
				kvs2 := make(map[string]string, len(kvs))
				for k, v := range kvs {
					kvs2[strings.ToLower(k)] = v
				}
				kvs = kvs2
			}

			log.Infof("%d pairs of key-value loaded", len(kvs))
		}

		files := getFileList(args)

		outfh, err := xopen.Wopen(outFile)
		checkError(err)
		defer outfh.Close()

		var r []byte
		var found [][]byte
		var k string
		var ok bool
		for _, file := range files {

			fastxReader, err := fastx.NewReader(alphabet, file, idRegexp)
			checkError(err)
			nr := 0
			for {
				record, err := fastxReader.Read()
				if err != nil {
					if err == io.EOF {
						break
					}
					checkError(err)
					break
				}

				nr++
				if bySeq {
					record.Seq.Seq = patternRegexp.ReplaceAll(record.Seq.Seq, replacement)
				} else {
					r = replacement

					if replaceWithNR {
						r = reNR.ReplaceAll(r, []byte(strconv.Itoa(nr)))
					}

					if replaceWithKV {
						found = patternRegexp.FindSubmatch(record.Name)
						if len(found) > 0 {
							k = string(found[1])
							if ignoreCase {
								k = strings.ToLower(k)
							}
							if _, ok = kvs[k]; ok {
								r = reKV.ReplaceAll(r, []byte(kvs[k]))
							} else {
								r = reKV.ReplaceAll(r, found[1])
							}
						}
					}

					record.Name = patternRegexp.ReplaceAll(record.Name, r)
				}

				record.FormatToWriter(outfh, lineWidth)
			}

		}
	},
}

func init() {
	RootCmd.AddCommand(replaceCmd)
	replaceCmd.Flags().StringP("pattern", "p", "", "search regular expression")
	replaceCmd.Flags().StringP("replacement", "r", "",
		"replacement. supporting capture variables. "+
			" e.g. $1 represents the text of the first submatch. "+
			"ATTENTION: use SINGLE quote NOT double quotes in *nix OS or "+
			`use the \ escape character. Record number is also supported by "{nr}"`)
	// replaceCmd.Flags().BoolP("by-name", "n", false, "replace full name instead of just id")
	replaceCmd.Flags().BoolP("by-seq", "s", false, "replace seq")
	replaceCmd.Flags().BoolP("ignore-case", "i", false, "ignore case")
	replaceCmd.Flags().StringP("kv-file", "k", "",
		`tab-delimited key-value file for replacing key with value when using "{kv}" in -r (--replacement)`)
}

var reNR = regexp.MustCompile(`\{(NR|nr)\}`)
var reKV = regexp.MustCompile(`\{(KV|kv)\}`)
<|endoftext|>


<commit_before>package relay

import (
	"bytes"
	"reflect"
	"testing"
	"time"
)

func TestGOBSerializer(t *testing.T) {
	type basic struct {
		Key   string
		Value string
		When  time.Time
	}

	g := &GOBSerializer{}

	if g.ContentType() != "binary/gob" {
		t.Fatalf("bad content type")
	}

	obj := basic{"test", "this is a value", time.Now()}
	var buf bytes.Buffer

	// Encode the struct
	if err := g.Encode(&buf, &obj); err != nil {
		t.Fatalf("unexpected err %s", err)
	}

	// Try to decode
	out := basic{}
	if err := g.Decode(&buf, &out); err != nil {
		t.Fatalf("unexpected err %s", err)
	}

	// Ensure equal
	if !reflect.DeepEqual(obj, out) {
		t.Fatalf("not equal. %v %v", obj, out)
	}
}
<commit_msg>Fix test<commit_after>package relay

import (
	"bytes"
	"reflect"
	"testing"
	"time"
)

func TestGOBSerializer(t *testing.T) {
	type basic struct {
		Key   string
		Value string
		When  time.Time
	}

	g := &GOBSerializer{}

	if g.ContentType() != "binary/gob" {
		t.Fatalf("bad content type")
	}

	obj := basic{"test", "this is a value", time.Now()}
	var buf bytes.Buffer

	// Encode the struct
	if err := g.RelayEncode(&buf, &obj); err != nil {
		t.Fatalf("unexpected err %s", err)
	}

	// Try to decode
	out := basic{}
	if err := g.RelayDecode(&buf, &out); err != nil {
		t.Fatalf("unexpected err %s", err)
	}

	// Ensure equal
	if !reflect.DeepEqual(obj, out) {
		t.Fatalf("not equal. %v %v", obj, out)
	}
}
<|endoftext|>


<commit_before>// Package feature provides access to the system features tables.
//
// These features are represented by the webframework.osm classes Feature,
// FeatureGroup and FeatureApplication.
//
// Features supported by the system are organized in groups such as
// "Administrator", "Area", "Errand Handling", etc. These are provided by the
// osm classes FeatureGroup and Feature. FeatureApplication represents the
// feature value.
//
// Features are associated with context. Context is a colon-separated pair of
// strings.
//
// Features can be queried via tags. Once loaded, features are cached.
// Subsequent query of the feature value using State, Str, Bool and Int using
// either uses the cached feature.
package feature

import (
	"log"
	wf "osm/webframework"
	"strings"
)

var DefaultContexts []string
var DefaultGlobalContext = ""
var featureCache = make(map[string]*wf.FeatureApplication)

// ClearCache clears the cached features that were loaded from the object server.
func ClearCache() {
	featureCache = make(map[string]*wf.FeatureApplication)
}

// Preload loads all the feature for the current default contexts.
func Preload() {
	States([]string{}, DefaultContexts)
}

// SetGlobalContext sets the default global context to the given context.
func SetGlobalContext(ctx string) {
	DefaultGlobalContext = ctx
}

// SetDefaultContexts sets the default contexts to the given contexts.
func SetDefaultContexts(ctxs []string) {
	DefaultContexts = ctxs
}

// SetDefaultContext sets the default context to the given context string. More
// than one context can be specified by separating them with semicolon.
func SetDefaultContext(ctx string) {
	SetDefaultContexts(strings.Split(ctx, ";"))
}

// DefaultContext returns the semicolon-separated default contexts.
func DefaultContext() string {
	return DefaultGlobalContext + ";" + wf.FeatureApplication_DEFAULT_CONTEXT
}

// States implements the Go equivalent of webframework/Core/Features.feh's
// state().
func States(featureTags []string, contexts []string) (map[string]*wf.FeatureApplication, error) {
	// TODO rename processList to byPriorityDesc
	var processList wf.FeatureApplicationSlice
	var err error
	var featureList []string
	// TODO rename list to featureMap
	list := map[string]*wf.FeatureApplication{}

	contextList := []string{DefaultContext()}
	for _, v := range contexts {
		contextList = append(contextList, DefaultGlobalContext+`;`+v)
	}

	if len(featureTags) == 0 {
		processList, err = wf.QueryFeatureApplicationFetchInContext(contextList)
		if err != nil {
			return nil, err
		}
	} else {
		for _, tag := range featureTags {
			if fa, exists := featureCache[tag]; exists {
				list[tag] = fa
			} else {
				featureList = append(featureList, tag)
			}
		}
		if len(featureList) > 0 {
			processList, err = wf.QueryFeatureApplicationFetchByFeaturesInContext(featureList, contextList)
			if err != nil {
				// FIXME don't panic
				panic(err)
			}
		}
	}

	for _, featureApplication := range processList {
		if _, exists := list[featureApplication.FeatureTag]; !exists {
			list[featureApplication.FeatureTag] = featureApplication
		}
	}

	if len(featureTags) == 0 {
		featureCache = list
	} else {
		for _, tag := range featureList {
			//fmt.Printf("%8v => %v\n", tag, list[tag].Val())
			if v, exists := list[tag]; exists {
				featureCache[tag] = v
			}
		}
	}

	return list, nil
}

func stateForTagWithDefaultContexts(tag string) (*wf.FeatureApplication, error) {
	m, err := States([]string{tag}, DefaultContexts)
	if err != nil {
		return nil, err
	}
	for _, v := range m {
		return v, nil
	}
	return nil, nil
}

// State returns the (possibly cached) feature application state for the given
// tag.
func State(tag string) (*wf.FeatureApplication, error) {
	if v, exists := featureCache[tag]; exists {
		return v, nil
	}
	fa, err := stateForTagWithDefaultContexts(tag)
	if err != nil {
		return nil, err
	}
	if fa != nil {
		return fa, nil
	}
	return nil, nil
}

// Bool returns the boolean value of the feature application state for the
// given tag.
func Bool(tag string) bool {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Bool(`%v`): %v", tag, err)
	}
	return fa != nil && fa.Bool()
}

// Int returns the integer value of the feature application state for the given
// tag.
func Int(tag string) int {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Int(`%v`): %v", tag, err)
	}
	if fa == nil {
		return -1
	}
	return fa.Int()
}

// Str returns the string value of the feature application state for the given
// tag.
func Str(tag string) string {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Str(`%v`): %v", tag, err)
	}
	if fa == nil {
		return ""
	}
	return fa.Str()
}
<commit_msg>go: Rename query function to follow the new format<commit_after>// Package feature provides access to the system features tables.
//
// These features are represented by the webframework.osm classes Feature,
// FeatureGroup and FeatureApplication.
//
// Features supported by the system are organized in groups such as
// "Administrator", "Area", "Errand Handling", etc. These are provided by the
// osm classes FeatureGroup and Feature. FeatureApplication represents the
// feature value.
//
// Features are associated with context. Context is a colon-separated pair of
// strings.
//
// Features can be queried via tags. Once loaded, features are cached.
// Subsequent query of the feature value using State, Str, Bool and Int using
// either uses the cached feature.
package feature

import (
	"log"
	wf "osm/webframework"
	"strings"
)

var DefaultContexts []string
var DefaultGlobalContext = ""
var featureCache = make(map[string]*wf.FeatureApplication)

// ClearCache clears the cached features that were loaded from the object server.
func ClearCache() {
	featureCache = make(map[string]*wf.FeatureApplication)
}

// Preload loads all the feature for the current default contexts.
func Preload() {
	States([]string{}, DefaultContexts)
}

// SetGlobalContext sets the default global context to the given context.
func SetGlobalContext(ctx string) {
	DefaultGlobalContext = ctx
}

// SetDefaultContexts sets the default contexts to the given contexts.
func SetDefaultContexts(ctxs []string) {
	DefaultContexts = ctxs
}

// SetDefaultContext sets the default context to the given context string. More
// than one context can be specified by separating them with semicolon.
func SetDefaultContext(ctx string) {
	SetDefaultContexts(strings.Split(ctx, ";"))
}

// DefaultContext returns the semicolon-separated default contexts.
func DefaultContext() string {
	return DefaultGlobalContext + ";" + wf.FeatureApplication_DEFAULT_CONTEXT
}

// States implements the Go equivalent of webframework/Core/Features.feh's
// state().
func States(featureTags []string, contexts []string) (map[string]*wf.FeatureApplication, error) {
	// TODO rename processList to byPriorityDesc
	var processList wf.FeatureApplicationSlice
	var err error
	var featureList []string
	// TODO rename list to featureMap
	list := map[string]*wf.FeatureApplication{}

	contextList := []string{DefaultContext()}
	for _, v := range contexts {
		contextList = append(contextList, DefaultGlobalContext+`;`+v)
	}

	if len(featureTags) == 0 {
		processList, err = wf.QueryFeatureApplication_fetchInContext(contextList)
		if err != nil {
			return nil, err
		}
	} else {
		for _, tag := range featureTags {
			if fa, exists := featureCache[tag]; exists {
				list[tag] = fa
			} else {
				featureList = append(featureList, tag)
			}
		}
		if len(featureList) > 0 {
			processList, err = wf.QueryFeatureApplication_fetchByFeaturesInContext(featureList, contextList)
			if err != nil {
				// FIXME don't panic
				panic(err)
			}
		}
	}

	for _, featureApplication := range processList {
		if _, exists := list[featureApplication.FeatureTag]; !exists {
			list[featureApplication.FeatureTag] = featureApplication
		}
	}

	if len(featureTags) == 0 {
		featureCache = list
	} else {
		for _, tag := range featureList {
			//fmt.Printf("%8v => %v\n", tag, list[tag].Val())
			if v, exists := list[tag]; exists {
				featureCache[tag] = v
			}
		}
	}

	return list, nil
}

func stateForTagWithDefaultContexts(tag string) (*wf.FeatureApplication, error) {
	m, err := States([]string{tag}, DefaultContexts)
	if err != nil {
		return nil, err
	}
	for _, v := range m {
		return v, nil
	}
	return nil, nil
}

// State returns the (possibly cached) feature application state for the given
// tag.
func State(tag string) (*wf.FeatureApplication, error) {
	if v, exists := featureCache[tag]; exists {
		return v, nil
	}
	fa, err := stateForTagWithDefaultContexts(tag)
	if err != nil {
		return nil, err
	}
	if fa != nil {
		return fa, nil
	}
	return nil, nil
}

// Bool returns the boolean value of the feature application state for the
// given tag.
func Bool(tag string) bool {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Bool(`%v`): %v", tag, err)
	}
	return fa != nil && fa.Bool()
}

// Int returns the integer value of the feature application state for the given
// tag.
func Int(tag string) int {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Int(`%v`): %v", tag, err)
	}
	if fa == nil {
		return -1
	}
	return fa.Int()
}

// Str returns the string value of the feature application state for the given
// tag.
func Str(tag string) string {
	fa, err := State(tag)
	if err != nil {
		log.Printf("feature.Str(`%v`): %v", tag, err)
	}
	if fa == nil {
		return ""
	}
	return fa.Str()
}
<|endoftext|>