<commit_before>// Utility to create and manage chromium builds.
package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/buildskia"
	"go.skia.org/infra/go/util"

	"strings"
)

// Construct the name of a directory to store a chromium build. For generic clean builds, runID
// should be empty.
func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string {
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		return fmt.Sprintf("%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash))
	} else {
		return fmt.Sprintf("%s-%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash),
			runID)
	}
}

// CreateChromiumBuildOnSwarming creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
// uploadSingleBuild if true does not upload a 2nd build of Chromium.
func CreateChromiumBuildOnSwarming(runID, targetPlatform, chromiumHash, skiaHash, pathToPyFiles string, applyPatches, uploadSingleBuild bool) (string, string, error) {
	chromiumBuildDir, _ := filepath.Split(ChromiumSrcDir)
	// Determine which fetch target to use.
	var fetchTarget string
	if targetPlatform == "Android" {
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = buildskia.GetSkiaHash(nil)
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	err = ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil, nil)
	if err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		err := ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil,
			nil)
		if err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := ResetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := ChromiumBuildDir(chromiumHash, skiaHash, runID)
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Hack: Use the "-DSK_WHITELIST_SERIALIZED_TYPEFACES" flag only when *runID is
	// empty i.e. when invoked by the build_chromium task.
	useWhitelistedFonts := (runID == "")
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform, useWhitelistedFonts); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Create and upload another chromium build if the uploadSingleBuild flag is false. This build
	// will be created without applying patches.
	if !uploadSingleBuild {
		// Make sure we are starting from a clean slate.
		if err := ResetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform, useWhitelistedFonts); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-nopatch", ChromiumBuildDir(chromiumHash, skiaHash, runID))
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	err = ExecuteCmd(BINARY_GIT, totArgs, []string{}, GIT_LS_REMOTE_TIMEOUT, stdoutFile, nil)
	if err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.
		util.MkdirAll(ChromiumBuildsDir, 0755)

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		// Make sure the tmp dir is empty.
		util.RemoveAll(genTmpDir)
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		// Make sure the tmp dir is empty.
		util.RemoveAll(objTmpDir)
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string, useWhitelistedFonts bool) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		buildTarget = "chrome_public_apk"
	}

	gn_args := []string{"is_debug=false", "treat_warnings_as_errors=false"}
	if targetPlatform == "Android" {
		gn_args = append(gn_args, "target_os=\"android\"")
	}
	if useWhitelistedFonts {
		gn_args = append(gn_args, "skia_whitelist_serialized_typefaces=true")
	}

	// Run "gn gen out/Release --args=...".
	if err := ExecuteCmd("gn", []string{"gen", "out/Release", fmt.Sprintf("--args=%s", strings.Join(gn_args, " "))}, os.Environ(), GN_CHROMIUM_TIMEOUT, nil, nil); err != nil {
		return fmt.Errorf("Error while running gn: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), NINJA_TIMEOUT, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func ResetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Catapult.
	catapultDir := filepath.Join(chromiumSrcDir, RelativeCatapultSrcDir)
	if err := ResetCheckout(catapultDir); err != nil {
		return fmt.Errorf("Could not reset Catapult's checkout in %s: %s", catapultDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch if it exists.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	if _, err := os.Stat(skiaPatch); err == nil {
		skiaPatchFile, _ := os.Open(skiaPatch)
		skiaPatchFileInfo, _ := skiaPatchFile.Stat()
		if skiaPatchFileInfo.Size() > 10 {
			if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
				return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
			}
		}
	}
	// Apply Chromium patch if it exists.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	if _, err := os.Stat(chromiumPatch); err == nil {
		chromiumPatchFile, _ := os.Open(chromiumPatch)
		chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
		if chromiumPatchFileInfo.Size() > 10 {
			if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
				return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
			}
		}
	}
	return nil
}

func InstallChromeAPK(chromiumBuildName string) error {
	// Install the APK on the Android device.
	chromiumApk := filepath.Join(ChromiumBuildsDir, chromiumBuildName, ApkName)
	glog.Infof("Installing the APK at %s", chromiumApk)
	err := ExecuteCmd(BINARY_ADB, []string{"install", "-r", chromiumApk}, []string{},
		ADB_INSTALL_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Could not install the chromium APK at %s: %s", chromiumBuildName, err)
	}
	return nil
}
<commit_msg>Ensure depot_tools/ninja is used<commit_after>// Utility to create and manage chromium builds.
package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/buildskia"
	"go.skia.org/infra/go/util"

	"strings"
)

// Construct the name of a directory to store a chromium build. For generic clean builds, runID
// should be empty.
func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string {
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		return fmt.Sprintf("%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash))
	} else {
		return fmt.Sprintf("%s-%s-%s",
			getTruncatedHash(chromiumHash),
			getTruncatedHash(skiaHash),
			runID)
	}
}

// CreateChromiumBuildOnSwarming creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
// uploadSingleBuild if true does not upload a 2nd build of Chromium.
func CreateChromiumBuildOnSwarming(runID, targetPlatform, chromiumHash, skiaHash, pathToPyFiles string, applyPatches, uploadSingleBuild bool) (string, string, error) {
	chromiumBuildDir, _ := filepath.Split(ChromiumSrcDir)
	// Determine which fetch target to use.
	var fetchTarget string
	if targetPlatform == "Android" {
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = buildskia.GetSkiaHash(nil)
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	err = ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil, nil)
	if err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		err := ExecuteCmd("python", syncArgs, []string{}, SYNC_SKIA_IN_CHROME_TIMEOUT, nil,
			nil)
		if err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := ResetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := ChromiumBuildDir(chromiumHash, skiaHash, runID)
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Hack: Use the "-DSK_WHITELIST_SERIALIZED_TYPEFACES" flag only when *runID is
	// empty i.e. when invoked by the build_chromium task.
	useWhitelistedFonts := (runID == "")
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform, useWhitelistedFonts); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Create and upload another chromium build if the uploadSingleBuild flag is false. This build
	// will be created without applying patches.
	if !uploadSingleBuild {
		// Make sure we are starting from a clean slate.
		if err := ResetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform, useWhitelistedFonts); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-nopatch", ChromiumBuildDir(chromiumHash, skiaHash, runID))
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	err = ExecuteCmd(BINARY_GIT, totArgs, []string{}, GIT_LS_REMOTE_TIMEOUT, stdoutFile, nil)
	if err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.
		util.MkdirAll(ChromiumBuildsDir, 0755)

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		// Make sure the tmp dir is empty.
		util.RemoveAll(genTmpDir)
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		// Make sure the tmp dir is empty.
		util.RemoveAll(objTmpDir)
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string, useWhitelistedFonts bool) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		buildTarget = "chrome_public_apk"
	}

	gn_args := []string{"is_debug=false", "treat_warnings_as_errors=false"}
	if targetPlatform == "Android" {
		gn_args = append(gn_args, "target_os=\"android\"")
	}
	if useWhitelistedFonts {
		gn_args = append(gn_args, "skia_whitelist_serialized_typefaces=true")
	}

	// Run "gn gen out/Release --args=...".
	if err := ExecuteCmd("gn", []string{"gen", "out/Release", fmt.Sprintf("--args=%s", strings.Join(gn_args, " "))}, os.Environ(), GN_CHROMIUM_TIMEOUT, nil, nil); err != nil {
		return fmt.Errorf("Error while running gn: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd(filepath.Join(DepotToolsDir, "ninja"), args, os.Environ(), NINJA_TIMEOUT, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func ResetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Catapult.
	catapultDir := filepath.Join(chromiumSrcDir, RelativeCatapultSrcDir)
	if err := ResetCheckout(catapultDir); err != nil {
		return fmt.Errorf("Could not reset Catapult's checkout in %s: %s", catapultDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch if it exists.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	if _, err := os.Stat(skiaPatch); err == nil {
		skiaPatchFile, _ := os.Open(skiaPatch)
		skiaPatchFileInfo, _ := skiaPatchFile.Stat()
		if skiaPatchFileInfo.Size() > 10 {
			if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
				return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
			}
		}
	}
	// Apply Chromium patch if it exists.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	if _, err := os.Stat(chromiumPatch); err == nil {
		chromiumPatchFile, _ := os.Open(chromiumPatch)
		chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
		if chromiumPatchFileInfo.Size() > 10 {
			if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
				return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
			}
		}
	}
	return nil
}

func InstallChromeAPK(chromiumBuildName string) error {
	// Install the APK on the Android device.
	chromiumApk := filepath.Join(ChromiumBuildsDir, chromiumBuildName, ApkName)
	glog.Infof("Installing the APK at %s", chromiumApk)
	err := ExecuteCmd(BINARY_ADB, []string{"install", "-r", chromiumApk}, []string{},
		ADB_INSTALL_TIMEOUT, nil, nil)
	if err != nil {
		return fmt.Errorf("Could not install the chromium APK at %s: %s", chromiumBuildName, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"strings"
	"testing"
)

func TestRmiWithContainerFails(t *testing.T) {
	errSubstr := "is using it"

	// create a container
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	if err != nil {
		t.Fatalf("failed to create a container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	// try to delete the image
	runCmd = exec.Command(dockerBinary, "rmi", "busybox")
	out, _, err = runCommandWithOutput(runCmd)
	if err == nil {
		t.Fatalf("Container %q is using image, should not be able to rmi: %q", cleanedContainerID, out)
	}
	if !strings.Contains(out, errSubstr) {
		t.Fatalf("Container %q is using image, error message should contain %q: %v", cleanedContainerID, errSubstr, out)
	}

	// make sure it didn't delete the busybox name
	images, _, _ := dockerCmd(t, "images")
	if !strings.Contains(images, "busybox") {
		t.Fatalf("The name 'busybox' should not have been removed from images: %q", images)
	}

	deleteContainer(cleanedContainerID)

	logDone("rmi- container using image while rmi, should not remove image name")
}

func TestRmiTag(t *testing.T) {
	imagesBefore, _, _ := dockerCmd(t, "images", "-a")
	dockerCmd(t, "tag", "busybox", "utest:tag1")
	dockerCmd(t, "tag", "busybox", "utest/docker:tag2")
	dockerCmd(t, "tag", "busybox", "utest:5000/docker:tag3")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+3 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}
	}
	dockerCmd(t, "rmi", "utest/docker:tag2")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+2 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	dockerCmd(t, "rmi", "utest:5000/docker:tag3")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+1 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	dockerCmd(t, "rmi", "utest:tag1")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+0 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	logDone("rmi - tag,rmi- tagging the same images multiple times then removing tags")
}

func TestRmiTagWithExistingContainers(t *testing.T) {
	container := "test-delete-tag"
	newtag := "busybox:newtag"
	bb := "busybox:latest"
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", bb, newtag)); err != nil {
		t.Fatalf("Could not tag busybox: %v: %s", err, out)
	}
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "--name", container, bb, "/bin/true")); err != nil {
		t.Fatalf("Could not run busybox: %v: %s", err, out)
	}
	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "rmi", newtag))
	if err != nil {
		t.Fatalf("Could not remove tag %s: %v: %s", newtag, err, out)
	}
	if d := strings.Count(out, "Untagged: "); d != 1 {
		t.Fatalf("Expected 1 untagged entry got %d: %q", d, out)
	}

	deleteAllContainers()

	logDone("rmi - delete tag with existing containers")
}

func TestRmiForceWithExistingContainers(t *testing.T) {
	image := "busybox-clone"
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "build", "--no-cache", "-t", image, "/docker-busybox")); err != nil {
		t.Fatalf("Could not build %s: %s, %v", image, out, err)
	}

	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "--name", "test-force-rmi", image, "/bin/true")); err != nil {
		t.Fatalf("Could not run container: %s, %v", out, err)
	}

	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "rmi", "-f", image))
	if err != nil {
		t.Fatalf("Could not remove image %s:  %s, %v", image, out, err)
	}

	deleteAllContainers()

	logDone("rmi - force delete with existing containers")
}
<commit_msg>integration-cli: fix test to use busybox outside container<commit_after>package main

import (
	"os/exec"
	"strings"
	"testing"
)

func TestRmiWithContainerFails(t *testing.T) {
	errSubstr := "is using it"

	// create a container
	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
	out, _, err := runCommandWithOutput(runCmd)
	if err != nil {
		t.Fatalf("failed to create a container: %s, %v", out, err)
	}

	cleanedContainerID := stripTrailingCharacters(out)

	// try to delete the image
	runCmd = exec.Command(dockerBinary, "rmi", "busybox")
	out, _, err = runCommandWithOutput(runCmd)
	if err == nil {
		t.Fatalf("Container %q is using image, should not be able to rmi: %q", cleanedContainerID, out)
	}
	if !strings.Contains(out, errSubstr) {
		t.Fatalf("Container %q is using image, error message should contain %q: %v", cleanedContainerID, errSubstr, out)
	}

	// make sure it didn't delete the busybox name
	images, _, _ := dockerCmd(t, "images")
	if !strings.Contains(images, "busybox") {
		t.Fatalf("The name 'busybox' should not have been removed from images: %q", images)
	}

	deleteContainer(cleanedContainerID)

	logDone("rmi- container using image while rmi, should not remove image name")
}

func TestRmiTag(t *testing.T) {
	imagesBefore, _, _ := dockerCmd(t, "images", "-a")
	dockerCmd(t, "tag", "busybox", "utest:tag1")
	dockerCmd(t, "tag", "busybox", "utest/docker:tag2")
	dockerCmd(t, "tag", "busybox", "utest:5000/docker:tag3")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+3 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}
	}
	dockerCmd(t, "rmi", "utest/docker:tag2")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+2 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	dockerCmd(t, "rmi", "utest:5000/docker:tag3")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+1 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	dockerCmd(t, "rmi", "utest:tag1")
	{
		imagesAfter, _, _ := dockerCmd(t, "images", "-a")
		if strings.Count(imagesAfter, "\n") != strings.Count(imagesBefore, "\n")+0 {
			t.Fatalf("before: %q\n\nafter: %q\n", imagesBefore, imagesAfter)
		}

	}
	logDone("rmi - tag,rmi- tagging the same images multiple times then removing tags")
}

func TestRmiTagWithExistingContainers(t *testing.T) {
	container := "test-delete-tag"
	newtag := "busybox:newtag"
	bb := "busybox:latest"
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", bb, newtag)); err != nil {
		t.Fatalf("Could not tag busybox: %v: %s", err, out)
	}
	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "--name", container, bb, "/bin/true")); err != nil {
		t.Fatalf("Could not run busybox: %v: %s", err, out)
	}
	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "rmi", newtag))
	if err != nil {
		t.Fatalf("Could not remove tag %s: %v: %s", newtag, err, out)
	}
	if d := strings.Count(out, "Untagged: "); d != 1 {
		t.Fatalf("Expected 1 untagged entry got %d: %q", d, out)
	}

	deleteAllContainers()

	logDone("rmi - delete tag with existing containers")
}

func TestRmiForceWithExistingContainers(t *testing.T) {
	image := "busybox-clone"

	cmd := exec.Command(dockerBinary, "build", "--no-cache", "-t", image, "-")
	cmd.Stdin = strings.NewReader(`FROM busybox
MAINTAINER foo`)

	if out, _, err := runCommandWithOutput(cmd); err != nil {
		t.Fatalf("Could not build %s: %s, %v", image, out, err)
	}

	if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "run", "--name", "test-force-rmi", image, "/bin/true")); err != nil {
		t.Fatalf("Could not run container: %s, %v", out, err)
	}

	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "rmi", "-f", image))
	if err != nil {
		t.Fatalf("Could not remove image %s:  %s, %v", image, out, err)
	}

	deleteAllContainers()

	logDone("rmi - force delete with existing containers")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
	"flag"

	"github.com/couchbaselabs/clog"
	"github.com/russross/blackfriday"
)

type Index map[string]int

func setup(cacheDir string, src string, srcF os.FileInfo, idx Index, err error) error {
	if strings.HasPrefix(srcF.Name(), ".") || strings.Contains(src, "/.") {
		return nil
	}

	dst := cacheDir + src
	if strings.HasSuffix(dst, ".md") {
		dst = strings.TrimSuffix(dst, ".md") + ".html"
	}
	dstF, dstE := os.Stat(dst)

	// if up to date, skip
	if !os.IsNotExist(dstE) &&
		(srcF.IsDir() || dstF.ModTime().After(srcF.ModTime())) {
		return nil
	}

	// copy to cache
	if os.IsNotExist(dstE) {
		clog.Log("Copying %s", srcF.Name())
	} else {
		clog.Log("Updating %s", srcF.Name())
	}

	if srcF.IsDir() {
		if err := os.MkdirAll(dst, srcF.Mode()); err != nil {
			return err
		}
		return nil
	}

	sbuf, err := ioutil.ReadFile(src)
	if err != nil {
		return err
	}
	dbuf := sbuf
	if strings.HasSuffix(src, ".md") {
		dbuf = blackfriday.MarkdownCommon(sbuf)
	}
	if err := ioutil.WriteFile(dst, dbuf, srcF.Mode()); err != nil {
		return err
	}

	if strings.HasSuffix(src, ".md") {
		search := regexp.MustCompile("##[^\n]+\n")
		found := search.FindString(string(sbuf))
		if len(found) > 0 {
			found = strings.TrimSpace(strings.TrimPrefix(found, "##"))
			nsearch := regexp.MustCompile("/slide-(\\d+).md")
			num := nsearch.FindStringSubmatch(src)
			if len(num) > 0 {
				idx[found], _ = strconv.Atoi(num[1])
			}
		}
	}

	return nil
}

func main() {
	xsrc := flag.String("src", "./content", "Source directory to read markdown from")
	xdst := flag.String("dst", "", "Destination to write translated content")
	flag.Parse()

	srcD, srcE := os.Stat(*xsrc)
	if os.IsNotExist(srcE) || !srcD.IsDir() {
    	clog.Fatalf("Source directory does not exist: %s", *xsrc)
 	}

 	if len(*xdst) > 0 {
		dstD, dstE := os.Stat(*xdst)
		if os.IsNotExist(dstE) || !dstD.IsDir() {
	    	clog.Fatalf("Target directory does not exist: %s", *xdst)
 		}
 	}

 	tld := strings.Trim(*xsrc, "/")
 	dirpos := strings.LastIndex(tld, "/")
 	if dirpos < 0 {
	 	tld = tld[dirpos + 1:]
 	}
 	if (len(tld) < 1) {
	 	clog.Fatalf("Source directory path must be at least one level deep, example: ./content")
	}

	if len(*xdst) > 0 {
		translate(*xsrc, *xdst, tld)
	} else {
		serve(*xsrc, tld)
	}
}

func translate(src, dst, tld string) {
	var idx Index = make(map[string]int)
	walker := func(fsrc string, f os.FileInfo, err error) error {
		return setup(dst, fsrc, f, idx, err)
	}
	if err := filepath.Walk(src, walker); err != nil {
		clog.Fatalf("Filewalk %v", err)
	}

	json, err := json.Marshal(idx)
	if err != nil {
		clog.Fatalf("During Index JSON Marshal %v", err)
	}

	jfile := strings.TrimRight(dst, "/")
	jfile += "/" + tld + "/index.json"
	clog.Print("Writing index file", jfile, " data ", idx)
    err = ioutil.WriteFile(jfile, json, 0666)
	if err != nil {
		clog.Fatalf("Error writing json file: %s", jfile)
	}
}

func serve(cdir string, tld string) {
	tempDir, _ := ioutil.TempDir("", "tut")
	tempDir += string(os.PathSeparator)
	defer os.RemoveAll(tempDir)
	clog.Log("Workdir %s", tempDir)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		os.RemoveAll(tempDir)
		clog.Fatal("Stopped")
	}()

	var idx Index
	idx = make(map[string]int)

	walker := func(src string, f os.FileInfo, err error) error {
		return setup(tempDir, src, f, idx, err)
	}

	if err := filepath.Walk(cdir, walker); err != nil {
		clog.Fatalf("Filewalk %v", err)
	}

	getindex := func(w http.ResponseWriter, r *http.Request) {
		json, err := json.Marshal(idx)
		if err != nil {
			clog.Fatalf("During Index JSON Marshal %v", err)
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write(json)
	}
	http.HandleFunc("/tutorial/index.json", getindex)

	url, _ := url.Parse("http://localhost:8093")
	rp := httputil.NewSingleHostReverseProxy(url)
	http.Handle("/query", rp)

	fs := http.FileServer(http.Dir(tempDir + "/" + tld + "/"))
	http.Handle("/tutorial/", http.StripPrefix("/tutorial/", fs))

	http.Handle("/", http.RedirectHandler("/tutorial/tutorial.html#1", 302))

	clog.Log("Running at http://localhost:8000/")
	go func() {
		for {
			filepath.Walk(cdir, walker)
			time.Sleep(2 * time.Second)
		}
	}()

	// last step
	if err := http.ListenAndServe(":8000", nil); err != nil {
		clog.Fatalf("ListenAndServe %v", err)
	}
}
<commit_msg>Check to ensure destination does not have any files<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/couchbaselabs/clog"
	"github.com/russross/blackfriday"
)

type Index map[string]int

func setup(cacheDir string, src string, srcF os.FileInfo, idx Index, err error) error {
	if strings.HasPrefix(srcF.Name(), ".") || strings.Contains(src, "/.") {
		return nil
	}

	dst := cacheDir + src
	if strings.HasSuffix(dst, ".md") {
		dst = strings.TrimSuffix(dst, ".md") + ".html"
	}
	dstF, dstE := os.Stat(dst)

	// if up to date, skip
	if !os.IsNotExist(dstE) &&
		(srcF.IsDir() || dstF.ModTime().After(srcF.ModTime())) {
		return nil
	}

	// copy to cache
	if os.IsNotExist(dstE) {
		clog.Log("Copying %s", srcF.Name())
	} else {
		clog.Log("Updating %s", srcF.Name())
	}

	if srcF.IsDir() {
		if err := os.MkdirAll(dst, srcF.Mode()); err != nil {
			return err
		}
		return nil
	}

	sbuf, err := ioutil.ReadFile(src)
	if err != nil {
		return err
	}
	dbuf := sbuf
	if strings.HasSuffix(src, ".md") {
		dbuf = blackfriday.MarkdownCommon(sbuf)
	}
	if err := ioutil.WriteFile(dst, dbuf, srcF.Mode()); err != nil {
		return err
	}

	if strings.HasSuffix(src, ".md") {
		search := regexp.MustCompile("##[^\n]+\n")
		found := search.FindString(string(sbuf))
		if len(found) > 0 {
			found = strings.TrimSpace(strings.TrimPrefix(found, "##"))
			nsearch := regexp.MustCompile("/slide-(\\d+).md")
			num := nsearch.FindStringSubmatch(src)
			if len(num) > 0 {
				idx[found], _ = strconv.Atoi(num[1])
			}
		}
	}

	return nil
}

func main() {
	xsrc := flag.String("src", "./content", "Source directory to read markdown from")
	xdst := flag.String("dst", "", "Destination to write translated content")
	flag.Parse()

	srcD, srcE := os.Stat(*xsrc)
	if os.IsNotExist(srcE) || !srcD.IsDir() {
		clog.Fatalf("Source directory does not exist: %s", *xsrc)
	}

	if len(*xdst) > 0 {
		dstD, dstE := os.Stat(*xdst)
		if os.IsNotExist(dstE) || !dstD.IsDir() {
			clog.Fatalf("Target directory does not exist: %s", *xdst)
		}
	}

	tld := strings.Trim(*xsrc, "/")
	dirpos := strings.LastIndex(tld, "/")
	if dirpos < 0 {
		tld = tld[dirpos+1:]
	}
	if len(tld) < 1 {
		clog.Fatalf("Source directory path must be at least one level deep, example: ./content")
	}

	if len(*xdst) > 0 {
		translate(*xsrc, *xdst, tld)
	} else {
		serve(*xsrc, tld)
	}
}

func isEmpty(dir string) bool {
	empty := true
	walker := func(dir string, f os.FileInfo, err error) error {
		if f.IsDir() {
			return nil
		}
		empty = false
		return nil
	}
	if err := filepath.Walk(dir, walker); err != nil {
		clog.Fatalf("Filewalk %v", err)
	}
	return empty
}

func translate(src, dst, tld string) {

	if !isEmpty(dst) {
		clog.Fatalf("Target directory %s is not empty", dst)
	}

	var idx Index = make(map[string]int)
	walker := func(fsrc string, f os.FileInfo, err error) error {
		return setup(dst, fsrc, f, idx, err)
	}
	if err := filepath.Walk(src, walker); err != nil {
		clog.Fatalf("Filewalk %v", err)
	}

	json, err := json.Marshal(idx)
	if err != nil {
		clog.Fatalf("During Index JSON Marshal %v", err)
	}

	jfile := strings.TrimRight(dst, "/")
	jfile += "/" + tld + "/index.json"
	clog.Print("Writing index file", jfile, " data ", idx)
	err = ioutil.WriteFile(jfile, json, 0666)
	if err != nil {
		clog.Fatalf("Error writing json file: %s", jfile)
	}
}

func serve(cdir string, tld string) {
	tempDir, _ := ioutil.TempDir("", "tut")
	tempDir += string(os.PathSeparator)
	defer os.RemoveAll(tempDir)
	clog.Log("Workdir %s", tempDir)

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		os.RemoveAll(tempDir)
		clog.Fatal("Stopped")
	}()

	var idx Index
	idx = make(map[string]int)

	walker := func(src string, f os.FileInfo, err error) error {
		return setup(tempDir, src, f, idx, err)
	}

	if err := filepath.Walk(cdir, walker); err != nil {
		clog.Fatalf("Filewalk %v", err)
	}

	getindex := func(w http.ResponseWriter, r *http.Request) {
		json, err := json.Marshal(idx)
		if err != nil {
			clog.Fatalf("During Index JSON Marshal %v", err)
		}
		w.Header().Set("Content-Type", "application/json")
		w.Write(json)
	}
	http.HandleFunc("/tutorial/index.json", getindex)

	url, _ := url.Parse("http://localhost:8093")
	rp := httputil.NewSingleHostReverseProxy(url)
	http.Handle("/query", rp)

	fs := http.FileServer(http.Dir(tempDir + "/" + tld + "/"))
	http.Handle("/tutorial/", http.StripPrefix("/tutorial/", fs))

	http.Handle("/", http.RedirectHandler("/tutorial/tutorial.html#1", 302))

	clog.Log("Running at http://localhost:8000/")
	go func() {
		for {
			filepath.Walk(cdir, walker)
			time.Sleep(2 * time.Second)
		}
	}()

	// last step
	if err := http.ListenAndServe(":8000", nil); err != nil {
		clog.Fatalf("ListenAndServe %v", err)
	}
}
<|endoftext|>


<commit_before>// Utility to create and manage chromium builds.
package util

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/util"

	"strings"
)

// CreateChromiumBuild creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Blink/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) {
	// Determine which build dir and fetch target to use.
	var chromiumBuildDir, fetchTarget string
	if targetPlatform == "Android" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base")
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base")
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = getSkiaHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))),
		"py")
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := fmt.Sprintf("%s-%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		googleStorageDirName = fmt.Sprintf("%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash))
	}
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Check for the applypatch flag and reset and then build again and copy to
	// google storage.
	if applyPatches {
		// Now build chromium without the patches and upload it to Google Storage.

		// Make sure we are starting from a clean slate.
		if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	if err := ExecuteCmd(BINARY_GIT, totArgs, []string{}, time.Minute*5, stdoutFile, nil); err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func getSkiaHash() (string, error) {
	// Find Skia's LKGR commit hash.
	resp, err := http.Get("http://chromium.googlesource.com/chromium/src/+/master/DEPS?format=TEXT")
	if err != nil {
		return "", fmt.Errorf("Could not get Skia's LKGR: %s", err)
	}
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("Got statuscode %d while accessing Chromium's DEPS file", resp.StatusCode)
	}
	defer util.Close(resp.Body)
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("Could not read Skia's LKGR: %s", err)
	}
	base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(string(body))))
	l, _ := base64.StdEncoding.Decode(base64Text, []byte(string(body)))
	chromiumDepsText := string(base64Text[:l])
	if strings.Contains(chromiumDepsText, "skia_revision") {
		reg, _ := regexp.Compile(".*'skia_revision': '(?P<revision>[0-9a-fA-F]{2,40})'.*")
		return reg.FindStringSubmatch(chromiumDepsText)[1], nil
	}
	return "", fmt.Errorf("Could not find skia_revision in Chromium DEPS file")
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		util.LogErr(
			ioutil.WriteFile(filepath.Join(chromiumDir, "chromium.gyp_env"), []byte("{ 'GYP_DEFINES': 'OS=android', }\n"), 0777))
		buildTarget = "chrome_shell_apk"
	}

	// Run "GYP_DEFINES='gomadir=/b/build/goma' GYP_GENERATORS='ninja' build/gyp_chromium -Duse_goma=1".
	env := []string{fmt.Sprintf("GYP_DEFINES=gomadir=%s", GomaDir), "GYP_GENERATORS=ninja"}
	if err := ExecuteCmd(filepath.Join("build", "gyp_chromium"), []string{"-Duse_goma=1"}, env, 30*time.Minute, nil, nil); err != nil {
		return fmt.Errorf("Error while running gyp_chromium: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), 2*time.Hour, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func resetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Blink.
	blinkDir := filepath.Join(chromiumSrcDir, "third_party", "WebKit")
	if err := ResetCheckout(blinkDir); err != nil {
		return fmt.Errorf("Could not reset Blink's checkout in %s: %s", blinkDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	skiaPatchFile, _ := os.Open(skiaPatch)
	skiaPatchFileInfo, _ := skiaPatchFile.Stat()
	if skiaPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
			return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
		}
	}
	// Apply Blink patch.
	blinkDir := filepath.Join(chromiumSrcDir, "third_party", "WebKit")
	blinkPatch := filepath.Join(os.TempDir(), runID+".blink.patch")
	blinkPatchFile, _ := os.Open(blinkPatch)
	blinkPatchFileInfo, _ := blinkPatchFile.Stat()
	if blinkPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(blinkPatch, blinkDir); err != nil {
			return fmt.Errorf("Could not apply Blink's patch in %s: %s", blinkDir, err)
		}
	}
	// Apply Chromium patch.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	chromiumPatchFile, _ := os.Open(chromiumPatch)
	chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
	if chromiumPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
			return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
		}
	}
	return nil
}
<commit_msg>[CT] Restart goma's compiler proxy prior to each chromium build<commit_after>// Utility to create and manage chromium builds.
package util

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/util"

	"strings"
)

// CreateChromiumBuild creates a chromium build using the specified arguments.

// runID is the unique id of the current run (typically requester + timestamp).
// targetPlatform is the platform the benchmark will run on (Android / Linux ).
// chromiumHash is the hash the checkout should be synced to. If not specified then
// Chromium's Tot hash is used.
// skiaHash is the hash the checkout should be synced to. If not specified then
// Skia's LKGR hash is used (the hash in Chromium's DEPS file).
// applyPatches if true looks for Chromium/Blink/Skia patches in the temp dir and
// runs once with the patch applied and once without the patch applied.
func CreateChromiumBuild(runID, targetPlatform, chromiumHash, skiaHash string, applyPatches bool) (string, string, error) {
	// Determine which build dir and fetch target to use.
	var chromiumBuildDir, fetchTarget string
	if targetPlatform == "Android" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "android_base")
		fetchTarget = "android"
	} else if targetPlatform == "Linux" {
		chromiumBuildDir = filepath.Join(ChromiumBuildsDir, "linux_base")
		fetchTarget = "chromium"
	} else {
		return "", "", fmt.Errorf("Unrecognized target_platform %s", targetPlatform)
	}
	util.MkdirAll(chromiumBuildDir, 0700)

	// Find which Chromium commit hash should be used.
	var err error
	if chromiumHash == "" {
		chromiumHash, err = getChromiumHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Chromium's Hash: %s", err)
		}
	}

	// Find which Skia commit hash should be used.
	if skiaHash == "" {
		skiaHash, err = getSkiaHash()
		if err != nil {
			return "", "", fmt.Errorf("Error while finding Skia's Hash: %s", err)
		}
	}

	// Run chromium sync command using the above commit hashes.
	// Construct path to the sync_skia_in_chrome python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(currentFile)))),
		"py")
	syncArgs := []string{
		filepath.Join(pathToPyFiles, "sync_skia_in_chrome.py"),
		"--destination=" + chromiumBuildDir,
		"--fetch_target=" + fetchTarget,
		"--chrome_revision=" + chromiumHash,
		"--skia_revision=" + skiaHash,
	}
	if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
		glog.Warning("There was an error. Deleting base directory and trying again.")
		util.RemoveAll(chromiumBuildDir)
		util.MkdirAll(chromiumBuildDir, 0700)
		if err := ExecuteCmd("python", syncArgs, []string{}, 2*time.Hour, nil, nil); err != nil {
			return "", "", fmt.Errorf("There was an error checking out chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
	}

	// Make sure we are starting from a clean slate.
	if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
		return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
	}
	googleStorageDirName := fmt.Sprintf("%s-%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
	if runID == "" {
		// Do not include the runID in the dir name if it is not specified.
		googleStorageDirName = fmt.Sprintf("%s-%s", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash))
	}
	if applyPatches {
		if err := applyRepoPatches(filepath.Join(chromiumBuildDir, "src"), runID); err != nil {
			return "", "", fmt.Errorf("Could not apply patches in the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Add "try" prefix and "withpatch" suffix.
		googleStorageDirName = fmt.Sprintf("try-%s-withpatch", googleStorageDirName)
	}
	// Build chromium.
	if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
		return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
	}

	// Upload to Google Storage.
	gs, err := NewGsUtil(nil)
	if err != nil {
		return "", "", fmt.Errorf("Could not create GS object: %s", err)
	}
	if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
		return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
	}

	// Check for the applypatch flag and reset and then build again and copy to
	// google storage.
	if applyPatches {
		// Now build chromium without the patches and upload it to Google Storage.

		// Make sure we are starting from a clean slate.
		if err := resetChromiumCheckout(filepath.Join(chromiumBuildDir, "src")); err != nil {
			return "", "", fmt.Errorf("Could not reset the chromium checkout in %s: %s", chromiumBuildDir, err)
		}
		// Build chromium.
		if err := buildChromium(chromiumBuildDir, targetPlatform); err != nil {
			return "", "", fmt.Errorf("There was an error building chromium %s + skia %s: %s", chromiumHash, skiaHash, err)
		}
		// Upload to Google Storage.
		googleStorageDirName = fmt.Sprintf("try-%s-%s-%s-nopatch", getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), runID)
		if err := uploadChromiumBuild(filepath.Join(chromiumBuildDir, "src", "out", "Release"), filepath.Join(CHROMIUM_BUILDS_DIR_NAME, googleStorageDirName), targetPlatform, gs); err != nil {
			return "", "", fmt.Errorf("There was an error uploaded the chromium build dir %s: %s", filepath.Join(chromiumBuildDir, "src", "out", "Release"), err)
		}
	}
	return getTruncatedHash(chromiumHash), getTruncatedHash(skiaHash), nil
}

func getChromiumHash() (string, error) {
	// Find Chromium's Tot commit hash.
	stdoutFilePath := filepath.Join(os.TempDir(), "chromium-tot")
	stdoutFile, err := os.Create(stdoutFilePath)
	defer util.Close(stdoutFile)
	defer util.Remove(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Could not create %s: %s", stdoutFilePath, err)
	}
	totArgs := []string{"ls-remote", "https://chromium.googlesource.com/chromium/src.git", "--verify", "refs/heads/master"}
	if err := ExecuteCmd(BINARY_GIT, totArgs, []string{}, time.Minute*5, stdoutFile, nil); err != nil {
		return "", fmt.Errorf("Error while finding Chromium's ToT: %s", err)
	}
	output, err := ioutil.ReadFile(stdoutFilePath)
	if err != nil {
		return "", fmt.Errorf("Cannot read %s: %s", stdoutFilePath, err)
	}
	tokens := strings.Split(string(output), "\t")
	return tokens[0], nil
}

func getSkiaHash() (string, error) {
	// Find Skia's LKGR commit hash.
	resp, err := http.Get("http://chromium.googlesource.com/chromium/src/+/master/DEPS?format=TEXT")
	if err != nil {
		return "", fmt.Errorf("Could not get Skia's LKGR: %s", err)
	}
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("Got statuscode %d while accessing Chromium's DEPS file", resp.StatusCode)
	}
	defer util.Close(resp.Body)
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("Could not read Skia's LKGR: %s", err)
	}
	base64Text := make([]byte, base64.StdEncoding.EncodedLen(len(string(body))))
	l, _ := base64.StdEncoding.Decode(base64Text, []byte(string(body)))
	chromiumDepsText := string(base64Text[:l])
	if strings.Contains(chromiumDepsText, "skia_revision") {
		reg, _ := regexp.Compile(".*'skia_revision': '(?P<revision>[0-9a-fA-F]{2,40})'.*")
		return reg.FindStringSubmatch(chromiumDepsText)[1], nil
	}
	return "", fmt.Errorf("Could not find skia_revision in Chromium DEPS file")
}

func uploadChromiumBuild(localOutDir, gsDir, targetPlatform string, gs *GsUtil) error {
	localUploadDir := localOutDir
	if targetPlatform == "Android" {
		localUploadDir = filepath.Join(localUploadDir, "apks")
	} else {
		// Temporarily move the not needed large "gen" and "obj" directories so
		// that they do not get uploaded to Google Storage. Move them back after
		// the method completes.

		genDir := filepath.Join(localOutDir, "gen")
		genTmpDir := filepath.Join(ChromiumBuildsDir, "gen")
		if err := os.Rename(genDir, genTmpDir); err != nil {
			return fmt.Errorf("Could not rename gen dir: %s", err)
		}
		defer util.Rename(genTmpDir, genDir)

		objDir := filepath.Join(localOutDir, "obj")
		objTmpDir := filepath.Join(ChromiumBuildsDir, "obj")
		if err := os.Rename(objDir, objTmpDir); err != nil {
			return fmt.Errorf("Could not rename obj dir: %s", err)
		}
		defer util.Rename(objTmpDir, objDir)
	}
	return gs.UploadDir(localUploadDir, gsDir, true)
}

func buildChromium(chromiumDir, targetPlatform string) error {
	if err := os.Chdir(filepath.Join(chromiumDir, "src")); err != nil {
		return fmt.Errorf("Could not chdir to %s/src: %s", chromiumDir, err)
	}

	// Find the build target to use while building chromium.
	buildTarget := "chrome"
	if targetPlatform == "Android" {
		util.LogErr(
			ioutil.WriteFile(filepath.Join(chromiumDir, "chromium.gyp_env"), []byte("{ 'GYP_DEFINES': 'OS=android', }\n"), 0777))
		buildTarget = "chrome_shell_apk"
	}

	// Restart Goma's compiler proxy right before building the checkout.
	if err := ExecuteCmd("python", []string{filepath.Join(GomaDir, "goma_ctl.py"), "restart"}, os.Environ(), 10*time.Minute, nil, nil); err != nil {
		return fmt.Errorf("Error while restarting goma compiler proxy: %s", err)
	}

	// Run "GYP_DEFINES='gomadir=/b/build/goma' GYP_GENERATORS='ninja' build/gyp_chromium -Duse_goma=1".
	env := []string{fmt.Sprintf("GYP_DEFINES=gomadir=%s", GomaDir), "GYP_GENERATORS=ninja"}
	if err := ExecuteCmd(filepath.Join("build", "gyp_chromium"), []string{"-Duse_goma=1"}, env, 30*time.Minute, nil, nil); err != nil {
		return fmt.Errorf("Error while running gyp_chromium: %s", err)
	}
	// Run "ninja -C out/Release -j100 ${build_target}".
	// Use the full system env while building chromium.
	args := []string{"-C", "out/Release", "-j100", buildTarget}
	return ExecuteCmd("ninja", args, os.Environ(), 2*time.Hour, nil, nil)
}

func getTruncatedHash(commitHash string) string {
	return commitHash[0:7]
}

func resetChromiumCheckout(chromiumSrcDir string) error {
	// Reset Skia.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	if err := ResetCheckout(skiaDir); err != nil {
		return fmt.Errorf("Could not reset Skia's checkout in %s: %s", skiaDir, err)
	}
	// Reset Blink.
	blinkDir := filepath.Join(chromiumSrcDir, "third_party", "WebKit")
	if err := ResetCheckout(blinkDir); err != nil {
		return fmt.Errorf("Could not reset Blink's checkout in %s: %s", blinkDir, err)
	}
	// Reset Chromium.
	if err := ResetCheckout(chromiumSrcDir); err != nil {
		return fmt.Errorf("Could not reset Chromium's checkout in %s: %s", chromiumSrcDir, err)
	}
	return nil
}

func applyRepoPatches(chromiumSrcDir, runID string) error {
	// Apply Skia patch.
	skiaDir := filepath.Join(chromiumSrcDir, "third_party", "skia")
	skiaPatch := filepath.Join(os.TempDir(), runID+".skia.patch")
	skiaPatchFile, _ := os.Open(skiaPatch)
	skiaPatchFileInfo, _ := skiaPatchFile.Stat()
	if skiaPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(skiaPatch, skiaDir); err != nil {
			return fmt.Errorf("Could not apply Skia's patch in %s: %s", skiaDir, err)
		}
	}
	// Apply Blink patch.
	blinkDir := filepath.Join(chromiumSrcDir, "third_party", "WebKit")
	blinkPatch := filepath.Join(os.TempDir(), runID+".blink.patch")
	blinkPatchFile, _ := os.Open(blinkPatch)
	blinkPatchFileInfo, _ := blinkPatchFile.Stat()
	if blinkPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(blinkPatch, blinkDir); err != nil {
			return fmt.Errorf("Could not apply Blink's patch in %s: %s", blinkDir, err)
		}
	}
	// Apply Chromium patch.
	chromiumPatch := filepath.Join(os.TempDir(), runID+".chromium.patch")
	chromiumPatchFile, _ := os.Open(chromiumPatch)
	chromiumPatchFileInfo, _ := chromiumPatchFile.Stat()
	if chromiumPatchFileInfo.Size() > 10 {
		if err := ApplyPatch(chromiumPatch, chromiumSrcDir); err != nil {
			return fmt.Errorf("Could not apply Chromium's patch in %s: %s", chromiumSrcDir, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2020 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package z

import (
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"sort"

	"github.com/pkg/errors"
)

// Buffer is equivalent of bytes.Buffer without the ability to read. It is NOT thread-safe.
//
// In UseCalloc mode, z.Calloc is used to allocate memory, which depending upon how the code is
// compiled could use jemalloc for allocations.
//
// In UseMmap mode, Buffer  uses file mmap to allocate memory. This allows us to store big data
// structures without using physical memory.
//
// MaxSize can be set to limit the memory usage.
type Buffer struct {
	buf           []byte
	offset        int
	curSz         int
	maxSz         int
	fd            *os.File
	bufType       BufferType
	autoMmapAfter int
}

type BufferType int

func (t BufferType) String() string {
	switch t {
	case UseCalloc:
		return "UseCalloc"
	case UseMmap:
		return "UseMmap"
	}
	return "invalid"
}

const (
	UseCalloc BufferType = iota
	UseMmap
	UseInvalid
)

// smallBufferSize is an initial allocation minimal capacity.
const smallBufferSize = 64

// Newbuffer is a helper utility, which creates a virtually unlimited Buffer in UseCalloc mode.
func NewBuffer(sz int) *Buffer {
	buf, err := NewBufferWith(sz, math.MaxInt64, UseCalloc)
	if err != nil {
		log.Fatalf("while creating buffer: %v", err)
	}
	return buf
}

func (b *Buffer) doMmap() error {
	curBuf := b.buf
	fd, err := ioutil.TempFile("", "buffer")
	if err != nil {
		return err
	}
	if err := fd.Truncate(int64(b.curSz)); err != nil {
		return errors.Wrapf(err, "while truncating %s to size: %d", fd.Name(), b.curSz)
	}

	buf, err := Mmap(fd, true, int64(b.maxSz)) // Mmap up to max size.
	if err != nil {
		return errors.Wrapf(err, "while mmapping %s with size: %d", fd.Name(), b.maxSz)
	}
	if len(curBuf) > 0 {
		assert(b.offset == copy(buf, curBuf[:b.offset]))
		Free(curBuf)
	}
	b.buf = buf
	b.bufType = UseMmap
	b.fd = fd
	return nil
}

// NewBufferWith would allocate a buffer of size sz upfront, with the total size of the buffer not
// exceeding maxSz. Both sz and maxSz can be set to zero, in which case reasonable defaults would be
// used. Buffer can't be used without initialization via NewBuffer.
func NewBufferWith(sz, maxSz int, bufType BufferType) (*Buffer, error) {
	if sz == 0 {
		sz = smallBufferSize
	}
	if maxSz == 0 {
		maxSz = math.MaxInt32
	}

	b := &Buffer{
		offset:  1,
		curSz:   sz,
		maxSz:   maxSz,
		bufType: UseCalloc, // by default.
	}

	switch bufType {
	case UseCalloc:
		b.buf = Calloc(sz)
	case UseMmap:
		if err := b.doMmap(); err != nil {
			return nil, err
		}
	default:
		log.Fatalf("Invalid bufType: %q\n", bufType)
	}

	b.buf[0] = 0x00
	return b, nil
}

func (b *Buffer) IsEmpty() bool {
	return b.offset == 1
}

// Len would return the number of bytes written to the buffer so far.
func (b *Buffer) Len() int {
	return b.offset
}

// Bytes would return all the written bytes as a slice.
func (b *Buffer) Bytes() []byte {
	return b.buf[1:b.offset]
}

func (b *Buffer) AutoMmapAfter(size int) {
	b.autoMmapAfter = size
}

// Grow would grow the buffer to have at least n more bytes. In case the buffer is at capacity, it
// would reallocate twice the size of current capacity + n, to ensure n bytes can be written to the
// buffer without further allocation. In UseMmap mode, this might result in underlying file
// expansion.
func (b *Buffer) Grow(n int) {
	// In this case, len and cap are the same.
	if b.buf == nil {
		panic("z.Buffer needs to be initialized before using")
	}
	if b.maxSz-b.offset < n {
		panic(fmt.Sprintf("Buffer max size exceeded: %d."+
			" Offset: %d. Grow: %d", b.maxSz, b.offset, n))
	}
	if b.curSz-b.offset > n {
		return
	}

	growBy := b.curSz + n
	if growBy > 1<<30 {
		growBy = 1 << 30
	}
	if n > growBy {
		// Always at least allocate n, even if it exceeds the 1GB limit above.
		growBy = n
	}
	b.curSz += growBy

	switch b.bufType {
	case UseCalloc:
		if b.autoMmapAfter > 0 && b.curSz > b.autoMmapAfter {
			// This would do copy as well.
			check(b.doMmap())

		} else {
			newBuf := Calloc(b.curSz)
			copy(newBuf, b.buf[:b.offset])
			Free(b.buf)
			b.buf = newBuf
		}
	case UseMmap:
		if err := b.fd.Truncate(int64(b.curSz)); err != nil {
			log.Fatalf("While trying to truncate file %s to size: %d error: %v\n",
				b.fd.Name(), b.curSz, err)
		}
	}
}

// Allocate is a way to get a slice of size n back from the buffer. This slice can be directly
// written to. Warning: Allocate is not thread-safe. The byte slice returned MUST be used before
// further calls to Buffer.
func (b *Buffer) Allocate(n int) []byte {
	b.Grow(n)
	off := b.offset
	b.offset += n
	return b.buf[off:b.offset]
}

// AllocateOffset works the same way as allocate, but instead of returning a byte slice, it returns
// the offset of the allocation.
func (b *Buffer) AllocateOffset(n int) int {
	b.Grow(n)
	b.offset += n
	return b.offset - n
}

func (b *Buffer) writeLen(sz int) {
	buf := b.Allocate(4)
	binary.BigEndian.PutUint32(buf, uint32(sz))
}

// SliceAllocate would encode the size provided into the buffer, followed by a call to Allocate,
// hence returning the slice of size sz. This can be used to allocate a lot of small buffers into
// this big buffer.
// Note that SliceAllocate should NOT be mixed with normal calls to Write.
func (b *Buffer) SliceAllocate(sz int) []byte {
	b.Grow(4 + sz)
	b.writeLen(sz)
	return b.Allocate(sz)
}

type LessFunc func(a, b []byte) bool
type sortHelper struct {
	offsets []int
	b       *Buffer
	tmp     *Buffer
	less    LessFunc
	small   []int
}

func (s *sortHelper) sortSmall(start, end int) {
	s.tmp.Reset()
	s.small = s.small[:0]
	next := start
	for next != 0 && next < end {
		s.small = append(s.small, next)
		_, next = s.b.Slice(next)
	}

	// We are sorting the slices pointed to by s.small offsets, but only moving the offsets around.
	sort.Slice(s.small, func(i, j int) bool {
		left, _ := s.b.Slice(s.small[i])
		right, _ := s.b.Slice(s.small[j])
		return s.less(left, right)
	})
	// Now we iterate over the s.small offsets and copy over the slices. The result is now in order.
	for _, off := range s.small {
		s.tmp.Write(rawSlice(s.b.buf[off:]))
	}
	assert(end-start == copy(s.b.buf[start:end], s.tmp.Bytes()))
}

func assert(b bool) {
	if !b {
		log.Fatalf("Assertion failure")
	}
}
func check(err error) {
	assert(err == nil)
}
func check2(_ interface{}, err error) {
	check(err)
}

func (s *sortHelper) merge(left, right []byte, start, end int) {
	if len(left) == 0 || len(right) == 0 {
		return
	}
	s.tmp.Reset()
	check2(s.tmp.Write(left))
	left = s.tmp.Bytes()

	var ls, rs []byte

	copyLeft := func() {
		assert(len(ls) == copy(s.b.buf[start:], ls))
		left = left[len(ls):]
		start += len(ls)
	}
	copyRight := func() {
		assert(len(rs) == copy(s.b.buf[start:], rs))
		right = right[len(rs):]
		start += len(rs)
	}

	for start < end {
		if len(left) == 0 {
			assert(len(right) == copy(s.b.buf[start:end], right))
			return
		}
		if len(right) == 0 {
			assert(len(left) == copy(s.b.buf[start:end], left))
			return
		}
		ls = rawSlice(left)
		rs = rawSlice(right)

		// We skip the first 4 bytes in the rawSlice, because that stores the length.
		if s.less(ls[4:], rs[4:]) {
			copyLeft()
		} else {
			copyRight()
		}
	}
}

func (s *sortHelper) sort(lo, hi int) []byte {
	assert(lo <= hi)

	mid := lo + (hi-lo)/2
	loff, hoff := s.offsets[lo], s.offsets[hi]
	if lo == mid {
		// No need to sort, just return the buffer.
		return s.b.buf[loff:hoff]
	}

	// lo, mid would sort from [offset[lo], offset[mid]) .
	left := s.sort(lo, mid)
	// Typically we'd use mid+1, but here mid represents an offset in the buffer. Each offset
	// contains a thousand entries. So, if we do mid+1, we'd skip over those entries.
	right := s.sort(mid, hi)

	s.merge(left, right, loff, hoff)
	return s.b.buf[loff:hoff]
}

// SortSlice is like SortSliceBetween but sorting over the entire buffer.
func (b *Buffer) SortSlice(less func(left, right []byte) bool) {
	b.SortSliceBetween(1, b.offset, less)
}
func (b *Buffer) SortSliceBetween(start, end int, less LessFunc) {
	if start >= end {
		return
	}
	if start == 0 {
		panic("start can never be zero")
	}

	var offsets []int
	next, count := start, 0
	for next != 0 && next < end {
		if count%1024 == 0 {
			offsets = append(offsets, next)
		}
		_, next = b.Slice(next)
		count++
	}
	assert(len(offsets) > 0)
	if offsets[len(offsets)-1] != end {
		offsets = append(offsets, end)
	}

	szTmp := int(float64((end-start)/2) * 1.1)
	s := &sortHelper{
		offsets: offsets,
		b:       b,
		less:    less,
		small:   make([]int, 0, 1024),
		tmp:     NewBuffer(szTmp),
	}
	defer s.tmp.Release()

	left := offsets[0]
	for _, off := range offsets[1:] {
		s.sortSmall(left, off)
		left = off
	}
	s.sort(0, len(offsets)-1)
}

func rawSlice(buf []byte) []byte {
	sz := binary.BigEndian.Uint32(buf)
	return buf[:4+int(sz)]
}

// Slice would return the slice written at offset.
func (b *Buffer) Slice(offset int) ([]byte, int) {
	if offset >= b.offset {
		return nil, 0
	}

	sz := binary.BigEndian.Uint32(b.buf[offset:])
	start := offset + 4
	next := start + int(sz)
	res := b.buf[start:next]
	if next >= b.offset {
		next = 0
	}
	return res, next
}

func (b *Buffer) Data(offset int) []byte {
	if offset > b.curSz {
		panic("offset beyond current size")
	}
	return b.buf[offset:b.curSz]
}

// Write would write p bytes to the buffer.
func (b *Buffer) Write(p []byte) (n int, err error) {
	b.Grow(len(p))
	n = copy(b.buf[b.offset:], p)
	b.offset += n
	return n, nil
}

// Reset would reset the buffer to be reused.
func (b *Buffer) Reset() {
	b.offset = 1
}

// Release would free up the memory allocated by the buffer. Once the usage of buffer is done, it is
// important to call Release, otherwise a memory leak can happen.
func (b *Buffer) Release() error {
	switch b.bufType {
	case UseCalloc:
		Free(b.buf)

	case UseMmap:
		fname := b.fd.Name()
		if err := Munmap(b.buf); err != nil {
			return errors.Wrapf(err, "while munmap file %s", fname)
		}
		if err := b.fd.Truncate(0); err != nil {
			return errors.Wrapf(err, "while truncating file %s", fname)
		}
		if err := b.fd.Close(); err != nil {
			return errors.Wrapf(err, "while closing file %s", fname)
		}
		if err := os.Remove(b.fd.Name()); err != nil {
			return errors.Wrapf(err, "while deleting file %s", fname)
		}
	}
	return nil
}
<commit_msg>Buffer: Use 256GB mmap size instead of MaxInt64 (#198)<commit_after>/*
 * Copyright 2020 Dgraph Labs, Inc. and Contributors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package z

import (
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"sort"

	"github.com/pkg/errors"
)

// Buffer is equivalent of bytes.Buffer without the ability to read. It is NOT thread-safe.
//
// In UseCalloc mode, z.Calloc is used to allocate memory, which depending upon how the code is
// compiled could use jemalloc for allocations.
//
// In UseMmap mode, Buffer  uses file mmap to allocate memory. This allows us to store big data
// structures without using physical memory.
//
// MaxSize can be set to limit the memory usage.
type Buffer struct {
	buf           []byte
	offset        int
	curSz         int
	maxSz         int
	fd            *os.File
	bufType       BufferType
	autoMmapAfter int
}

type BufferType int

func (t BufferType) String() string {
	switch t {
	case UseCalloc:
		return "UseCalloc"
	case UseMmap:
		return "UseMmap"
	}
	return "invalid"
}

const (
	UseCalloc BufferType = iota
	UseMmap
	UseInvalid
)

// smallBufferSize is an initial allocation minimal capacity.
const smallBufferSize = 64

// Newbuffer is a helper utility, which creates a virtually unlimited Buffer in UseCalloc mode.
func NewBuffer(sz int) *Buffer {
	buf, err := NewBufferWith(sz, 256<<30, UseCalloc)
	if err != nil {
		log.Fatalf("while creating buffer: %v", err)
	}
	return buf
}

func (b *Buffer) doMmap() error {
	curBuf := b.buf
	fd, err := ioutil.TempFile("", "buffer")
	if err != nil {
		return err
	}
	if err := fd.Truncate(int64(b.curSz)); err != nil {
		return errors.Wrapf(err, "while truncating %s to size: %d", fd.Name(), b.curSz)
	}

	buf, err := Mmap(fd, true, int64(b.maxSz)) // Mmap up to max size.
	if err != nil {
		return errors.Wrapf(err, "while mmapping %s with size: %d", fd.Name(), b.maxSz)
	}
	if len(curBuf) > 0 {
		assert(b.offset == copy(buf, curBuf[:b.offset]))
		Free(curBuf)
	}
	b.buf = buf
	b.bufType = UseMmap
	b.fd = fd
	return nil
}

// NewBufferWith would allocate a buffer of size sz upfront, with the total size of the buffer not
// exceeding maxSz. Both sz and maxSz can be set to zero, in which case reasonable defaults would be
// used. Buffer can't be used without initialization via NewBuffer.
func NewBufferWith(sz, maxSz int, bufType BufferType) (*Buffer, error) {
	if sz == 0 {
		sz = smallBufferSize
	}
	if maxSz == 0 {
		maxSz = math.MaxInt32
	}

	b := &Buffer{
		offset:  1,
		curSz:   sz,
		maxSz:   maxSz,
		bufType: UseCalloc, // by default.
	}

	switch bufType {
	case UseCalloc:
		b.buf = Calloc(sz)
	case UseMmap:
		if err := b.doMmap(); err != nil {
			return nil, err
		}
	default:
		log.Fatalf("Invalid bufType: %q\n", bufType)
	}

	b.buf[0] = 0x00
	return b, nil
}

func (b *Buffer) IsEmpty() bool {
	return b.offset == 1
}

// Len would return the number of bytes written to the buffer so far.
func (b *Buffer) Len() int {
	return b.offset
}

// Bytes would return all the written bytes as a slice.
func (b *Buffer) Bytes() []byte {
	return b.buf[1:b.offset]
}

func (b *Buffer) AutoMmapAfter(size int) {
	b.autoMmapAfter = size
}

// Grow would grow the buffer to have at least n more bytes. In case the buffer is at capacity, it
// would reallocate twice the size of current capacity + n, to ensure n bytes can be written to the
// buffer without further allocation. In UseMmap mode, this might result in underlying file
// expansion.
func (b *Buffer) Grow(n int) {
	// In this case, len and cap are the same.
	if b.buf == nil {
		panic("z.Buffer needs to be initialized before using")
	}
	if b.maxSz-b.offset < n {
		panic(fmt.Sprintf("Buffer max size exceeded: %d."+
			" Offset: %d. Grow: %d", b.maxSz, b.offset, n))
	}
	if b.curSz-b.offset > n {
		return
	}

	growBy := b.curSz + n
	if growBy > 1<<30 {
		growBy = 1 << 30
	}
	if n > growBy {
		// Always at least allocate n, even if it exceeds the 1GB limit above.
		growBy = n
	}
	b.curSz += growBy

	switch b.bufType {
	case UseCalloc:
		if b.autoMmapAfter > 0 && b.curSz > b.autoMmapAfter {
			// This would do copy as well.
			check(b.doMmap())

		} else {
			newBuf := Calloc(b.curSz)
			copy(newBuf, b.buf[:b.offset])
			Free(b.buf)
			b.buf = newBuf
		}
	case UseMmap:
		if err := b.fd.Truncate(int64(b.curSz)); err != nil {
			log.Fatalf("While trying to truncate file %s to size: %d error: %v\n",
				b.fd.Name(), b.curSz, err)
		}
	}
}

// Allocate is a way to get a slice of size n back from the buffer. This slice can be directly
// written to. Warning: Allocate is not thread-safe. The byte slice returned MUST be used before
// further calls to Buffer.
func (b *Buffer) Allocate(n int) []byte {
	b.Grow(n)
	off := b.offset
	b.offset += n
	return b.buf[off:b.offset]
}

// AllocateOffset works the same way as allocate, but instead of returning a byte slice, it returns
// the offset of the allocation.
func (b *Buffer) AllocateOffset(n int) int {
	b.Grow(n)
	b.offset += n
	return b.offset - n
}

func (b *Buffer) writeLen(sz int) {
	buf := b.Allocate(4)
	binary.BigEndian.PutUint32(buf, uint32(sz))
}

// SliceAllocate would encode the size provided into the buffer, followed by a call to Allocate,
// hence returning the slice of size sz. This can be used to allocate a lot of small buffers into
// this big buffer.
// Note that SliceAllocate should NOT be mixed with normal calls to Write.
func (b *Buffer) SliceAllocate(sz int) []byte {
	b.Grow(4 + sz)
	b.writeLen(sz)
	return b.Allocate(sz)
}

type LessFunc func(a, b []byte) bool
type sortHelper struct {
	offsets []int
	b       *Buffer
	tmp     *Buffer
	less    LessFunc
	small   []int
}

func (s *sortHelper) sortSmall(start, end int) {
	s.tmp.Reset()
	s.small = s.small[:0]
	next := start
	for next != 0 && next < end {
		s.small = append(s.small, next)
		_, next = s.b.Slice(next)
	}

	// We are sorting the slices pointed to by s.small offsets, but only moving the offsets around.
	sort.Slice(s.small, func(i, j int) bool {
		left, _ := s.b.Slice(s.small[i])
		right, _ := s.b.Slice(s.small[j])
		return s.less(left, right)
	})
	// Now we iterate over the s.small offsets and copy over the slices. The result is now in order.
	for _, off := range s.small {
		s.tmp.Write(rawSlice(s.b.buf[off:]))
	}
	assert(end-start == copy(s.b.buf[start:end], s.tmp.Bytes()))
}

func assert(b bool) {
	if !b {
		log.Fatalf("%+v", errors.Errorf("Assertion failure"))
	}
}
func check(err error) {
	if err != nil {
		log.Fatalf("%+v", err)
	}
}
func check2(_ interface{}, err error) {
	check(err)
}

func (s *sortHelper) merge(left, right []byte, start, end int) {
	if len(left) == 0 || len(right) == 0 {
		return
	}
	s.tmp.Reset()
	check2(s.tmp.Write(left))
	left = s.tmp.Bytes()

	var ls, rs []byte

	copyLeft := func() {
		assert(len(ls) == copy(s.b.buf[start:], ls))
		left = left[len(ls):]
		start += len(ls)
	}
	copyRight := func() {
		assert(len(rs) == copy(s.b.buf[start:], rs))
		right = right[len(rs):]
		start += len(rs)
	}

	for start < end {
		if len(left) == 0 {
			assert(len(right) == copy(s.b.buf[start:end], right))
			return
		}
		if len(right) == 0 {
			assert(len(left) == copy(s.b.buf[start:end], left))
			return
		}
		ls = rawSlice(left)
		rs = rawSlice(right)

		// We skip the first 4 bytes in the rawSlice, because that stores the length.
		if s.less(ls[4:], rs[4:]) {
			copyLeft()
		} else {
			copyRight()
		}
	}
}

func (s *sortHelper) sort(lo, hi int) []byte {
	assert(lo <= hi)

	mid := lo + (hi-lo)/2
	loff, hoff := s.offsets[lo], s.offsets[hi]
	if lo == mid {
		// No need to sort, just return the buffer.
		return s.b.buf[loff:hoff]
	}

	// lo, mid would sort from [offset[lo], offset[mid]) .
	left := s.sort(lo, mid)
	// Typically we'd use mid+1, but here mid represents an offset in the buffer. Each offset
	// contains a thousand entries. So, if we do mid+1, we'd skip over those entries.
	right := s.sort(mid, hi)

	s.merge(left, right, loff, hoff)
	return s.b.buf[loff:hoff]
}

// SortSlice is like SortSliceBetween but sorting over the entire buffer.
func (b *Buffer) SortSlice(less func(left, right []byte) bool) {
	b.SortSliceBetween(1, b.offset, less)
}
func (b *Buffer) SortSliceBetween(start, end int, less LessFunc) {
	if start >= end {
		return
	}
	if start == 0 {
		panic("start can never be zero")
	}

	var offsets []int
	next, count := start, 0
	for next != 0 && next < end {
		if count%1024 == 0 {
			offsets = append(offsets, next)
		}
		_, next = b.Slice(next)
		count++
	}
	assert(len(offsets) > 0)
	if offsets[len(offsets)-1] != end {
		offsets = append(offsets, end)
	}

	szTmp := int(float64((end-start)/2) * 1.1)
	s := &sortHelper{
		offsets: offsets,
		b:       b,
		less:    less,
		small:   make([]int, 0, 1024),
		tmp:     NewBuffer(szTmp),
	}
	defer s.tmp.Release()

	left := offsets[0]
	for _, off := range offsets[1:] {
		s.sortSmall(left, off)
		left = off
	}
	s.sort(0, len(offsets)-1)
}

func rawSlice(buf []byte) []byte {
	sz := binary.BigEndian.Uint32(buf)
	return buf[:4+int(sz)]
}

// Slice would return the slice written at offset.
func (b *Buffer) Slice(offset int) ([]byte, int) {
	if offset >= b.offset {
		return nil, 0
	}

	sz := binary.BigEndian.Uint32(b.buf[offset:])
	start := offset + 4
	next := start + int(sz)
	res := b.buf[start:next]
	if next >= b.offset {
		next = 0
	}
	return res, next
}

func (b *Buffer) Data(offset int) []byte {
	if offset > b.curSz {
		panic("offset beyond current size")
	}
	return b.buf[offset:b.curSz]
}

// Write would write p bytes to the buffer.
func (b *Buffer) Write(p []byte) (n int, err error) {
	b.Grow(len(p))
	n = copy(b.buf[b.offset:], p)
	b.offset += n
	return n, nil
}

// Reset would reset the buffer to be reused.
func (b *Buffer) Reset() {
	b.offset = 1
}

// Release would free up the memory allocated by the buffer. Once the usage of buffer is done, it is
// important to call Release, otherwise a memory leak can happen.
func (b *Buffer) Release() error {
	switch b.bufType {
	case UseCalloc:
		Free(b.buf)

	case UseMmap:
		fname := b.fd.Name()
		if err := Munmap(b.buf); err != nil {
			return errors.Wrapf(err, "while munmap file %s", fname)
		}
		if err := b.fd.Truncate(0); err != nil {
			return errors.Wrapf(err, "while truncating file %s", fname)
		}
		if err := b.fd.Close(); err != nil {
			return errors.Wrapf(err, "while closing file %s", fname)
		}
		if err := os.Remove(b.fd.Name()); err != nil {
			return errors.Wrapf(err, "while deleting file %s", fname)
		}
	}
	return nil
}
<|endoftext|>