<commit_before>// build_and_deploy_cipd performs a Bazel build of the given targets and uploads
// a CIPD package including the given build products.
package main

import (
	"context"
	"flag"
	"fmt"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	cipd_pkg "go.chromium.org/luci/cipd/client/cipd/pkg"
	cipd_common "go.chromium.org/luci/cipd/common"
	"golang.org/x/oauth2"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/task_driver/go/lib/auth_steps"
	"go.skia.org/infra/task_driver/go/lib/bazel"
	"go.skia.org/infra/task_driver/go/lib/os_steps"
	"go.skia.org/infra/task_driver/go/td"
)

var (
	// Required properties for this task.
	projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
	taskId    = flag.String("task_id", "", "ID of this task.")
	taskName  = flag.String("task_name", "", "Name of the task.")

	pkgName       = flag.String("package_name", "", "Name of the CIPD package.")
	targets       = common.NewMultiStringFlag("target", nil, "Bazel build targets.")
	platformsList = common.NewMultiStringFlag("platform", nil, "Pairs of Bazel build platform and CIPD platform in <bazel platform>=<cipd platform> format.")
	includePaths  = common.NewMultiStringFlag("include_path", nil, "Paths to include, relative to //_bazel_bin.  Use [.exe] for optional suffix, eg. \"program[.exe]\"")

	bazelCacheDir     = flag.String("bazel_cache_dir", "", "Path to the Bazel cache directory.")
	bazelRepoCacheDir = flag.String("bazel_repo_cache_dir", "", "Path to the Bazel repository cache directory.")

	// Optional flags.
	buildDir       = flag.String("build_dir", ".", "Directory containing the Bazel workspace to build.")
	cipdServiceURL = flag.String("cipd_service_url", cipd.DefaultServiceURL, "CIPD service URL.")
	tags           = common.NewMultiStringFlag("tag", nil, "Tags to apply to the package, in key:value format.")
	refs           = common.NewMultiStringFlag("ref", nil, "Refs to apply to the package.")
	metadata       = common.NewMultiStringFlag("metadata", nil, "Metadata to apply to the package, in key:value format.")
	rbe            = flag.Bool("rbe", false, "Whether to run Bazel on RBE or locally.")
	rbeKey         = flag.String("rbe_key", "", "Path to the service account key to use for RBE.")
	local          = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
	output         = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)

var (
	// executableSuffixRegex is used to parse an --include_path which uses the
	// path[.extension] format.
	executableSuffixRegex = regexp.MustCompile(`(.+)\[(.+)\]`)
)

func main() {
	// Setup.
	ctx := td.StartRun(projectId, taskId, taskName, output, local)
	defer td.EndRun(ctx)

	if *pkgName == "" {
		td.Fatalf(ctx, "--package_name is required.")
	}
	if len(*includePaths) == 0 {
		td.Fatalf(ctx, "At least one --include_path is required.")
	}
	if len(*targets) == 0 {
		td.Fatalf(ctx, "At least one --target is required.")
	}
	if len(*platformsList) == 0 {
		td.Fatalf(ctx, "At least one --platform is required.")
	}
	for _, tag := range *tags {
		splitPair(ctx, tag, ":")
	}
	metadataMap := make(map[string]string, len(*metadata))
	for _, md := range *metadata {
		k, v := splitPair(ctx, md, ":")
		metadataMap[k] = v
	}

	// Create directories for each of the build platforms.
	pkgs := make([]*pkgSpec, 0, len(*platformsList))
	var ts oauth2.TokenSource
	var cipdClient cipd.CIPDClient
	if err := td.Do(ctx, td.Props("Setup").Infra(), func(ctx context.Context) error {
		var err error
		ts, err = auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail)
		if err != nil {
			return err
		}
		httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
		cipdClient, err = cipd.NewClient(httpClient, ".", *cipdServiceURL)
		if err != nil {
			return err
		}
		for _, platform := range *platformsList {
			bzlPlatform, cipdPlatform := splitPair(ctx, platform, "=")
			tmpDir, err := os_steps.TempDir(ctx, "", cipdPlatform)
			if err != nil {
				return err
			}
			pkgs = append(pkgs, &pkgSpec{
				bazelPlatform: bzlPlatform,
				cipdPlatform:  cipdPlatform,
				cipdPkgPath:   path.Join(*pkgName, cipdPlatform),
				tmpDir:        tmpDir,
			})
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}
	defer func() {
		if err := td.Do(ctx, td.Props("Cleanup").Infra(), func(ctx context.Context) error {
			var rvErr error
			for _, pkg := range pkgs {
				tmpDir := pkg.tmpDir
				if err := os_steps.RemoveAll(ctx, tmpDir); err != nil {
					rvErr = err
				}
			}
			return rvErr
		}); err != nil {
			td.Fatal(ctx, err)
		}
	}()

	// Perform the build(s).
	if err := td.Do(ctx, td.Props("Build"), func(ctx context.Context) (rvErr error) {
		opts := bazel.BazelOptions{
			CachePath:           *bazelCacheDir,
			RepositoryCachePath: *bazelRepoCacheDir,
		}
		bzl, err := bazel.New(ctx, *buildDir, *rbeKey, opts)
		if err != nil {
			return err
		}

		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props("Build "+pkg.cipdPlatform), func(ctx context.Context) error {
				// We're building for multiple platforms, and Bazel writes all
				// of the build products into the same directory regardless of
				// platform, so there's a potential for accidental inclusion of
				// incompatible binaries in the CIPD package, eg. "app.exe" vs
				// "app". "bazel clean" prevents that by emptying the output
				// directory between builds.
				if _, err := bzl.Do(ctx, "clean"); err != nil {
					return err
				}

				// Perform the build.
				args := []string{fmt.Sprintf("--platforms=%s", pkg.bazelPlatform)}
				args = append(args, *targets...)
				doFunc := bzl.Do
				if *rbe {
					doFunc = bzl.DoOnRBE
				}
				if _, err := doFunc(ctx, "build", args...); err != nil {
					return err
				}

				// Copy the outputs to the destination dir.
				for _, path := range *includePaths {
					paths := []string{path}
					m := executableSuffixRegex.FindAllStringSubmatch(path, -1)
					if m != nil {
						paths = []string{m[0][1], m[0][1] + m[0][2]}
					}
					found := false
					for _, path := range paths {
						path := filepath.Join(*buildDir, path)
						if _, err := os_steps.Stat(ctx, path); err == nil {
							dest := filepath.Join(pkg.tmpDir, filepath.Base(path))
							if err := os_steps.CopyFile(ctx, path, dest); err != nil {
								return err
							}
							found = true
							break
						}
					}
					if !found {
						return fmt.Errorf("Unable to find %q; tried %v", path, paths)
					}
				}
				return nil
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}

	// Upload the package(s) to CIPD.
	// TODO(borenet): See if we can use the CIPD Go code directly, rather than
	// having to ship a separate binary.
	if err := td.Do(ctx, td.Props("Upload to CIPD"), func(ctx context.Context) error {
		// Upload all of the package instances.
		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props(fmt.Sprintf("Upload %s", pkg.cipdPlatform)), func(ctx context.Context) error {
				pin, err := cipdClient.Create(ctx, pkg.cipdPkgPath, pkg.tmpDir, cipd_pkg.InstallModeCopy, nil, nil, nil, nil)
				if err != nil {
					return err
				}
				pkg.pin = pin
				return nil
			}); err != nil {
				return err
			}
		}
		// Apply refs, tags, and metadata. Do this after all platforms have been
		// built and uploaded to increase the likelihood that the refs and tags
		// get applied to all packages or none. Otherwise it's possible for some
		// platforms to be missing when querying by ref or tag.
		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props(fmt.Sprintf("Attach %s", pkg.cipdPlatform)), func(ctx context.Context) error {
				// If any of the provided tags is already attached to a
				// different instance, stop and return an error.
				for _, tag := range *tags {
					found, err := cipdClient.SearchInstances(ctx, pkg.cipdPkgPath, []string{tag})
					if err != nil {
						return err
					}
					if len(found) == 1 && found[0].InstanceID != pkg.pin.InstanceID {
						return skerr.Fmt("Found existing instance %s of package %s with tag %s", found[0].InstanceID, pkg.cipdPkgPath, tag)
					}
					if len(found) > 1 {
						return skerr.Fmt("Found more than one instance of package %s with tag %s. This may result in failure to retrieve the package by tag due to ambiguity. Please contact the current infra gardener to investigate. To detach tags, see http://go/luci-cipd#detachtags", pkg.cipdPkgPath, tag)
					}
				}
				return cipdClient.Attach(ctx, pkg.pin, *refs, *tags, metadataMap)
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}
}

// splitPair splits a key and value from a command line flag and Fatals if it
// does not follow the expected format.
func splitPair(ctx context.Context, elem, sep string) (string, string) {
	split := strings.SplitN(elem, sep, 2)
	if len(split) != 2 {
		td.Fatalf(ctx, "Expected <key>%s<value> format for %q", sep, elem)
	}
	return split[0], split[1]
}

// pkgSpec contains information about how to build and upload an indivdual CIPD
// package instance.
type pkgSpec struct {
	bazelPlatform string
	cipdPlatform  string
	cipdPkgPath   string
	tmpDir        string
	pin           cipd_common.Pin
}
<commit_msg>[build-and-deploy-cipd] Add instance ID to step name<commit_after>// build_and_deploy_cipd performs a Bazel build of the given targets and uploads
// a CIPD package including the given build products.
package main

import (
	"context"
	"flag"
	"fmt"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	cipd_pkg "go.chromium.org/luci/cipd/client/cipd/pkg"
	cipd_common "go.chromium.org/luci/cipd/common"
	"golang.org/x/oauth2"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/task_driver/go/lib/auth_steps"
	"go.skia.org/infra/task_driver/go/lib/bazel"
	"go.skia.org/infra/task_driver/go/lib/os_steps"
	"go.skia.org/infra/task_driver/go/td"
)

var (
	// Required properties for this task.
	projectId = flag.String("project_id", "", "ID of the Google Cloud project.")
	taskId    = flag.String("task_id", "", "ID of this task.")
	taskName  = flag.String("task_name", "", "Name of the task.")

	pkgName       = flag.String("package_name", "", "Name of the CIPD package.")
	targets       = common.NewMultiStringFlag("target", nil, "Bazel build targets.")
	platformsList = common.NewMultiStringFlag("platform", nil, "Pairs of Bazel build platform and CIPD platform in <bazel platform>=<cipd platform> format.")
	includePaths  = common.NewMultiStringFlag("include_path", nil, "Paths to include, relative to //_bazel_bin.  Use [.exe] for optional suffix, eg. \"program[.exe]\"")

	bazelCacheDir     = flag.String("bazel_cache_dir", "", "Path to the Bazel cache directory.")
	bazelRepoCacheDir = flag.String("bazel_repo_cache_dir", "", "Path to the Bazel repository cache directory.")

	// Optional flags.
	buildDir       = flag.String("build_dir", ".", "Directory containing the Bazel workspace to build.")
	cipdServiceURL = flag.String("cipd_service_url", cipd.DefaultServiceURL, "CIPD service URL.")
	tags           = common.NewMultiStringFlag("tag", nil, "Tags to apply to the package, in key:value format.")
	refs           = common.NewMultiStringFlag("ref", nil, "Refs to apply to the package.")
	metadata       = common.NewMultiStringFlag("metadata", nil, "Metadata to apply to the package, in key:value format.")
	rbe            = flag.Bool("rbe", false, "Whether to run Bazel on RBE or locally.")
	rbeKey         = flag.String("rbe_key", "", "Path to the service account key to use for RBE.")
	local          = flag.Bool("local", false, "True if running locally (as opposed to on the bots)")
	output         = flag.String("o", "", "If provided, dump a JSON blob of step data to the given file. Prints to stdout if '-' is given.")
)

var (
	// executableSuffixRegex is used to parse an --include_path which uses the
	// path[.extension] format.
	executableSuffixRegex = regexp.MustCompile(`(.+)\[(.+)\]`)
)

func main() {
	// Setup.
	ctx := td.StartRun(projectId, taskId, taskName, output, local)
	defer td.EndRun(ctx)

	if *pkgName == "" {
		td.Fatalf(ctx, "--package_name is required.")
	}
	if len(*includePaths) == 0 {
		td.Fatalf(ctx, "At least one --include_path is required.")
	}
	if len(*targets) == 0 {
		td.Fatalf(ctx, "At least one --target is required.")
	}
	if len(*platformsList) == 0 {
		td.Fatalf(ctx, "At least one --platform is required.")
	}
	for _, tag := range *tags {
		splitPair(ctx, tag, ":")
	}
	metadataMap := make(map[string]string, len(*metadata))
	for _, md := range *metadata {
		k, v := splitPair(ctx, md, ":")
		metadataMap[k] = v
	}

	// Create directories for each of the build platforms.
	pkgs := make([]*pkgSpec, 0, len(*platformsList))
	var ts oauth2.TokenSource
	var cipdClient cipd.CIPDClient
	if err := td.Do(ctx, td.Props("Setup").Infra(), func(ctx context.Context) error {
		var err error
		ts, err = auth_steps.Init(ctx, *local, auth.ScopeUserinfoEmail)
		if err != nil {
			return err
		}
		httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
		cipdClient, err = cipd.NewClient(httpClient, ".", *cipdServiceURL)
		if err != nil {
			return err
		}
		for _, platform := range *platformsList {
			bzlPlatform, cipdPlatform := splitPair(ctx, platform, "=")
			tmpDir, err := os_steps.TempDir(ctx, "", cipdPlatform)
			if err != nil {
				return err
			}
			pkgs = append(pkgs, &pkgSpec{
				bazelPlatform: bzlPlatform,
				cipdPlatform:  cipdPlatform,
				cipdPkgPath:   path.Join(*pkgName, cipdPlatform),
				tmpDir:        tmpDir,
			})
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}
	defer func() {
		if err := td.Do(ctx, td.Props("Cleanup").Infra(), func(ctx context.Context) error {
			var rvErr error
			for _, pkg := range pkgs {
				tmpDir := pkg.tmpDir
				if err := os_steps.RemoveAll(ctx, tmpDir); err != nil {
					rvErr = err
				}
			}
			return rvErr
		}); err != nil {
			td.Fatal(ctx, err)
		}
	}()

	// Perform the build(s).
	if err := td.Do(ctx, td.Props("Build"), func(ctx context.Context) (rvErr error) {
		opts := bazel.BazelOptions{
			CachePath:           *bazelCacheDir,
			RepositoryCachePath: *bazelRepoCacheDir,
		}
		bzl, err := bazel.New(ctx, *buildDir, *rbeKey, opts)
		if err != nil {
			return err
		}

		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props("Build "+pkg.cipdPlatform), func(ctx context.Context) error {
				// We're building for multiple platforms, and Bazel writes all
				// of the build products into the same directory regardless of
				// platform, so there's a potential for accidental inclusion of
				// incompatible binaries in the CIPD package, eg. "app.exe" vs
				// "app". "bazel clean" prevents that by emptying the output
				// directory between builds.
				if _, err := bzl.Do(ctx, "clean"); err != nil {
					return err
				}

				// Perform the build.
				args := []string{fmt.Sprintf("--platforms=%s", pkg.bazelPlatform)}
				args = append(args, *targets...)
				doFunc := bzl.Do
				if *rbe {
					doFunc = bzl.DoOnRBE
				}
				if _, err := doFunc(ctx, "build", args...); err != nil {
					return err
				}

				// Copy the outputs to the destination dir.
				for _, path := range *includePaths {
					paths := []string{path}
					m := executableSuffixRegex.FindAllStringSubmatch(path, -1)
					if m != nil {
						paths = []string{m[0][1], m[0][1] + m[0][2]}
					}
					found := false
					for _, path := range paths {
						path := filepath.Join(*buildDir, path)
						if _, err := os_steps.Stat(ctx, path); err == nil {
							dest := filepath.Join(pkg.tmpDir, filepath.Base(path))
							if err := os_steps.CopyFile(ctx, path, dest); err != nil {
								return err
							}
							found = true
							break
						}
					}
					if !found {
						return fmt.Errorf("Unable to find %q; tried %v", path, paths)
					}
				}
				return nil
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}

	// Upload the package(s) to CIPD.
	// TODO(borenet): See if we can use the CIPD Go code directly, rather than
	// having to ship a separate binary.
	if err := td.Do(ctx, td.Props("Upload to CIPD"), func(ctx context.Context) error {
		// Upload all of the package instances.
		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props(fmt.Sprintf("Upload %s", pkg.cipdPlatform)), func(ctx context.Context) error {
				pin, err := cipdClient.Create(ctx, pkg.cipdPkgPath, pkg.tmpDir, cipd_pkg.InstallModeCopy, nil, nil, nil, nil)
				if err != nil {
					return err
				}
				pkg.pin = pin
				return nil
			}); err != nil {
				return err
			}
		}
		// Apply refs, tags, and metadata. Do this after all platforms have been
		// built and uploaded to increase the likelihood that the refs and tags
		// get applied to all packages or none. Otherwise it's possible for some
		// platforms to be missing when querying by ref or tag.
		for _, pkg := range pkgs {
			if err := td.Do(ctx, td.Props(fmt.Sprintf("Attach %s %s", pkg.cipdPlatform, pkg.pin.String())), func(ctx context.Context) error {
				// If any of the provided tags is already attached to a
				// different instance, stop and return an error.
				for _, tag := range *tags {
					found, err := cipdClient.SearchInstances(ctx, pkg.cipdPkgPath, []string{tag})
					if err != nil {
						return err
					}
					if len(found) == 1 && found[0].InstanceID != pkg.pin.InstanceID {
						return skerr.Fmt("Found existing instance %s of package %s with tag %s", found[0].InstanceID, pkg.cipdPkgPath, tag)
					}
					if len(found) > 1 {
						return skerr.Fmt("Found more than one instance of package %s with tag %s. This may result in failure to retrieve the package by tag due to ambiguity. Please contact the current infra gardener to investigate. To detach tags, see http://go/luci-cipd#detachtags", pkg.cipdPkgPath, tag)
					}
				}
				return cipdClient.Attach(ctx, pkg.pin, *refs, *tags, metadataMap)
			}); err != nil {
				return err
			}
		}
		return nil
	}); err != nil {
		td.Fatal(ctx, err)
	}
}

// splitPair splits a key and value from a command line flag and Fatals if it
// does not follow the expected format.
func splitPair(ctx context.Context, elem, sep string) (string, string) {
	split := strings.SplitN(elem, sep, 2)
	if len(split) != 2 {
		td.Fatalf(ctx, "Expected <key>%s<value> format for %q", sep, elem)
	}
	return split[0], split[1]
}

// pkgSpec contains information about how to build and upload an indivdual CIPD
// package instance.
type pkgSpec struct {
	bazelPlatform string
	cipdPlatform  string
	cipdPkgPath   string
	tmpDir        string
	pin           cipd_common.Pin
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// +build darwin freebsd linux netbsd openbsd plan9

package storage

import (
	"os"
	"syscall"
)

type unixFileLock struct {
	f *os.File
}

func (fl *unixFileLock) release() error {
	if err := setFileLock(fl.f, false); err != nil {
		return err
	}
	return fl.f.Close()
}

func newFileLock(path string) (fl fileLock, err error) {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return
	}
	err = setFileLock(f, true)
	if err != nil {
		f.Close()
		return
	}
	fl = &unixFileLock{f: f}
	return
}

func setFileLock(f *os.File, lock bool) (err error) {
	how := syscall.LOCK_UN
	if lock {
		how = syscall.LOCK_EX
	}
	return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
}

func rename(oldpath, newpath string) (err error) {
	return os.Rename(oldpath, newpath)
}
<commit_msg>Remove naked returns from leveldb/storage/file_storage_unix.go.<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// +build darwin freebsd linux netbsd openbsd plan9

package storage

import (
	"os"
	"syscall"
)

type unixFileLock struct {
	f *os.File
}

func (fl *unixFileLock) release() error {
	if err := setFileLock(fl.f, false); err != nil {
		return err
	}
	return fl.f.Close()
}

func newFileLock(path string) (fileLock, error) {
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}
	if err := setFileLock(f, true); err != nil {
		f.Close()
		return nil, err
	}
	return &unixFileLock{f: f}, nil
}

func setFileLock(f *os.File, lock bool) error {
	how := syscall.LOCK_UN
	if lock {
		how = syscall.LOCK_EX
	}
	return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
}

func rename(oldpath, newpath string) (err error) {
	return os.Rename(oldpath, newpath)
}
<|endoftext|>


<commit_before>// (C) 1990-2016.  Robert W Solomon.  All rights reserved.
// ShowUtf-8 codes.  Based on utf8toascii, based on nocr.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode/utf8"
	//
)

const lastCompiled = "6 May 17"

//const openQuoteRune = 0xe2809c
//const closeQuoteRune = 0xe2809d
//const squoteRune = 0xe28099
//const emdashRune = 0xe28094
const openQuoteRune = 8220
const closeQuoteRune = 8221
const squoteRune = 8217
const opensquoteRune = 8216
const emdashRune = 8212
const endashRune = 8211
const bulletpointRune = 8226
const threedotsRune = 8230
const hyphenRune = 8208

const quoteString = "\""
const squoteString = "'"
const emdashStr = " -- "
const bulletpointStr = "--"

/*
   REVISION HISTORY
   ----------------
   17 Apr 17 -- Started writing nocr, based on rpn.go
   18 Apr 17 -- It worked yesterday.  Now I'll rename files as in Modula-2.
    5 May 17 -- Now will convert utf8 to ascii, based on nocr.go
	6 May 17 -- Need to know the utf8 codes before I can convert 'em.
	6 May 17 -- Added a flag -a for after to see the rest of the string to give me context for a new rune.
*/

func main() {
	var instr string
	//	var err error

	fmt.Println(" ShowUtf8.  Last compiled ", lastCompiled)
	fmt.Println()

	if len(os.Args) <= 1 {
		fmt.Println(" Usage: utf8toascii <filename> ")
		os.Exit(1)
	}

	var afterflag = flag.Bool("a", false, "afterflag -- show string after rune.")
	var AfterFlag bool
	flag.BoolVar(&AfterFlag, "A", false, "AfterFlag -- show string after rune.")

	flag.Parse()

	After := *afterflag || AfterFlag

	commandline := flag.Arg(0)
	BaseFilename := filepath.Clean(commandline)
	InFilename := ""
	InFileExists := false
	Ext1Default := ".txt"
	//	OutFileSuffix := ".out"

	if strings.Contains(BaseFilename, ".") {
		InFilename = BaseFilename
		_, err := os.Stat(InFilename)
		if err == nil {
			InFileExists = true
		}
	} else {
		InFilename = BaseFilename + Ext1Default
		_, err := os.Stat(InFilename)
		if err == nil {
			InFileExists = true
		}
	}

	if !InFileExists {
		fmt.Println(" File ", BaseFilename, " or ", InFilename, " does not exist.  Exiting.")
		os.Exit(1)
	}

	InputFile, err := os.Open(InFilename)
	if err != nil {
		fmt.Println(" Error while opening ", InFilename, ".  Exiting.")
		os.Exit(1)
	}
	defer InputFile.Close()
	InBufioScanner := bufio.NewScanner(InputFile)
	linecounter := 0
	for InBufioScanner.Scan() {
		instr = InBufioScanner.Text() // does not include the trailing EOL char
		linecounter++
		runecount := utf8.RuneCountInString(instr)
		if len(instr) == runecount {
			continue
		} else { // a mismatch btwn instr length and rune count means that a multibyte rune is in this instr
			fmt.Print(" Line ", linecounter, " : ")
			for dnctr := runecount; dnctr > 0; dnctr-- {
				r, siz := utf8.DecodeRuneInString(instr) // front rune in r
				instr = instr[siz:]                      // chop off the first rune
				if r > 128 {
					fmt.Print(" r: ", r, ", siz: ", siz, "; ")
					if r == openQuoteRune {
						fmt.Print(" rune is opening", quoteString, "; ")
					} else if r == closeQuoteRune {
						fmt.Print(" rune is closing", quoteString, "; ")
					} else if r == squoteRune || r == opensquoteRune {
						fmt.Print(" rune is ", squoteString, "; ")
					} else if r == emdashRune || r == endashRune {
						fmt.Print(" rune is ", emdashStr, "; ")
					} else if r == bulletpointRune {
						fmt.Print(" rune is bulletpoint; ")
					} else if r == threedotsRune {
						fmt.Print(" rune is ... ; ")
					} else if r == hyphenRune {
						fmt.Print(" rune is hyphen; ")
					} else {
						fmt.Print(" rune is new.")
						if After {
							fmt.Print("  Rest of input line is: ")
							fmt.Println(instr)
						}
					}
				}
			}
			fmt.Println()
		}
	}

	//	InputFile.Close()
	fmt.Println()

} // main in ShowUtf8.go
/*
func check(e error) {
	if e != nil {
		panic(e)
	}
}
*/
<commit_msg>modified:   showutf8/showutf8.go -- added -h flag<commit_after>// (C) 1990-2016.  Robert W Solomon.  All rights reserved.
// ShowUtf-8 codes.  Based on utf8toascii, based on nocr.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode/utf8"
	//
)

const lastCompiled = "7 May 17"

//const openQuoteRune = 0xe2809c
//const closeQuoteRune = 0xe2809d
//const squoteRune = 0xe28099
//const emdashRune = 0xe28094
const openQuoteRune = 8220
const closeQuoteRune = 8221
const squoteRune = 8217
const opensquoteRune = 8216
const emdashRune = 8212
const endashRune = 8211
const bulletpointRune = 8226
const threedotsRune = 8230
const hyphenRune = 8208

const quoteString = "\""
const squoteString = "'"
const emdashStr = " -- "
const bulletpointStr = "--"

/*
   REVISION HISTORY
   ----------------
   17 Apr 17 -- Started writing nocr, based on rpn.go
   18 Apr 17 -- It worked yesterday.  Now I'll rename files as in Modula-2.
    5 May 17 -- Now will convert utf8 to ascii, based on nocr.go
	6 May 17 -- Need to know the utf8 codes before I can convert 'em.
	6 May 17 -- Added a flag -a for after to see the rest of the string to give me context for a new rune.
	7 May 17 -- Added help flag.
*/

func main() {
	var instr string
	//	var err error

	fmt.Println(" ShowUtf8.  Last compiled ", lastCompiled)
	fmt.Println()

	var afterflag = flag.Bool("a", false, "afterflag -- show string after rune.")
	var AfterFlag bool
	flag.BoolVar(&AfterFlag, "A", false, "AfterFlag -- show string after rune.")
	var helpflag = flag.Bool("h", false, "print help message") // pointer
	var HelpFlag bool
	flag.BoolVar(&HelpFlag, "H", false, "print help message")

	flag.Parse()

	if len(os.Args) <= 1 {
		fmt.Println(" Usage: showutf8 <filename> ")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *helpflag || HelpFlag {
		flag.PrintDefaults()
	}

	After := *afterflag || AfterFlag

	commandline := flag.Arg(0)
	BaseFilename := filepath.Clean(commandline)
	InFilename := ""
	InFileExists := false
	Ext1Default := ".txt"
	//	OutFileSuffix := ".out"

	if strings.Contains(BaseFilename, ".") {
		InFilename = BaseFilename
		_, err := os.Stat(InFilename)
		if err == nil {
			InFileExists = true
		}
	} else {
		InFilename = BaseFilename + Ext1Default
		_, err := os.Stat(InFilename)
		if err == nil {
			InFileExists = true
		}
	}

	if !InFileExists {
		fmt.Println(" File ", BaseFilename, " or ", InFilename, " does not exist.  Exiting.")
		os.Exit(1)
	}

	InputFile, err := os.Open(InFilename)
	if err != nil {
		fmt.Println(" Error while opening ", InFilename, ".  Exiting.")
		os.Exit(1)
	}
	defer InputFile.Close()
	InBufioScanner := bufio.NewScanner(InputFile)
	linecounter := 0
	for InBufioScanner.Scan() {
		instr = InBufioScanner.Text() // does not include the trailing EOL char
		linecounter++
		runecount := utf8.RuneCountInString(instr)
		if len(instr) == runecount {
			continue
		} else { // a mismatch btwn instr length and rune count means that a multibyte rune is in this instr
			fmt.Print(" Line ", linecounter, " : ")
			for dnctr := runecount; dnctr > 0; dnctr-- {
				r, siz := utf8.DecodeRuneInString(instr) // front rune in r
				instr = instr[siz:]                      // chop off the first rune
				if r > 128 {
					fmt.Print(" r: ", r, ", siz: ", siz, "; ")
					if r == openQuoteRune {
						fmt.Print(" rune is opening", quoteString, "; ")
					} else if r == closeQuoteRune {
						fmt.Print(" rune is closing", quoteString, "; ")
					} else if r == squoteRune || r == opensquoteRune {
						fmt.Print(" rune is ", squoteString, "; ")
					} else if r == emdashRune || r == endashRune {
						fmt.Print(" rune is ", emdashStr, "; ")
					} else if r == bulletpointRune {
						fmt.Print(" rune is bulletpoint; ")
					} else if r == threedotsRune {
						fmt.Print(" rune is ... ; ")
					} else if r == hyphenRune {
						fmt.Print(" rune is hyphen; ")
					} else {
						fmt.Print(" rune is new.")
						if After {
							fmt.Print("  Rest of input line is: ")
							fmt.Println(instr)
						}
					}
				}
			}
			fmt.Println()
		}
	}

	//	InputFile.Close()
	fmt.Println()

} // main in ShowUtf8.go
/*
func check(e error) {
	if e != nil {
		panic(e)
	}
}
*/
<|endoftext|>


<commit_before>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// Package ardop provides means of establishing a connection to a remote node using ARDOP TNC
package ardop

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

// The default address Ardop TNC listens on
const DefaultAddr = "localhost:8515"

var (
	ErrConnectTimeout = errors.New("Connect timeout")
	ErrInvalidAddr    = errors.New("Invalid address format")
)

type Bandwidth struct {
	Forced bool
	Max    uint
}

var (
	Bandwidth200Max  = Bandwidth{false, 200}
	Bandwidth500Max  = Bandwidth{false, 500}
	Bandwidth1000Max = Bandwidth{false, 1000}
	Bandwidth2000Max = Bandwidth{false, 2000}

	Bandwidth200Forced  = Bandwidth{true, 200}
	Bandwidth500Forced  = Bandwidth{true, 500}
	Bandwidth1000Forced = Bandwidth{true, 1000}
	Bandwidth2000Forced = Bandwidth{true, 2000}
)

func (bw Bandwidth) String() string {
	str := fmt.Sprintf("%d", bw.Max)
	if bw.Forced {
		str += "FORCED"
	} else {
		str += "MAX"
	}
	return str
}

func (bw Bandwidth) IsZero() bool { return bw.Max == 0 }

type State uint8

//go:generate stringer -type=State .
const (
	Unknown      State = iota
	Offline            // Sound card disabled and all sound card resources are released
	Disconnected       // The session is disconnected, the sound card remains active
	ISS                // Information Sending Station (Sending Data)
	IRS                // Information Receiving Station (Receiving data)
	Idle               // ??
	FECSend            // ??
	FECReceive         // Receiving FEC (unproto) data
)

var stateMap = map[string]State{
	"":        Unknown,
	"OFFLINE": Offline,
	"DISC":    Disconnected,
	"ISS":     ISS,
	"IRS":     IRS,
	"IDLE":    Idle,
	"FECRcv":  FECReceive,
	"FECSend": FECSend,
}

func strToState(str string) (State, bool) {
	state, ok := stateMap[strings.ToUpper(str)]
	return state, ok
}

func debugEnabled() bool {
	return os.Getenv("ardop_debug") != ""
}
<commit_msg>ardop: Remove obsolete error<commit_after>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// Package ardop provides means of establishing a connection to a remote node using ARDOP TNC
package ardop

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

// The default address Ardop TNC listens on
const DefaultAddr = "localhost:8515"

var ErrConnectTimeout = errors.New("Connect timeout")

type Bandwidth struct {
	Forced bool
	Max    uint
}

var (
	Bandwidth200Max  = Bandwidth{false, 200}
	Bandwidth500Max  = Bandwidth{false, 500}
	Bandwidth1000Max = Bandwidth{false, 1000}
	Bandwidth2000Max = Bandwidth{false, 2000}

	Bandwidth200Forced  = Bandwidth{true, 200}
	Bandwidth500Forced  = Bandwidth{true, 500}
	Bandwidth1000Forced = Bandwidth{true, 1000}
	Bandwidth2000Forced = Bandwidth{true, 2000}
)

func (bw Bandwidth) String() string {
	str := fmt.Sprintf("%d", bw.Max)
	if bw.Forced {
		str += "FORCED"
	} else {
		str += "MAX"
	}
	return str
}

func (bw Bandwidth) IsZero() bool { return bw.Max == 0 }

type State uint8

//go:generate stringer -type=State .
const (
	Unknown      State = iota
	Offline            // Sound card disabled and all sound card resources are released
	Disconnected       // The session is disconnected, the sound card remains active
	ISS                // Information Sending Station (Sending Data)
	IRS                // Information Receiving Station (Receiving data)
	Idle               // ??
	FECSend            // ??
	FECReceive         // Receiving FEC (unproto) data
)

var stateMap = map[string]State{
	"":        Unknown,
	"OFFLINE": Offline,
	"DISC":    Disconnected,
	"ISS":     ISS,
	"IRS":     IRS,
	"IDLE":    Idle,
	"FECRcv":  FECReceive,
	"FECSend": FECSend,
}

func strToState(str string) (State, bool) {
	state, ok := stateMap[strings.ToUpper(str)]
	return state, ok
}

func debugEnabled() bool {
	return os.Getenv("ardop_debug") != ""
}
<|endoftext|>


<commit_before>// This file is part of netbackup (http://github.com/marcopaganini/netbackup)
// See instructions in the README.md file that accompanies this program.
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package transports

import (
	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"regexp"
	"testing"
)

const (
	rsyncTestCmd = "rsync -av --delete --numeric-ids"
)

// Common rsync tests: Initialize an rsync instance with the config passed
// and test fail if the generated command doesn't match the expected command
// line regexp. The value of dryRun is passed to the initializon of rsync. If
// must_error is true, the call to NewRsyncTransport *must* fail, or the test
// will fail.
func rsyncTest(t *testing.T, cfg *config.Config, expect string, dryRun bool, mustError bool) {
	fakeExecute := NewFakeExecute()

	log := logger.New("")

	// Create a new rsync object with our fakeExecute and a sinking outLogWriter.
	rsync, err := NewRsyncTransport(cfg, fakeExecute, log, dryRun)
	if err != nil {
		if mustError {
			return
		}
		t.Fatalf("rsync.NewRsyncTransport failed: %v", err)
	}
	if mustError {
		t.Fatalf("rsync.NewRsyncTransport should return have failed, but got nil error")
	}
	if err := rsync.Run(); err != nil {
		t.Fatalf("rsync.Run failed: %v", err)
	}
	matched, err := regexp.MatchString(expect, fakeExecute.Cmd())
	if err != nil {
		t.Fatalf("error during regexp match: %v", err)
	}
	if !matched {
		t.Fatalf("name should match %s; is %s", expect, fakeExecute.Cmd())
	}
}

// Dry run: No command should be executed.
func TestRsyncDryRun(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", true, false)
}

// Same machine (local copy).
func TestRsyncSameMachine(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" /tmp/a/ /tmp/b", false, false)
}

// Local source, remote destination.
func TestRsyncLocalSourceRemoteDest(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		DestHost:  "desthost",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" /tmp/a/ desthost:/tmp/b", false, false)
}

// Remote source, local destination.
func TestRsyncRemoteSourceLocalDest(t *testing.T) {
	cfg := &config.Config{
		Name:       "fake",
		SourceHost: "srchost",
		SourceDir:  "/tmp/a",
		DestDir:    "/tmp/b",
		Transport:  "rsync",
		Logfile:    "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" srchost:/tmp/a/ /tmp/b", false, false)
}

// Remote source, Remote destination (server side copy)not supported by rsync.
func TestRsyncRemoteSourceRemoteDest(t *testing.T) {
	cfg := &config.Config{
		Name:       "fake",
		SourceHost: "srchost",
		SourceDir:  "/tmp/a",
		DestHost:   "desthost",
		DestDir:    "/tmp/b",
		Transport:  "rsync",
		Logfile:    "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}

// Sources ending in a slash should not have another slash added.
func TestRsyncDoubleSlash(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" / /tmp/b", false, false)
}

// Exclude list only.
func TestRsyncExcludeListOnly(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Exclude:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --exclude-from=[^ ]+ --delete-excluded /tmp/a/ /tmp/b", false, false)
}

// Include list only.
func TestRsyncIncludeListOnly(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Include:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --include-from=[^ ]+ /tmp/a/ /tmp/b", false, false)
}

// Include & Exclude lists.
func TestRsyncIncludeAndExclude(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Exclude:   []string{"x/foo", "x/bar"},
		Include:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --exclude-from=[^ ]+ --delete-excluded --include-from=[^ ]+ /tmp/a/ /tmp/b", false, false)
}

// Test that an empty source dir results in error.
func TestRsyncEmptySourceDir(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}

// Test that an empty destination dir results in error.
func TestRsyncEmptyDestDir(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}
<commit_msg>Fix rsync unit tests.<commit_after>// This file is part of netbackup (http://github.com/marcopaganini/netbackup)
// See instructions in the README.md file that accompanies this program.
// (C) 2015 by Marco Paganini <paganini AT paganini DOT net>

package transports

import (
	"github.com/marcopaganini/logger"
	"github.com/marcopaganini/netbackup/config"
	"regexp"
	"testing"
)

const (
	rsyncTestCmd = "rsync -avXH --delete --numeric-ids"
)

// Common rsync tests: Initialize an rsync instance with the config passed
// and test fail if the generated command doesn't match the expected command
// line regexp. The value of dryRun is passed to the initializon of rsync. If
// must_error is true, the call to NewRsyncTransport *must* fail, or the test
// will fail.
func rsyncTest(t *testing.T, cfg *config.Config, expect string, dryRun bool, mustError bool) {
	fakeExecute := NewFakeExecute()

	log := logger.New("")

	// Create a new rsync object with our fakeExecute and a sinking outLogWriter.
	rsync, err := NewRsyncTransport(cfg, fakeExecute, log, dryRun)
	if err != nil {
		if mustError {
			return
		}
		t.Fatalf("rsync.NewRsyncTransport failed: %v", err)
	}
	if mustError {
		t.Fatalf("rsync.NewRsyncTransport should return have failed, but got nil error")
	}
	if err := rsync.Run(); err != nil {
		t.Fatalf("rsync.Run failed: %v", err)
	}
	matched, err := regexp.MatchString(expect, fakeExecute.Cmd())
	if err != nil {
		t.Fatalf("error during regexp match: %v", err)
	}
	if !matched {
		t.Fatalf("name should match %s; is %s", expect, fakeExecute.Cmd())
	}
}

// Dry run: No command should be executed.
func TestRsyncDryRun(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", true, false)
}

// Same machine (local copy).
func TestRsyncSameMachine(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" /tmp/a/ /tmp/b", false, false)
}

// Local source, remote destination.
func TestRsyncLocalSourceRemoteDest(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		DestHost:  "desthost",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" /tmp/a/ desthost:/tmp/b", false, false)
}

// Remote source, local destination.
func TestRsyncRemoteSourceLocalDest(t *testing.T) {
	cfg := &config.Config{
		Name:       "fake",
		SourceHost: "srchost",
		SourceDir:  "/tmp/a",
		DestDir:    "/tmp/b",
		Transport:  "rsync",
		Logfile:    "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" srchost:/tmp/a/ /tmp/b", false, false)
}

// Remote source, Remote destination (server side copy)not supported by rsync.
func TestRsyncRemoteSourceRemoteDest(t *testing.T) {
	cfg := &config.Config{
		Name:       "fake",
		SourceHost: "srchost",
		SourceDir:  "/tmp/a",
		DestHost:   "desthost",
		DestDir:    "/tmp/b",
		Transport:  "rsync",
		Logfile:    "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}

// Sources ending in a slash should not have another slash added.
func TestRsyncDoubleSlash(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" / /tmp/b", false, false)
}

// Exclude list only.
func TestRsyncExcludeListOnly(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Exclude:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --exclude-from=[^ ]+ --delete-excluded /tmp/a/ /tmp/b", false, false)
}

// Include list only.
func TestRsyncIncludeListOnly(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Include:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --include-from=[^ ]+ /tmp/a/ /tmp/b", false, false)
}

// Include & Exclude lists.
func TestRsyncIncludeAndExclude(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		DestDir:   "/tmp/b",
		Exclude:   []string{"x/foo", "x/bar"},
		Include:   []string{"x/foo", "x/bar"},
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, rsyncTestCmd+" --exclude-from=[^ ]+ --delete-excluded --include-from=[^ ]+ /tmp/a/ /tmp/b", false, false)
}

// Test that an empty source dir results in error.
func TestRsyncEmptySourceDir(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		DestDir:   "/tmp/b",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}

// Test that an empty destination dir results in error.
func TestRsyncEmptyDestDir(t *testing.T) {
	cfg := &config.Config{
		Name:      "fake",
		SourceDir: "/tmp/a",
		Transport: "rsync",
		Logfile:   "/dev/null",
	}
	rsyncTest(t, cfg, "", false, true)
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary main uses the provided service account key to duplicate all of
// the files in the indicated bucket. It uses several concurrent copiers and
// provides for a naive retry mechanism.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/storage/v1"
)

const (
	numCopiers = 10
	numFiles   = 1000
	usage      = `
Usage:
	go run generate_files.go BUCKET PATH/TO/IMAGE
Where BUCKET is the GCS bucket in which to generate files and PATH/TO/IMAGE is
the path to the image file we wish to duplicate.
`
)

//var (
//	imageFile = flag.String("image-file", "", "The path to the image file to duplicate in GCS.")
//	bucket    = flag.String("bucket", "", "The bucket in which to generate files.")
//)

type GCSCopyReq struct {
	SourceBucket, SourceFile, DestBucket, DestFile string
}

func buildName(prefix int, name string) string {
	return strings.Join([]string{strconv.Itoa(prefix), name}, "-")
}

// copyObjects takes copy requests from the input channel and attempts to use
// the GCS Storage API to perform the action. It incorporates naive retry logic
// and will output failures to the outut channel.
func copyObjects(s *storage.Service, in <-chan *GCSCopyReq, out chan<- string) {
	var err error
	for o := range in {
		for i := 0; i < 3; i++ {
			if _, err = s.Objects.Copy(o.SourceBucket, o.SourceFile, o.DestBucket, o.DestFile, nil).Do(); err == nil {
				break
			}
		}
		if err != nil {
			out <- o.DestFile
		}
	}
}

func main() {
	flag.Parse()
	if flag.NArg() != 2 {
		log.Fatalf("Please specify both required arguments." + usage)
	}
	bucket := flag.Arg(0)
	imagePath := flag.Arg(1)
	//	flag.Parse()
	//	if *imageFile == "" || *bucket == "" {
	//		log.Fatal("Please specify both of the required flags. See -help for instructions.")
	//	}
	file, err := os.Open(imagePath)
	if err != nil {
		log.Fatalf("Error opening image file: %v", err)
	}
	fileName := path.Base(imagePath)
	defer file.Close()
	service, err := storage.New(oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource("")))
	if err != nil {
		log.Fatalf("Failed to create GCS client: %v", err)
	}
	// Insert the image into GCS.
	baseFileName := buildName(0, fileName)
	_, err = service.Objects.Insert(bucket, &storage.Object{Name: baseFileName}).Media(file).Do()
	if err != nil {
		log.Fatalf("Unable to upload initial file to bucket: %v", err)
	}
	c := make(chan *GCSCopyReq, 999)
	f := make(chan string)
	wg := &sync.WaitGroup{}
	wg.Add(numCopiers)
	for i := 0; i < numCopiers; i++ {
		go func() {
			copyObjects(service, c, f)
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(f)
	}()
	for i := 1; i < numFiles; i++ {
		c <- &GCSCopyReq{
			SourceBucket: bucket,
			SourceFile:   baseFileName,
			DestBucket:   bucket,
			DestFile:     buildName(i, fileName),
		}
	}
	close(c)
	for errFile := range f {
		fmt.Printf("Could not copy to %v\n", errFile)
	}
}
<commit_msg>Remove some commented lines.<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary main uses the provided service account key to duplicate all of
// the files in the indicated bucket. It uses several concurrent copiers and
// provides for a naive retry mechanism.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/storage/v1"
)

const (
	numCopiers = 10
	numFiles   = 1000
	usage      = `
Usage:
	go run generate_files.go BUCKET PATH/TO/IMAGE
Where BUCKET is the GCS bucket in which to generate files and PATH/TO/IMAGE is
the path to the image file we wish to duplicate.
`
)

type GCSCopyReq struct {
	SourceBucket, SourceFile, DestBucket, DestFile string
}

func buildName(prefix int, name string) string {
	return strings.Join([]string{strconv.Itoa(prefix), name}, "-")
}

// copyObjects takes copy requests from the input channel and attempts to use
// the GCS Storage API to perform the action. It incorporates naive retry logic
// and will output failures to the outut channel.
func copyObjects(s *storage.Service, in <-chan *GCSCopyReq, out chan<- string) {
	var err error
	for o := range in {
		for i := 0; i < 3; i++ {
			if _, err = s.Objects.Copy(o.SourceBucket, o.SourceFile, o.DestBucket, o.DestFile, nil).Do(); err == nil {
				break
			}
		}
		if err != nil {
			out <- o.DestFile
		}
	}
}

func main() {
	flag.Parse()
	if flag.NArg() != 2 {
		log.Fatalf("Please specify both required arguments." + usage)
	}
	bucket := flag.Arg(0)
	imagePath := flag.Arg(1)
	file, err := os.Open(imagePath)
	if err != nil {
		log.Fatalf("Error opening image file: %v", err)
	}
	fileName := path.Base(imagePath)
	defer file.Close()
	service, err := storage.New(oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource("")))
	if err != nil {
		log.Fatalf("Failed to create GCS client: %v", err)
	}
	// Insert the image into GCS.
	baseFileName := buildName(0, fileName)
	_, err = service.Objects.Insert(bucket, &storage.Object{Name: baseFileName}).Media(file).Do()
	if err != nil {
		log.Fatalf("Unable to upload initial file to bucket: %v", err)
	}
	c := make(chan *GCSCopyReq, 999)
	f := make(chan string)
	wg := &sync.WaitGroup{}
	wg.Add(numCopiers)
	for i := 0; i < numCopiers; i++ {
		go func() {
			copyObjects(service, c, f)
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(f)
	}()
	for i := 1; i < numFiles; i++ {
		c <- &GCSCopyReq{
			SourceBucket: bucket,
			SourceFile:   baseFileName,
			DestBucket:   bucket,
			DestFile:     buildName(i, fileName),
		}
	}
	close(c)
	for errFile := range f {
		fmt.Printf("Could not copy to %v\n", errFile)
	}
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package results

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/mozilla/mig/modules"
)

type MockPersistResults struct {
	PersistFn func(float64, []modules.Result) PersistError
}

func TestUpload(t *testing.T) {
	testCases := []struct {
		Description    string
		ShouldError    bool
		ExpectedStatus int
		RequestBody    string
		PersistFn      func(float64, []modules.Result) PersistError
	}{
		{
			Description:    `Should get status 200 if persisting succeeds`,
			ShouldError:    false,
			ExpectedStatus: http.StatusOK,
			RequestBody: `{
        "action": 12351,
        "results": [
          {
            "foundAnything": true,
            "success": true,
            "elements": {
              "data": [
                {
                  "file": "/Users/test/.ssh/unauthorized.key"
                }
              ]
            },
            "statistics": {
              "directoriesScanned": 9001,
              "findings": 1
            },
            "errors": []
          }
        ]
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNil },
		},
		{
			Description:    `Should get an error if request is missing data`,
			ShouldError:    true,
			ExpectedStatus: http.StatusBadRequest,
			RequestBody: `{
        "results": []
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNil },
		},
		{
			Description:    `Should get an error if persisting fails`,
			ShouldError:    true,
			ExpectedStatus: http.StatusInternalServerError,
			RequestBody: `{
        "action": 172341,
        "results": []
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorMediumFailure },
		},
		{
			Description:    `Should get an error if persisting fails because the action is invalid`,
			ShouldError:    true,
			ExpectedStatus: http.StatusBadRequest,
			RequestBody: `{
        "action": 12341,
        "results": []
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorInvalidAction },
		},
		{
			Description:    `Should get an error if the agent is determined to not be allowed to save results`,
			ShouldError:    true,
			ExpectedStatus: http.StatusUnauthorized,
			RequestBody: `{
        "action": 184234,
        "results": []
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNotAuthorized },
		},
	}

	for caseNum, testCase := range testCases {
		t.Logf("Running TestUpload case #%d: %s", caseNum, testCase.Description)

		func() {
			server := httptest.NewServer(NewUpload(MockPersistResults{
				PersistFn: testCase.PersistFn,
			}))
			defer server.Close()

			response, err := http.Post(server.URL, "application/json", strings.NewReader(testCase.RequestBody))
			if err != nil {
				t.Fatalf("Error making request: %v", err)
			}

			respData := uploadResponse{}
			decoder := json.NewDecoder(response.Body)
			decodeErr := decoder.Decode(&respData)

			defer response.Body.Close()

			if decodeErr != nil {
				t.Fatalf("Error decoding response from server: %v", decodeErr)
			}

			if response.StatusCode != testCase.ExpectedStatus {
				t.Errorf("Expected status code %d but got %d", testCase.ExpectedStatus, response.StatusCode)
			}

			gotErr := respData.Error != nil
			if gotErr && !testCase.ShouldError {
				t.Errorf("Did not expect to get an error but got '%s'", *respData.Error)
			} else if !gotErr && testCase.ShouldError {
				t.Errorf("Expected to get an error but did not")
			}
		}()
	}
}

func (mock MockPersistResults) PersistResults(actionID float64, results []modules.Result) PersistError {
	return mock.PersistFn(actionID, results)
}
<commit_msg>Fixed up tests so they fail in the right place<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Zack Mullaly zmullaly@mozilla.com [:zack]

package results

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/mozilla/mig/modules"
)

type MockPersistResults struct {
	PersistFn func(float64, []modules.Result) PersistError
}

func TestUpload(t *testing.T) {
	testCases := []struct {
		Description    string
		ShouldError    bool
		ExpectedStatus int
		RequestBody    string
		PersistFn      func(float64, []modules.Result) PersistError
	}{
		{
			Description:    `Should get status 200 if persisting succeeds`,
			ShouldError:    false,
			ExpectedStatus: http.StatusOK,
			RequestBody: `{
        "action": 12351,
        "results": [
          {
            "foundAnything": true,
            "success": true,
            "elements": {
              "data": [
                {
                  "file": "/Users/test/.ssh/unauthorized.key"
                }
              ]
            },
            "statistics": {
              "directoriesScanned": 9001,
              "findings": 1
            },
            "errors": []
          }
        ]
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNil },
		},
		{
			Description:    `Should get an error if request is missing data`,
			ShouldError:    true,
			ExpectedStatus: http.StatusBadRequest,
			RequestBody: `{
        "results": []
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNil },
		},
		{
			Description:    `Should get an error if persisting fails`,
			ShouldError:    true,
			ExpectedStatus: http.StatusInternalServerError,
			RequestBody: `{
        "action": 172341,
        "results": [
          {
            "foundAnything": false,
            "success": true,
            "elements": {},
            "statistics": {},
            "errors": []
          }
        ]
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorMediumFailure },
		},
		{
			Description:    `Should get an error if persisting fails because the action is invalid`,
			ShouldError:    true,
			ExpectedStatus: http.StatusBadRequest,
			RequestBody: `{
        "action": 12341,
        "results": [
          {
            "foundAnything": false,
            "success": true,
            "elements": {},
            "statistics": {},
            "errors": []
          }
        ]
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorInvalidAction },
		},
		{
			Description:    `Should get an error if the agent is determined to not be allowed to save results`,
			ShouldError:    true,
			ExpectedStatus: http.StatusUnauthorized,
			RequestBody: `{
        "action": 184234,
        "results": [
          {
            "foundAnything": false,
            "success": true,
            "elements": {},
            "statistics": {},
            "errors": []
          }
        ]
      }`,
			PersistFn: func(_ float64, _ []modules.Result) PersistError { return PersistErrorNotAuthorized },
		},
	}

	for caseNum, testCase := range testCases {
		t.Logf("Running TestUpload case #%d: %s", caseNum, testCase.Description)

		func() {
			server := httptest.NewServer(NewUpload(MockPersistResults{
				PersistFn: testCase.PersistFn,
			}))
			defer server.Close()

			response, err := http.Post(server.URL, "application/json", strings.NewReader(testCase.RequestBody))
			if err != nil {
				t.Fatalf("Error making request: %v", err)
			}

			respData := uploadResponse{}
			decoder := json.NewDecoder(response.Body)
			decodeErr := decoder.Decode(&respData)

			defer response.Body.Close()

			if decodeErr != nil {
				t.Fatalf("Error decoding response from server: %v", decodeErr)
			}

			if response.StatusCode != testCase.ExpectedStatus {
				t.Errorf("Expected status code %d but got %d", testCase.ExpectedStatus, response.StatusCode)
			}

			gotErr := respData.Error != nil
			if gotErr && !testCase.ShouldError {
				t.Errorf("Did not expect to get an error but got '%s'", *respData.Error)
			} else if !gotErr && testCase.ShouldError {
				t.Errorf("Expected to get an error but did not")
			}
		}()
	}
}

func (mock MockPersistResults) PersistResults(actionID float64, results []modules.Result) PersistError {
	return mock.PersistFn(actionID, results)
}
<|endoftext|>


<commit_before>package glfw3

// Not sure about the darwin flag

//#cgo windows LDFLAGS: -lglfw3 -lopengl32 -lgdi32
//#cgo linux LDGLAGS: -lglfw
//#cgo darwin LDFLAGS: -lglfw
//#include <GLFW/glfw3.h>
import "C"

const (
	VersionMajor    = C.GLFW_VERSION_MAJOR    //This is incremented when the API is changed in non-compatible ways.
	VersionMinor    = C.GLFW_VERSION_MINOR    //This is incremented when features are added to the API but it remains backward-compatible.
	VersionRevision = C.GLFW_VERSION_REVISION //This is incremented when a bug fix release is made that does not contain any API changes.
)

//Init initializes the GLFW library. Before most GLFW functions can be used,
//GLFW must be initialized, and before a program terminates GLFW should be
//terminated in order to free any resources allocated during or after
//initialization.
//
//If this function fails, it calls Terminate before returning. If it succeeds,
//you should call Terminate before the program exits.
//
//Additional calls to this function after successful initialization but before
//termination will succeed but will do nothing.
//
//This function may take several seconds to complete on some systems, while on
//other systems it may take only a fraction of a second to complete.
//
//On Mac OS X, this function will change the current directory of the
//application to the Contents/Resources subdirectory of the application's
//bundle, if present.
func Init() bool {
	r := C.glfwInit()

	if r == C.GL_TRUE {
		return true
	}
	return false
}

//Terminate destroys all remaining windows, frees any allocated resources and
//sets the library to an uninitialized state. Once this is called, you must
//again call Init successfully before you will be able to use most GLFW
//functions.
//
//If GLFW has been successfully initialized, this function should be called
//before the program exits. If initialization fails, there is no need to call
//this function, as it is called by Init before it returns failure.
func Terminate() {
	C.glfwTerminate()
}

//GetVersion retrieves the major, minor and revision numbers of the GLFW
//library. It is intended for when you are using GLFW as a shared library and
//want to ensure that you are using the minimum required version.
//
//This function may be called before Init.
func GetVersion() (int, int, int) {
	var (
		major C.int
		minor C.int
		rev   C.int
	)

	C.glfwGetVersion(&major, &minor, &rev)
	return int(major), int(minor), int(rev)
}

//GetVersionString returns a static string generated at compile-time according
//to which configuration macros were defined. This is intended for use when
//submitting bug reports, to allow developers to see which code paths are
//enabled in a binary.
//
//This function may be called before Init.
func GetVersionString() string {
	return C.GoString(C.glfwGetVersionString())
}
<commit_msg>Programs that include this package on windows now run, make sure to rename the .a file as mentioned in the new comment<commit_after>package glfw3

// Not sure about the darwin flag

// Windows users: If you download the GLFW 64-bit binaries, when you copy over the contents of the lib folder make sure to rename
// glfw3dll.a to libglfw3dll.a, it doesn't work otherwise.

//#cgo windows LDFLAGS: -lglfw3dll -lopengl32 -lgdi32
//#cgo linux LDGLAGS: -lglfw
//#cgo darwin LDFLAGS: -lglfw
//#ifdef _WIN32
//  #define GLFW_DLL
//#endif
//#include <GLFW/glfw3.h>
import "C"

const (
	VersionMajor    = C.GLFW_VERSION_MAJOR    //This is incremented when the API is changed in non-compatible ways.
	VersionMinor    = C.GLFW_VERSION_MINOR    //This is incremented when features are added to the API but it remains backward-compatible.
	VersionRevision = C.GLFW_VERSION_REVISION //This is incremented when a bug fix release is made that does not contain any API changes.
)

//Init initializes the GLFW library. Before most GLFW functions can be used,
//GLFW must be initialized, and before a program terminates GLFW should be
//terminated in order to free any resources allocated during or after
//initialization.
//
//If this function fails, it calls Terminate before returning. If it succeeds,
//you should call Terminate before the program exits.
//
//Additional calls to this function after successful initialization but before
//termination will succeed but will do nothing.
//
//This function may take several seconds to complete on some systems, while on
//other systems it may take only a fraction of a second to complete.
//
//On Mac OS X, this function will change the current directory of the
//application to the Contents/Resources subdirectory of the application's
//bundle, if present.
func Init() bool {
	r := C.glfwInit()

	if r == C.GL_TRUE {
		return true
	}
	return false
}

//Terminate destroys all remaining windows, frees any allocated resources and
//sets the library to an uninitialized state. Once this is called, you must
//again call Init successfully before you will be able to use most GLFW
//functions.
//
//If GLFW has been successfully initialized, this function should be called
//before the program exits. If initialization fails, there is no need to call
//this function, as it is called by Init before it returns failure.
func Terminate() {
	C.glfwTerminate()
}

//GetVersion retrieves the major, minor and revision numbers of the GLFW
//library. It is intended for when you are using GLFW as a shared library and
//want to ensure that you are using the minimum required version.
//
//This function may be called before Init.
func GetVersion() (int, int, int) {
	var (
		major C.int
		minor C.int
		rev   C.int
	)

	C.glfwGetVersion(&major, &minor, &rev)
	return int(major), int(minor), int(rev)
}

//GetVersionString returns a static string generated at compile-time according
//to which configuration macros were defined. This is intended for use when
//submitting bug reports, to allow developers to see which code paths are
//enabled in a binary.
//
//This function may be called before Init.
func GetVersionString() string {
	return C.GoString(C.glfwGetVersionString())
}
<|endoftext|>


<commit_before>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unquoteCStyle(ss), ""
	}
	return unquoteCStyle(ss[:tabi]), ss[tabi+1:]
}

// C-style name unquoting.
// it is from https://github.com/git/git/blob/77556354bb7ac50450e3b28999e3576969869068/quote.c#L345-L413
func unquoteCStyle(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unescape
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, string(line))
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := (lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew)
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<commit_msg>s/unescape/unquote/<commit_after>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unquoteCStyle(ss), ""
	}
	return unquoteCStyle(ss[:tabi]), ss[tabi+1:]
}

// C-style name unquoting.
// it is from https://github.com/git/git/blob/77556354bb7ac50450e3b28999e3576969869068/quote.c#L345-L413
func unquoteCStyle(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unquote
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, string(line))
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := (lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew)
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<|endoftext|>