<commit_before>package vcs

import (
	"errors"

	"golang.org/x/tools/godoc/vfs"
)

// A Repository is a VCS repository.
type Repository interface {
	// ResolveRevision returns the revision that the given revision
	// specifier resolves to, or a non-nil error if there is no such
	// revision.
	//
	// Implementations may choose to return ErrRevisionNotFound in all
	// cases where the revision is not found, or more specific errors
	// (such as ErrCommitNotFound) if spec can be partially resolved
	// or determined to be a certain kind of revision specifier.
	ResolveRevision(spec string) (CommitID, error)

	// ResolveTag returns the tag with the given name, or
	// ErrTagNotFound if no such tag exists.
	ResolveTag(name string) (CommitID, error)

	// ResolveBranch returns the branch with the given name, or
	// ErrBranchNotFound if no such branch exists.
	ResolveBranch(name string) (CommitID, error)

	// Branches returns a list of all branches in the repository.
	Branches(BranchesOptions) ([]*Branch, error)

	// Tags returns a list of all tags in the repository.
	Tags() ([]*Tag, error)

	// GetCommit returns the commit with the given commit ID, or
	// ErrCommitNotFound if no such commit exists.
	GetCommit(CommitID) (*Commit, error)

	// Commits returns all commits matching the options, as well as
	// the total number of commits (the count of which is not subject
	// to the N/Skip options).
	//
	// Optionally, the caller can request the total not to be computed,
	// as this can be expensive.
	Commits(CommitsOptions) (commits []*Commit, total uint, err error)

	// FileSystem opens the repository file tree at a given commit ID.
	//
	// Implementations may choose to check that the commit exists
	// before FileSystem returns or to defer the check until
	// operations are performed on the filesystem. (For example, an
	// implementation proxying a remote filesystem may not want to
	// incur the round-trip to check that the commit exists.)
	FileSystem(at CommitID) (vfs.FileSystem, error)
}

// A Blamer is a repository that can blame portions of a file.
type Blamer interface {
	BlameFile(path string, opt *BlameOptions) ([]*Hunk, error)
}

// BlameOptions configures a blame.
type BlameOptions struct {
	NewestCommit CommitID `json:",omitempty" url:",omitempty"`
	OldestCommit CommitID `json:",omitempty" url:",omitempty"` // or "" for the root commit

	StartLine int `json:",omitempty" url:",omitempty"` // 1-indexed start byte (or 0 for beginning of file)
	EndLine   int `json:",omitempty" url:",omitempty"` // 1-indexed end byte (or 0 for end of file)
}

// A Hunk is a contiguous portion of a file associated with a commit.
type Hunk struct {
	StartLine int // 1-indexed start line number
	EndLine   int // 1-indexed end line number
	StartByte int // 0-indexed start byte position (inclusive)
	EndByte   int // 0-indexed end byte position (exclusive)
	CommitID
	Author Signature
}

// A Differ is a repository that can compute diffs between two
// commits.
type Differ interface {
	// Diff shows changes between two commits. If base or head do not
	// exist, an error is returned.
	Diff(base, head CommitID, opt *DiffOptions) (*Diff, error)
}

// A CrossRepoDiffer is a repository that can compute diffs with
// respect to a commit in a different repository.
type CrossRepoDiffer interface {
	// CrossRepoDiff shows changes between two commits in different
	// repositories. If base or head do not exist, an error is
	// returned.
	CrossRepoDiff(base CommitID, headRepo Repository, head CommitID, opt *DiffOptions) (*Diff, error)
}

var (
	ErrBranchNotFound   = errors.New("branch not found")
	ErrCommitNotFound   = errors.New("commit not found")
	ErrRevisionNotFound = errors.New("revision not found")
	ErrTagNotFound      = errors.New("tag not found")
)

type CommitID string

// Marshal implements proto.Marshaler.
func (c CommitID) Marshal() ([]byte, error) {
	return []byte(c), nil
}

// Unmarshal implements proto.Unmarshaler.
func (c *CommitID) Unmarshal(data []byte) error {
	*c = CommitID(data)
	return nil
}

// CommitsOptions specifies limits on the list of commits returned by
// (Repository).Commits.
type CommitsOptions struct {
	Head CommitID // include all commits reachable from this commit (required)
	Base CommitID // exlude all commits reachable from this commit (optional, like `git log Base..Head`)

	N    uint // limit the number of returned commits to this many (0 means no limit)
	Skip uint // skip this many commits at the beginning

	Path string // only commits modifying the given path are selected (optional)

	NoTotal bool // avoid counting the total number of commits
}

// DiffOptions configures a diff.
type DiffOptions struct {
	Paths                 []string // constrain diff to these pathspecs
	DetectRenames         bool
	OrigPrefix, NewPrefix string // prefixes for orig and new filenames (e.g., "a/", "b/")

	ExcludeReachableFromBoth bool // like "<rev1>...<rev2>" (see `git rev-parse --help`)
}

// A Diff represents changes between two commits.
type Diff struct {
	Raw string // the raw diff output
}

type Branches []*Branch

func (p Branches) Len() int           { return len(p) }
func (p Branches) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p Branches) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// ByAuthorDate sorts by author date. Requires full commit information to be included.
type ByAuthorDate []*Branch

func (p ByAuthorDate) Len() int { return len(p) }
func (p ByAuthorDate) Less(i, j int) bool {
	return p[i].Commit.Author.Date.Time().Before(p[j].Commit.Author.Date.Time())
}
func (p ByAuthorDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

type Tags []*Tag

func (p Tags) Len() int           { return len(p) }
func (p Tags) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p Tags) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<commit_msg>Elaborate in docs on why computing total might be expensive.<commit_after>package vcs

import (
	"errors"

	"golang.org/x/tools/godoc/vfs"
)

// A Repository is a VCS repository.
type Repository interface {
	// ResolveRevision returns the revision that the given revision
	// specifier resolves to, or a non-nil error if there is no such
	// revision.
	//
	// Implementations may choose to return ErrRevisionNotFound in all
	// cases where the revision is not found, or more specific errors
	// (such as ErrCommitNotFound) if spec can be partially resolved
	// or determined to be a certain kind of revision specifier.
	ResolveRevision(spec string) (CommitID, error)

	// ResolveTag returns the tag with the given name, or
	// ErrTagNotFound if no such tag exists.
	ResolveTag(name string) (CommitID, error)

	// ResolveBranch returns the branch with the given name, or
	// ErrBranchNotFound if no such branch exists.
	ResolveBranch(name string) (CommitID, error)

	// Branches returns a list of all branches in the repository.
	Branches(BranchesOptions) ([]*Branch, error)

	// Tags returns a list of all tags in the repository.
	Tags() ([]*Tag, error)

	// GetCommit returns the commit with the given commit ID, or
	// ErrCommitNotFound if no such commit exists.
	GetCommit(CommitID) (*Commit, error)

	// Commits returns all commits matching the options, as well as
	// the total number of commits (the count of which is not subject
	// to the N/Skip options).
	//
	// Optionally, the caller can request the total not to be computed,
	// as this can be expensive for large branches.
	Commits(CommitsOptions) (commits []*Commit, total uint, err error)

	// FileSystem opens the repository file tree at a given commit ID.
	//
	// Implementations may choose to check that the commit exists
	// before FileSystem returns or to defer the check until
	// operations are performed on the filesystem. (For example, an
	// implementation proxying a remote filesystem may not want to
	// incur the round-trip to check that the commit exists.)
	FileSystem(at CommitID) (vfs.FileSystem, error)
}

// A Blamer is a repository that can blame portions of a file.
type Blamer interface {
	BlameFile(path string, opt *BlameOptions) ([]*Hunk, error)
}

// BlameOptions configures a blame.
type BlameOptions struct {
	NewestCommit CommitID `json:",omitempty" url:",omitempty"`
	OldestCommit CommitID `json:",omitempty" url:",omitempty"` // or "" for the root commit

	StartLine int `json:",omitempty" url:",omitempty"` // 1-indexed start byte (or 0 for beginning of file)
	EndLine   int `json:",omitempty" url:",omitempty"` // 1-indexed end byte (or 0 for end of file)
}

// A Hunk is a contiguous portion of a file associated with a commit.
type Hunk struct {
	StartLine int // 1-indexed start line number
	EndLine   int // 1-indexed end line number
	StartByte int // 0-indexed start byte position (inclusive)
	EndByte   int // 0-indexed end byte position (exclusive)
	CommitID
	Author Signature
}

// A Differ is a repository that can compute diffs between two
// commits.
type Differ interface {
	// Diff shows changes between two commits. If base or head do not
	// exist, an error is returned.
	Diff(base, head CommitID, opt *DiffOptions) (*Diff, error)
}

// A CrossRepoDiffer is a repository that can compute diffs with
// respect to a commit in a different repository.
type CrossRepoDiffer interface {
	// CrossRepoDiff shows changes between two commits in different
	// repositories. If base or head do not exist, an error is
	// returned.
	CrossRepoDiff(base CommitID, headRepo Repository, head CommitID, opt *DiffOptions) (*Diff, error)
}

var (
	ErrBranchNotFound   = errors.New("branch not found")
	ErrCommitNotFound   = errors.New("commit not found")
	ErrRevisionNotFound = errors.New("revision not found")
	ErrTagNotFound      = errors.New("tag not found")
)

type CommitID string

// Marshal implements proto.Marshaler.
func (c CommitID) Marshal() ([]byte, error) {
	return []byte(c), nil
}

// Unmarshal implements proto.Unmarshaler.
func (c *CommitID) Unmarshal(data []byte) error {
	*c = CommitID(data)
	return nil
}

// CommitsOptions specifies limits on the list of commits returned by
// (Repository).Commits.
type CommitsOptions struct {
	Head CommitID // include all commits reachable from this commit (required)
	Base CommitID // exlude all commits reachable from this commit (optional, like `git log Base..Head`)

	N    uint // limit the number of returned commits to this many (0 means no limit)
	Skip uint // skip this many commits at the beginning

	Path string // only commits modifying the given path are selected (optional)

	NoTotal bool // avoid counting the total number of commits
}

// DiffOptions configures a diff.
type DiffOptions struct {
	Paths                 []string // constrain diff to these pathspecs
	DetectRenames         bool
	OrigPrefix, NewPrefix string // prefixes for orig and new filenames (e.g., "a/", "b/")

	ExcludeReachableFromBoth bool // like "<rev1>...<rev2>" (see `git rev-parse --help`)
}

// A Diff represents changes between two commits.
type Diff struct {
	Raw string // the raw diff output
}

type Branches []*Branch

func (p Branches) Len() int           { return len(p) }
func (p Branches) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p Branches) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// ByAuthorDate sorts by author date. Requires full commit information to be included.
type ByAuthorDate []*Branch

func (p ByAuthorDate) Len() int { return len(p) }
func (p ByAuthorDate) Less(i, j int) bool {
	return p[i].Commit.Author.Date.Time().Before(p[j].Commit.Author.Date.Time())
}
func (p ByAuthorDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] }

type Tags []*Tag

func (p Tags) Len() int           { return len(p) }
func (p Tags) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p Tags) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/hyperhq/runv/api"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/lib/linuxsignal"
	"github.com/opencontainers/runc/libcontainer/system"
	"github.com/opencontainers/runtime-spec/specs-go"
)

func startContainer(vm *hypervisor.Vm, root, container string, spec *specs.Spec, state *State) error {
	err := vm.StartContainer(container)
	if err != nil {
		glog.V(1).Infof("Start Container fail: fail to start container with err: %#v\n", err)
		return err
	}

	err = syscall.Kill(state.Pid, syscall.SIGUSR1)
	if err != nil {
		glog.V(1).Infof("failed to notify the shim to work", err.Error())
		return err
	}

	glog.V(3).Infof("change the status of container %s to `running`", container)
	state.Status = "running"
	state.ContainerCreateTime = time.Now().UTC().Unix()
	if err = saveStateFile(root, container, state); err != nil {
		return err
	}

	err = execPoststartHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Poststart hooks failed %s\n", err.Error())
	}

	return err
}

func createContainer(options runvOptions, vm *hypervisor.Vm, container, bundle, stateRoot string, spec *specs.Spec) (shim *os.Process, err error) {
	if err = setupContainerFs(vm, bundle, container, spec); err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			removeContainerFs(vm, container)
		}
	}()

	glog.V(3).Infof("vm.AddContainer()")
	config := api.ContainerDescriptionFromOCF(container, spec)
	r := vm.AddContainer(config)
	if !r.IsSuccess() {
		return nil, fmt.Errorf("add container %s failed: %s", container, r.Message())
	}
	defer func() {
		if err != nil {
			vm.RemoveContainer(container)
		}
	}()

	// Prepare container state directory
	stateDir := filepath.Join(stateRoot, container)
	_, err = os.Stat(stateDir)
	if err == nil {
		glog.Errorf("Container %s exists\n", container)
		return nil, fmt.Errorf("Container %s exists", container)
	}
	err = os.MkdirAll(stateDir, 0644)
	if err != nil {
		glog.V(1).Infof("%s\n", err.Error())
		return nil, err
	}
	defer func() {
		if err != nil {
			os.RemoveAll(stateDir)
		}
	}()

	// Create sandbox dir symbol link in container root dir
	vmRootLinkPath := filepath.Join(stateDir, "sandbox")
	vmRootPath := sandboxPath(vm)
	if err = os.Symlink(vmRootPath, vmRootLinkPath); err != nil {
		return nil, fmt.Errorf("failed to create symbol link %q: %v", vmRootLinkPath, err)
	}

	// create shim and save the state
	shim, err = createShim(options, container, "init")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			shim.Kill()
		}
	}()

	var stat system.Stat_t
	stat, err = system.Stat(shim.Pid)
	if err != nil {
		return nil, err
	}

	state := &State{
		State: specs.State{
			Version: spec.Version,
			ID:      container,
			Status:  "created",
			Pid:     shim.Pid,
			Bundle:  bundle,
		},
		ShimCreateTime: stat.StartTime,
	}
	glog.V(3).Infof("save state id %s, boundle %s", container, bundle)
	if err = saveStateFile(stateRoot, container, state); err != nil {
		return nil, err
	}

	err = execPrestartHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Prestart hooks failed, %s\n", err.Error())
		return nil, err
	}

	// If runv is launched via docker/containerd, we start netlistener to watch/collect network changes.
	// TODO: if runv is launched by cni compatible tools, the cni script can use `runv cni` cmdline to update the network.
	// Create the listener process which will enters into the netns of the shim
	options.withContainer = state
	if err = startNsListener(options, vm); err != nil {
		glog.Errorf("start ns listener fail: %v", err)
		return nil, err
	}

	return shim, nil
}

func deleteContainer(vm *hypervisor.Vm, root, container string, force bool, spec *specs.Spec, state *State) error {

	// todo: check the container from vm.ContainerList()

	// non-force killing can only be performed when at least one of the realProcess and shimProcess exited
	exitedVM := vm.SignalProcess(container, "init", syscall.Signal(0)) != nil // todo: is this check reliable?
	exitedHost := !containerShimAlive(state)
	if !exitedVM && !exitedHost && !force {
		// don't perform deleting
		return fmt.Errorf("the container %s is still alive, use -f to force kill it?", container)
	}

	if !exitedVM { // force kill the real init process inside the vm
		for i := 0; i < 100; i++ {
			vm.SignalProcess(container, "init", linuxsignal.SIGKILL)
			time.Sleep(100 * time.Millisecond)
			if vm.SignalProcess(container, "init", syscall.Signal(0)) != nil {
				break
			}
		}
	}

	if !exitedHost { // force kill the shim process in the host
		time.Sleep(200 * time.Millisecond) // the shim might be going to exit, wait it
		for i := 0; i < 100; i++ {
			syscall.Kill(state.Pid, syscall.SIGKILL)
			time.Sleep(100 * time.Millisecond)
			if !containerShimAlive(state) {
				break
			}
		}
	}

	vm.RemoveContainer(container)
	err := execPoststopHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Poststop hooks failed %s\n", err.Error())
		removeContainerFs(vm, container)
		os.RemoveAll(filepath.Join(root, container))
		return err // return err of the hooks
	}

	removeContainerFs(vm, container)
	return os.RemoveAll(filepath.Join(root, container))
}

func addProcess(options runvOptions, vm *hypervisor.Vm, container, process string, spec *specs.Process) (shim *os.Process, err error) {
	p := &api.Process{
		Container: container,
		Id:        process,
		Terminal:  spec.Terminal,
		Args:      spec.Args,
		Envs:      spec.Env,
		Workdir:   spec.Cwd,
	}
	if spec.User.UID != 0 {
		p.User = strconv.FormatUint(uint64(spec.User.UID), 10)
	}
	if spec.User.GID != 0 {
		p.Group = strconv.FormatUint(uint64(spec.User.GID), 10)
	}
	if len(spec.User.AdditionalGids) > 0 {
		ag := []string{}
		for _, g := range spec.User.AdditionalGids {
			ag = append(ag, strconv.FormatUint(uint64(g), 10))
		}
	}
	err = vm.AddProcess(p, nil)

	if err != nil {
		glog.V(1).Infof("add process to container failed: %v\n", err)
		return nil, err
	}
	defer func() {
		if err != nil {
			vm.SignalProcess(container, process, linuxsignal.SIGKILL)
		}
	}()

	shim, err = createShim(options, container, process)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			shim.Kill()
		}
	}()

	// cli refactor todo (for the purpose of 'runv ps` command) save <container, process, shim-pid, spec> to persist file.

	return shim, nil
}

func execHook(hook specs.Hook, state *State) error {
	b, err := json.Marshal(state)
	if err != nil {
		return err
	}
	cmd := exec.Cmd{
		Path:  hook.Path,
		Args:  hook.Args,
		Env:   hook.Env,
		Stdin: bytes.NewReader(b),
	}
	return cmd.Run()
}

func execPrestartHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Prestart {
		err := execHook(hook, state)
		if err != nil {
			return err
		}
	}

	return nil
}

func execPoststartHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Poststart {
		err := execHook(hook, state)
		if err != nil {
			glog.V(1).Infof("exec Poststart hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func execPoststopHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Poststop {
		err := execHook(hook, state)
		if err != nil {
			glog.V(1).Infof("exec Poststop hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func containerShimAlive(state *State) bool {
	stat, err := system.Stat(state.Pid)
	return err == nil && stat.StartTime == state.ShimCreateTime && stat.State != system.Zombie && stat.State != system.Dead
}
<commit_msg>check container from vm.ContainerList in delete<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"
	"time"

	"github.com/golang/glog"
	"github.com/hyperhq/runv/api"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/lib/linuxsignal"
	"github.com/opencontainers/runc/libcontainer/system"
	"github.com/opencontainers/runtime-spec/specs-go"
)

func startContainer(vm *hypervisor.Vm, root, container string, spec *specs.Spec, state *State) error {
	err := vm.StartContainer(container)
	if err != nil {
		glog.V(1).Infof("Start Container fail: fail to start container with err: %#v\n", err)
		return err
	}

	err = syscall.Kill(state.Pid, syscall.SIGUSR1)
	if err != nil {
		glog.V(1).Infof("failed to notify the shim to work", err.Error())
		return err
	}

	glog.V(3).Infof("change the status of container %s to `running`", container)
	state.Status = "running"
	state.ContainerCreateTime = time.Now().UTC().Unix()
	if err = saveStateFile(root, container, state); err != nil {
		return err
	}

	err = execPoststartHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Poststart hooks failed %s\n", err.Error())
	}

	return err
}

func createContainer(options runvOptions, vm *hypervisor.Vm, container, bundle, stateRoot string, spec *specs.Spec) (shim *os.Process, err error) {
	if err = setupContainerFs(vm, bundle, container, spec); err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			removeContainerFs(vm, container)
		}
	}()

	glog.V(3).Infof("vm.AddContainer()")
	config := api.ContainerDescriptionFromOCF(container, spec)
	r := vm.AddContainer(config)
	if !r.IsSuccess() {
		return nil, fmt.Errorf("add container %s failed: %s", container, r.Message())
	}
	defer func() {
		if err != nil {
			vm.RemoveContainer(container)
		}
	}()

	// Prepare container state directory
	stateDir := filepath.Join(stateRoot, container)
	_, err = os.Stat(stateDir)
	if err == nil {
		glog.Errorf("Container %s exists\n", container)
		return nil, fmt.Errorf("Container %s exists", container)
	}
	err = os.MkdirAll(stateDir, 0644)
	if err != nil {
		glog.V(1).Infof("%s\n", err.Error())
		return nil, err
	}
	defer func() {
		if err != nil {
			os.RemoveAll(stateDir)
		}
	}()

	// Create sandbox dir symbol link in container root dir
	vmRootLinkPath := filepath.Join(stateDir, "sandbox")
	vmRootPath := sandboxPath(vm)
	if err = os.Symlink(vmRootPath, vmRootLinkPath); err != nil {
		return nil, fmt.Errorf("failed to create symbol link %q: %v", vmRootLinkPath, err)
	}

	// create shim and save the state
	shim, err = createShim(options, container, "init")
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			shim.Kill()
		}
	}()

	var stat system.Stat_t
	stat, err = system.Stat(shim.Pid)
	if err != nil {
		return nil, err
	}

	state := &State{
		State: specs.State{
			Version: spec.Version,
			ID:      container,
			Status:  "created",
			Pid:     shim.Pid,
			Bundle:  bundle,
		},
		ShimCreateTime: stat.StartTime,
	}
	glog.V(3).Infof("save state id %s, boundle %s", container, bundle)
	if err = saveStateFile(stateRoot, container, state); err != nil {
		return nil, err
	}

	err = execPrestartHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Prestart hooks failed, %s\n", err.Error())
		return nil, err
	}

	// If runv is launched via docker/containerd, we start netlistener to watch/collect network changes.
	// TODO: if runv is launched by cni compatible tools, the cni script can use `runv cni` cmdline to update the network.
	// Create the listener process which will enters into the netns of the shim
	options.withContainer = state
	if err = startNsListener(options, vm); err != nil {
		glog.Errorf("start ns listener fail: %v", err)
		return nil, err
	}

	return shim, nil
}

func deleteContainer(vm *hypervisor.Vm, root, container string, force bool, spec *specs.Spec, state *State) error {
	// non-force killing can only be performed when at least one of the realProcess and shimProcess exited
	exitedVM := true
	for _, c := range vm.ContainerList() {
		if c == container {
			exitedVM = vm.SignalProcess(container, "init", syscall.Signal(0)) != nil // todo: is this check reliable?
			break
		}
	}
	exitedHost := !containerShimAlive(state)
	if !exitedVM && !exitedHost && !force {
		// don't perform deleting
		return fmt.Errorf("the container %s is still alive, use -f to force kill it?", container)
	}

	if !exitedVM { // force kill the real init process inside the vm
		for i := 0; i < 100; i++ {
			vm.SignalProcess(container, "init", linuxsignal.SIGKILL)
			time.Sleep(100 * time.Millisecond)
			if vm.SignalProcess(container, "init", syscall.Signal(0)) != nil {
				break
			}
		}
	}

	if !exitedHost { // force kill the shim process in the host
		time.Sleep(200 * time.Millisecond) // the shim might be going to exit, wait it
		for i := 0; i < 100; i++ {
			syscall.Kill(state.Pid, syscall.SIGKILL)
			time.Sleep(100 * time.Millisecond)
			if !containerShimAlive(state) {
				break
			}
		}
	}

	vm.RemoveContainer(container)
	err := execPoststopHooks(spec, state)
	if err != nil {
		glog.V(1).Infof("execute Poststop hooks failed %s\n", err.Error())
		removeContainerFs(vm, container)
		os.RemoveAll(filepath.Join(root, container))
		return err // return err of the hooks
	}

	removeContainerFs(vm, container)
	return os.RemoveAll(filepath.Join(root, container))
}

func addProcess(options runvOptions, vm *hypervisor.Vm, container, process string, spec *specs.Process) (shim *os.Process, err error) {
	p := &api.Process{
		Container: container,
		Id:        process,
		Terminal:  spec.Terminal,
		Args:      spec.Args,
		Envs:      spec.Env,
		Workdir:   spec.Cwd,
	}
	if spec.User.UID != 0 {
		p.User = strconv.FormatUint(uint64(spec.User.UID), 10)
	}
	if spec.User.GID != 0 {
		p.Group = strconv.FormatUint(uint64(spec.User.GID), 10)
	}
	if len(spec.User.AdditionalGids) > 0 {
		ag := []string{}
		for _, g := range spec.User.AdditionalGids {
			ag = append(ag, strconv.FormatUint(uint64(g), 10))
		}
	}
	err = vm.AddProcess(p, nil)

	if err != nil {
		glog.V(1).Infof("add process to container failed: %v\n", err)
		return nil, err
	}
	defer func() {
		if err != nil {
			vm.SignalProcess(container, process, linuxsignal.SIGKILL)
		}
	}()

	shim, err = createShim(options, container, process)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err != nil {
			shim.Kill()
		}
	}()

	// cli refactor todo (for the purpose of 'runv ps` command) save <container, process, shim-pid, spec> to persist file.

	return shim, nil
}

func execHook(hook specs.Hook, state *State) error {
	b, err := json.Marshal(state)
	if err != nil {
		return err
	}
	cmd := exec.Cmd{
		Path:  hook.Path,
		Args:  hook.Args,
		Env:   hook.Env,
		Stdin: bytes.NewReader(b),
	}
	return cmd.Run()
}

func execPrestartHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Prestart {
		err := execHook(hook, state)
		if err != nil {
			return err
		}
	}

	return nil
}

func execPoststartHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Poststart {
		err := execHook(hook, state)
		if err != nil {
			glog.V(1).Infof("exec Poststart hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func execPoststopHooks(rt *specs.Spec, state *State) error {
	if rt.Hooks == nil {
		return nil
	}
	for _, hook := range rt.Hooks.Poststop {
		err := execHook(hook, state)
		if err != nil {
			glog.V(1).Infof("exec Poststop hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func containerShimAlive(state *State) bool {
	stat, err := system.Stat(state.Pid)
	return err == nil && stat.StartTime == state.ShimCreateTime && stat.State != system.Zombie && stat.State != system.Dead
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc.  All Rights Reserved.
// This file is available under the Apache license.

package exporter

import (
	"expvar"
	"fmt"
	"net/http"
	"sort"
	"strings"

	"github.com/google/mtail/metrics"
)

var (
	exportVarzTotal = expvar.NewInt("exporter_varz_total")
)

const varzFormat = "%s{%s} %s"

// HandleVarz exports the metrics in Varz format via HTTP.
func (e *Exporter) HandleVarz(w http.ResponseWriter, r *http.Request) {
	e.store.RLock()
	defer e.store.RUnlock()

	w.Header().Add("Content-type", "text/plain")

	for _, m := range e.store.Metrics {
		m.RLock()
		m.RUnlock()
		exportVarzTotal.Add(1)
		lc := make(chan *metrics.LabelSet)
		go m.EmitLabelSets(lc)
		for l := range lc {
			line := metricToVarz(e.hostname, m, l)
			fmt.Fprint(w, line)
		}
	}
}

func metricToVarz(hostname string, m *metrics.Metric, l *metrics.LabelSet) string {
	var s []string
	for k, v := range l.Labels {
		s = append(s, fmt.Sprintf("%s=%s", k, v))
	}
	sort.Strings(s)
	s = append(s, fmt.Sprintf("prog=\"%s\"", m.Program))
	s = append(s, fmt.Sprintf("instance=\"%s\"", hostname))
	return fmt.Sprintf(varzFormat,
		m.Name,
		strings.Join(s, ","),
		l.Datum.Get())
}
<commit_msg>fix formatting<commit_after>// Copyright 2015 Google Inc.  All Rights Reserved.
// This file is available under the Apache license.

package exporter

import (
	"expvar"
	"fmt"
	"net/http"
	"sort"
	"strings"

	"github.com/google/mtail/metrics"
)

var (
	exportVarzTotal = expvar.NewInt("exporter_varz_total")
)

const varzFormat = "%s{%s} %d\n"

// HandleVarz exports the metrics in Varz format via HTTP.
func (e *Exporter) HandleVarz(w http.ResponseWriter, r *http.Request) {
	e.store.RLock()
	defer e.store.RUnlock()

	w.Header().Add("Content-type", "text/plain")

	for _, m := range e.store.Metrics {
		m.RLock()
		m.RUnlock()
		exportVarzTotal.Add(1)
		lc := make(chan *metrics.LabelSet)
		go m.EmitLabelSets(lc)
		for l := range lc {
			line := metricToVarz(e.hostname, m, l)
			fmt.Fprint(w, line)
		}
	}
}

func metricToVarz(hostname string, m *metrics.Metric, l *metrics.LabelSet) string {
	var s []string
	for k, v := range l.Labels {
		s = append(s, fmt.Sprintf("%s=%s", k, v))
	}
	sort.Strings(s)
	s = append(s, fmt.Sprintf("prog=%s", m.Program))
	s = append(s, fmt.Sprintf("instance=%s", hostname))
	return fmt.Sprintf(varzFormat,
		m.Name,
		strings.Join(s, ","),
		l.Datum.Get())
}
<|endoftext|>


<commit_before>// Package include implements a zcl extension that allows inclusion of
// one HCL body into another using blocks of type "include", with the following
// structure:
//
//     include {
//       path = "./foo.hcl"
//     }
//
// The processing of the given path is delegated to the calling application,
// allowing it to decide how to interpret the path and which syntaxes to
// support for referenced files.
package include
<commit_msg>ext/include: update stale reference to "zcl"<commit_after>// Package include implements a HCL extension that allows inclusion of
// one HCL body into another using blocks of type "include", with the following
// structure:
//
//     include {
//       path = "./foo.hcl"
//     }
//
// The processing of the given path is delegated to the calling application,
// allowing it to decide how to interpret the path and which syntaxes to
// support for referenced files.
package include
<|endoftext|>


<commit_before>// Copyright 2015 Colin Stewart.  All rights reserved.
// Use of this source code is governed by an MIT
// license that can be found in the LICENSE.txt file.

/*
Package tal implements the TAL template language for generating HTML5 output.

TAL templates are HTML5 documents containing additional attributes that, using the TALES expression language, change the structure and content of the resulting document.

Here is a basic example (without error handling) that prints "
Raising Steam
 by Author: Terry Pratchett"

	type Book struct {
		Title  string
		Author string
	}

	book := Book{"Raising Steam", "Terry Pratchett"}
	tmpl, _ := tal.CompileTemplate(strings.NewReader(`<html><h1 tal:content="title">Title goes here</h1> by <b tal:replace="string: Author: $Author"></b>`))
	tmpl.Render(book, os.Stdout)

The tal package also supports METAL macros as described later on.

TAL Commands

The tal package supports all TAL commands defined by Zope (see http://docs.zope.org/zope2/zope2book/AppendixC.html).

Define

tal:define sets a local or global variable:

	tal:define="[local | global] name expression [; define-expression...]

Description: Sets the value of "name" to "expression".  By default the name will be applicable in the "local" scope, which consists of this tag, and all other tags nested inside this tag.  If the "global" keyword is used then this name will keep its value for the rest of the document.

Example:

Condition

tal:condition makes output of an element conditional:

	tal:condition="expression"

Description:  If the expression evaluates to true then this tag and all its children will be output.  If the expression evaluates to false then this tag and all its children will not be included in the output.

Example:

Welcome to this page!

Repeat

tal:repeat replicates an element a number of times:

	tal:repeat="name expression"

Description:  Evaluates "expression", and if it is a sequence, repeats this tag and all children once for each item in the sequence.  The "name" will be set to the value of the item in the current iteration, and is also the name of the repeat variable.  The repeat variable is accessible using the TAL path: repeat/name and has the following properties:

    index 		- Iteration number starting from zero
    number 		- Iteration number starting from one
    even 		- True if this is an even iteration
    odd 		- True if this is an odd iteration
    start 		- True if this is the first item in the sequence
    end 		- True if this is the last item in the sequence
    length 		- The length of the sequence
    letter 		- The lower case letter for this iteration, starting at "a"
    Letter		- Upper case version of letter
    roman 		- Iteration number in Roman numerals, starting at i
    Roman	 	- Upper case version of roman

Note that letterUpper and romanUpper are used instead of the standard TAL Letter and Roman.  The "first" and "last" properties are not supported.

Example:

Content

tal:content replaces the content of an element:

	tal:content="[text | structure] expression"

Description:  Replaces the contents of the tag with the value of "expression".  By default, and if the "text" keyword is present, then the value of the expression will be escaped as required (i.e. characters "&<> will be escaped).  If the "structure" keyword is present then the value will be output with no escaping performed.

Example:

Replace

tal:replace replaces the whole element:

	tal:replace="[text | structure] expression"

Description: Behaves identically to tal:content, except that the tag is removed from the output (as if tal:omit-tag had been used).

Example:

Welcome

Attributes

tal:attributes sets or removes attributes on the element:

	tal:attributes="name expression[;attributes-expression]"

Description:  Evaluates each "expression" and replaces the tag's attribute "name".  If the expression evaluates to nothing then the attribute is removed from the tag.  If the expression evaluates to default then the original tag's attribute is kept.  If the "expression" requires a semi-colon then it must be escaped by using ";;".

Example:

	Your Homepage

Omit Tag

tal:omit-tag removes the start and end tags:

	tal:omit-tag="expression"

Description: Removes the tag (leaving the tags content) if the expression evaluates to true.  If expression is empty then it is taken as true.

Example:

Welcome to this page!

TALES Expressions

The expressions used in TAL are called TALES expressions.  The simplest TALES expression is a path which references a value, e.g. page/body references the body property of the page object.  Objects are passed as the first argument of the Render method on a compiled template and must be either a struct, pointer to a struct or a map with strings as keys.

The tal package does not support the python: and nocall: expression types.

Path

path: provides access to properties on objects.

	Syntax: [path:]string[|TALES Expression]

Description: A path, optionally starting with the modifier 'path:', references a property of an object.  The '/' delimiter is used to end the name of an object and start of the property name.  Properties themselves may be objects that in turn have properties.  The '|' ("or") character is used to find an alternative value to a path if the first path evaluates does not exist.

Example:

There are several built in variables that can be used in paths:

    nothing	- acts as nil in Go
    default	- keeps the existing value of the node (tag content or attribute value)
    repeat	- access to repeat variables (see tal:repeat)
    attrs	- a dictionary of original attributes of the current tag

Path Variables

Path variables allows for indirection in paths.

	Syntax: [path:]object/?attribute

Description:  The "attribute" is evaluated as a local or global variable, and it's value is used as the attribute to lookup on the object.  Useful for accessing the contents of a map within a loop:

Example:

Exists

exists: Tests whether a path exists.

	Syntax: exists:path

Description: Returns true if the path exists, false otherwise.  Particularly useful for eliminating entire subtrees if a particular object is not available.

Example:

...

Not

not: Returns the inverse boolean value of a path.

	Syntax: not:tales-path

Description: Returns the inverse of the tales-path.  If the path returns true, not:path will return false.

Example:

Welcome to the site!

String

string: Basic string substitution

	Syntax: string:text

Description:  Evaluates to a string with value text while substituting variables with the form ${pathName} and $pathName

Example:

METAL Macro Language

METAL is a macro language commonly used with TAL & TALES.  METAL allows part of a template to be used as a macro in later parts of the template, or shared across templates.

Macros defined in a template can be retrieved from a compiled template and added to the context by calling the Macros() method.

Define Macro

metal:define-macro marks an element and it's subtree as being a reusable macro.

	Syntax: metal:define-macro="name"

Description: Defines a new macro that can be reference later as "name".

Example:

Copyright 2004

Use Macro

metal:use-macro expands a macro into the template.

	Syntax: metal:use-macro="expression"

Description: Evaluates "expression" and uses this as a macro.

Example:

Define Slot

metal:define-slot creates a customisation point within a macro.

	Syntax: metal:define-slot="name"

Description: Defines a customisation point in a macro with the given name.

Example:

		Standard disclaimer for the site.
		Contact admin@site.com

Fill Slot

metal:fill-slot overwrites the contents of a slot when using a macro.

Syntax: metal:fill-slot="name"

Description: Replaces the content of a slot with this element.

Example:

		Contact someone else

*/
package tal
<commit_msg>Updated documentation to reflect behaviour of |<commit_after>// Copyright 2015 Colin Stewart.  All rights reserved.
// Use of this source code is governed by an MIT
// license that can be found in the LICENSE.txt file.

/*
Package tal implements the TAL template language for generating HTML5 output.

TAL templates are HTML5 documents containing additional attributes that, using the TALES expression language, change the structure and content of the resulting document.

Here is a basic example (without error handling) that prints "
Raising Steam
 by Author: Terry Pratchett"

	type Book struct {
		Title  string
		Author string
	}

	book := Book{"Raising Steam", "Terry Pratchett"}
	tmpl, _ := tal.CompileTemplate(strings.NewReader(`<html><h1 tal:content="title">Title goes here</h1> by <b tal:replace="string: Author: $Author"></b>`))
	tmpl.Render(book, os.Stdout)

The tal package also supports METAL macros as described later on.

TAL Commands

The tal package supports all TAL commands defined by Zope (see http://docs.zope.org/zope2/zope2book/AppendixC.html).

Define

tal:define sets a local or global variable:

	tal:define="[local | global] name expression [; define-expression...]

Description: Sets the value of "name" to "expression".  By default the name will be applicable in the "local" scope, which consists of this tag, and all other tags nested inside this tag.  If the "global" keyword is used then this name will keep its value for the rest of the document.

Example:

Condition

tal:condition makes output of an element conditional:

	tal:condition="expression"

Description:  If the expression evaluates to true then this tag and all its children will be output.  If the expression evaluates to false then this tag and all its children will not be included in the output.

Example:

Welcome to this page!

Repeat

tal:repeat replicates an element a number of times:

	tal:repeat="name expression"

Description:  Evaluates "expression", and if it is a sequence, repeats this tag and all children once for each item in the sequence.  The "name" will be set to the value of the item in the current iteration, and is also the name of the repeat variable.  The repeat variable is accessible using the TAL path: repeat/name and has the following properties:

    index 		- Iteration number starting from zero
    number 		- Iteration number starting from one
    even 		- True if this is an even iteration
    odd 		- True if this is an odd iteration
    start 		- True if this is the first item in the sequence
    end 		- True if this is the last item in the sequence
    length 		- The length of the sequence
    letter 		- The lower case letter for this iteration, starting at "a"
    Letter		- Upper case version of letter
    roman 		- Iteration number in Roman numerals, starting at i
    Roman	 	- Upper case version of roman

Note that letterUpper and romanUpper are used instead of the standard TAL Letter and Roman.  The "first" and "last" properties are not supported.

Example:

Content

tal:content replaces the content of an element:

	tal:content="[text | structure] expression"

Description:  Replaces the contents of the tag with the value of "expression".  By default, and if the "text" keyword is present, then the value of the expression will be escaped as required (i.e. characters "&<> will be escaped).  If the "structure" keyword is present then the value will be output with no escaping performed.

Example:

Replace

tal:replace replaces the whole element:

	tal:replace="[text | structure] expression"

Description: Behaves identically to tal:content, except that the tag is removed from the output (as if tal:omit-tag had been used).

Example:

Welcome

Attributes

tal:attributes sets or removes attributes on the element:

	tal:attributes="name expression[;attributes-expression]"

Description:  Evaluates each "expression" and replaces the tag's attribute "name".  If the expression evaluates to nothing then the attribute is removed from the tag.  If the expression evaluates to default then the original tag's attribute is kept.  If the "expression" requires a semi-colon then it must be escaped by using ";;".

Example:

	Your Homepage

Omit Tag

tal:omit-tag removes the start and end tags:

	tal:omit-tag="expression"

Description: Removes the tag (leaving the tags content) if the expression evaluates to true.  If expression is empty then it is taken as true.

Example:

Welcome to this page!

TALES Expressions

The expressions used in TAL are called TALES expressions.  The simplest TALES expression is a path which references a value, e.g. page/body references the body property of the page object.  Objects are passed as the first argument of the Render method on a compiled template and must be either a struct, pointer to a struct or a map with strings as keys.

The tal package does not support the python: and nocall: expression types.

Path

path: provides access to properties on objects.

	Syntax: [path:]string[|TALES Expression]

Description: A path, optionally starting with the modifier 'path:', references a property of an object.  The '/' delimiter is used to end the name of an object and start of the property name.  Properties themselves may be objects that in turn have properties.  The '|' ("or") character is used to find an alternative value to a path if the first path evaluates to nil or does not exist.

Example:

There are several built in variables that can be used in paths:

    nothing	- acts as nil in Go
    default	- keeps the existing value of the node (tag content or attribute value)
    repeat	- access to repeat variables (see tal:repeat)
    attrs	- a dictionary of original attributes of the current tag

Path Variables

Path variables allows for indirection in paths.

	Syntax: [path:]object/?attribute

Description:  The "attribute" is evaluated as a local or global variable, and it's value is used as the attribute to lookup on the object.  Useful for accessing the contents of a map within a loop:

Example:

Exists

exists: Tests whether a path exists.

	Syntax: exists:path

Description: Returns true if the path exists, false otherwise.  Particularly useful for eliminating entire subtrees if a particular object is not available.

Example:

...

Not

not: Returns the inverse boolean value of a path.

	Syntax: not:tales-path

Description: Returns the inverse of the tales-path.  If the path returns true, not:path will return false.

Example:

Welcome to the site!

String

string: Basic string substitution

	Syntax: string:text

Description:  Evaluates to a string with value text while substituting variables with the form ${pathName} and $pathName

Example:

METAL Macro Language

METAL is a macro language commonly used with TAL & TALES.  METAL allows part of a template to be used as a macro in later parts of the template, or shared across templates.

Macros defined in a template can be retrieved from a compiled template and added to the context by calling the Macros() method.

Define Macro

metal:define-macro marks an element and it's subtree as being a reusable macro.

	Syntax: metal:define-macro="name"

Description: Defines a new macro that can be reference later as "name".

Example:

Copyright 2004

Use Macro

metal:use-macro expands a macro into the template.

	Syntax: metal:use-macro="expression"

Description: Evaluates "expression" and uses this as a macro.

Example:

Define Slot

metal:define-slot creates a customisation point within a macro.

	Syntax: metal:define-slot="name"

Description: Defines a customisation point in a macro with the given name.

Example:

		Standard disclaimer for the site.
		Contact admin@site.com

Fill Slot

metal:fill-slot overwrites the contents of a slot when using a macro.

Syntax: metal:fill-slot="name"

Description: Replaces the content of a slot with this element.

Example:

		Contact someone else

*/
package tal
<|endoftext|>


<commit_before>// Package str is a comprehensive set of string functions to build more
// awesomeness. Str is a port of many of the functions from [stringjs](http://stringjs.com),
// which includes my contributions to the project.
//
// Package str differs from string.js in that str is based on simple functions
// not an intermediate String object to be consistent with Go.
//
//      str.Between("foo", "", "") == "foo"
//
// Package str is designed to be pipelineable.
//
//      s := str.Pipe(
//          "\nabcdef\n",
//          Clean,
//          BetweenF("a", "f"),
//          ChompLeftF("bc")
//      ) // "de"
//
// User defined filters can be added to the pipeline if they have a signature
// of
//
//      func(string) string
//
package str
<commit_msg>update README<commit_after>// Package str is a comprehensive set of string functions to build more
// awesomeness. Str is a port of many of the functions from the JavaScript
// [string.js](http://stringjs.com), which includes my contributions to
// the project.
//
// Package str differs from string.js in that str is based on simple functions
// not an intermediate String object to be consistent with Go.
//
//      str.Between("foo", "", "") == "foo"
//
// Package str is designed to be pipelined.
//
//      s := str.Pipe(
//          "\nabcdef\n",
//          Clean,
//          BetweenF("a", "f"),
//          ChompLeftF("bc")
//      ) // "de"
//
// User defined filters can be added to the pipeline if they have a signature
// of
//
//      func(string) string
//
package str
<|endoftext|>


<commit_before>/*
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

/*
Package cml implments the Count-Min-Log datastructure
It is based on
[Count-Min-Log sketch: Approximately counting with approximate counters - Guillaume Pitel & Geoffroy Fouquier](http://iswag-symposium.org/2015/pdfs/shortpaper1.pdf)

<b>TL;DR:</b> Count-Min-Log Sketch for improved Average Relative Error on low frequency events

Count-Min Sketch is a widely adopted algorithm for approximate event counting in large scale processing. However, the original version of the Count-Min-Sketch (CMS) suffers of some deficiences, especially if one is interested in the low-frequency items, such as in text- mining related tasks. Several variants of CMS have been proposed to compensate for the high relative error for low-frequency events, but the proposed solutions tend to correct the errors instead of preventing them. In this paper, we propose the Count-Min-Log sketch, which uses logarithm-based, approximate counters instead of linear counters to improve the average relative error of CMS at constant memory footprint.
*/
package cml
<commit_msg>Update doc.go<commit_after>/*
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

/*
Package cml implments the Count-Min-Log datastructure
It is based on Count-Min-Log sketch: Approximately counting with approximate counters - Guillaume Pitel & Geoffroy Fouquier
http://iswag-symposium.org/2015/pdfs/shortpaper1.pdf

TL;DR: Count-Min-Log Sketch for improved Average Relative Error on low frequency events

Count-Min Sketch is a widely adopted algorithm for approximate event counting in large scale processing. However, the original version of the Count-Min-Sketch (CMS) suffers of some deficiences, especially if one is interested in the low-frequency items, such as in text- mining related tasks. Several variants of CMS have been proposed to compensate for the high relative error for low-frequency events, but the proposed solutions tend to correct the errors instead of preventing them. In this paper, we propose the Count-Min-Log sketch, which uses logarithm-based, approximate counters instead of linear counters to improve the average relative error of CMS at constant memory footprint.
*/
package cml
<|endoftext|>


<commit_before>/*
Package money provides a way of dealing with money in type safe way, including parsing from strings.

Parsing

To turn a money amount represented as a string into a money type, do something like the follow.

	canadianMoney, err := money.ParseCAD("$123.45")

	usMoney, err := money.ParseUSD("$123.45")

Formatting

To turn a money amount into a specific string format, do something like the following.

	strCAD := canadianMoney.Format("")           // $123.45

	strCAD := canadianMoney.Format("$")          // $123.45

	strCAD := canadianMoney.Format("\uFE69")     // ﹩123.45
	strCAD := canadianMoney.Format("﹩")          // ﹩123.45

	strCAD := canadianMoney.Format("\uFF04")     // ＄123.45
	strCAD := canadianMoney.Format("＄")         // ＄123.45

	strCAD := canadianMoney.Format("\U0001F4B2") // 💲123.45
	strCAD := canadianMoney.Format("💲")         // 💲123.45

	strCAD := canadianMoney.Format("\u00A2")     // 123.45¢
	strCAD := canadianMoney.Format("¢")          // 123.45¢

	strCAD := canadianMoney.Format("\uFFE0")     // 12345￠
	strCAD := canadianMoney.Format("￠")         // 12345￠

	strUSD := usMoney.Format("")           // $123.45

	strUSD := usMoney.Format("$")          // $123.45

	strUSD := usMoney.Format("\uFE69")     // ﹩123.45
	strUSD := usMoney.Format("﹩")          // ﹩123.45

	strUSD := usMoney.Format("\uFF04")     // ＄123.45
	strUSD := usMoney.Format("＄")         // ＄123.45

	strUSD := usMoney.Format("\U0001F4B2") // 💲123.45
	strUSD := usMoney.Format("💲")         // 💲123.45

	strUSD := usMoney.Format("\u00A2")     // 12345¢
	strUSD := usMoney.Format("¢")          // 12345¢

	strUSD := usMoney.Format("\uFFE0")     // 12345￠
	strUSD := usMoney.Format("￠")         // 12345￠

Constants

Constants exist that can be used a money "literals". For example:

	fiveCanadianDollars := 5 * CAD_DOLLAR

	fiveUsDollars := 5 * USD_DOLLAR

	twentyCanadianCents := 20 * CAD_CENT

	twentyUsCents := 20 * USD_CENT
*/
package money
<commit_msg>correction<commit_after>/*
Package money provides a way of dealing with money in type safe way, including parsing from strings.

Parsing

To turn a money amount represented as a string into a money type, do something like the follow.

	canadianMoney, err := money.ParseCAD("$123.45")

	usMoney, err := money.ParseUSD("$123.45")

Formatting

To turn a money amount into a specific string format, do something like the following.

	strCAD := canadianMoney.Format("")           // $123.45

	strCAD := canadianMoney.Format("$")          // $123.45

	strCAD := canadianMoney.Format("\uFE69")     // ﹩123.45
	strCAD := canadianMoney.Format("﹩")          // ﹩123.45

	strCAD := canadianMoney.Format("\uFF04")     // ＄123.45
	strCAD := canadianMoney.Format("＄")         // ＄123.45

	strCAD := canadianMoney.Format("\U0001F4B2") // 💲123.45
	strCAD := canadianMoney.Format("💲")         // 💲123.45

	strCAD := canadianMoney.Format("\u00A2")     // 123.45¢
	strCAD := canadianMoney.Format("¢")          // 123.45¢

	strCAD := canadianMoney.Format("\uFFE0")     // 12345￠
	strCAD := canadianMoney.Format("￠")         // 12345￠

	strUSD := usMoney.Format("")           // $123.45

	strUSD := usMoney.Format("$")          // $123.45

	strUSD := usMoney.Format("\uFE69")     // ﹩123.45
	strUSD := usMoney.Format("﹩")          // ﹩123.45

	strUSD := usMoney.Format("\uFF04")     // ＄123.45
	strUSD := usMoney.Format("＄")         // ＄123.45

	strUSD := usMoney.Format("\U0001F4B2") // 💲123.45
	strUSD := usMoney.Format("💲")         // 💲123.45

	strUSD := usMoney.Format("\u00A2")     // 12345¢
	strUSD := usMoney.Format("¢")          // 12345¢

	strUSD := usMoney.Format("\uFFE0")     // 12345￠
	strUSD := usMoney.Format("￠")         // 12345￠

Constants

Constants exist that can be used a money "literals". For example:

	fiveCanadianDollars := 5 * money.CAD_DOLLAR

	fiveUsDollars := 5 * money.USD_DOLLAR

	twentyCanadianCents := 20 * money.CAD_CENT

	twentyUsCents := 20 * money.USD_CENT
*/
package money
<|endoftext|>


<commit_before>/*
Package equalfile provides facilities for comparing files.

Equalfile is similar to Python's filecmp:

        import filecmp
        if filecmp.cmp(filename1, filename2, shallow=False):

Comparing only two files

In single mode, equalfile compares files byte-by-byte.

        import "github.com/udhos/equalfile"
        // ...
       cmp := equalfile.New(nil, equalfile.Options{}) // compare using single mode
       equal, err := cmp.CompareFile("file1", "file2")

Comparing multiple files

In multiple mode, equalfile records files hashes in order to speedup repeated comparisons.
You must provide the hashing function.

        import "crypto/sha256"
        import "github.com/udhos/equalfile"
        // ...
        cmp := equalfile.NewMultiple(nil, equalfile.Options{}, sha256.New(), true) // enable multiple mode
        equal, err := cmp.CompareFile("file1", "file2")

*/
package equalfile
<commit_msg>Markdown fix.<commit_after>/*
Package equalfile provides facilities for comparing files.

Equalfile is similar to Python's filecmp:

        import filecmp
        if filecmp.cmp(filename1, filename2, shallow=False):

Comparing only two files

In single mode, equalfile compares files byte-by-byte.

       import "github.com/udhos/equalfile"
       // ...
       cmp := equalfile.New(nil, equalfile.Options{}) // compare using single mode
       equal, err := cmp.CompareFile("file1", "file2")

Comparing multiple files

In multiple mode, equalfile records files hashes in order to speedup repeated comparisons.
You must provide the hashing function.

        import "crypto/sha256"
        import "github.com/udhos/equalfile"
        // ...
        cmp := equalfile.NewMultiple(nil, equalfile.Options{}, sha256.New(), true) // enable multiple mode
        equal, err := cmp.CompareFile("file1", "file2")

*/
package equalfile
<|endoftext|>


<commit_before>// Objx is a package for dealing with maps, slices, JSON and other data in Go.
//
// Reading data
//
// Objx follows a fluent design allowing you to quickly and intuitively access
// data from within otherwise difficult or deep nested places.
//
//     // parse some JSON
//     data := objx.MustFromJSON(`{
//       "name": "Tyler",
//       "age": 29,
//       "tags": ["musician","coder","singer"],
//       "deep": {
//	       "nested": {
//	         "item": "easy"
//         }
//       }
//     }`)
//
//     // get some fields
//     name := data.Get("name").Str()
//     age := data.Get("age").Int()
//
//     // access an array element
//     secondTag := data.Get("tags[2]").Str()
//
//     // get at some deep data using dot notation
//     deepItem := data.Get("deep.nested.item").Str()
//
// Type strong
//
// Objx is type strong, meaning that it attempts to remove the need to
// work so loosely with interface{}.  In that spirit, it providers
// strongly typed accessors that make reading data easy.
//
// Example: Int(), Str(), Bool(), Float64(), UInt64() etc.
//
// If the data is missing or of the wrong type, the methods will
// return a default value, or optionally, you can specify the default value.
//
// Objx also providers stricter Must methods for each supported type (like MustStr(),
// MustBool(), MustInt(), MustFloat64() etc.)
package objx
<commit_msg>fixed whitespace<commit_after>// Objx is a package for dealing with maps, slices, JSON and other data in Go.
//
// Reading data
//
// Objx follows a fluent design allowing you to quickly and intuitively access
// data from within otherwise difficult or deep nested places.
//
//     // parse some JSON
//     data := objx.MustFromJSON(`{
//       "name": "Tyler",
//       "age": 29,
//       "tags": ["musician","coder","singer"],
//       "deep": {
//         "nested": {
//	         "item": "easy"
//         }
//       }
//     }`)
//
//     // get some fields
//     name := data.Get("name").Str()
//     age := data.Get("age").Int()
//
//     // access an array element
//     secondTag := data.Get("tags[2]").Str()
//
//     // get at some deep data using dot notation
//     deepItem := data.Get("deep.nested.item").Str()
//
// Type strong
//
// Objx is type strong, meaning that it attempts to remove the need to
// work so loosely with interface{}.  In that spirit, it providers
// strongly typed accessors that make reading data easy.
//
// Example: Int(), Str(), Bool(), Float64(), UInt64() etc.
//
// If the data is missing or of the wrong type, the methods will
// return a default value, or optionally, you can specify the default value.
//
// Objx also providers stricter Must methods for each supported type (like MustStr(),
// MustBool(), MustInt(), MustFloat64() etc.)
package objx
<|endoftext|>


<commit_before>package docconv

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"

	"github.com/richardlehane/mscfb"
	"github.com/richardlehane/msoleps"
)

// ConvertDoc converts an MS Word .doc to text.
func ConvertDoc(r io.Reader) (string, map[string]string, error) {
	f, err := NewLocalFile(r)
	if err != nil {
		return "", nil, fmt.Errorf("error creating local file: %v", err)
	}
	defer f.Done()

	// Meta data
	mc := make(chan map[string]string, 1)
	go func() {
		defer func() {
			if e := recover(); e != nil {
				log.Printf("panic when reading doc format: %v", e)
			}
		}()

		meta := make(map[string]string)

		doc, err := mscfb.New(f)
		if err != nil {
			log.Printf("ConvertDoc: could not read doc: %v", err)
		}

		props := msoleps.New()
		for entry, err := doc.Next(); err == nil; entry, err = doc.Next() {
			if msoleps.IsMSOLEPS(entry.Initial) {
				if oerr := props.Reset(doc); oerr != nil {
					log.Printf("ConvertDoc: could not reset props: %v", oerr)
					break
				}

				for _, prop := range props.Property {
					meta[prop.Name] = prop.String()
				}
			}
		}

		const defaultTimeFormat = "2006-01-02 15:04:05.999999999 -0700 MST"

		// Convert parsed meta
		if tmp, ok := meta["LastSaveTime"]; ok {
			if t, err := time.Parse(defaultTimeFormat, tmp); err == nil {
				meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if tmp, ok := meta["CreateTime"]; ok {
			if t, err := time.Parse(defaultTimeFormat, tmp); err == nil {
				meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}

		mc <- meta
	}()

	// Document body
	bc := make(chan string, 1)
	go func() {

		// Save output to a file
		outputFile, err := ioutil.TempFile("/tmp", "sajari-convert-")
		if err != nil {
			// TODO: Remove this.
			log.Println("TempFile Out:", err)
			return
		}
		defer os.Remove(outputFile.Name())

		err = exec.Command("wvText", f.Name(), outputFile.Name()).Run()
		if err != nil {
			// TODO: Remove this.
			log.Println("wvText:", err)
		}

		var buf bytes.Buffer
		_, err = buf.ReadFrom(outputFile)
		if err != nil {
			// TODO: Remove this.
			log.Println("wvText:", err)
		}

		bc <- buf.String()
	}()

	// TODO: Should errors in either of the above Goroutines stop things from progressing?
	body := <-bc
	meta := <-mc

	// TODO: Check for errors instead of len(body) == 0?
	if len(body) == 0 {
		f.Seek(0, 0)
		return ConvertDocx(f)
	}
	return body, meta, nil
}
<commit_msg>convert-doc: don't use doc if there was an error (#117)<commit_after>package docconv

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"time"

	"github.com/richardlehane/mscfb"
	"github.com/richardlehane/msoleps"
)

// ConvertDoc converts an MS Word .doc to text.
func ConvertDoc(r io.Reader) (string, map[string]string, error) {
	f, err := NewLocalFile(r)
	if err != nil {
		return "", nil, fmt.Errorf("error creating local file: %v", err)
	}
	defer f.Done()

	// Meta data
	mc := make(chan map[string]string, 1)
	go func() {
		defer func() {
			if e := recover(); e != nil {
				log.Printf("panic when reading doc format: %v", e)
			}
		}()

		meta := make(map[string]string)

		doc, err := mscfb.New(f)
		if err != nil {
			log.Printf("ConvertDoc: could not read doc: %v", err)
			return
		}

		props := msoleps.New()
		for entry, err := doc.Next(); err == nil; entry, err = doc.Next() {
			if msoleps.IsMSOLEPS(entry.Initial) {
				if oerr := props.Reset(doc); oerr != nil {
					log.Printf("ConvertDoc: could not reset props: %v", oerr)
					break
				}

				for _, prop := range props.Property {
					meta[prop.Name] = prop.String()
				}
			}
		}

		const defaultTimeFormat = "2006-01-02 15:04:05.999999999 -0700 MST"

		// Convert parsed meta
		if tmp, ok := meta["LastSaveTime"]; ok {
			if t, err := time.Parse(defaultTimeFormat, tmp); err == nil {
				meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if tmp, ok := meta["CreateTime"]; ok {
			if t, err := time.Parse(defaultTimeFormat, tmp); err == nil {
				meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}

		mc <- meta
	}()

	// Document body
	bc := make(chan string, 1)
	go func() {

		// Save output to a file
		outputFile, err := ioutil.TempFile("/tmp", "sajari-convert-")
		if err != nil {
			// TODO: Remove this.
			log.Println("TempFile Out:", err)
			return
		}
		defer os.Remove(outputFile.Name())

		err = exec.Command("wvText", f.Name(), outputFile.Name()).Run()
		if err != nil {
			// TODO: Remove this.
			log.Println("wvText:", err)
		}

		var buf bytes.Buffer
		_, err = buf.ReadFrom(outputFile)
		if err != nil {
			// TODO: Remove this.
			log.Println("wvText:", err)
		}

		bc <- buf.String()
	}()

	// TODO: Should errors in either of the above Goroutines stop things from progressing?
	body := <-bc
	meta := <-mc

	// TODO: Check for errors instead of len(body) == 0?
	if len(body) == 0 {
		f.Seek(0, 0)
		return ConvertDocx(f)
	}
	return body, meta, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The rspace Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc is a simple document printer that produces the doc comments for its
// argument symbols, plus a link to the full documentation and a pointer to
// the source. It has a more Go-like UI than godoc. It can also search for
// symbols by looking in all packages, and case is ignored. For instance:
//	doc isupper
// will find unicode.IsUpper.
//
// The -pkg flag retrieves package-level doc comments only.
//
// Usage:
//	doc pkg.name   # "doc io.Writer"
//	doc pkg name   # "doc fmt Printf"
//	doc name       # "doc isupper" (finds unicode.IsUpper)
//	doc -pkg pkg   # "doc fmt"
//
// The pkg is the last element of the package path;
// no slashes (ast.Node not go/ast.Node).
//
// Flags
//	-c(onst) -f(unc) -i(nterface) -m(ethod) -s(truct) -t(ype) -v(ar)
// restrict hits to declarations of the corresponding kind.
// Flags
//	-doc -src -url
// restrict printing to the documentation, source path, or godoc URL.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"
)

const usageDoc = `Find documentation for names.
usage:
	doc pkg.name   # "doc io.Writer"
	doc pkg name   # "doc fmt Printf"
	doc name       # "doc isupper" finds unicode.IsUpper
	doc -pkg pkg   # "doc fmt"
pkg is the last component of any package, e.g. fmt, parser
name is the name of an exported symbol; case is ignored in matches.
Flags
	-c(onst) -f(unc) -i(nterface) -m(ethod) -s(truct) -t(ype) -v(ar)
restrict hits to declarations of the corresponding kind.
Flags
	-doc -src -url
restrict printing to the documentation, source path, or godoc URL.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

var (
	// If none is set, all are set.
	constantFlag  = flag.Bool("const", false, "show doc for consts only")
	functionFlag  = flag.Bool("func", false, "show doc for funcs only")
	interfaceFlag = flag.Bool("interface", false, "show doc for interfaces only")
	methodFlag    = flag.Bool("method", false, "show doc for methods only")
	packageFlag   = flag.Bool("package", false, "show top-level package doc only")
	structFlag    = flag.Bool("struct", false, "show doc for structs only")
	typeFlag      = flag.Bool("type", false, "show doc for types only")
	variableFlag  = flag.Bool("var", false, "show  doc for vars only")
)

var (
	// If none is set, all are set.
	docFlag = flag.Bool("doc", false, "restrict output to documentation only")
	srcFlag = flag.Bool("src", false, "restrict output to source file only")
	urlFlag = flag.Bool("url", false, "restrict output to godoc URL only")
)

func init() {
	flag.BoolVar(constantFlag, "c", false, "alias for -const")
	flag.BoolVar(functionFlag, "f", false, "alias for -func")
	flag.BoolVar(interfaceFlag, "i", false, "alias for -interface")
	flag.BoolVar(methodFlag, "m", false, "alias for -method")
	flag.BoolVar(packageFlag, "pkg", false, "alias for -package")
	flag.BoolVar(structFlag, "s", false, "alias for -struct")
	flag.BoolVar(typeFlag, "t", false, "alias for -type")
	flag.BoolVar(variableFlag, "v", false, "alias for -var")
}

func main() {
	flag.Parse()
	if !(*constantFlag || *functionFlag || *interfaceFlag || *methodFlag || *packageFlag || *structFlag || *typeFlag || *variableFlag) { // none set
		*constantFlag = true
		*functionFlag = true
		*methodFlag = true
		// Not package! It's special.
		*typeFlag = true
		*variableFlag = true
	}
	if !(*docFlag || *srcFlag || *urlFlag) {
		*docFlag = true
		*srcFlag = true
		*urlFlag = true
	}
	var pkg, name string
	switch flag.NArg() {
	case 1:
		if *packageFlag {
			pkg = flag.Arg(0)
		} else if strings.Contains(flag.Arg(0), ".") {
			pkg, name = split(flag.Arg(0))
		} else {
			name = flag.Arg(0)
		}
	case 2:
		if *packageFlag {
			usage()
		}
		pkg, name = flag.Arg(0), flag.Arg(1)
	default:
		usage()
	}
	for _, path := range paths(pkg) {
		lookInDirectory(path, name)
	}
}

var slash = string(filepath.Separator)
var slashDot = string(filepath.Separator) + "."
var goRootSrcPkg = filepath.Join(runtime.GOROOT(), "src", "pkg")
var goRootSrcCmd = filepath.Join(runtime.GOROOT(), "src", "cmd")
var goPaths = splitGopath()

func split(arg string) (pkg, name string) {
	str := strings.Split(arg, ".")
	if len(str) != 2 {
		usage()
	}
	return str[0], str[1]
}

func paths(pkg string) []string {
	pkgs := pathsFor(runtime.GOROOT(), pkg)
	for _, root := range goPaths {
		pkgs = append(pkgs, pathsFor(root, pkg)...)
	}
	return pkgs
}

func splitGopath() []string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return nil
	}
	return strings.Split(gopath, string(os.PathListSeparator))
}

// pathsFor recursively walks the tree looking for possible directories for the package:
// those whose basename is pkg.
func pathsFor(root, pkg string) []string {
	root = path.Join(root, "src")
	pkgPaths := make([]string, 0, 10)
	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .hg or other dot nonsense please.
		if strings.Contains(pathName, slashDot) {
			return filepath.SkipDir
		}
		// Is the last element of the path correct
		if pkg == "" || filepath.Base(pathName) == pkg {
			pkgPaths = append(pkgPaths, pathName)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return pkgPaths
}

// lookInDirectory looks in the package (if any) in the directory for the named exported identifier.
func lookInDirectory(directory, name string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		// If it's just that there are no go source files, that's fine.
		if _, nogo := err.(*build.NoGoError); nogo {
			return
		}
		// Non-fatal: we are doing a recursive walk and there may be other directories.
		return
	}
	var fileNames []string
	fileNames = append(fileNames, pkg.GoFiles...)
	prefixDirectory(directory, fileNames)
	doPackage(fileNames, name)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) {
	if directory != "." {
		for i, name := range names {
			names[i] = filepath.Join(directory, name)
		}
	}
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	fset       *token.FileSet
	name       string // Name of file.
	ident      string // Identifier we are searching for.
	pathPrefix string // Prefix from GOROOT/GOPATH.
	urlPrefix  string // Start of corresponding URL for golang.org or godoc.org.
	file       *ast.File
	comments   ast.CommentMap
}

const godocOrg = "http://godoc.org"

// doPackage analyzes the single package constructed from the named files, looking for
// the definition of ident.
func doPackage(fileNames []string, ident string) {
	var files []*File
	var astFiles []*ast.File
	fs := token.NewFileSet()
	for _, name := range fileNames {
		f, err := os.Open(name)
		if err != nil {
			// Warn but continue to next package.
			fmt.Fprintf(os.Stderr, "%s: %s", name, err)
			return
		}
		defer f.Close()
		data, err := ioutil.ReadAll(f)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: %s", name, err)
			return
		}
		parsedFile, err := parser.ParseFile(fs, name, bytes.NewReader(data), parser.ParseComments)
		if err != nil {
			// Noisy - just ignore.
			// fmt.Fprintf(os.Stderr, "%s: %s", name, err)
			return
		}
		thisFile := &File{
			fset:     fs,
			name:     name,
			ident:    ident,
			file:     parsedFile,
			comments: ast.NewCommentMap(fs, parsedFile, parsedFile.Comments),
		}
		switch {
		case strings.HasPrefix(name, goRootSrcPkg):
			thisFile.urlPrefix = "http://golang.org/pkg"
			thisFile.pathPrefix = goRootSrcPkg
		case strings.HasPrefix(name, goRootSrcCmd):
			thisFile.urlPrefix = "http://golang.org/cmd"
			thisFile.pathPrefix = goRootSrcCmd
		default:
			thisFile.urlPrefix = godocOrg
			for _, path := range goPaths {
				p := filepath.Join(path, "src")
				if strings.HasPrefix(name, p) {
					thisFile.pathPrefix = p
					break
				}
			}
		}
		files = append(files, thisFile)
		astFiles = append(astFiles, parsedFile)
	}
	for _, file := range files {
		if *packageFlag {
			file.pkgComments()
		} else {
			ast.Walk(file, file.file)
		}
	}
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.GenDecl:
		// Variables, constants, types.
		for _, spec := range n.Specs {
			switch spec := spec.(type) {
			case *ast.ValueSpec:
				if *constantFlag && n.Tok == token.CONST || *variableFlag && n.Tok == token.VAR {
					for _, ident := range spec.Names {
						if equal(ident.Name, f.ident) {
							f.printNode(n, ident, f.nameURL(ident.Name))
							break
						}
					}
				}
			case *ast.TypeSpec:
				// If there is only one Spec, there are probably no parens and the
				// comment we want appears before the type keyword, bound to
				// the GenDecl. If the Specs are parenthesized, the comment we want
				// is bound to the Spec. Hence we dig into the GenDecl to the Spec,
				// but only if there are no parens.
				node := ast.Node(n)
				if n.Lparen.IsValid() {
					node = spec
				}
				if equal(spec.Name.Name, f.ident) {
					if *typeFlag {
						f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						break
					}
					switch spec.Type.(type) {
					case *ast.InterfaceType:
						if *interfaceFlag {
							f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						}
					case *ast.StructType:
						if *structFlag {
							f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						}
					}
				}
			case *ast.ImportSpec:
				continue // Don't care.
			}
		}
	case *ast.FuncDecl:
		// Methods, top-level functions.
		if equal(n.Name.Name, f.ident) {
			n.Body = nil // Do not print the function body.
			if *methodFlag && n.Recv != nil {
				f.printNode(n, n.Name, f.methodURL(n.Recv.List[0].Type, n.Name.Name))
			} else if *functionFlag && n.Recv == nil {
				f.printNode(n, n.Name, f.nameURL(n.Name.Name))
			}
		}
	}
	return f
}

func equal(n1, n2 string) bool {
	// n1 must  be exported.
	r, _ := utf8.DecodeRuneInString(n1)
	if !unicode.IsUpper(r) {
		return false
	}
	return strings.ToLower(n1) == strings.ToLower(n2)
}

func (f *File) printNode(node, ident ast.Node, url string) {
	fmt.Printf("%s%s%s", url, f.sourcePos(f.fset.Position(ident.Pos())), f.docs(node))
}

func (f *File) docs(node ast.Node) []byte {
	if !*docFlag {
		return nil
	}
	commentedNode := printer.CommentedNode{Node: node}
	if comments := f.comments.Filter(node).Comments(); comments != nil {
		commentedNode.Comments = comments
	}
	var b bytes.Buffer
	printer.Fprint(&b, f.fset, &commentedNode)
	b.Write([]byte("\n\n")) // Add a blank line between entries if we print documentation.
	return b.Bytes()
}

func (f *File) pkgComments() {
	doc := f.file.Doc
	if doc == nil {
		return
	}
	url := ""
	if *urlFlag {
		url = f.packageURL() + "\n"
	}
	docText := ""
	if *docFlag {
		docText = fmt.Sprintf("package %s\n%s\n\n", f.file.Name.Name, doc.Text())
	}
	fmt.Printf("%s%s%s", url, f.sourcePos(f.fset.Position(doc.Pos())), docText)
}

func (f *File) packageURL() string {
	s := strings.TrimPrefix(f.name, f.pathPrefix)
	// Now we have a path with a final file name. Drop it.
	if i := strings.LastIndex(s, slash); i > 0 {
		s = s[:i+1]
	}
	return f.urlPrefix + s
}

func (f *File) sourcePos(posn token.Position) string {
	if !*srcFlag {
		return ""
	}
	return fmt.Sprintf("%s:%d:\n", posn.Filename, posn.Line)
}

func (f *File) nameURL(name string) string {
	if !*urlFlag {
		return ""
	}
	return fmt.Sprintf("%s#%s\n", f.packageURL(), name)
}

func (f *File) methodURL(typ ast.Expr, name string) string {
	if !*urlFlag {
		return ""
	}
	var b bytes.Buffer
	printer.Fprint(&b, f.fset, typ)
	typeName := b.Bytes()
	if len(typeName) > 0 && typeName[0] == '*' {
		typeName = typeName[1:]
	}
	return fmt.Sprintf("%s#%s.%s\n", f.packageURL(), typeName, name)
}
<commit_msg>rspace.cmd/doc: switch to package-level instead of file-level parsing Simpler code; avoids dependence on go/build; a step towards better docs.<commit_after>// Copyright 2013 The rspace Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc is a simple document printer that produces the doc comments for its
// argument symbols, plus a link to the full documentation and a pointer to
// the source. It has a more Go-like UI than godoc. It can also search for
// symbols by looking in all packages, and case is ignored. For instance:
//	doc isupper
// will find unicode.IsUpper.
//
// The -pkg flag retrieves package-level doc comments only.
//
// Usage:
//	doc pkg.name   # "doc io.Writer"
//	doc pkg name   # "doc fmt Printf"
//	doc name       # "doc isupper" (finds unicode.IsUpper)
//	doc -pkg pkg   # "doc fmt"
//
// The pkg is the last element of the package path;
// no slashes (ast.Node not go/ast.Node).
//
// Flags
//	-c(onst) -f(unc) -i(nterface) -m(ethod) -s(truct) -t(ype) -v(ar)
// restrict hits to declarations of the corresponding kind.
// Flags
//	-doc -src -url
// restrict printing to the documentation, source path, or godoc URL.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"
)

const usageDoc = `Find documentation for names.
usage:
	doc pkg.name   # "doc io.Writer"
	doc pkg name   # "doc fmt Printf"
	doc name       # "doc isupper" finds unicode.IsUpper
	doc -pkg pkg   # "doc fmt"
pkg is the last component of any package, e.g. fmt, parser
name is the name of an exported symbol; case is ignored in matches.
Flags
	-c(onst) -f(unc) -i(nterface) -m(ethod) -s(truct) -t(ype) -v(ar)
restrict hits to declarations of the corresponding kind.
Flags
	-doc -src -url
restrict printing to the documentation, source path, or godoc URL.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

var (
	// If none is set, all are set.
	constantFlag  = flag.Bool("const", false, "show doc for consts only")
	functionFlag  = flag.Bool("func", false, "show doc for funcs only")
	interfaceFlag = flag.Bool("interface", false, "show doc for interfaces only")
	methodFlag    = flag.Bool("method", false, "show doc for methods only")
	packageFlag   = flag.Bool("package", false, "show top-level package doc only")
	structFlag    = flag.Bool("struct", false, "show doc for structs only")
	typeFlag      = flag.Bool("type", false, "show doc for types only")
	variableFlag  = flag.Bool("var", false, "show  doc for vars only")
)

var (
	// If none is set, all are set.
	docFlag = flag.Bool("doc", false, "restrict output to documentation only")
	srcFlag = flag.Bool("src", false, "restrict output to source file only")
	urlFlag = flag.Bool("url", false, "restrict output to godoc URL only")
)

func init() {
	flag.BoolVar(constantFlag, "c", false, "alias for -const")
	flag.BoolVar(functionFlag, "f", false, "alias for -func")
	flag.BoolVar(interfaceFlag, "i", false, "alias for -interface")
	flag.BoolVar(methodFlag, "m", false, "alias for -method")
	flag.BoolVar(packageFlag, "pkg", false, "alias for -package")
	flag.BoolVar(structFlag, "s", false, "alias for -struct")
	flag.BoolVar(typeFlag, "t", false, "alias for -type")
	flag.BoolVar(variableFlag, "v", false, "alias for -var")
}

func main() {
	flag.Parse()
	if !(*constantFlag || *functionFlag || *interfaceFlag || *methodFlag || *packageFlag || *structFlag || *typeFlag || *variableFlag) { // none set
		*constantFlag = true
		*functionFlag = true
		*methodFlag = true
		// Not package! It's special.
		*typeFlag = true
		*variableFlag = true
	}
	if !(*docFlag || *srcFlag || *urlFlag) {
		*docFlag = true
		*srcFlag = true
		*urlFlag = true
	}
	var pkg, name string
	switch flag.NArg() {
	case 1:
		if *packageFlag {
			pkg = flag.Arg(0)
		} else if strings.Contains(flag.Arg(0), ".") {
			pkg, name = split(flag.Arg(0))
		} else {
			name = flag.Arg(0)
		}
	case 2:
		if *packageFlag {
			usage()
		}
		pkg, name = flag.Arg(0), flag.Arg(1)
	default:
		usage()
	}
	for _, path := range paths(pkg) {
		lookInDirectory(path, name)
	}
}

var slash = string(filepath.Separator)
var slashDot = string(filepath.Separator) + "."
var goRootSrcPkg = filepath.Join(runtime.GOROOT(), "src", "pkg")
var goRootSrcCmd = filepath.Join(runtime.GOROOT(), "src", "cmd")
var goPaths = splitGopath()

func split(arg string) (pkg, name string) {
	str := strings.Split(arg, ".")
	if len(str) != 2 {
		usage()
	}
	return str[0], str[1]
}

func paths(pkg string) []string {
	pkgs := pathsFor(runtime.GOROOT(), pkg)
	for _, root := range goPaths {
		pkgs = append(pkgs, pathsFor(root, pkg)...)
	}
	return pkgs
}

func splitGopath() []string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return nil
	}
	return strings.Split(gopath, string(os.PathListSeparator))
}

// pathsFor recursively walks the tree looking for possible directories for the package:
// those whose basename is pkg.
func pathsFor(root, pkg string) []string {
	root = path.Join(root, "src")
	pkgPaths := make([]string, 0, 10)
	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .hg or other dot nonsense please.
		if strings.Contains(pathName, slashDot) {
			return filepath.SkipDir
		}
		// Is the last element of the path correct
		if pkg == "" || filepath.Base(pathName) == pkg {
			pkgPaths = append(pkgPaths, pathName)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return pkgPaths
}

// lookInDirectory looks in the package (if any) in the directory for the named exported identifier.
func lookInDirectory(directory, name string) {
	fset := token.NewFileSet()
	pkgs, _ := parser.ParseDir(fset, directory, nil, parser.ParseComments) // Ignore the error.
	for _, pkg := range pkgs {
		doPackage(pkg, fset, name)
	}
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) {
	if directory != "." {
		for i, name := range names {
			names[i] = filepath.Join(directory, name)
		}
	}
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	fset       *token.FileSet
	name       string // Name of file.
	ident      string // Identifier we are searching for.
	pathPrefix string // Prefix from GOROOT/GOPATH.
	urlPrefix  string // Start of corresponding URL for golang.org or godoc.org.
	file       *ast.File
	comments   ast.CommentMap
}

const godocOrg = "http://godoc.org"

// doPackage analyzes the single package constructed from the named files, looking for
// the definition of ident.
func doPackage(pkg *ast.Package, fset *token.FileSet, ident string) {
	for name, astFile := range pkg.Files {
		if *packageFlag && astFile.Doc == nil {
			continue
		}
		file := &File{
			fset:     fset,
			name:     name,
			ident:    ident,
			file:     astFile,
			comments: ast.NewCommentMap(fset, astFile, astFile.Comments),
		}
		switch {
		case strings.HasPrefix(name, goRootSrcPkg):
			file.urlPrefix = "http://golang.org/pkg"
			file.pathPrefix = goRootSrcPkg
		case strings.HasPrefix(name, goRootSrcCmd):
			file.urlPrefix = "http://golang.org/cmd"
			file.pathPrefix = goRootSrcCmd
		default:
			file.urlPrefix = godocOrg
			for _, path := range goPaths {
				p := filepath.Join(path, "src")
				if strings.HasPrefix(name, p) {
					file.pathPrefix = p
					break
				}
			}
		}
		if *packageFlag {
			file.pkgComments()
		} else {
			ast.Walk(file, file.file)
		}
	}
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.GenDecl:
		// Variables, constants, types.
		for _, spec := range n.Specs {
			switch spec := spec.(type) {
			case *ast.ValueSpec:
				if *constantFlag && n.Tok == token.CONST || *variableFlag && n.Tok == token.VAR {
					for _, ident := range spec.Names {
						if equal(ident.Name, f.ident) {
							f.printNode(n, ident, f.nameURL(ident.Name))
							break
						}
					}
				}
			case *ast.TypeSpec:
				// If there is only one Spec, there are probably no parens and the
				// comment we want appears before the type keyword, bound to
				// the GenDecl. If the Specs are parenthesized, the comment we want
				// is bound to the Spec. Hence we dig into the GenDecl to the Spec,
				// but only if there are no parens.
				node := ast.Node(n)
				if n.Lparen.IsValid() {
					node = spec
				}
				if equal(spec.Name.Name, f.ident) {
					if *typeFlag {
						f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						break
					}
					switch spec.Type.(type) {
					case *ast.InterfaceType:
						if *interfaceFlag {
							f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						}
					case *ast.StructType:
						if *structFlag {
							f.printNode(node, spec.Name, f.nameURL(spec.Name.Name))
						}
					}
				}
			case *ast.ImportSpec:
				continue // Don't care.
			}
		}
	case *ast.FuncDecl:
		// Methods, top-level functions.
		if equal(n.Name.Name, f.ident) {
			n.Body = nil // Do not print the function body.
			if *methodFlag && n.Recv != nil {
				f.printNode(n, n.Name, f.methodURL(n.Recv.List[0].Type, n.Name.Name))
			} else if *functionFlag && n.Recv == nil {
				f.printNode(n, n.Name, f.nameURL(n.Name.Name))
			}
		}
	}
	return f
}

func equal(n1, n2 string) bool {
	// n1 must  be exported.
	r, _ := utf8.DecodeRuneInString(n1)
	if !unicode.IsUpper(r) {
		return false
	}
	return strings.ToLower(n1) == strings.ToLower(n2)
}

func (f *File) printNode(node, ident ast.Node, url string) {
	fmt.Printf("%s%s%s", url, f.sourcePos(f.fset.Position(ident.Pos())), f.docs(node))
}

func (f *File) docs(node ast.Node) []byte {
	if !*docFlag {
		return nil
	}
	commentedNode := printer.CommentedNode{Node: node}
	if comments := f.comments.Filter(node).Comments(); comments != nil {
		commentedNode.Comments = comments
	}
	var b bytes.Buffer
	printer.Fprint(&b, f.fset, &commentedNode)
	b.Write([]byte("\n\n")) // Add a blank line between entries if we print documentation.
	return b.Bytes()
}

func (f *File) pkgComments() {
	doc := f.file.Doc
	if doc == nil {
		return
	}
	url := ""
	if *urlFlag {
		url = f.packageURL() + "\n"
	}
	docText := ""
	if *docFlag {
		docText = fmt.Sprintf("package %s\n%s\n\n", f.file.Name.Name, doc.Text())
	}
	fmt.Printf("%s%s%s", url, f.sourcePos(f.fset.Position(doc.Pos())), docText)
}

func (f *File) packageURL() string {
	s := strings.TrimPrefix(f.name, f.pathPrefix)
	// Now we have a path with a final file name. Drop it.
	if i := strings.LastIndex(s, slash); i > 0 {
		s = s[:i+1]
	}
	return f.urlPrefix + s
}

func (f *File) sourcePos(posn token.Position) string {
	if !*srcFlag {
		return ""
	}
	return fmt.Sprintf("%s:%d:\n", posn.Filename, posn.Line)
}

func (f *File) nameURL(name string) string {
	if !*urlFlag {
		return ""
	}
	return fmt.Sprintf("%s#%s\n", f.packageURL(), name)
}

func (f *File) methodURL(typ ast.Expr, name string) string {
	if !*urlFlag {
		return ""
	}
	var b bytes.Buffer
	printer.Fprint(&b, f.fset, typ)
	typeName := b.Bytes()
	if len(typeName) > 0 && typeName[0] == '*' {
		typeName = typeName[1:]
	}
	return fmt.Sprintf("%s#%s.%s\n", f.packageURL(), typeName, name)
}
<|endoftext|>


<commit_before>/*
Package errorutil allows errors to be tagged, allowing calling code to make decisions,
such as "is the error retryable ?" or "what kind of HTTP status code should I return ?".

Retryable errors

Mark errors as retryable :

  func foo() error {
    err := bar()
    if err != nil {
      // should be retried
      return errorutil.RetryableError(err)
    }
    return baz() // should not be retried
  }

  func main() {
    err := foo()
    if errorutil.IsRetryable(err) {
      // retry !
    }
  }

HTTP Aware errors

Build an error based on a http.Response. It will be retryable of status code is http.StatusBadGateway, http.StatusGatewayTimeout, http.StatusServiceUnavailable, http.StatusInternalServerError or 429 (Too many requests).

  resp, err := http.Get("http://www.example.com")
  if err != nil {
    // return error
  }
  defer resp.Body.Close()
  if err := errorutil.HTTPError(resp); err != nil {
    // return error
  }
  // handle response
  Find the most appropriate status code for an error :

  w.WriteHeader(errorutil.HTTPStatusCode(err))

Generate specific error types :

  err := errors.New("some error")
  err = errorutil.NotFoundError(err)
  w.WriteHeader(errorutil.HTTPStatusCode(err)) // returns http.StatusNotFound

Exponential backoff

see backoffutil sub package

Notes

errorutil is compatible with https://github.com/pkg/errors :

  err = errors.Wrap(errorutil.RetryableError(err), "some message")
  errorutil.IsRetryable(err) // returns true
*/
package errorutil
<commit_msg>README, comments<commit_after>/*
Package errorutil allows errors to be tagged, allowing calling code to make decisions,
such as "is the error retryable ?" or "what kind of HTTP status code should I return ?".

Retryable errors

Mark errors as retryable :

  func foo() error {
    err := bar()
    if err != nil {
      // should be retried
      return errorutil.RetryableError(err)
    }
    return baz() // should not be retried
  }

  func main() {
    err := foo()
    if errorutil.IsRetryable(err) {
      // retry !
    }
  }

HTTP Aware errors

Build an error based on a http.Response. It will be retryable of status code is http.StatusBadGateway, http.StatusGatewayTimeout, http.StatusServiceUnavailable, http.StatusInternalServerError or 429 (Too many requests).

  resp, err := http.Get("http://www.example.com")
  if err != nil {
    // return error
  }
  defer resp.Body.Close()
  if err := errorutil.HTTPError(resp); err != nil {
    // return error
  }
  // handle response

Find the most appropriate status code for an error :

  w.WriteHeader(errorutil.HTTPStatusCode(err))

Generate specific error types :

  err := errors.New("some error")
  err = errorutil.NotFoundError(err)
  w.WriteHeader(errorutil.HTTPStatusCode(err)) // returns http.StatusNotFound

Exponential backoff

see backoffutil sub package

Notes

errorutil is compatible with https://github.com/pkg/errors :

  err = errors.Wrap(errorutil.RetryableError(err), "some message")
  errorutil.IsRetryable(err) // returns true
*/
package errorutil
<|endoftext|>