<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc (usually run as go doc) accepts zero, one or two arguments.
//
// Zero arguments:
//	go doc
// Show the documentation for the package in the current directory.
//
// One argument:
//	go doc <pkg>
//	go doc <sym>[.<methodOrField>]
//	go doc [<pkg>.]<sym>[.<methodOrField>]
//	go doc [<pkg>.][<sym>.]<methodOrField>
// The first item in this list that succeeds is the one whose documentation
// is printed. If there is a symbol but no package, the package in the current
// directory is chosen. However, if the argument begins with a capital
// letter it is always assumed to be a symbol in the current directory.
//
// Two arguments:
//	go doc <pkg> <sym>[.<methodOrField>]
//
// Show the documentation for the package, symbol, and method or field. The
// first argument must be a full package path. This is similar to the
// command-line usage for the godoc command.
//
// For commands, unless the -cmd flag is present "go doc command"
// shows only the package-level docs for the package.
//
// The -src flag causes doc to print the full source code for the symbol, such
// as the body of a struct, function or method.
//
// The -all flag causes doc to print all documentation for the package and
// all its visible symbols. The argument must identify a package.
//
// For complete documentation, run "go help doc".
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	unexported bool // -u flag
	matchCase  bool // -c flag
	showAll    bool // -all flag
	showCmd    bool // -cmd flag
	showSrc    bool // -src flag
	short      bool // -short flag
)

// usage is a replacement usage function for the flags package.
func usage() {
	fmt.Fprintf(os.Stderr, "Usage of [go] doc:\n")
	fmt.Fprintf(os.Stderr, "\tgo doc\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg>\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc [<pkg>].<sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg> <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "For more information run\n")
	fmt.Fprintf(os.Stderr, "\tgo help doc\n\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("doc: ")
	dirsInit()
	err := do(os.Stdout, flag.CommandLine, os.Args[1:])
	if err != nil {
		log.Fatal(err)
	}
}

// do is the workhorse, broken out of main to make testing easier.
func do(writer io.Writer, flagSet *flag.FlagSet, args []string) (err error) {
	flagSet.Usage = usage
	unexported = false
	matchCase = false
	flagSet.BoolVar(&unexported, "u", false, "show unexported symbols as well as exported")
	flagSet.BoolVar(&matchCase, "c", false, "symbol matching honors case (paths not affected)")
	flagSet.BoolVar(&showAll, "all", false, "show all documentation for package")
	flagSet.BoolVar(&showCmd, "cmd", false, "show symbols with package docs even if package is a command")
	flagSet.BoolVar(&showSrc, "src", false, "show source code for symbol")
	flagSet.BoolVar(&short, "short", false, "one-line representation for each symbol")
	flagSet.Parse(args)
	var paths []string
	var symbol, method string
	// Loop until something is printed.
	dirs.Reset()
	for i := 0; ; i++ {
		buildPackage, userPath, sym, more := parseArgs(flagSet.Args())
		if i > 0 && !more { // Ignore the "more" bit on the first iteration.
			return failMessage(paths, symbol, method)
		}
		if buildPackage == nil {
			return fmt.Errorf("no such package: %s", userPath)
		}
		symbol, method = parseSymbol(sym)
		pkg := parsePackage(writer, buildPackage, userPath)
		paths = append(paths, pkg.prettyPath())

		defer func() {
			pkg.flush()
			e := recover()
			if e == nil {
				return
			}
			pkgError, ok := e.(PackageError)
			if ok {
				err = pkgError
				return
			}
			panic(e)
		}()

		// The builtin package needs special treatment: its symbols are lower
		// case but we want to see them, always.
		if pkg.build.ImportPath == "builtin" {
			unexported = true
		}

		// We have a package.
		if showAll && symbol == "" {
			pkg.allDoc()
			return
		}

		switch {
		case symbol == "":
			pkg.packageDoc() // The package exists, so we got some output.
			return
		case method == "":
			if pkg.symbolDoc(symbol) {
				return
			}
		default:
			if pkg.methodDoc(symbol, method) {
				return
			}
			if pkg.fieldDoc(symbol, method) {
				return
			}
		}
	}
}

// failMessage creates a nicely formatted error message when there is no result to show.
func failMessage(paths []string, symbol, method string) error {
	var b bytes.Buffer
	if len(paths) > 1 {
		b.WriteString("s")
	}
	b.WriteString(" ")
	for i, path := range paths {
		if i > 0 {
			b.WriteString(", ")
		}
		b.WriteString(path)
	}
	if method == "" {
		return fmt.Errorf("no symbol %s in package%s", symbol, &b)
	}
	return fmt.Errorf("no method or field %s.%s in package%s", symbol, method, &b)
}

// parseArgs analyzes the arguments (if any) and returns the package
// it represents, the part of the argument the user used to identify
// the path (or "" if it's the current package) and the symbol
// (possibly with a .method) within that package.
// parseSymbol is used to analyze the symbol itself.
// The boolean final argument reports whether it is possible that
// there may be more directories worth looking at. It will only
// be true if the package path is a partial match for some directory
// and there may be more matches. For example, if the argument
// is rand.Float64, we must scan both crypto/rand and math/rand
// to find the symbol, and the first call will return crypto/rand, true.
func parseArgs(args []string) (pkg *build.Package, path, symbol string, more bool) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	if len(args) == 0 {
		// Easy: current directory.
		return importDir(wd), "", "", false
	}
	arg := args[0]
	// We have an argument. If it is a directory name beginning with . or ..,
	// use the absolute path name. This discriminates "./errors" from "errors"
	// if the current directory contains a non-standard errors package.
	if isDotSlash(arg) {
		arg = filepath.Join(wd, arg)
	}
	switch len(args) {
	default:
		usage()
	case 1:
		// Done below.
	case 2:
		// Package must be findable and importable.
		pkg, err := build.Import(args[0], wd, build.ImportComment)
		if err == nil {
			return pkg, args[0], args[1], false
		}
		for {
			packagePath, ok := findNextPackage(arg)
			if !ok {
				break
			}
			if pkg, err := build.ImportDir(packagePath, build.ImportComment); err == nil {
				return pkg, arg, args[1], true
			}
		}
		return nil, args[0], args[1], false
	}
	// Usual case: one argument.
	// If it contains slashes, it begins with either a package path
	// or an absolute directory.
	// First, is it a complete package path as it is? If so, we are done.
	// This avoids confusion over package paths that have other
	// package paths as their prefix.
	var importErr error
	if filepath.IsAbs(arg) {
		pkg, importErr = build.ImportDir(arg, build.ImportComment)
		if importErr == nil {
			return pkg, arg, "", false
		}
	} else {
		pkg, importErr = build.Import(arg, wd, build.ImportComment)
		if importErr == nil {
			return pkg, arg, "", false
		}
	}
	// Another disambiguator: If the argument starts with an upper
	// case letter, it can only be a symbol in the current directory.
	// Kills the problem caused by case-insensitive file systems
	// matching an upper case name as a package name.
	if !strings.ContainsAny(arg, `/\`) && token.IsExported(arg) {
		pkg, err := build.ImportDir(".", build.ImportComment)
		if err == nil {
			return pkg, "", arg, false
		}
	}
	// If it has a slash, it must be a package path but there is a symbol.
	// It's the last package path we care about.
	slash := strings.LastIndex(arg, "/")
	// There may be periods in the package path before or after the slash
	// and between a symbol and method.
	// Split the string at various periods to see what we find.
	// In general there may be ambiguities but this should almost always
	// work.
	var period int
	// slash+1: if there's no slash, the value is -1 and start is 0; otherwise
	// start is the byte after the slash.
	for start := slash + 1; start < len(arg); start = period + 1 {
		period = strings.Index(arg[start:], ".")
		symbol := ""
		if period < 0 {
			period = len(arg)
		} else {
			period += start
			symbol = arg[period+1:]
		}
		// Have we identified a package already?
		pkg, err := build.Import(arg[0:period], wd, build.ImportComment)
		if err == nil {
			return pkg, arg[0:period], symbol, false
		}
		// See if we have the basename or tail of a package, as in json for encoding/json
		// or ivy/value for robpike.io/ivy/value.
		pkgName := arg[:period]
		for {
			path, ok := findNextPackage(pkgName)
			if !ok {
				break
			}
			if pkg, err = build.ImportDir(path, build.ImportComment); err == nil {
				return pkg, arg[0:period], symbol, true
			}
		}
		dirs.Reset() // Next iteration of for loop must scan all the directories again.
	}
	// If it has a slash, we've failed.
	if slash >= 0 {
		// build.Import should always include the path in its error message,
		// and we should avoid repeating it. Unfortunately, build.Import doesn't
		// return a structured error. That can't easily be fixed, since it
		// invokes 'go list' and returns the error text from the loaded package.
		// TODO(golang.org/issue/34750): load using golang.org/x/tools/go/packages
		// instead of go/build.
		importErrStr := importErr.Error()
		if strings.Contains(importErrStr, arg[:period]) {
			log.Fatal(importErrStr)
		} else {
			log.Fatalf("no such package %s: %s", arg[:period], importErrStr)
		}
	}
	// Guess it's a symbol in the current directory.
	return importDir(wd), "", arg, false
}

// dotPaths lists all the dotted paths legal on Unix-like and
// Windows-like file systems. We check them all, as the chance
// of error is minute and even on Windows people will use ./
// sometimes.
var dotPaths = []string{
	`./`,
	`../`,
	`.\`,
	`..\`,
}

// isDotSlash reports whether the path begins with a reference
// to the local . or .. directory.
func isDotSlash(arg string) bool {
	if arg == "." || arg == ".." {
		return true
	}
	for _, dotPath := range dotPaths {
		if strings.HasPrefix(arg, dotPath) {
			return true
		}
	}
	return false
}

// importDir is just an error-catching wrapper for build.ImportDir.
func importDir(dir string) *build.Package {
	pkg, err := build.ImportDir(dir, build.ImportComment)
	if err != nil {
		log.Fatal(err)
	}
	return pkg
}

// parseSymbol breaks str apart into a symbol and method.
// Both may be missing or the method may be missing.
// If present, each must be a valid Go identifier.
func parseSymbol(str string) (symbol, method string) {
	if str == "" {
		return
	}
	elem := strings.Split(str, ".")
	switch len(elem) {
	case 1:
	case 2:
		method = elem[1]
		if !token.IsIdentifier(method) {
			log.Fatalf("invalid identifier %q", method)
		}
	default:
		log.Printf("too many periods in symbol specification")
		usage()
	}
	symbol = elem[0]
	if !token.IsIdentifier(symbol) {
		log.Fatalf("invalid identifier %q", symbol)
	}
	return
}

// isExported reports whether the name is an exported identifier.
// If the unexported flag (-u) is true, isExported returns true because
// it means that we treat the name as if it is exported.
func isExported(name string) bool {
	return unexported || token.IsExported(name)
}

// findNextPackage returns the next full file name path that matches the
// (perhaps partial) package path pkg. The boolean reports if any match was found.
func findNextPackage(pkg string) (string, bool) {
	if filepath.IsAbs(pkg) {
		if dirs.offset == 0 {
			dirs.offset = -1
			return pkg, true
		}
		return "", false
	}
	if pkg == "" || token.IsExported(pkg) { // Upper case symbol cannot be a package name.
		return "", false
	}
	pkg = path.Clean(pkg)
	pkgSuffix := "/" + pkg
	for {
		d, ok := dirs.Next()
		if !ok {
			return "", false
		}
		if d.importPath == pkg || strings.HasSuffix(d.importPath, pkgSuffix) {
			return d.dir, true
		}
	}
}

var buildCtx = build.Default

// splitGopath splits $GOPATH into a list of roots.
func splitGopath() []string {
	return filepath.SplitList(buildCtx.GOPATH)
}
<commit_msg>cmd/doc: don't bother checking validity of symbols at top level<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc (usually run as go doc) accepts zero, one or two arguments.
//
// Zero arguments:
//	go doc
// Show the documentation for the package in the current directory.
//
// One argument:
//	go doc <pkg>
//	go doc <sym>[.<methodOrField>]
//	go doc [<pkg>.]<sym>[.<methodOrField>]
//	go doc [<pkg>.][<sym>.]<methodOrField>
// The first item in this list that succeeds is the one whose documentation
// is printed. If there is a symbol but no package, the package in the current
// directory is chosen. However, if the argument begins with a capital
// letter it is always assumed to be a symbol in the current directory.
//
// Two arguments:
//	go doc <pkg> <sym>[.<methodOrField>]
//
// Show the documentation for the package, symbol, and method or field. The
// first argument must be a full package path. This is similar to the
// command-line usage for the godoc command.
//
// For commands, unless the -cmd flag is present "go doc command"
// shows only the package-level docs for the package.
//
// The -src flag causes doc to print the full source code for the symbol, such
// as the body of a struct, function or method.
//
// The -all flag causes doc to print all documentation for the package and
// all its visible symbols. The argument must identify a package.
//
// For complete documentation, run "go help doc".
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	unexported bool // -u flag
	matchCase  bool // -c flag
	showAll    bool // -all flag
	showCmd    bool // -cmd flag
	showSrc    bool // -src flag
	short      bool // -short flag
)

// usage is a replacement usage function for the flags package.
func usage() {
	fmt.Fprintf(os.Stderr, "Usage of [go] doc:\n")
	fmt.Fprintf(os.Stderr, "\tgo doc\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg>\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc [<pkg>].<sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "\tgo doc <pkg> <sym>[.<method>]\n")
	fmt.Fprintf(os.Stderr, "For more information run\n")
	fmt.Fprintf(os.Stderr, "\tgo help doc\n\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("doc: ")
	dirsInit()
	err := do(os.Stdout, flag.CommandLine, os.Args[1:])
	if err != nil {
		log.Fatal(err)
	}
}

// do is the workhorse, broken out of main to make testing easier.
func do(writer io.Writer, flagSet *flag.FlagSet, args []string) (err error) {
	flagSet.Usage = usage
	unexported = false
	matchCase = false
	flagSet.BoolVar(&unexported, "u", false, "show unexported symbols as well as exported")
	flagSet.BoolVar(&matchCase, "c", false, "symbol matching honors case (paths not affected)")
	flagSet.BoolVar(&showAll, "all", false, "show all documentation for package")
	flagSet.BoolVar(&showCmd, "cmd", false, "show symbols with package docs even if package is a command")
	flagSet.BoolVar(&showSrc, "src", false, "show source code for symbol")
	flagSet.BoolVar(&short, "short", false, "one-line representation for each symbol")
	flagSet.Parse(args)
	var paths []string
	var symbol, method string
	// Loop until something is printed.
	dirs.Reset()
	for i := 0; ; i++ {
		buildPackage, userPath, sym, more := parseArgs(flagSet.Args())
		if i > 0 && !more { // Ignore the "more" bit on the first iteration.
			return failMessage(paths, symbol, method)
		}
		if buildPackage == nil {
			return fmt.Errorf("no such package: %s", userPath)
		}
		symbol, method = parseSymbol(sym)
		pkg := parsePackage(writer, buildPackage, userPath)
		paths = append(paths, pkg.prettyPath())

		defer func() {
			pkg.flush()
			e := recover()
			if e == nil {
				return
			}
			pkgError, ok := e.(PackageError)
			if ok {
				err = pkgError
				return
			}
			panic(e)
		}()

		// The builtin package needs special treatment: its symbols are lower
		// case but we want to see them, always.
		if pkg.build.ImportPath == "builtin" {
			unexported = true
		}

		// We have a package.
		if showAll && symbol == "" {
			pkg.allDoc()
			return
		}

		switch {
		case symbol == "":
			pkg.packageDoc() // The package exists, so we got some output.
			return
		case method == "":
			if pkg.symbolDoc(symbol) {
				return
			}
		default:
			if pkg.methodDoc(symbol, method) {
				return
			}
			if pkg.fieldDoc(symbol, method) {
				return
			}
		}
	}
}

// failMessage creates a nicely formatted error message when there is no result to show.
func failMessage(paths []string, symbol, method string) error {
	var b bytes.Buffer
	if len(paths) > 1 {
		b.WriteString("s")
	}
	b.WriteString(" ")
	for i, path := range paths {
		if i > 0 {
			b.WriteString(", ")
		}
		b.WriteString(path)
	}
	if method == "" {
		return fmt.Errorf("no symbol %s in package%s", symbol, &b)
	}
	return fmt.Errorf("no method or field %s.%s in package%s", symbol, method, &b)
}

// parseArgs analyzes the arguments (if any) and returns the package
// it represents, the part of the argument the user used to identify
// the path (or "" if it's the current package) and the symbol
// (possibly with a .method) within that package.
// parseSymbol is used to analyze the symbol itself.
// The boolean final argument reports whether it is possible that
// there may be more directories worth looking at. It will only
// be true if the package path is a partial match for some directory
// and there may be more matches. For example, if the argument
// is rand.Float64, we must scan both crypto/rand and math/rand
// to find the symbol, and the first call will return crypto/rand, true.
func parseArgs(args []string) (pkg *build.Package, path, symbol string, more bool) {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}
	if len(args) == 0 {
		// Easy: current directory.
		return importDir(wd), "", "", false
	}
	arg := args[0]
	// We have an argument. If it is a directory name beginning with . or ..,
	// use the absolute path name. This discriminates "./errors" from "errors"
	// if the current directory contains a non-standard errors package.
	if isDotSlash(arg) {
		arg = filepath.Join(wd, arg)
	}
	switch len(args) {
	default:
		usage()
	case 1:
		// Done below.
	case 2:
		// Package must be findable and importable.
		pkg, err := build.Import(args[0], wd, build.ImportComment)
		if err == nil {
			return pkg, args[0], args[1], false
		}
		for {
			packagePath, ok := findNextPackage(arg)
			if !ok {
				break
			}
			if pkg, err := build.ImportDir(packagePath, build.ImportComment); err == nil {
				return pkg, arg, args[1], true
			}
		}
		return nil, args[0], args[1], false
	}
	// Usual case: one argument.
	// If it contains slashes, it begins with either a package path
	// or an absolute directory.
	// First, is it a complete package path as it is? If so, we are done.
	// This avoids confusion over package paths that have other
	// package paths as their prefix.
	var importErr error
	if filepath.IsAbs(arg) {
		pkg, importErr = build.ImportDir(arg, build.ImportComment)
		if importErr == nil {
			return pkg, arg, "", false
		}
	} else {
		pkg, importErr = build.Import(arg, wd, build.ImportComment)
		if importErr == nil {
			return pkg, arg, "", false
		}
	}
	// Another disambiguator: If the argument starts with an upper
	// case letter, it can only be a symbol in the current directory.
	// Kills the problem caused by case-insensitive file systems
	// matching an upper case name as a package name.
	if !strings.ContainsAny(arg, `/\`) && token.IsExported(arg) {
		pkg, err := build.ImportDir(".", build.ImportComment)
		if err == nil {
			return pkg, "", arg, false
		}
	}
	// If it has a slash, it must be a package path but there is a symbol.
	// It's the last package path we care about.
	slash := strings.LastIndex(arg, "/")
	// There may be periods in the package path before or after the slash
	// and between a symbol and method.
	// Split the string at various periods to see what we find.
	// In general there may be ambiguities but this should almost always
	// work.
	var period int
	// slash+1: if there's no slash, the value is -1 and start is 0; otherwise
	// start is the byte after the slash.
	for start := slash + 1; start < len(arg); start = period + 1 {
		period = strings.Index(arg[start:], ".")
		symbol := ""
		if period < 0 {
			period = len(arg)
		} else {
			period += start
			symbol = arg[period+1:]
		}
		// Have we identified a package already?
		pkg, err := build.Import(arg[0:period], wd, build.ImportComment)
		if err == nil {
			return pkg, arg[0:period], symbol, false
		}
		// See if we have the basename or tail of a package, as in json for encoding/json
		// or ivy/value for robpike.io/ivy/value.
		pkgName := arg[:period]
		for {
			path, ok := findNextPackage(pkgName)
			if !ok {
				break
			}
			if pkg, err = build.ImportDir(path, build.ImportComment); err == nil {
				return pkg, arg[0:period], symbol, true
			}
		}
		dirs.Reset() // Next iteration of for loop must scan all the directories again.
	}
	// If it has a slash, we've failed.
	if slash >= 0 {
		// build.Import should always include the path in its error message,
		// and we should avoid repeating it. Unfortunately, build.Import doesn't
		// return a structured error. That can't easily be fixed, since it
		// invokes 'go list' and returns the error text from the loaded package.
		// TODO(golang.org/issue/34750): load using golang.org/x/tools/go/packages
		// instead of go/build.
		importErrStr := importErr.Error()
		if strings.Contains(importErrStr, arg[:period]) {
			log.Fatal(importErrStr)
		} else {
			log.Fatalf("no such package %s: %s", arg[:period], importErrStr)
		}
	}
	// Guess it's a symbol in the current directory.
	return importDir(wd), "", arg, false
}

// dotPaths lists all the dotted paths legal on Unix-like and
// Windows-like file systems. We check them all, as the chance
// of error is minute and even on Windows people will use ./
// sometimes.
var dotPaths = []string{
	`./`,
	`../`,
	`.\`,
	`..\`,
}

// isDotSlash reports whether the path begins with a reference
// to the local . or .. directory.
func isDotSlash(arg string) bool {
	if arg == "." || arg == ".." {
		return true
	}
	for _, dotPath := range dotPaths {
		if strings.HasPrefix(arg, dotPath) {
			return true
		}
	}
	return false
}

// importDir is just an error-catching wrapper for build.ImportDir.
func importDir(dir string) *build.Package {
	pkg, err := build.ImportDir(dir, build.ImportComment)
	if err != nil {
		log.Fatal(err)
	}
	return pkg
}

// parseSymbol breaks str apart into a symbol and method.
// Both may be missing or the method may be missing.
// If present, each must be a valid Go identifier.
func parseSymbol(str string) (symbol, method string) {
	if str == "" {
		return
	}
	elem := strings.Split(str, ".")
	switch len(elem) {
	case 1:
	case 2:
		method = elem[1]
	default:
		log.Printf("too many periods in symbol specification")
		usage()
	}
	symbol = elem[0]
	return
}

// isExported reports whether the name is an exported identifier.
// If the unexported flag (-u) is true, isExported returns true because
// it means that we treat the name as if it is exported.
func isExported(name string) bool {
	return unexported || token.IsExported(name)
}

// findNextPackage returns the next full file name path that matches the
// (perhaps partial) package path pkg. The boolean reports if any match was found.
func findNextPackage(pkg string) (string, bool) {
	if filepath.IsAbs(pkg) {
		if dirs.offset == 0 {
			dirs.offset = -1
			return pkg, true
		}
		return "", false
	}
	if pkg == "" || token.IsExported(pkg) { // Upper case symbol cannot be a package name.
		return "", false
	}
	pkg = path.Clean(pkg)
	pkgSuffix := "/" + pkg
	for {
		d, ok := dirs.Next()
		if !ok {
			return "", false
		}
		if d.importPath == pkg || strings.HasSuffix(d.importPath, pkgSuffix) {
			return d.dir, true
		}
	}
}

var buildCtx = build.Default

// splitGopath splits $GOPATH into a list of roots.
func splitGopath() []string {
	return filepath.SplitList(buildCtx.GOPATH)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdClean = &Command{
	UsageLine: "clean [-i] [-r] [-n] [-x] [importpath...]",
	Short:     "remove object files",
	Long: `
Clean removes object files from package source directories.
The go command builds most objects in a temporary directory,
so go clean is mainly concerned with object files left by other
tools or by manual invocations of go build.

Specifically, clean removes the following files from each of the
source directories corresponding to the import paths:

	_obj/            old object directory, left from Makefiles
	_test/           old test directory, left from Makefiles
	_testmain.go     old gotest file, left from Makefiles
	test.out         old test log, left from Makefiles
	build.out        old test log, left from Makefiles
	*.[568ao]        object files, left from Makefiles

	DIR(.exe)        from go build
	DIR.test(.exe)   from go test -c
	MAINFILE(.exe)   from go build MAINFILE.go

In the list, DIR represents the final path element of the
directory, and MAINFILE is the base name of any Go source
file in the directory that is not included when building
the package.

The -i flag causes clean to remove the corresponding installed
archive or binary (what 'go install' would create).

The -n flag causes clean to print the remove commands it would execute,
but not run them.

The -r flag causes clean to be applied recursively to all the
dependencies of the packages named by the import paths.

The -x flag causes clean to print remove commands as it executes them.
	`,
}

var cleanI bool // clean -i flag
var cleanN bool // clean -n flag
var cleanR bool // clean -r flag
var cleanX bool // clean -x flag

func init() {
	// break init cycle
	cmdClean.Run = runClean

	cmdClean.Flag.BoolVar(&cleanI, "i", false, "")
	cmdClean.Flag.BoolVar(&cleanN, "n", false, "")
	cmdClean.Flag.BoolVar(&cleanR, "r", false, "")
	cmdClean.Flag.BoolVar(&cleanX, "x", false, "")
}

func runClean(cmd *Command, args []string) {
	for _, pkg := range packagesAndErrors(args) {
		clean(pkg)
	}
}

var cleaned = map[*Package]bool{}

// TODO: These are dregs left by Makefile-based builds.
// Eventually, can stop deleting these.
var cleanDir = map[string]bool{
	"_test": true,
	"_obj":  true,
}

var cleanFile = map[string]bool{
	"_testmain.go": true,
	"test.out":     true,
	"build.out":    true,
	"a.out":        true,
}

var cleanExt = map[string]bool{
	".5": true,
	".6": true,
	".8": true,
	".a": true,
	".o": true,
}

func clean(p *Package) {
	if cleaned[p] {
		return
	}
	if p.Dir == "" {
		errorf("can't load package: %v", p.Error)
		return
	}
	dirs, err := ioutil.ReadDir(p.Dir)
	if err != nil {
		errorf("%v", err)
		return
	}

	var b builder

	packageFile := map[string]bool{}
	if p.Name != "main" {
		// Record which files are not in package main.
		// The others are.
		keep := func(list []string) {
			for _, f := range list {
				packageFile[f] = true
			}
		}
		keep(p.GoFiles)
		keep(p.CgoFiles)
		keep(p.TestGoFiles)
		keep(p.XTestGoFiles)
	}

	_, elem := filepath.Split(p.Dir)
	allRemove := []string{
		elem,
		elem + ".exe",
		elem + ".test",
		elem + ".test.exe",
	}
	for _, dir := range dirs {
		name := dir.Name()
		if packageFile[name] {
			continue
		}
		if !dir.IsDir() && strings.HasSuffix(name, ".go") {
			base := name[:len(name)-len(".go")]
			allRemove = append(allRemove, base, base+".exe")
		}
	}
	if cleanN || cleanX {
		b.showcmd(p.Dir, "rm %s", strings.Join(allRemove, " "))
	}

	toRemove := map[string]bool{}
	for _, name := range allRemove {
		toRemove[name] = true
	}
	for _, dir := range dirs {
		name := dir.Name()
		if dir.IsDir() {
			// TODO: Remove once Makefiles are forgotten.
			if cleanDir[name] {
				if cleanN || cleanX {
					b.showcmd(p.Dir, "rm -r %s", name)
					if cleanN {
						continue
					}
				}
				os.RemoveAll(filepath.Join(p.Dir, name))
			}
			continue
		}

		if cleanN {
			continue
		}

		if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] {
			os.Remove(filepath.Join(p.Dir, name))
		}
	}

	if cleanI && p.target != "" {
		if cleanN || cleanX {
			b.showcmd("", "rm %s", p.target)
		}
		if !cleanN {
			os.Remove(p.target)
		}
	}

	if cleanR {
		for _, p1 := range p.imports {
			clean(p1)
		}
	}
}
<commit_msg>cmd/go: fixed panic on `go clean -n` and `go clean -x`.         also made generated scripts act more like running go clean itself<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var cmdClean = &Command{
	UsageLine: "clean [-i] [-r] [-n] [-x] [importpath...]",
	Short:     "remove object files",
	Long: `
Clean removes object files from package source directories.
The go command builds most objects in a temporary directory,
so go clean is mainly concerned with object files left by other
tools or by manual invocations of go build.

Specifically, clean removes the following files from each of the
source directories corresponding to the import paths:

	_obj/            old object directory, left from Makefiles
	_test/           old test directory, left from Makefiles
	_testmain.go     old gotest file, left from Makefiles
	test.out         old test log, left from Makefiles
	build.out        old test log, left from Makefiles
	*.[568ao]        object files, left from Makefiles

	DIR(.exe)        from go build
	DIR.test(.exe)   from go test -c
	MAINFILE(.exe)   from go build MAINFILE.go

In the list, DIR represents the final path element of the
directory, and MAINFILE is the base name of any Go source
file in the directory that is not included when building
the package.

The -i flag causes clean to remove the corresponding installed
archive or binary (what 'go install' would create).

The -n flag causes clean to print the remove commands it would execute,
but not run them.

The -r flag causes clean to be applied recursively to all the
dependencies of the packages named by the import paths.

The -x flag causes clean to print remove commands as it executes them.
	`,
}

var cleanI bool // clean -i flag
var cleanN bool // clean -n flag
var cleanR bool // clean -r flag
var cleanX bool // clean -x flag

func init() {
	// break init cycle
	cmdClean.Run = runClean

	cmdClean.Flag.BoolVar(&cleanI, "i", false, "")
	cmdClean.Flag.BoolVar(&cleanN, "n", false, "")
	cmdClean.Flag.BoolVar(&cleanR, "r", false, "")
	cmdClean.Flag.BoolVar(&cleanX, "x", false, "")
}

func runClean(cmd *Command, args []string) {
	for _, pkg := range packagesAndErrors(args) {
		clean(pkg)
	}
}

var cleaned = map[*Package]bool{}

// TODO: These are dregs left by Makefile-based builds.
// Eventually, can stop deleting these.
var cleanDir = map[string]bool{
	"_test": true,
	"_obj":  true,
}

var cleanFile = map[string]bool{
	"_testmain.go": true,
	"test.out":     true,
	"build.out":    true,
	"a.out":        true,
}

var cleanExt = map[string]bool{
	".5": true,
	".6": true,
	".8": true,
	".a": true,
	".o": true,
}

func clean(p *Package) {
	if cleaned[p] {
		return
	}
	if p.Dir == "" {
		errorf("can't load package: %v", p.Error)
		return
	}
	dirs, err := ioutil.ReadDir(p.Dir)
	if err != nil {
		errorf("%v", err)
		return
	}

	var b builder
	b.print = fmt.Print

	packageFile := map[string]bool{}
	if p.Name != "main" {
		// Record which files are not in package main.
		// The others are.
		keep := func(list []string) {
			for _, f := range list {
				packageFile[f] = true
			}
		}
		keep(p.GoFiles)
		keep(p.CgoFiles)
		keep(p.TestGoFiles)
		keep(p.XTestGoFiles)
	}

	_, elem := filepath.Split(p.Dir)
	allRemove := []string{
		elem,
		elem + ".exe",
		elem + ".test",
		elem + ".test.exe",
	}
	for _, dir := range dirs {
		name := dir.Name()
		if packageFile[name] {
			continue
		}
		if !dir.IsDir() && strings.HasSuffix(name, ".go") {
			base := name[:len(name)-len(".go")]
			allRemove = append(allRemove, base, base+".exe")
		}
	}
	if cleanN || cleanX {
		b.showcmd(p.Dir, "rm -f %s", strings.Join(allRemove, " "))
	}

	toRemove := map[string]bool{}
	for _, name := range allRemove {
		toRemove[name] = true
	}
	for _, dir := range dirs {
		name := dir.Name()
		if dir.IsDir() {
			// TODO: Remove once Makefiles are forgotten.
			if cleanDir[name] {
				if cleanN || cleanX {
					b.showcmd(p.Dir, "rm -r %s", name)
					if cleanN {
						continue
					}
				}
				os.RemoveAll(filepath.Join(p.Dir, name))
			}
			continue
		}

		if cleanN {
			continue
		}

		if cleanFile[name] || cleanExt[filepath.Ext(name)] || toRemove[name] {
			os.Remove(filepath.Join(p.Dir, name))
		}
	}

	if cleanI && p.target != "" {
		if cleanN || cleanX {
			b.showcmd("", "rm -f %s", p.target)
		}
		if !cleanN {
			os.Remove(p.target)
		}
	}

	if cleanR {
		for _, p1 := range p.imports {
			clean(p1)
		}
	}
}
<|endoftext|>


<commit_before>// Package pomf provides a simple file hosting pomf compatible web application
package pomf

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/dchest/uniuri"
	_ "github.com/go-sql-driver/mysql"
)

const (
	// Length is used to specify filename length
	LENGTH = 6
	// port is the port pomf will listen on
	PORT = ":8080"
	// UPDIRECTORY specifies what directory pomf will upload to
	UPDIRECTORY = "/tmp/"
	// Kawaii grill directory
	GRILLDIRECTORY = "pomf/build/img/"
	// Static pomf directory
	POMFDIRECTORY = "pomf/build"
	// Domain to serve static files from
	UPADDRESS = "http://localhost"
	// Database username
	dbUSERNAME = ""
	// database name
	dbNAME = ""
	// database password
	dbPASSWORD = ""
	// DATABASE connection constant
	DATABASE = dbUSERNAME + ":" + dbPASSWORD + "@/" + dbNAME + "?charset=utf8"
	// MAXSIZE in bytes
	MAXSIZE = 10 * 1024 * 1024
)

// Result information struct
type Result struct {
	URL  string `json:"url"`
	Name string `json:"name"`
	Hash string `json:"hash"`
	Size int64  `json:"size"`
}

// Pomf compatible Response struct
type Response struct {
	Success     bool     `json:"success"`
	ErrorCode   int      `json:"errorcode,omitempty"`
	Description string   `json:"description,omitempty"`
	Files       []Result `json:"files,omitempty"`
}

// generateName returns a random string that isn't in the database
func generateName() (string, error) {
	name := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		return "", err
	}
	var id string
	err = db.QueryRow("select id from files where id=?", name).Scan(&id)
	if err != sql.ErrNoRows {
		generateName()
	}
	db.Close()

	return name, nil
}

// respond
func respond(w http.ResponseWriter, output string, resp Response) {
	if resp.ErrorCode != 0 {
		resp.Files = []Result{}
		resp.Success = false
	} else {
		resp.Success = true
	}

	switch output {
	case "json":

		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	case "xml":
		x, err := xml.MarshalIndent(resp, "", "  ")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/xml")
		w.Write(x)

	case "html":
		w.Header().Set("Content-Type", "text/html")
		for _, file := range resp.Files {
			io.WriteString(w, ""+file.URL+"
")
		}

	case "gyazo", "text":
		w.Header().Set("Content-Type", "plain/text")
		for _, file := range resp.Files {
			io.WriteString(w, file.URL+"\n")
		}

	case "csv":
		w.Header().Set("Content-Type", "text/csv")
		io.WriteString(w, "name, url, hash, size\n")
		for _, file := range resp.Files {
			io.WriteString(w, file.Name+","+file.URL+","+file.Hash+","+strconv.FormatInt(file.Size, 10)+"\n")
		}

	default:
		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

	}

}

func grillHandler(w http.ResponseWriter, r *http.Request) {
	kawaii, err := ioutil.ReadDir(GRILLDIRECTORY)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	http.Redirect(w, r, "/img/"+kawaii[rand.Intn(len(kawaii))].Name(), http.StatusFound)
}

func uploadHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	output := r.FormValue("output")
	reader, err := r.MultipartReader()
	resp := Response{Files: []Result{}}
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		respond(w, output, resp)
		return
	}

	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		return
	}

	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}

		if part.FileName() == "" {
			continue
		}

		s, err := generateName()
		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			break
		}
		extName := filepath.Ext(part.FileName())
		filename := s + extName
		dst, err := os.Create(UPDIRECTORY + filename)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		defer dst.Close()

		h := sha1.New()
		t := io.TeeReader(part, h)
		_, err = io.Copy(dst, t)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		hash := h.Sum(nil)
		sha1 := base64.URLEncoding.EncodeToString(hash)
		stat, _ := dst.Stat()
		size := stat.Size()
		if size > MAXSIZE {
			resp.ErrorCode = http.StatusRequestEntityTooLarge
			resp.Description = err.Error()
			break
		}

		originalname := part.FileName()
		err = db.QueryRow("select originalname, filename, size from files where hash=?", sha1).Scan(&originalname, &filename, &size)
		res := Result{
			URL:  UPADDRESS + "/" + filename,
			Name: originalname,
			Hash: sha1,
			Size: size,
		}
		if err == sql.ErrNoRows {
			query, err := db.Prepare("INSERT into files(hash, originalname, filename, size, date) values(?, ?, ?, ?, ?)")
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
			_, err = query.Exec(res.Hash, res.Name, filename, res.Size, time.Now().Format("2016-01-02"))
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
		}
		resp.Files = append(resp.Files, res)
	}
	respond(w, output, resp)
}

func main() {
	http.HandleFunc("/upload.php", uploadHandler)
	http.HandleFunc("/grill.php", grillHandler)
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(POMFDIRECTORY))))
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<commit_msg>Add more documentation<commit_after>// Package pomf provides a simple file hosting pomf compatible web application
package pomf

import (
	"crypto/sha1"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"github.com/dchest/uniuri"
	_ "github.com/go-sql-driver/mysql"
)

const (
	// Length is used to specify filename length
	LENGTH = 6
	// port is the port pomf will listen on
	PORT = ":8080"
	// UPDIRECTORY specifies what directory pomf will upload to
	UPDIRECTORY = "/tmp/"
	// Kawaii grill directory
	GRILLDIRECTORY = "pomf/build/img/"
	// Static pomf directory
	POMFDIRECTORY = "pomf/build"
	// Domain to serve static files from
	UPADDRESS = "http://localhost"
	// Database username
	dbUSERNAME = ""
	// database name
	dbNAME = ""
	// database password
	dbPASSWORD = ""
	// DATABASE connection constant
	DATABASE = dbUSERNAME + ":" + dbPASSWORD + "@/" + dbNAME + "?charset=utf8"
	// MAXSIZE in bytes
	MAXSIZE = 10 * 1024 * 1024
)

// Result information struct
type Result struct {
	URL  string `json:"url"`
	Name string `json:"name"`
	Hash string `json:"hash"`
	Size int64  `json:"size"`
}

// Pomf compatible Response struct
type Response struct {
	Success     bool     `json:"success"`
	ErrorCode   int      `json:"errorcode,omitempty"`
	Description string   `json:"description,omitempty"`
	Files       []Result `json:"files,omitempty"`
}

// generateName returns a random string that isn't in the database
func generateName() (string, error) {
	name := uniuri.NewLen(LENGTH)
	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		return "", err
	}
	var id string
	err = db.QueryRow("select id from files where id=?", name).Scan(&id)
	if err != sql.ErrNoRows {
		generateName()
	}
	db.Close()

	return name, nil
}

// respond outputs Response struct in user specified formats
// supported formats are xml, json, text and html
func respond(w http.ResponseWriter, output string, resp Response) {
	if resp.ErrorCode != 0 {
		resp.Files = []Result{}
		resp.Success = false
	} else {
		resp.Success = true
	}

	switch output {
	case "json":

		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
	case "xml":
		x, err := xml.MarshalIndent(resp, "", "  ")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/xml")
		w.Write(x)

	case "html":
		w.Header().Set("Content-Type", "text/html")
		for _, file := range resp.Files {
			io.WriteString(w, ""+file.URL+"
")
		}

	case "gyazo", "text":
		w.Header().Set("Content-Type", "plain/text")
		for _, file := range resp.Files {
			io.WriteString(w, file.URL+"\n")
		}

	case "csv":
		w.Header().Set("Content-Type", "text/csv")
		io.WriteString(w, "name, url, hash, size\n")
		for _, file := range resp.Files {
			io.WriteString(w, file.Name+","+file.URL+","+file.Hash+","+strconv.FormatInt(file.Size, 10)+"\n")
		}

	default:
		w.Header().Set("Content-Type", "application/json")
		err := json.NewEncoder(w).Encode(resp)

		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

	}

}

// grillHandler randomly selects a kawaii grill
func grillHandler(w http.ResponseWriter, r *http.Request) {
	kawaii, err := ioutil.ReadDir(GRILLDIRECTORY)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	http.Redirect(w, r, "/img/"+kawaii[rand.Intn(len(kawaii))].Name(), http.StatusFound)
}

// uploadHandler constructs the response struct
func uploadHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	output := r.FormValue("output")
	reader, err := r.MultipartReader()
	resp := Response{Files: []Result{}}
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		respond(w, output, resp)
		return
	}

	db, err := sql.Open("mysql", DATABASE)
	if err != nil {
		resp.ErrorCode = http.StatusInternalServerError
		resp.Description = err.Error()
		return
	}

	for {
		part, err := reader.NextPart()
		if err == io.EOF {
			break
		}

		if part.FileName() == "" {
			continue
		}

		s, err := generateName()
		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			break
		}
		extName := filepath.Ext(part.FileName())
		filename := s + extName
		dst, err := os.Create(UPDIRECTORY + filename)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		defer dst.Close()

		h := sha1.New()
		t := io.TeeReader(part, h)
		_, err = io.Copy(dst, t)

		if err != nil {
			resp.ErrorCode = http.StatusInternalServerError
			resp.Description = err.Error()
			respond(w, output, resp)
			break
		}
		hash := h.Sum(nil)
		sha1 := base64.URLEncoding.EncodeToString(hash)
		stat, _ := dst.Stat()
		size := stat.Size()
		if size > MAXSIZE {
			resp.ErrorCode = http.StatusRequestEntityTooLarge
			resp.Description = err.Error()
			break
		}

		originalname := part.FileName()
		err = db.QueryRow("select originalname, filename, size from files where hash=?", sha1).Scan(&originalname, &filename, &size)
		res := Result{
			URL:  UPADDRESS + "/" + filename,
			Name: originalname,
			Hash: sha1,
			Size: size,
		}
		if err == sql.ErrNoRows {
			query, err := db.Prepare("INSERT into files(hash, originalname, filename, size, date) values(?, ?, ?, ?, ?)")
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
			_, err = query.Exec(res.Hash, res.Name, filename, res.Size, time.Now().Format("2016-01-02"))
			if err != nil {
				resp.ErrorCode = http.StatusInternalServerError
				resp.Description = err.Error()
				break
			}
		}
		resp.Files = append(resp.Files, res)
	}
	respond(w, output, resp)
}

func main() {
	http.HandleFunc("/upload.php", uploadHandler)
	http.HandleFunc("/grill.php", grillHandler)
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(POMFDIRECTORY))))
	err := http.ListenAndServe(PORT, nil)
	if err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>// vim: tabstop=2 shiftwidth=2

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net/mail"
	"os"
	"path"
	"strings"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/idlog"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

func poolOutboundSend() {
	var err error
	var filenames []string
	if flag_send || flag_client {
		// Read all the pool files
		filenames, err = readDir(cfg.Files.Pooldir, "m")
	} else {
		// Read dynamic mix of outbound files from the Pool
		filenames, err = poolRead()
	}
	if err != nil {
		Warn.Printf("Reading pool failed: %s", err)
		return
	}
	for _, file := range filenames {
		err = mailPoolFile(path.Join(cfg.Files.Pooldir, file))
		if err != nil {
			Warn.Printf("Pool mailing failed: %s", err)
		} else {
			poolDelete(file)
		}
	}
}

// poolRead returns a dynamic Mix of filenames from the outbound pool.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	if poolSize > 0 {
		Trace.Printf("Processing %d pool messages.\n", poolSize)
	}
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) (err error) {
	// Delete a pool file
	err = os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	}
	return
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(
	public *keymgr.Pubring,
	secret *keymgr.Secring,
	id idlog.IDLog,
	chunkDB Chunk) (err error) {

	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	if len(keys) > 0 {
		Trace.Printf("Reading %d messages from %s\n", len(keys), cfg.Files.Maildir)
	}
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err != nil {
				Info.Println(err)
			}
		} else {
			// It's not a remailer-foo request so assume a remailer message
			var mailMsg *mail.Message
			mailMsg, err := dir.Message(key)
			if err != nil {
				Warn.Printf("%s: Reading message failed with: %s", key, err)
				continue
			}
			var msg []byte
			// Convert the armored Yamn message to its byte components
			msg, err = stripArmor(mailMsg.Body)
			if err != nil {
				Info.Println(err)
			}
			if msg == nil {
				Warn.Println("Dearmor returned zero bytes")
				continue
			}
			err = decodeMsg(msg, public, secret, id, chunkDB)
			if err != nil {
				Info.Println(err)
			}
		} // End of remailer-foo or remailer message
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(
	prefix string,
	public *keymgr.Pubring,
	secret *keymgr.Secring,
	id idlog.IDLog,
	chunkDB Chunk) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, public, secret, id, chunkDB)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// PoolWrite writes a raw Byte Yamn message to the Outbound pool with a prefix
// string.
func poolWrite(yamnMsg []byte, prefix string) (poolFileName string) {
	/*
		Currently supported prefixs are:-
		[ m              Oubound message (final or intermediate) ]
		[ i          Inbound message (destined for this remailer ]
		[ p               Partial message chunk needing assembly ]
	*/
	digest := sha256.New()
	digest.Write(yamnMsg)
	poolFileName = prefix + hex.EncodeToString(digest.Sum(nil))[:14]
	fqfn := path.Join(cfg.Files.Pooldir, poolFileName)
	err := ioutil.WriteFile(fqfn, yamnMsg, 0600)
	if err != nil {
		panic(err)
	}
	return
}
<commit_msg>Revert to unconditionally deleting pool files<commit_after>// vim: tabstop=2 shiftwidth=2

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net/mail"
	"os"
	"path"
	"strings"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/idlog"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

func poolOutboundSend() {
	var err error
	var filenames []string
	if flag_send || flag_client {
		// Read all the pool files
		filenames, err = readDir(cfg.Files.Pooldir, "m")
	} else {
		// Read dynamic mix of outbound files from the Pool
		filenames, err = poolRead()
	}
	if err != nil {
		Warn.Printf("Reading pool failed: %s", err)
		return
	}
	for _, file := range filenames {
		err = mailPoolFile(path.Join(cfg.Files.Pooldir, file))
		if err != nil {
			Warn.Printf("Pool mailing failed: %s", err)
		}
		poolDelete(file)
	}
}

// poolRead returns a dynamic Mix of filenames from the outbound pool.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	if poolSize > 0 {
		Trace.Printf("Processing %d pool messages.\n", poolSize)
	}
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) (err error) {
	// Delete a pool file
	err = os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	}
	return
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(
	public *keymgr.Pubring,
	secret *keymgr.Secring,
	id idlog.IDLog,
	chunkDB Chunk) (err error) {

	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	if len(keys) > 0 {
		Trace.Printf("Reading %d messages from %s\n", len(keys), cfg.Files.Maildir)
	}
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err != nil {
				Info.Println(err)
			}
		} else {
			// It's not a remailer-foo request so assume a remailer message
			var mailMsg *mail.Message
			mailMsg, err := dir.Message(key)
			if err != nil {
				Warn.Printf("%s: Reading message failed with: %s", key, err)
				continue
			}
			var msg []byte
			// Convert the armored Yamn message to its byte components
			msg, err = stripArmor(mailMsg.Body)
			if err != nil {
				Info.Println(err)
			}
			if msg == nil {
				Warn.Println("Dearmor returned zero bytes")
				continue
			}
			err = decodeMsg(msg, public, secret, id, chunkDB)
			if err != nil {
				Info.Println(err)
			}
		} // End of remailer-foo or remailer message
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(
	prefix string,
	public *keymgr.Pubring,
	secret *keymgr.Secring,
	id idlog.IDLog,
	chunkDB Chunk) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, public, secret, id, chunkDB)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// PoolWrite writes a raw Byte Yamn message to the Outbound pool with a prefix
// string.
func poolWrite(yamnMsg []byte, prefix string) (poolFileName string) {
	/*
		Currently supported prefixs are:-
		[ m              Oubound message (final or intermediate) ]
		[ i          Inbound message (destined for this remailer ]
		[ p               Partial message chunk needing assembly ]
	*/
	digest := sha256.New()
	digest.Write(yamnMsg)
	poolFileName = prefix + hex.EncodeToString(digest.Sum(nil))[:14]
	fqfn := path.Join(cfg.Files.Pooldir, poolFileName)
	err := ioutil.WriteFile(fqfn, yamnMsg, 0600)
	if err != nil {
		panic(err)
	}
	return
}
<|endoftext|>


<commit_before>// vim: tabstop=2 shiftwidth=2

package main

import (
	"fmt"
	"io/ioutil"
	"net/mail"
	"os"
	"path"
	"strings"
	"time"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

func poolOutboundSend() {
	var err error
	var delFlag bool
	var filenames []string
	if flag_send || flag_client {
		// Read all the pool files
		filenames, err = readDir(cfg.Files.Pooldir, "m")
	} else {
		// Read dynamic mix of outbound files from the Pool
		filenames, err = poolRead()
	}
	if err != nil {
		Warn.Printf("Reading pool failed: %s", err)
		return
	}
	for _, file := range filenames {
		delFlag, err = mailPoolFile(path.Join(cfg.Files.Pooldir, file))
		if err != nil {
			Warn.Printf("Pool mailing failed: %s", err)
			if delFlag {
				// If delFlag is true, we delete the file, even though mailing
				// failed.
				poolDelete(file)
			}
		} else {
			stats.outMail++
			poolDelete(file)
		}
	}
}

// poolRead returns a dynamic Mix of filenames from the outbound pool.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	if poolSize > 0 {
		Trace.Printf("Processing %d pool messages.\n", poolSize)
	}
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) {
	// Delete a pool file
	err := os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	} else {
		Trace.Printf("Deleted %s from Pool", filename)
	}
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(secret *keymgr.Secring) (err error) {
	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	newMsgs := len(keys)
	if newMsgs == 0 {
		// Nothing to do, move along!
		return
	}
	Trace.Printf(
		"Reading %d messages from %s\n",
		newMsgs,
		cfg.Files.Maildir,
	)
	// Increment inbound Email counter
	stats.inMail += newMsgs
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err == nil {
				// Increments stats counter
				stats.inRemFoo++
			} else {
				Info.Println(err)
			}
			// Nothing else to do, move on to the next message
			continue
		}
		// It's not a remailer-foo request so assume a remailer message
		var mailMsg *mail.Message
		mailMsg, err := dir.Message(key)
		if err != nil {
			Warn.Printf(
				"%s: Reading message failed with: %s",
				key,
				err,
			)
			continue
		}
		var msg []byte
		// Convert the armored Yamn message to its byte components
		msg, err = stripArmor(mailMsg.Body)
		if err != nil {
			Info.Println(err)
		}
		if msg == nil {
			Warn.Println("Dearmor returned zero bytes")
			continue
		}
		err = decodeMsg(msg, secret)
		if err != nil {
			Info.Println(err)
		}
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(prefix string, secret *keymgr.Secring) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, secret)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// PoolWrite writes a raw Byte Yamn message to the Outbound pool with a prefix
// string.
func poolWrite(yamnMsg []byte, prefix string) (poolFileName string) {
	/*
		Currently supported prefixs are:-
		[ m              Oubound message (final or intermediate) ]
		[ i          Inbound message (destined for this remailer ]
		[ p               Partial message chunk needing assembly ]
	*/

	// Using a hash for the filename ensures that duplicate files are only
	// written once.  The hash is truncated so there is a tiny risk of
	// accidental collision but it's a tiny risk!
	fqfn := randPoolFilename(prefix)

	// Create a pool file for the message
	f, err := os.Create(fqfn)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Insert a Yamn internal header containing the date pooled.
	// This is useful for performing expiry on old messages.
	dateHeader := fmt.Sprintf(
		"Yamn-Pooled-Date: %s\n",
		time.Now().Format(shortdate),
	)
	f.WriteString(dateHeader)

	// Write the remainder of the message.
	f.Write(yamnMsg)
	f.Sync()
	_, poolFileName = path.Split(fqfn)
	return
}
<commit_msg>Stop processing message if dearmor fails<commit_after>// vim: tabstop=2 shiftwidth=2

package main

import (
	"fmt"
	"io/ioutil"
	"net/mail"
	"os"
	"path"
	"strings"
	"time"
	//"github.com/codahale/blake2"
	"github.com/crooks/yamn/keymgr"
	"github.com/luksen/maildir"
)

func poolOutboundSend() {
	var err error
	var delFlag bool
	var filenames []string
	if flag_send || flag_client {
		// Read all the pool files
		filenames, err = readDir(cfg.Files.Pooldir, "m")
	} else {
		// Read dynamic mix of outbound files from the Pool
		filenames, err = poolRead()
	}
	if err != nil {
		Warn.Printf("Reading pool failed: %s", err)
		return
	}
	for _, file := range filenames {
		delFlag, err = mailPoolFile(path.Join(cfg.Files.Pooldir, file))
		if err != nil {
			Warn.Printf("Pool mailing failed: %s", err)
			if delFlag {
				// If delFlag is true, we delete the file, even though mailing
				// failed.
				poolDelete(file)
			}
		} else {
			stats.outMail++
			poolDelete(file)
		}
	}
}

// poolRead returns a dynamic Mix of filenames from the outbound pool.
func poolRead() (selectedPoolFiles []string, err error) {
	poolFiles, err := readDir(cfg.Files.Pooldir, "m")
	if err != nil {
		Warn.Printf("Unable to access pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	var numToSend int // Number of files to return for processing
	if poolSize < cfg.Pool.Size {
		// Pool isn't sufficiently populated
		Trace.Println("Pool insufficiently populated to trigger sending.",
			fmt.Sprintf("Require=%d, Got=%d", cfg.Pool.Size, poolSize))
		return
	} else {
		// Normal pool processing condition
		numToSend = int((float32(poolSize) / 100.0) * float32(cfg.Pool.Rate))
	}
	keys := randInts(len(poolFiles))
	if poolSize > 0 {
		Trace.Printf("Processing %d pool messages.\n", poolSize)
	}
	for n := 0; n < numToSend; n++ {
		mykey := keys[n]
		selectedPoolFiles = append(selectedPoolFiles, poolFiles[mykey])
	}
	return
}

// Delete a given file from the pool
func poolDelete(filename string) {
	// Delete a pool file
	err := os.Remove(path.Join(cfg.Files.Pooldir, filename))
	if err != nil {
		Error.Printf("Failed to remove %s from %s\n", filename, cfg.Files.Pooldir)
	} else {
		Trace.Printf("Deleted %s from Pool", filename)
	}
}

// processMail reads the Remailer's Maildir and processes the content
func processMail(secret *keymgr.Secring) (err error) {
	dir := maildir.Dir(cfg.Files.Maildir)
	// Get a list of Maildir keys from the directory
	keys, err := dir.Unseen()
	if err != nil {
		return
	}
	newMsgs := len(keys)
	if newMsgs == 0 {
		// Nothing to do, move along!
		return
	}
	Trace.Printf(
		"Reading %d messages from %s\n",
		newMsgs,
		cfg.Files.Maildir,
	)
	// Increment inbound Email counter
	stats.inMail += newMsgs
	// Fetch headers for each Maildir key
	var head mail.Header
	for _, key := range keys {
		head, err = dir.Header(key)
		if err != nil {
			Warn.Printf("%s: Getting headers failed with: %s", key, err)
			continue
		}
		// The Subject determines if the message needs remailer-foo handling
		subject := strings.TrimSpace(strings.ToLower(head.Get("Subject")))
		if strings.HasPrefix(subject, "remailer-") {
			// It's a remailer-foo request
			err = remailerFoo(subject, head.Get("From"))
			if err == nil {
				// Increments stats counter
				stats.inRemFoo++
			} else {
				Info.Println(err)
			}
			// Nothing else to do, move on to the next message
			continue
		}
		// It's not a remailer-foo request so assume a remailer message
		var mailMsg *mail.Message
		mailMsg, err := dir.Message(key)
		if err != nil {
			Warn.Printf(
				"%s: Reading message failed with: %s",
				key,
				err,
			)
			continue
		}
		var msg []byte
		// Convert the armored Yamn message to its byte components
		msg, err = stripArmor(mailMsg.Body)
		if err != nil {
			Info.Println(err)
			continue
		}
		if msg == nil {
			Warn.Println("Dearmor returned zero bytes")
			continue
		}
		err = decodeMsg(msg, secret)
		if err != nil {
			Info.Println(err)
		}
		err = dir.Purge(key)
		if err != nil {
			Warn.Printf("Cannot delete mail: %s", err)
		}
	} // Maildir keys loop
	return
}

// processInpool is similar to processMail but reads the Inbound Pool
func processInpool(prefix string, secret *keymgr.Secring) {
	poolFiles, err := readDir(cfg.Files.Pooldir, prefix)
	if err != nil {
		Warn.Printf("Unable to access inbound pool: %s", err)
		return
	}
	poolSize := len(poolFiles)
	processed := 0
	for _, f := range poolFiles {
		filename := path.Join(cfg.Files.Pooldir, f)
		msg := make([]byte, messageBytes)
		msg, err = ioutil.ReadFile(filename)
		if err != nil {
			Warn.Printf("Failed to read %s from pool: %s", f, err)
			continue
		}
		err = decodeMsg(msg, secret)
		if err != nil {
			Warn.Println(err)
		}
		poolDelete(f)
		processed++
	}
	if poolSize > 0 {
		Trace.Printf("Inbound pool processing complete. Read=%d, Decoded=%d",
			poolSize, processed)
	}
}

// PoolWrite writes a raw Byte Yamn message to the Outbound pool with a prefix
// string.
func poolWrite(yamnMsg []byte, prefix string) (poolFileName string) {
	/*
		Currently supported prefixs are:-
		[ m              Oubound message (final or intermediate) ]
		[ i          Inbound message (destined for this remailer ]
		[ p               Partial message chunk needing assembly ]
	*/

	// Using a hash for the filename ensures that duplicate files are only
	// written once.  The hash is truncated so there is a tiny risk of
	// accidental collision but it's a tiny risk!
	fqfn := randPoolFilename(prefix)

	// Create a pool file for the message
	f, err := os.Create(fqfn)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	// Insert a Yamn internal header containing the date pooled.
	// This is useful for performing expiry on old messages.
	dateHeader := fmt.Sprintf(
		"Yamn-Pooled-Date: %s\n",
		time.Now().Format(shortdate),
	)
	f.WriteString(dateHeader)

	// Write the remainder of the message.
	f.Write(yamnMsg)
	f.Sync()
	_, poolFileName = path.Split(fqfn)
	return
}
<|endoftext|>


<commit_before>package buildah

import (
	"context"
	"fmt"
	"io"
	"strings"

	"github.com/containers/buildah/pkg/blobcache"
	"github.com/containers/buildah/util"
	cp "github.com/containers/image/copy"
	"github.com/containers/image/directory"
	"github.com/containers/image/docker"
	dockerarchive "github.com/containers/image/docker/archive"
	"github.com/containers/image/docker/reference"
	tarfile "github.com/containers/image/docker/tarfile"
	ociarchive "github.com/containers/image/oci/archive"
	oci "github.com/containers/image/oci/layout"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// PullOptions can be used to alter how an image is copied in from somewhere.
type PullOptions struct {
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
	// github.com/containers/image/types SystemContext to hold credentials
	// and other authentication/authorization information.
	SystemContext *types.SystemContext
	// BlobDirectory is the name of a directory in which we'll attempt to
	// store copies of layer blobs that we pull down, if any.  It should
	// already exist.
	BlobDirectory string
	// AllTags is a boolean value that determines if all tagged images
	// will be downloaded from the repository. The default is false.
	AllTags bool
}

func localImageNameForReference(ctx context.Context, store storage.Store, srcRef types.ImageReference, spec string) (string, error) {
	if srcRef == nil {
		return "", errors.Errorf("reference to image is empty")
	}
	split := strings.SplitN(spec, ":", 2)
	file := split[len(split)-1]
	var name string
	switch srcRef.Transport().Name() {
	case dockerarchive.Transport.Name():
		tarSource, err := tarfile.NewSourceFromFile(file)
		if err != nil {
			return "", errors.Wrapf(err, "error opening tarfile %q as a source image", file)
		}
		manifest, err := tarSource.LoadTarManifest()
		if err != nil {
			return "", errors.Errorf("error retrieving manifest.json from tarfile %q: %v", file, err)
		}
		// to pull the first image stored in the tar file
		if len(manifest) == 0 {
			// use the hex of the digest if no manifest is found
			name, err = getImageDigest(ctx, srcRef, nil)
			if err != nil {
				return "", err
			}
		} else {
			if len(manifest[0].RepoTags) > 0 {
				name = manifest[0].RepoTags[0]
			} else {
				// If the input image has no repotags, we need to feed it a dest anyways
				name, err = getImageDigest(ctx, srcRef, nil)
				if err != nil {
					return "", err
				}
			}
		}
	case ociarchive.Transport.Name():
		// retrieve the manifest from index.json to access the image name
		manifest, err := ociarchive.LoadManifestDescriptor(srcRef)
		if err != nil {
			return "", errors.Wrapf(err, "error loading manifest for %q", transports.ImageName(srcRef))
		}
		// if index.json has no reference name, compute the image digest instead
		if manifest.Annotations == nil || manifest.Annotations["org.opencontainers.image.ref.name"] == "" {
			name, err = getImageDigest(ctx, srcRef, nil)
			if err != nil {
				return "", err
			}
		} else {
			name = manifest.Annotations["org.opencontainers.image.ref.name"]
		}
	case directory.Transport.Name():
		// supports pull from a directory
		name = split[1]
		// remove leading "/"
		if name[:1] == "/" {
			name = name[1:]
		}
	case oci.Transport.Name():
		// supports pull from a directory
		name = split[1]
		// remove leading "/"
		if name[:1] == "/" {
			name = name[1:]
		}
	default:
		ref := srcRef.DockerReference()
		if ref == nil {
			name = srcRef.StringWithinTransport()
			_, err := is.Transport.ParseStoreReference(store, name)
			if err == nil {
				return name, nil
			}
			logrus.Debugf("error parsing local storage reference %q: %v", name, err)
			if strings.LastIndex(name, "/") != -1 {
				name = name[strings.LastIndex(name, "/")+1:]
				_, err = is.Transport.ParseStoreReference(store, name)
				if err == nil {
					return name, errors.Wrapf(err, "error parsing local storage reference %q", name)
				}
			}
			return "", errors.Errorf("reference to image %q is not a named reference", transports.ImageName(srcRef))
		}

		if named, ok := ref.(reference.Named); ok {
			name = named.Name()
			if namedTagged, ok := ref.(reference.NamedTagged); ok {
				name = name + ":" + namedTagged.Tag()
			}
			if canonical, ok := ref.(reference.Canonical); ok {
				name = name + "@" + canonical.Digest().String()
			}
		}
	}

	if _, err := is.Transport.ParseStoreReference(store, name); err != nil {
		return "", errors.Wrapf(err, "error parsing computed local image name %q", name)
	}
	return name, nil
}

// Pull copies the contents of the image from somewhere else to local storage.
func Pull(ctx context.Context, imageName string, options PullOptions) error {
	systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath)

	boptions := BuilderOptions{
		FromImage:           imageName,
		SignaturePolicyPath: options.SignaturePolicyPath,
		SystemContext:       systemContext,
		PullBlobDirectory:   options.BlobDirectory,
		ReportWriter:        options.ReportWriter,
	}

	storageRef, transport, img, err := resolveImage(ctx, systemContext, options.Store, boptions)
	if err != nil {
		return err
	}

	var errs *multierror.Error
	if options.AllTags {
		if transport != util.DefaultTransport {
			return errors.New("Non-docker transport is not supported, for --all-tags pulling")
		}

		repo := storageRef.DockerReference().Name()
		dockerRef, err := alltransports.ParseImageName(transport + repo)
		if err != nil {
			return errors.Wrapf(err, "error getting repository tags")
		}
		tags, err := docker.GetRepositoryTags(ctx, systemContext, dockerRef)
		if err != nil {
			return errors.Wrapf(err, "error getting repository tags")
		}
		for _, tag := range tags {
			name := repo + ":" + tag
			if options.ReportWriter != nil {
				options.ReportWriter.Write([]byte("Pulling " + name + "\n"))
			}
			ref, err := pullImage(ctx, options.Store, transport, name, options, systemContext)
			if err != nil {
				errs = multierror.Append(errs, err)
				continue
			}
			taggedImg, err := is.Transport.GetStoreImage(options.Store, ref)
			if err != nil {
				errs = multierror.Append(errs, err)
				continue
			}
			fmt.Printf("%s\n", taggedImg.ID)
		}
	} else {
		fmt.Printf("%s\n", img.ID)
	}

	return errs.ErrorOrNil()
}

func pullImage(ctx context.Context, store storage.Store, transport string, imageName string, options PullOptions, sc *types.SystemContext) (types.ImageReference, error) {
	spec := imageName
	srcRef, err := alltransports.ParseImageName(spec)
	if err != nil {
		logrus.Debugf("error parsing image name %q, trying with transport %q: %v", spec, transport, err)
		if transport == "" {
			transport = util.DefaultTransport
		} else {
			if transport != util.DefaultTransport {
				transport = transport + ":"
			}
		}
		spec = transport + spec
		srcRef2, err2 := alltransports.ParseImageName(spec)
		if err2 != nil {
			return nil, errors.Wrapf(err2, "error parsing image name %q", spec)
		}
		srcRef = srcRef2
	}
	logrus.Debugf("parsed image name %q", spec)

	blocked, err := isReferenceBlocked(srcRef, sc)
	if err != nil {
		return nil, errors.Wrapf(err, "error checking if pulling from registry for %q is blocked", transports.ImageName(srcRef))
	}
	if blocked {
		return nil, errors.Errorf("pull access to registry for %q is blocked by configuration", transports.ImageName(srcRef))
	}

	destName, err := localImageNameForReference(ctx, store, srcRef, spec)
	if err != nil {
		return nil, errors.Wrapf(err, "error computing local image name for %q", transports.ImageName(srcRef))
	}
	if destName == "" {
		return nil, errors.Errorf("error computing local image name for %q", transports.ImageName(srcRef))
	}

	destRef, err := is.Transport.ParseStoreReference(store, destName)
	if err != nil {
		return nil, errors.Wrapf(err, "error parsing image name %q", destName)
	}
	var maybeCachedDestRef types.ImageReference = destRef
	if options.BlobDirectory != "" {
		cachedRef, err := blobcache.NewBlobCache(destRef, options.BlobDirectory, types.PreserveOriginal)
		if err != nil {
			return nil, errors.Wrapf(err, "error wrapping image reference %q in blob cache at %q", transports.ImageName(destRef), options.BlobDirectory)
		}
		maybeCachedDestRef = cachedRef
	}

	policy, err := signature.DefaultPolicy(sc)
	if err != nil {
		return nil, errors.Wrapf(err, "error obtaining default signature policy")
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return nil, errors.Wrapf(err, "error creating new signature policy context")
	}

	defer func() {
		if err2 := policyContext.Destroy(); err2 != nil {
			logrus.Debugf("error destroying signature policy context: %v", err2)
		}
	}()

	logrus.Debugf("copying %q to %q", spec, destName)
	if _, err := cp.Image(ctx, policyContext, maybeCachedDestRef, srcRef, getCopyOptions(options.ReportWriter, srcRef, sc, maybeCachedDestRef, nil, "")); err != nil {
		logrus.Debugf("error copying src image [%q] to dest image [%q] err: %v", spec, destName, err)
		return nil, err
	}
	return destRef, nil
}

// getImageDigest creates an image object and uses the hex value of the digest as the image ID
// for parsing the store reference
func getImageDigest(ctx context.Context, src types.ImageReference, sc *types.SystemContext) (string, error) {
	newImg, err := src.NewImage(ctx, sc)
	if err != nil {
		return "", errors.Wrapf(err, "error opening image %q for reading", transports.ImageName(src))
	}
	defer newImg.Close()

	digest := newImg.ConfigInfo().Digest
	if err = digest.Validate(); err != nil {
		return "", errors.Wrapf(err, "error getting config info from image %q", transports.ImageName(src))
	}
	return "@" + digest.Hex(), nil
}
<commit_msg>Use reference.WithTag instead of manual string manipulation in Pull<commit_after>package buildah

import (
	"context"
	"fmt"
	"io"
	"strings"

	"github.com/containers/buildah/pkg/blobcache"
	"github.com/containers/buildah/util"
	cp "github.com/containers/image/copy"
	"github.com/containers/image/directory"
	"github.com/containers/image/docker"
	dockerarchive "github.com/containers/image/docker/archive"
	"github.com/containers/image/docker/reference"
	tarfile "github.com/containers/image/docker/tarfile"
	ociarchive "github.com/containers/image/oci/archive"
	oci "github.com/containers/image/oci/layout"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// PullOptions can be used to alter how an image is copied in from somewhere.
type PullOptions struct {
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
	// github.com/containers/image/types SystemContext to hold credentials
	// and other authentication/authorization information.
	SystemContext *types.SystemContext
	// BlobDirectory is the name of a directory in which we'll attempt to
	// store copies of layer blobs that we pull down, if any.  It should
	// already exist.
	BlobDirectory string
	// AllTags is a boolean value that determines if all tagged images
	// will be downloaded from the repository. The default is false.
	AllTags bool
}

func localImageNameForReference(ctx context.Context, store storage.Store, srcRef types.ImageReference, spec string) (string, error) {
	if srcRef == nil {
		return "", errors.Errorf("reference to image is empty")
	}
	split := strings.SplitN(spec, ":", 2)
	file := split[len(split)-1]
	var name string
	switch srcRef.Transport().Name() {
	case dockerarchive.Transport.Name():
		tarSource, err := tarfile.NewSourceFromFile(file)
		if err != nil {
			return "", errors.Wrapf(err, "error opening tarfile %q as a source image", file)
		}
		manifest, err := tarSource.LoadTarManifest()
		if err != nil {
			return "", errors.Errorf("error retrieving manifest.json from tarfile %q: %v", file, err)
		}
		// to pull the first image stored in the tar file
		if len(manifest) == 0 {
			// use the hex of the digest if no manifest is found
			name, err = getImageDigest(ctx, srcRef, nil)
			if err != nil {
				return "", err
			}
		} else {
			if len(manifest[0].RepoTags) > 0 {
				name = manifest[0].RepoTags[0]
			} else {
				// If the input image has no repotags, we need to feed it a dest anyways
				name, err = getImageDigest(ctx, srcRef, nil)
				if err != nil {
					return "", err
				}
			}
		}
	case ociarchive.Transport.Name():
		// retrieve the manifest from index.json to access the image name
		manifest, err := ociarchive.LoadManifestDescriptor(srcRef)
		if err != nil {
			return "", errors.Wrapf(err, "error loading manifest for %q", transports.ImageName(srcRef))
		}
		// if index.json has no reference name, compute the image digest instead
		if manifest.Annotations == nil || manifest.Annotations["org.opencontainers.image.ref.name"] == "" {
			name, err = getImageDigest(ctx, srcRef, nil)
			if err != nil {
				return "", err
			}
		} else {
			name = manifest.Annotations["org.opencontainers.image.ref.name"]
		}
	case directory.Transport.Name():
		// supports pull from a directory
		name = split[1]
		// remove leading "/"
		if name[:1] == "/" {
			name = name[1:]
		}
	case oci.Transport.Name():
		// supports pull from a directory
		name = split[1]
		// remove leading "/"
		if name[:1] == "/" {
			name = name[1:]
		}
	default:
		ref := srcRef.DockerReference()
		if ref == nil {
			name = srcRef.StringWithinTransport()
			_, err := is.Transport.ParseStoreReference(store, name)
			if err == nil {
				return name, nil
			}
			logrus.Debugf("error parsing local storage reference %q: %v", name, err)
			if strings.LastIndex(name, "/") != -1 {
				name = name[strings.LastIndex(name, "/")+1:]
				_, err = is.Transport.ParseStoreReference(store, name)
				if err == nil {
					return name, errors.Wrapf(err, "error parsing local storage reference %q", name)
				}
			}
			return "", errors.Errorf("reference to image %q is not a named reference", transports.ImageName(srcRef))
		}

		if named, ok := ref.(reference.Named); ok {
			name = named.Name()
			if namedTagged, ok := ref.(reference.NamedTagged); ok {
				name = name + ":" + namedTagged.Tag()
			}
			if canonical, ok := ref.(reference.Canonical); ok {
				name = name + "@" + canonical.Digest().String()
			}
		}
	}

	if _, err := is.Transport.ParseStoreReference(store, name); err != nil {
		return "", errors.Wrapf(err, "error parsing computed local image name %q", name)
	}
	return name, nil
}

// Pull copies the contents of the image from somewhere else to local storage.
func Pull(ctx context.Context, imageName string, options PullOptions) error {
	systemContext := getSystemContext(options.SystemContext, options.SignaturePolicyPath)

	boptions := BuilderOptions{
		FromImage:           imageName,
		SignaturePolicyPath: options.SignaturePolicyPath,
		SystemContext:       systemContext,
		PullBlobDirectory:   options.BlobDirectory,
		ReportWriter:        options.ReportWriter,
	}

	storageRef, transport, img, err := resolveImage(ctx, systemContext, options.Store, boptions)
	if err != nil {
		return err
	}

	var errs *multierror.Error
	if options.AllTags {
		if transport != util.DefaultTransport {
			return errors.New("Non-docker transport is not supported, for --all-tags pulling")
		}

		repo := reference.TrimNamed(storageRef.DockerReference())
		dockerRef, err := alltransports.ParseImageName(transport + repo.Name())
		if err != nil {
			return errors.Wrapf(err, "error getting repository tags")
		}
		tags, err := docker.GetRepositoryTags(ctx, systemContext, dockerRef)
		if err != nil {
			return errors.Wrapf(err, "error getting repository tags")
		}
		for _, tag := range tags {
			tagged, err := reference.WithTag(repo, tag)
			if err != nil {
				errs = multierror.Append(errs, err)
				continue
			}
			if options.ReportWriter != nil {
				options.ReportWriter.Write([]byte("Pulling " + tagged.String() + "\n"))
			}
			ref, err := pullImage(ctx, options.Store, transport, tagged.String(), options, systemContext)
			if err != nil {
				errs = multierror.Append(errs, err)
				continue
			}
			taggedImg, err := is.Transport.GetStoreImage(options.Store, ref)
			if err != nil {
				errs = multierror.Append(errs, err)
				continue
			}
			fmt.Printf("%s\n", taggedImg.ID)
		}
	} else {
		fmt.Printf("%s\n", img.ID)
	}

	return errs.ErrorOrNil()
}

func pullImage(ctx context.Context, store storage.Store, transport string, imageName string, options PullOptions, sc *types.SystemContext) (types.ImageReference, error) {
	spec := imageName
	srcRef, err := alltransports.ParseImageName(spec)
	if err != nil {
		logrus.Debugf("error parsing image name %q, trying with transport %q: %v", spec, transport, err)
		if transport == "" {
			transport = util.DefaultTransport
		} else {
			if transport != util.DefaultTransport {
				transport = transport + ":"
			}
		}
		spec = transport + spec
		srcRef2, err2 := alltransports.ParseImageName(spec)
		if err2 != nil {
			return nil, errors.Wrapf(err2, "error parsing image name %q", spec)
		}
		srcRef = srcRef2
	}
	logrus.Debugf("parsed image name %q", spec)

	blocked, err := isReferenceBlocked(srcRef, sc)
	if err != nil {
		return nil, errors.Wrapf(err, "error checking if pulling from registry for %q is blocked", transports.ImageName(srcRef))
	}
	if blocked {
		return nil, errors.Errorf("pull access to registry for %q is blocked by configuration", transports.ImageName(srcRef))
	}

	destName, err := localImageNameForReference(ctx, store, srcRef, spec)
	if err != nil {
		return nil, errors.Wrapf(err, "error computing local image name for %q", transports.ImageName(srcRef))
	}
	if destName == "" {
		return nil, errors.Errorf("error computing local image name for %q", transports.ImageName(srcRef))
	}

	destRef, err := is.Transport.ParseStoreReference(store, destName)
	if err != nil {
		return nil, errors.Wrapf(err, "error parsing image name %q", destName)
	}
	var maybeCachedDestRef types.ImageReference = destRef
	if options.BlobDirectory != "" {
		cachedRef, err := blobcache.NewBlobCache(destRef, options.BlobDirectory, types.PreserveOriginal)
		if err != nil {
			return nil, errors.Wrapf(err, "error wrapping image reference %q in blob cache at %q", transports.ImageName(destRef), options.BlobDirectory)
		}
		maybeCachedDestRef = cachedRef
	}

	policy, err := signature.DefaultPolicy(sc)
	if err != nil {
		return nil, errors.Wrapf(err, "error obtaining default signature policy")
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return nil, errors.Wrapf(err, "error creating new signature policy context")
	}

	defer func() {
		if err2 := policyContext.Destroy(); err2 != nil {
			logrus.Debugf("error destroying signature policy context: %v", err2)
		}
	}()

	logrus.Debugf("copying %q to %q", spec, destName)
	if _, err := cp.Image(ctx, policyContext, maybeCachedDestRef, srcRef, getCopyOptions(options.ReportWriter, srcRef, sc, maybeCachedDestRef, nil, "")); err != nil {
		logrus.Debugf("error copying src image [%q] to dest image [%q] err: %v", spec, destName, err)
		return nil, err
	}
	return destRef, nil
}

// getImageDigest creates an image object and uses the hex value of the digest as the image ID
// for parsing the store reference
func getImageDigest(ctx context.Context, src types.ImageReference, sc *types.SystemContext) (string, error) {
	newImg, err := src.NewImage(ctx, sc)
	if err != nil {
		return "", errors.Wrapf(err, "error opening image %q for reading", transports.ImageName(src))
	}
	defer newImg.Close()

	digest := newImg.ConfigInfo().Digest
	if err = digest.Validate(); err != nil {
		return "", errors.Wrapf(err, "error getting config info from image %q", transports.ImageName(src))
	}
	return "@" + digest.Hex(), nil
}
<|endoftext|>