<commit_before>package harness

import (
	"fmt"
	"github.com/revel/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	// First, clear the generated files (to avoid them messing with ProcessSource).
	cleanSource("tmp", "routes")

	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		appVersion := getAppVersion()
		versionLinkerFlags := fmt.Sprintf("-X %s/app.APP_VERSION \"%s\"", revel.ImportPath, appVersion)

		buildCmd := exec.Command(goPath, "build",
			"-ldflags", versionLinkerFlags,
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

// Try to define a version string for the compiled app
// The following is tried (first match returns):
// - Read a version explicitly specified in the APP_VERSION environment
//   variable
// - Read the output of "git describe" if the source is in a git repository
// If no version can be determined, an empty string is returned.
func getAppVersion() string {
	if version := os.Getenv("APP_VERSION"); version != "" {
		return version
	}

	// Check for the git binary
	if gitPath, err := exec.LookPath("git"); err == nil {
		// Check for the .git directory
		gitDir := path.Join(revel.BasePath, ".git")
		info, err := os.Stat(gitDir)
		if (err != nil && os.IsNotExist(err)) || !info.IsDir() {
			return ""
		}
		gitCmd := exec.Command(gitPath, "--git-dir=" + gitDir, "describe", "--always", "--dirty")
		revel.TRACE.Println("Exec:", gitCmd.Args)
		output, err := gitCmd.Output()

		if err != nil {
			revel.WARN.Println("Cannot determine git repository version:", err)
			return ""
		}

		return "git-" + strings.TrimSpace(string(output))
	}

	return ""
}

func cleanSource(dirs ...string) {
	for _, dir := range dirs {
		tmpPath := path.Join(revel.AppPath, dir)
		err := os.RemoveAll(tmpPath)
		if err != nil {
			revel.ERROR.Println("Failed to remove dir:", err)
		}
	}
}

// genSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/revel/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/revel/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (_ t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<commit_msg>Run go fmt<commit_after>package harness

import (
	"fmt"
	"github.com/revel/revel"
	"go/build"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"text/template"
)

var importErrorPattern = regexp.MustCompile("cannot find package \"([^\"]+)\"")

// Build the app:
// 1. Generate the the main.go file.
// 2. Run the appropriate "go build" command.
// Requires that revel.Init has been called previously.
// Returns the path to the built binary, and an error if there was a problem building it.
func Build() (app *App, compileError *revel.Error) {
	// First, clear the generated files (to avoid them messing with ProcessSource).
	cleanSource("tmp", "routes")

	sourceInfo, compileError := ProcessSource(revel.CodePaths)
	if compileError != nil {
		return nil, compileError
	}

	// Add the db.import to the import paths.
	if dbImportPath, found := revel.Config.String("db.import"); found {
		sourceInfo.InitImportPaths = append(sourceInfo.InitImportPaths, dbImportPath)
	}

	// Generate two source files.
	templateArgs := map[string]interface{}{
		"Controllers":    sourceInfo.ControllerSpecs(),
		"ValidationKeys": sourceInfo.ValidationKeys,
		"ImportPaths":    calcImportAliases(sourceInfo),
		"TestSuites":     sourceInfo.TestSuites(),
	}
	genSource("tmp", "main.go", MAIN, templateArgs)
	genSource("routes", "routes.go", ROUTES, templateArgs)

	// Read build config.
	buildTags := revel.Config.StringDefault("build.tags", "")

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		revel.ERROR.Fatalf("Go executable not found in PATH.")
	}

	pkg, err := build.Default.Import(revel.ImportPath, "", build.FindOnly)
	if err != nil {
		revel.ERROR.Fatalln("Failure importing", revel.ImportPath)
	}
	binName := path.Join(pkg.BinDir, path.Base(revel.BasePath))
	if runtime.GOOS == "windows" {
		binName += ".exe"
	}

	gotten := make(map[string]struct{})
	for {
		appVersion := getAppVersion()
		versionLinkerFlags := fmt.Sprintf("-X %s/app.APP_VERSION \"%s\"", revel.ImportPath, appVersion)

		buildCmd := exec.Command(goPath, "build",
			"-ldflags", versionLinkerFlags,
			"-tags", buildTags,
			"-o", binName, path.Join(revel.ImportPath, "app", "tmp"))
		revel.TRACE.Println("Exec:", buildCmd.Args)
		output, err := buildCmd.CombinedOutput()

		// If the build succeeded, we're done.
		if err == nil {
			return NewApp(binName), nil
		}
		revel.ERROR.Println(string(output))

		// See if it was an import error that we can go get.
		matches := importErrorPattern.FindStringSubmatch(string(output))
		if matches == nil {
			return nil, newCompileError(output)
		}

		// Ensure we haven't already tried to go get it.
		pkgName := matches[1]
		if _, alreadyTried := gotten[pkgName]; alreadyTried {
			return nil, newCompileError(output)
		}
		gotten[pkgName] = struct{}{}

		// Execute "go get <pkg>"
		getCmd := exec.Command(goPath, "get", pkgName)
		revel.TRACE.Println("Exec:", getCmd.Args)
		getOutput, err := getCmd.CombinedOutput()
		if err != nil {
			revel.ERROR.Println(string(getOutput))
			return nil, newCompileError(output)
		}

		// Success getting the import, attempt to build again.
	}
	revel.ERROR.Fatalf("Not reachable")
	return nil, nil
}

// Try to define a version string for the compiled app
// The following is tried (first match returns):
// - Read a version explicitly specified in the APP_VERSION environment
//   variable
// - Read the output of "git describe" if the source is in a git repository
// If no version can be determined, an empty string is returned.
func getAppVersion() string {
	if version := os.Getenv("APP_VERSION"); version != "" {
		return version
	}

	// Check for the git binary
	if gitPath, err := exec.LookPath("git"); err == nil {
		// Check for the .git directory
		gitDir := path.Join(revel.BasePath, ".git")
		info, err := os.Stat(gitDir)
		if (err != nil && os.IsNotExist(err)) || !info.IsDir() {
			return ""
		}
		gitCmd := exec.Command(gitPath, "--git-dir="+gitDir, "describe", "--always", "--dirty")
		revel.TRACE.Println("Exec:", gitCmd.Args)
		output, err := gitCmd.Output()

		if err != nil {
			revel.WARN.Println("Cannot determine git repository version:", err)
			return ""
		}

		return "git-" + strings.TrimSpace(string(output))
	}

	return ""
}

func cleanSource(dirs ...string) {
	for _, dir := range dirs {
		tmpPath := path.Join(revel.AppPath, dir)
		err := os.RemoveAll(tmpPath)
		if err != nil {
			revel.ERROR.Println("Failed to remove dir:", err)
		}
	}
}

// genSource renders the given template to produce source code, which it writes
// to the given directory and file.
func genSource(dir, filename, templateSource string, args map[string]interface{}) {
	sourceCode := revel.ExecuteTemplate(
		template.Must(template.New("").Parse(templateSource)),
		args)

	// Create a fresh dir.
	tmpPath := path.Join(revel.AppPath, dir)
	err := os.RemoveAll(tmpPath)
	if err != nil {
		revel.ERROR.Println("Failed to remove dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		revel.ERROR.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the file
	file, err := os.Create(path.Join(tmpPath, filename))
	defer file.Close()
	if err != nil {
		revel.ERROR.Fatalf("Failed to create file: %v", err)
	}
	_, err = file.WriteString(sourceCode)
	if err != nil {
		revel.ERROR.Fatalf("Failed to write to file: %v", err)
	}
}

// Looks through all the method args and returns a set of unique import paths
// that cover all the method arg types.
// Additionally, assign package aliases when necessary to resolve ambiguity.
func calcImportAliases(src *SourceInfo) map[string]string {
	aliases := make(map[string]string)
	typeArrays := [][]*TypeInfo{src.ControllerSpecs(), src.TestSuites()}
	for _, specs := range typeArrays {
		for _, spec := range specs {
			addAlias(aliases, spec.ImportPath, spec.PackageName)

			for _, methSpec := range spec.MethodSpecs {
				for _, methArg := range methSpec.Args {
					if methArg.ImportPath == "" {
						continue
					}

					addAlias(aliases, methArg.ImportPath, methArg.TypeExpr.PkgName)
				}
			}
		}
	}

	// Add the "InitImportPaths", with alias "_"
	for _, importPath := range src.InitImportPaths {
		if _, ok := aliases[importPath]; !ok {
			aliases[importPath] = "_"
		}
	}

	return aliases
}

func addAlias(aliases map[string]string, importPath, pkgName string) {
	alias, ok := aliases[importPath]
	if ok {
		return
	}
	alias = makePackageAlias(aliases, pkgName)
	aliases[importPath] = alias
}

func makePackageAlias(aliases map[string]string, pkgName string) string {
	i := 0
	alias := pkgName
	for containsValue(aliases, alias) {
		alias = fmt.Sprintf("%s%d", pkgName, i)
		i++
	}
	return alias
}

func containsValue(m map[string]string, val string) bool {
	for _, v := range m {
		if v == val {
			return true
		}
	}
	return false
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *revel.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		revel.ERROR.Println("Failed to parse build errors:\n", string(output))
		return &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &revel.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := revel.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		revel.ERROR.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}

const MAIN = `// GENERATED CODE - DO NOT EDIT
package main

import (
	"flag"
	"reflect"
	"github.com/revel/revel"{{range $k, $v := $.ImportPaths}}
	{{$v}} "{{$k}}"{{end}}
)

var (
	runMode    *string = flag.String("runMode", "", "Run mode.")
	port       *int    = flag.Int("port", 0, "By default, read from app.conf")
	importPath *string = flag.String("importPath", "", "Go Import Path for the app.")
	srcPath    *string = flag.String("srcPath", "", "Path to the source root.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	flag.Parse()
	revel.Init(*runMode, *importPath, *srcPath)
	revel.INFO.Println("Running revel server")
	{{range $i, $c := .Controllers}}
	revel.RegisterController((*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),
		[]*revel.MethodType{
			{{range .MethodSpecs}}&revel.MethodType{
				Name: "{{.Name}}",
				Args: []*revel.MethodArg{ {{range .Args}}
					&revel.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{index $.ImportPaths .ImportPath | .TypeExpr.TypeName}})(nil)) },{{end}}
				},
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	revel.DefaultValidationKeys = map[string]map[int]string{ {{range $path, $lines := .ValidationKeys}}
		"{{$path}}": { {{range $line, $key := $lines}}
			{{$line}}: "{{$key}}",{{end}}
		},{{end}}
	}
	revel.TestSuites = []interface{}{ {{range .TestSuites}}
		(*{{index $.ImportPaths .ImportPath}}.{{.StructName}})(nil),{{end}}
	}

	revel.Run(*port)
}
`
const ROUTES = `// GENERATED CODE - DO NOT EDIT
package routes

import "github.com/revel/revel"

{{range $i, $c := .Controllers}}
type t{{.StructName}} struct {}
var {{.StructName}} t{{.StructName}}

{{range .MethodSpecs}}
func (_ t{{$c.StructName}}) {{.Name}}({{range .Args}}
		{{.Name}} {{if .ImportPath}}interface{}{{else}}{{.TypeExpr.TypeName ""}}{{end}},{{end}}
		) string {
	args := make(map[string]string)
	{{range .Args}}
	revel.Unbind(args, "{{.Name}}", {{.Name}}){{end}}
	return revel.MainRouter.Reverse("{{$c.StructName}}.{{.Name}}", args).Url
}
{{end}}
{{end}}
`
<|endoftext|>


<commit_before>package reviewdog

import "io/ioutil"

import "os/exec"
import "strings"
import "testing"

func TestDiffString(t *testing.T) {
	difftext := `diff --git a/golint.old.go b/golint.new.go
index 34cacb9..a727dd3 100644
--- a/golint.old.go
+++ b/golint.new.go
@@ -2,6 +2,12 @@ package test

 var V int

+var NewError1 int
+
 // invalid func comment
 func F() {
 }
+
+// invalid func comment2
+func F2() {
+}
`
	d := NewDiffString(difftext, 1)
	b, err := d.Diff()
	if err != nil {
		t.Error(err)
	}
	got := string(b)
	if got != difftext {
		t.Errorf("got:\n%v\nwant:\n%v", got, difftext)
	}
}

func TestDiffCmd(t *testing.T) {
	wantb, err := ioutil.ReadFile("./diff/testdata/golint.diff")
	if err != nil {
		t.Fatal(err)
	}
	want := strings.SplitN(string(wantb), "\n", 5)[4] // strip extended header
	cmd := exec.Command("git", "diff", "--no-index", "./diff/testdata/golint.old.go", "./diff/testdata/golint.new.go")
	d := NewDiffCmd(cmd, 1)
	// ensure it supports multiple use
	for i := 0; i < 3; i++ {
		b, err := d.Diff()
		if err != nil {
			t.Fatal(string(b), err)
		}
		got := strings.SplitN(string(b), "\n", 5)[4]
		if got != want {
			t.Errorf("got:\n%v\nwant:\n%v", got, want)
		}
	}
}
<commit_msg>test: add test for data race of DiffCmd<commit_after>package reviewdog

import (
	"io/ioutil"
	"os/exec"
	"sync"
	"testing"
)

import "strings"

func TestDiffString(t *testing.T) {
	difftext := `diff --git a/golint.old.go b/golint.new.go
index 34cacb9..a727dd3 100644
--- a/golint.old.go
+++ b/golint.new.go
@@ -2,6 +2,12 @@ package test

 var V int

+var NewError1 int
+
 // invalid func comment
 func F() {
 }
+
+// invalid func comment2
+func F2() {
+}
`
	d := NewDiffString(difftext, 1)
	b, err := d.Diff()
	if err != nil {
		t.Error(err)
	}
	got := string(b)
	if got != difftext {
		t.Errorf("got:\n%v\nwant:\n%v", got, difftext)
	}
}

func TestDiffCmd(t *testing.T) {
	wantb, err := ioutil.ReadFile("./diff/testdata/golint.diff")
	if err != nil {
		t.Fatal(err)
	}
	want := strings.SplitN(string(wantb), "\n", 5)[4] // strip extended header
	cmd := exec.Command("git", "diff", "--no-index", "./diff/testdata/golint.old.go", "./diff/testdata/golint.new.go")
	d := NewDiffCmd(cmd, 1)
	// ensure it supports multiple use
	var wg sync.WaitGroup
	for i := 0; i < 3; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			b, err := d.Diff()
			if err != nil {
				t.Error(string(b), err)
			}
			got := strings.SplitN(string(b), "\n", 5)[4]
			if got != want {
				t.Errorf("got:\n%v\nwant:\n%v", got, want)
			}
		}()
	}
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2013 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.

package alto

const (
	MediaTypeDirectory = "application/alto-directory+json" // media type for ALTO directory service
)

// A Directory represents an information resource directory.
type Directory struct {
	Meta      Meta                `json:"meta"`
	Resources []DirectoryResource `json:"resources"`
}

// A DirResource represents a list of information resources.
type DirectoryResource struct {
	URI          string                 `json:"uri"`
	MediaType    string                 `json:"media-type"`
	Accepts      string                 `json:"accepts,omitempty"`
	Capabilities map[string]interface{} `json:"capabilities,omitempty"`
}
<commit_msg>alto: update doc<commit_after>// Copyright 2013 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.

package alto

const (
	MediaTypeDirectory = "application/alto-directory+json" // media type for ALTO directory service
)

// A Directory represents an information resource directory.
type Directory struct {
	Meta      Meta                `json:"meta"`
	Resources []DirectoryResource `json:"resources"`
}

// A DirectoryResource represents a list of information resources.
type DirectoryResource struct {
	URI          string                 `json:"uri"`
	MediaType    string                 `json:"media-type"`
	Accepts      string                 `json:"accepts,omitempty"`
	Capabilities map[string]interface{} `json:"capabilities,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1
	log.Printf("[Scanner.findBlock] cur %d, remaining %d\n", s.idx, remaining)

	if remaining <= 0 {
		log.Println("[Scanner.findBlock] last block")
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		log.Println("[Scanner.findBlock] next block is past key")
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		log.Printf("[Scanner.blockFor] curBlock after key %s (cur: %d, start: %s)\n",
			hex.EncodeToString(key),
			s.idx,
			hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
		)
		return nil, nil, false
	}

	idx := s.findBlock(key)
	log.Printf("[Scanner.blockFor] findBlock (key: %s) picked %d (starts: %s). Cur: %d (starts: %s)\n",
		hex.EncodeToString(key),
		idx,
		hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
		s.idx,
		hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
	)

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			log.Printf("[Scanner.blockFor] read err %s (key: %s, idx: %d, start: %s)\n",
				err,
				hex.EncodeToString(key),
				idx,
				hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
			)
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	} else {
		log.Println("[Scanner.blockFor] Re-using current block")
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetFirst] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetAll] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	log.Printf("[Scanner.getValuesFromBuffer] buf before %d\n", buf.Len())
	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		cmp := bytes.Compare(keyBytes, key)
		if cmp == 0 {
			if first {
				log.Printf("[Scanner.getValuesFromBuffer] buf after %d\n", buf.Len())
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
		if cmp > 0 {
			log.Printf("[Scanner.getValuesFromBuffer] past key %s vs %s. buf remaining %d\n",
				hex.EncodeToString(key),
				hex.EncodeToString(keyBytes),
				buf.Len(),
			)
			buf.Seek(-(int64(keyLen + valLen + 8)), 1)
			return nil, acc, len(acc) > 0
		}
	}
	log.Printf("[Scanner.getValuesFromBuffer] walked off block\n")
	return nil, acc, len(acc) > 0
}
<commit_msg>quiter scanner<commit_after>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	// "encoding/hex"
	"fmt"
	// "log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1
	// log.Printf("[Scanner.findBlock] cur %d, remaining %d\n", s.idx, remaining)

	if remaining <= 0 {
		// log.Println("[Scanner.findBlock] last block")
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		// log.Println("[Scanner.findBlock] next block is past key")
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		// log.Printf("[Scanner.blockFor] curBlock after key %s (cur: %d, start: %s)\n",
		// 	hex.EncodeToString(key),
		// 	s.idx,
		// 	hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
		// )
		return nil, nil, false
	}

	idx := s.findBlock(key)
	// log.Printf("[Scanner.blockFor] findBlock (key: %s) picked %d (starts: %s). Cur: %d (starts: %s)\n",
	// 	hex.EncodeToString(key),
	// 	idx,
	// 	hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
	// 	s.idx,
	// 	hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
	// )

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			// log.Printf("[Scanner.blockFor] read err %s (key: %s, idx: %d, start: %s)\n",
			// 	err,
			// 	hex.EncodeToString(key),
			// 	idx,
			// 	hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
			// )
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	} else {
		// log.Println("[Scanner.blockFor] Re-using current block")
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		// log.Printf("[Scanner.GetFirst] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		// log.Printf("[Scanner.GetAll] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	// log.Printf("[Scanner.getValuesFromBuffer] buf before %d\n", buf.Len())
	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		cmp := bytes.Compare(keyBytes, key)
		if cmp == 0 {
			if first {
				// log.Printf("[Scanner.getValuesFromBuffer] buf after %d\n", buf.Len())
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
		if cmp > 0 {
			// log.Printf("[Scanner.getValuesFromBuffer] past key %s vs %s. buf remaining %d\n",
			// 	hex.EncodeToString(key),
			// 	hex.EncodeToString(keyBytes),
			// 	buf.Len(),
			// )
			buf.Seek(-(int64(keyLen + valLen + 8)), 1)
			return nil, acc, len(acc) > 0
		}
	}
	// log.Printf("[Scanner.getValuesFromBuffer] walked off block\n")
	return nil, acc, len(acc) > 0
}
<|endoftext|>


<commit_before>package blocks

import (
  "log"
  "github.com/mattbaird/elastigo/api"
  "github.com/mattbaird/elastigo/core"
)

// Posts a message to a specified Elasticsearch index with the given type.
func ToElasticsearch(b *Block) {

	type toElasticsearchRule struct {
		Host string
		Port   string
		Index  string
		Type   string
	}

	var rule *toElasticsearchRule

	// TODO check the endpoint for happiness
	for {
		select {
		case msg := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &toElasticsearchRule{}
			}
			unmarshal(msg, rule)
      // Set the Elasticsearch Host/Port to Connect to
      api.Domain = rule.Host
      api.Port = rule.Port

		case msg := <-b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &toElasticsearchRule{})
			} else {
				marshal(msg, rule)
			}
		case <-b.QuitChan:
			quit(b)
			return
		case msg := <-b.InChan:
			if rule == nil {
				break
			}

      log.Println(msg.Msg)
      response, err := core.Index(true, rule.Index, rule.Type, "", msg.Msg)
			if err != nil {
				log.Println(err.Error())
				break
			}
      log.Printf("Index OK: %v", response.Ok)
  	}
	}
}
<commit_msg>Ditched the debugging log message<commit_after>package blocks

import (
  "log"
  "github.com/mattbaird/elastigo/api"
  "github.com/mattbaird/elastigo/core"
)

// Posts a message to a specified Elasticsearch index with the given type.
func ToElasticsearch(b *Block) {

	type toElasticsearchRule struct {
		Host string
		Port   string
		Index  string
		Type   string
	}

	var rule *toElasticsearchRule

	// TODO check the endpoint for happiness
	for {
		select {
		case msg := <-b.Routes["set_rule"]:
			if rule == nil {
				rule = &toElasticsearchRule{}
			}
			unmarshal(msg, rule)
      // Set the Elasticsearch Host/Port to Connect to
      api.Domain = rule.Host
      api.Port = rule.Port

		case msg := <-b.Routes["get_rule"]:
			if rule == nil {
				marshal(msg, &toElasticsearchRule{})
			} else {
				marshal(msg, rule)
			}
		case <-b.QuitChan:
			quit(b)
			return
		case msg := <-b.InChan:
			if rule == nil {
				break
			}

      response, err := core.Index(true, rule.Index, rule.Type, "", msg.Msg)
			if err != nil {
				log.Println(err.Error())
				break
			}
      log.Printf("Index OK: %v", response.Ok)
  	}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/http/cookiejar"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type (
	// Phantom 基于Phantomjs的下载器实现，作为surfer的补充
	// 效率较surfer会慢很多，但是因为模拟浏览器，破防性更好
	// 支持UserAgent/TryTimes/RetryPause/自定义js
	Phantom struct {
		PhantomjsFile string            //Phantomjs完整文件名
		TempJsDir     string            //临时js存放目录
		jsFileMap     map[string]string //已存在的js文件
		CookieJar     *cookiejar.Jar
	}
	// Response 用于解析Phantomjs的响应内容
	Response struct {
		Cookies []string
		Body    string
		Error   string
		Header  []struct {
			Name  string
			Value string
		}
	}

	//给phantomjs传输cookie用
	Cookie struct {
		Name   string `json:"name"`
		Value  string `json:"value"`
		Domain string `json:"domain"`
		Path   string `json:"path"`
	}
)

// NewPhantom 创建一个Phantomjs下载器
func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) Surfer {
	phantom := &Phantom{
		PhantomjsFile: phantomjsFile,
		TempJsDir:     tempJsDir,
		jsFileMap:     make(map[string]string),
	}
	if len(jar) != 0 {
		phantom.CookieJar = jar[0]
	} else {
		phantom.CookieJar, _ = cookiejar.New(nil)
	}
	if !filepath.IsAbs(phantom.PhantomjsFile) {
		phantom.PhantomjsFile, _ = filepath.Abs(phantom.PhantomjsFile)
	}
	if !filepath.IsAbs(phantom.TempJsDir) {
		phantom.TempJsDir, _ = filepath.Abs(phantom.TempJsDir)
	}
	// 创建/打开目录
	err := os.MkdirAll(phantom.TempJsDir, 0777)
	if err != nil {
		log.Printf("[E] Surfer: %v\n", err)
		return phantom
	}
	phantom.createJsFile("js", js)
	return phantom
}

// Download 实现surfer下载器接口
func (phantom *Phantom) Download(req *Request) (resp *http.Response, err error) {
	err = req.prepare()
	if err != nil {
		return resp, err
	}
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.Header.Del("Content-Type")

	cookie := ""
	if req.EnableCookie {
		httpCookies := phantom.CookieJar.Cookies(req.url)
		if len(httpCookies) > 0 {
			surferCookies := make([]*Cookie, len(httpCookies))

			for n, c := range httpCookies {
				surferCookie := &Cookie{Name: c.Name, Value: c.Value, Domain: req.url.Host, Path: "/"}
				surferCookies[n] = surferCookie
			}

			c, err := json.Marshal(surferCookies)
			if err != nil {
				log.Printf("cookie marshal error:%v", err)
			}
			cookie = string(c)
		}
	}

	var b, _ = req.ReadBody()
	resp = req.writeback(resp)
	resp.Request.URL = req.url

	var args = []string{
		phantom.jsFileMap["js"],
		req.Url,
		cookie,
		encoding,
		req.Header.Get("User-Agent"),
		string(b),
		strings.ToLower(req.Method),
		fmt.Sprint(int(req.DialTimeout / time.Millisecond)),
	}

	for i := 0; i < req.TryTimes; i++ {
		if i != 0 {
			time.Sleep(req.RetryPause)
		}

		cmd := exec.Command(phantom.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			continue
		}

		if retResp.Error != "" {
			log.Printf("phantomjs response error:%s", retResp.Error)
			continue
		}

		//设置header
		for _, h := range retResp.Header {
			resp.Header.Add(h.Name, h.Value)
		}

		//设置cookie
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		if req.EnableCookie {
			if rc := resp.Cookies(); len(rc) > 0 {
				phantom.CookieJar.SetCookies(resp.Request.URL, rc)
			}
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}

	return resp, err
}

// DestroyJsFiles 销毁js临时文件
func (phantom *Phantom) DestroyJsFiles() {
	p, _ := filepath.Split(phantom.TempJsDir)
	if p == "" {
		return
	}
	for _, filename := range phantom.jsFileMap {
		os.Remove(filename)
	}
	if len(WalkDir(p)) == 1 {
		os.Remove(p)
	}
}

func (phantom *Phantom) createJsFile(fileName, jsCode string) {
	fullFileName := filepath.Join(phantom.TempJsDir, fileName)
	// 创建并写入文件
	f, _ := os.Create(fullFileName)
	f.Write([]byte(jsCode))
	f.Close()
	phantom.jsFileMap[fileName] = fullFileName
}

/*
* system.args[0] == js
* system.args[1] == url
* system.args[2] == cookie
* system.args[3] == pageEncode
* system.args[4] == userAgent
* system.args[5] == postdata
* system.args[6] == method
* system.args[7] == timeout
 */
const js string = `
var system = require('system');
var page = require('webpage').create();
var url = system.args[1];
var cookie = system.args[2];
var pageEncode = system.args[3];
var userAgent = system.args[4];
var postdata = system.args[5];
var method = system.args[6];
var timeout = system.args[7];

var ret = new Object();
var exit = function () {
    console.log(JSON.stringify(ret));
    phantom.exit();
};

//输出参数
// console.log("url=" + url);
// console.log("cookie=" + cookie);
// console.log("pageEncode=" + pageEncode);
// console.log("userAgent=" + userAgent);
// console.log("postdata=" + postdata);
// console.log("method=" + method);
// console.log("timeout=" + timeout);

// ret += (url + "\n");
// ret += (cookie + "\n");
// ret += (pageEncode + "\n");
// ret += (userAgent + "\n");
// ret += (postdata + "\n");
// ret += (method + "\n");
// ret += (timeout + "\n");
// exit();

phantom.outputEncoding = pageEncode;
page.settings.userAgent = userAgent;
page.settings.resourceTimeout = timeout;
page.settings.XSSAuditingEnabled = true;

function addCookie() {
    if (cookie != "") {
        var cookies = JSON.parse(cookie);
        for (var i = 0; i < cookies.length; i++) {
            var c = cookies[i];

            phantom.addCookie({
                'name': c.name, /* required property */
                'value': c.value, /* required property */
                'domain': c.domain,
                'path': c.path, /* required property */
            });
        }
    }
}

addCookie();

page.onResourceRequested = function (requestData, request) {

};
page.onResourceReceived = function (response) {
    if (response.stage === "end") {
        // console.log("liguoqinjim received1------------------------------------------------");
        // console.log("url=" + response.url);
        //
        // for (var j in response.headers) {//用javascript的for/in循环遍历对象的属性
        //     // var m = sprintf("AttrId[%d]Value[%d]", j, result.Attrs[j]);
        //     // message += m;
        //     // console.log(response.headers[j]);
        //     console.log(response.headers[j]["name"] + ":" + response.headers[j]["value"]);
        // }
        //
        // console.log("liguoqinjim received2------------------------------------------------");

        //在ret中加入header
        ret["Header"] = response.headers;
    }
};
page.onError = function (msg, trace) {
    ret["Error"] = msg;
    exit();
};
page.onResourceTimeout = function (e) {
    // console.log("phantomjs onResourceTimeout error");
    // console.log(e.errorCode);   // it'll probably be 408
    // console.log(e.errorString); // it'll probably be 'Network timeout on resource'
    // console.log(e.url);         // the url whose request timed out
    // phantom.exit(1);
    ret["Error"] = "onResourceTimeout";
    exit();
};
page.onResourceError = function (e) {
    // console.log("onResourceError");
    // console.log("1:" + e.errorCode + "," + e.errorString);

    if (e.errorCode != 5) { //errorCode=5的情况和onResourceTimeout冲突
        ret["Error"] = "onResourceError";
        exit();
    }
};
page.onLoadFinished = function (status) {
    if (status !== 'success') {
        ret["Error"] = "status=" + status;
        exit();
    } else {
        var cookies = new Array();
        for (var i in page.cookies) {
            var cookie = page.cookies[i];
            var c = cookie["name"] + "=" + cookie["value"];
            for (var obj in cookie) {
                if (obj == 'name' || obj == 'value') {
                    continue;
                }
                if (obj == "httponly" || obj == "secure") {
                    if (cookie[obj] == true) {
                        c += ";" + obj;
                    }
                } else {
                    c += "; " + obj + "=" + cookie[obj];
                }
            }
            cookies[i] = c;
        }
        if (page.content.indexOf("body") != -1) {
            ret["Cookies"] = cookies;
            ret["Body"] = page.content;

            // ret = JSON.stringify(resp);
            exit();
        }
    }
};

page.open(url, method, postdata, function (status) {
});
`
<commit_msg>Fix phantom bug<commit_after>// Copyright 2015 henrylee2cn Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package surfer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"mime"
	"net/http"
	"net/http/cookiejar"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

type (
	// Phantom 基于Phantomjs的下载器实现，作为surfer的补充
	// 效率较surfer会慢很多，但是因为模拟浏览器，破防性更好
	// 支持UserAgent/TryTimes/RetryPause/自定义js
	Phantom struct {
		PhantomjsFile string            //Phantomjs完整文件名
		TempJsDir     string            //临时js存放目录
		jsFileMap     map[string]string //已存在的js文件
		CookieJar     *cookiejar.Jar
	}
	// Response 用于解析Phantomjs的响应内容
	Response struct {
		Cookies []string
		Body    string
		Error   string
		Header  []struct {
			Name  string
			Value string
		}
	}

	//给phantomjs传输cookie用
	Cookie struct {
		Name   string `json:"name"`
		Value  string `json:"value"`
		Domain string `json:"domain"`
		Path   string `json:"path"`
	}
)

// NewPhantom 创建一个Phantomjs下载器
func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) Surfer {
	phantom := &Phantom{
		PhantomjsFile: phantomjsFile,
		TempJsDir:     tempJsDir,
		jsFileMap:     make(map[string]string),
	}
	if len(jar) != 0 {
		phantom.CookieJar = jar[0]
	} else {
		phantom.CookieJar, _ = cookiejar.New(nil)
	}
	if !filepath.IsAbs(phantom.PhantomjsFile) {
		phantom.PhantomjsFile, _ = filepath.Abs(phantom.PhantomjsFile)
	}
	if !filepath.IsAbs(phantom.TempJsDir) {
		phantom.TempJsDir, _ = filepath.Abs(phantom.TempJsDir)
	}
	// 创建/打开目录
	err := os.MkdirAll(phantom.TempJsDir, 0777)
	if err != nil {
		log.Printf("[E] Surfer: %v\n", err)
		return phantom
	}
	phantom.createJsFile("js", js)
	return phantom
}

// Download 实现surfer下载器接口
func (phantom *Phantom) Download(req *Request) (resp *http.Response, err error) {
	err = req.prepare()
	if err != nil {
		return resp, err
	}
	var encoding = "utf-8"
	if _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err == nil {
		if cs, ok := params["charset"]; ok {
			encoding = strings.ToLower(strings.TrimSpace(cs))
		}
	}

	req.Header.Del("Content-Type")

	cookie := ""
	if req.EnableCookie {
		httpCookies := phantom.CookieJar.Cookies(req.url)
		if len(httpCookies) > 0 {
			surferCookies := make([]*Cookie, len(httpCookies))

			for n, c := range httpCookies {
				surferCookie := &Cookie{Name: c.Name, Value: c.Value, Domain: req.url.Host, Path: "/"}
				surferCookies[n] = surferCookie
			}

			c, err := json.Marshal(surferCookies)
			if err != nil {
				log.Printf("cookie marshal error:%v", err)
			}
			cookie = string(c)
		}
	}

	var b, _ = req.ReadBody()
	urlObj := req.url
	resp = req.writeback(resp)
	resp.Request.URL = urlObj

	var args = []string{
		phantom.jsFileMap["js"],
		req.Url,
		cookie,
		encoding,
		req.Header.Get("User-Agent"),
		string(b),
		strings.ToLower(req.Method),
		fmt.Sprint(int(req.DialTimeout / time.Millisecond)),
	}

	for i := 0; i < req.TryTimes; i++ {
		if i != 0 {
			time.Sleep(req.RetryPause)
		}

		cmd := exec.Command(phantom.PhantomjsFile, args...)
		if resp.Body, err = cmd.StdoutPipe(); err != nil {
			continue
		}
		err = cmd.Start()
		if err != nil || resp.Body == nil {
			continue
		}
		var b []byte
		b, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			continue
		}
		retResp := Response{}
		err = json.Unmarshal(b, &retResp)
		if err != nil {
			continue
		}

		if retResp.Error != "" {
			log.Printf("phantomjs response error:%s", retResp.Error)
			continue
		}

		//设置header
		for _, h := range retResp.Header {
			resp.Header.Add(h.Name, h.Value)
		}

		//设置cookie
		for _, c := range retResp.Cookies {
			resp.Header.Add("Set-Cookie", c)
		}
		if req.EnableCookie {
			if rc := resp.Cookies(); len(rc) > 0 {
				phantom.CookieJar.SetCookies(urlObj, rc)
			}
		}
		resp.Body = ioutil.NopCloser(strings.NewReader(retResp.Body))
		break
	}

	if err == nil {
		resp.StatusCode = http.StatusOK
		resp.Status = http.StatusText(http.StatusOK)
	} else {
		resp.StatusCode = http.StatusBadGateway
		resp.Status = err.Error()
	}

	return resp, err
}

// DestroyJsFiles 销毁js临时文件
func (phantom *Phantom) DestroyJsFiles() {
	p, _ := filepath.Split(phantom.TempJsDir)
	if p == "" {
		return
	}
	for _, filename := range phantom.jsFileMap {
		os.Remove(filename)
	}
	if len(WalkDir(p)) == 1 {
		os.Remove(p)
	}
}

func (phantom *Phantom) createJsFile(fileName, jsCode string) {
	fullFileName := filepath.Join(phantom.TempJsDir, fileName)
	// 创建并写入文件
	f, _ := os.Create(fullFileName)
	f.Write([]byte(jsCode))
	f.Close()
	phantom.jsFileMap[fileName] = fullFileName
}

/*
* system.args[0] == js
* system.args[1] == url
* system.args[2] == cookie
* system.args[3] == pageEncode
* system.args[4] == userAgent
* system.args[5] == postdata
* system.args[6] == method
* system.args[7] == timeout
 */
const js string = `
var system = require('system');
var page = require('webpage').create();
var url = system.args[1];
var cookie = system.args[2];
var pageEncode = system.args[3];
var userAgent = system.args[4];
var postdata = system.args[5];
var method = system.args[6];
var timeout = system.args[7];

var ret = new Object();
var exit = function () {
    console.log(JSON.stringify(ret));
    phantom.exit();
};

//输出参数
// console.log("url=" + url);
// console.log("cookie=" + cookie);
// console.log("pageEncode=" + pageEncode);
// console.log("userAgent=" + userAgent);
// console.log("postdata=" + postdata);
// console.log("method=" + method);
// console.log("timeout=" + timeout);

// ret += (url + "\n");
// ret += (cookie + "\n");
// ret += (pageEncode + "\n");
// ret += (userAgent + "\n");
// ret += (postdata + "\n");
// ret += (method + "\n");
// ret += (timeout + "\n");
// exit();

phantom.outputEncoding = pageEncode;
page.settings.userAgent = userAgent;
page.settings.resourceTimeout = timeout;
page.settings.XSSAuditingEnabled = true;

function addCookie() {
    if (cookie != "") {
        var cookies = JSON.parse(cookie);
        for (var i = 0; i < cookies.length; i++) {
            var c = cookies[i];

            phantom.addCookie({
                'name': c.name, /* required property */
                'value': c.value, /* required property */
                'domain': c.domain,
                'path': c.path, /* required property */
            });
        }
    }
}

addCookie();

page.onResourceRequested = function (requestData, request) {

};
page.onResourceReceived = function (response) {
    if (response.stage === "end") {
        // console.log("liguoqinjim received1------------------------------------------------");
        // console.log("url=" + response.url);
        //
        // for (var j in response.headers) {//用javascript的for/in循环遍历对象的属性
        //     // var m = sprintf("AttrId[%d]Value[%d]", j, result.Attrs[j]);
        //     // message += m;
        //     // console.log(response.headers[j]);
        //     console.log(response.headers[j]["name"] + ":" + response.headers[j]["value"]);
        // }
        //
        // console.log("liguoqinjim received2------------------------------------------------");

        //在ret中加入header
        ret["Header"] = response.headers;
    }
};
page.onError = function (msg, trace) {
    ret["Error"] = msg;
    exit();
};
page.onResourceTimeout = function (e) {
    // console.log("phantomjs onResourceTimeout error");
    // console.log(e.errorCode);   // it'll probably be 408
    // console.log(e.errorString); // it'll probably be 'Network timeout on resource'
    // console.log(e.url);         // the url whose request timed out
    // phantom.exit(1);
    ret["Error"] = "onResourceTimeout";
    exit();
};
page.onResourceError = function (e) {
    // console.log("onResourceError");
    // console.log("1:" + e.errorCode + "," + e.errorString);

    if (e.errorCode != 5) { //errorCode=5的情况和onResourceTimeout冲突
        ret["Error"] = "onResourceError";
        exit();
    }
};
page.onLoadFinished = function (status) {
    if (status !== 'success') {
        ret["Error"] = "status=" + status;
        exit();
    } else {
        var cookies = new Array();
        for (var i in page.cookies) {
            var cookie = page.cookies[i];
            var c = cookie["name"] + "=" + cookie["value"];
            for (var obj in cookie) {
                if (obj == 'name' || obj == 'value') {
                    continue;
                }
                if (obj == "httponly" || obj == "secure") {
                    if (cookie[obj] == true) {
                        c += ";" + obj;
                    }
                } else {
                    c += "; " + obj + "=" + cookie[obj];
                }
            }
            cookies[i] = c;
        }
        if (page.content.indexOf("body") != -1) {
            ret["Cookies"] = cookies;
            ret["Body"] = page.content;

            // ret = JSON.stringify(resp);
            exit();
        }
    }
};

page.open(url, method, postdata, function (status) {
});
`
<|endoftext|>


<commit_before>//
// Copyright 2016 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package http provides HTTP functionality for querying TS data as
// well as submitting data points to a receiver.
package http

import (
	"fmt"
	"log"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/tgres/tgres/dsl"
	"github.com/tgres/tgres/misc"
)

func GraphiteMetricsFindHandler(rcache dsl.NamedDSFetcher) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "[\n")
		nodes := rcache.FsFind(r.FormValue("query"))
		for n, node := range nodes {
			parts := strings.Split(node.Name, ".")
			if node.Leaf {
				fmt.Fprintf(w, `{"leaf": 1, "context": {}, "text": "%s", "expandable": 0, "id": "%s", "allowChildren": 0}`, parts[len(parts)-1], node.Name)
			} else {
				fmt.Fprintf(w, `{"leaf": 0, "context": {}, "text": "%s", "expandable": 1, "id": "%s", "allowChildren": 1}`, parts[len(parts)-1], node.Name)
			}
			if n < len(nodes)-1 {
				fmt.Fprintf(w, ",\n")
			}
		}
		fmt.Fprintf(w, "\n]\n")
	}
}

func GraphiteRenderHandler(rcache dsl.NamedDSFetcher) http.HandlerFunc {

	return func(w http.ResponseWriter, r *http.Request) {

		from, err := parseTime(r.FormValue("from"))
		if err != nil {
			log.Printf("RenderHandler(): (from) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		to, err := parseTime(r.FormValue("until"))
		if err != nil {
			log.Printf("RenderHandler(): (unitl) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		} else if to == nil {
			tmp := time.Now()
			to = &tmp
		}
		points, err := strconv.Atoi(r.FormValue("maxDataPoints"))
		if err != nil {
			log.Printf("RenderHandler(): (maxDataPoints) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		fmt.Fprintf(w, "[")

		for tn, target := range r.Form["target"] {

			seriesMap, err := processTarget(rcache, target, from.Unix(), to.Unix(), int64(points))

			if err != nil {
				log.Printf("RenderHandler(): %v", err)
				break // Graphite behaviour is empty list
			}

			nn := 0
			for _, name := range seriesMap.SortedKeys() {
				series := seriesMap[name]

				alias := series.Alias()
				if alias != "" {
					name = alias
				}

				fmt.Fprintf(w, "\n"+`{"target": "%s", "datapoints": [`+"\n", name)

				n := 0
				for series.Next() {
					if n > 0 {
						fmt.Fprintf(w, ",")
					}
					value := series.CurrentValue()
					ts := series.CurrentTime().Add(-series.Step()).Unix() // NOTE: Graphite protocol marks the *beginning* of the point
					if ts > 0 {
						if math.IsNaN(value) || math.IsInf(value, 0) {
							fmt.Fprintf(w, "[null, %v]", ts)
						} else {
							fmt.Fprintf(w, "[%v, %v]", value, ts)
						}
						n++
					}
				}
				if nn < len(seriesMap)-1 || tn < len(r.Form["target"])-1 {
					fmt.Fprintf(w, "]},\n")
				} else {
					fmt.Fprintf(w, "]}")
				}
				series.Close()
				nn++
			}
		}
		fmt.Fprintf(w, "]\n")
	}
}

func parseTime(s string) (*time.Time, error) {

	if len(s) == 0 {
		return nil, nil
	}

	if s[0] == '-' { // relative
		if dur, err := misc.BetterParseDuration(s[1:len(s)]); err == nil {
			t := time.Now().Add(-dur)
			return &t, nil
		} else {
			return nil, fmt.Errorf("parseTime(): Error parsing relative time %q: %v", s, err)
		}
	} else { // absolute
		if s == "now" {
			t := time.Now()
			return &t, nil
		} else if i, err := strconv.ParseInt(s, 10, 64); err == nil {
			t := time.Unix(i, 0)
			return &t, nil
		} else {
			return nil, fmt.Errorf("parseTime(): Error parsing absolute time %q: %v", s, err)
		}
	}
}

// This is not perfect, but it's better than nothing. It seeks
// identifiers containing a dot and surrounds them with quotes - this
// prevents errors for series names parts of which begin with a digit,
// which is not valid Go syntax.
func quoteIdentifiers(target string) string {
	result := target
	// Note that commas are only allowed inside {} (aka "value expression")
	parts := regexp.MustCompile(`("?[\w*][\w\-.*]*({[\w\-.*,]*})?[\w\-.*]*[\w*]"?)`).FindAllString(target, -1)
	for _, part := range parts {
		if strings.Contains(part, ".") && !strings.HasPrefix(part, "\"") {
			result = quoteIdentifiers(strings.Replace(result, part, fmt.Sprintf("%q", part), -1))
			break
		}
	}
	return result
}

func processTarget(rcache dsl.NamedDSFetcher, target string, from, to, maxPoints int64) (dsl.SeriesMap, error) {
	target = quoteIdentifiers(target)
	// In our DSL everything must be a function call, so we wrap everything in group()
	query := fmt.Sprintf("group(%s)", target)
	return dsl.ParseDsl(rcache, query, time.Unix(from, 0), time.Unix(to, 0), maxPoints)
}
<commit_msg>Even though it is technically correct, lets not adjust graphite api points back one period for less confusion<commit_after>//
// Copyright 2016 Gregory Trubetskoy. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package http provides HTTP functionality for querying TS data as
// well as submitting data points to a receiver.
package http

import (
	"fmt"
	"log"
	"math"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/tgres/tgres/dsl"
	"github.com/tgres/tgres/misc"
)

func GraphiteMetricsFindHandler(rcache dsl.NamedDSFetcher) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "[\n")
		nodes := rcache.FsFind(r.FormValue("query"))
		for n, node := range nodes {
			parts := strings.Split(node.Name, ".")
			if node.Leaf {
				fmt.Fprintf(w, `{"leaf": 1, "context": {}, "text": "%s", "expandable": 0, "id": "%s", "allowChildren": 0}`, parts[len(parts)-1], node.Name)
			} else {
				fmt.Fprintf(w, `{"leaf": 0, "context": {}, "text": "%s", "expandable": 1, "id": "%s", "allowChildren": 1}`, parts[len(parts)-1], node.Name)
			}
			if n < len(nodes)-1 {
				fmt.Fprintf(w, ",\n")
			}
		}
		fmt.Fprintf(w, "\n]\n")
	}
}

func GraphiteRenderHandler(rcache dsl.NamedDSFetcher) http.HandlerFunc {

	return func(w http.ResponseWriter, r *http.Request) {

		from, err := parseTime(r.FormValue("from"))
		if err != nil {
			log.Printf("RenderHandler(): (from) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}
		to, err := parseTime(r.FormValue("until"))
		if err != nil {
			log.Printf("RenderHandler(): (unitl) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		} else if to == nil {
			tmp := time.Now()
			to = &tmp
		}
		points, err := strconv.Atoi(r.FormValue("maxDataPoints"))
		if err != nil {
			log.Printf("RenderHandler(): (maxDataPoints) %v", err)
			w.WriteHeader(http.StatusBadRequest)
			return
		}

		fmt.Fprintf(w, "[")

		for tn, target := range r.Form["target"] {

			seriesMap, err := processTarget(rcache, target, from.Unix(), to.Unix(), int64(points))

			if err != nil {
				log.Printf("RenderHandler(): %v", err)
				break // Graphite behaviour is empty list
			}

			nn := 0
			for _, name := range seriesMap.SortedKeys() {
				series := seriesMap[name]

				alias := series.Alias()
				if alias != "" {
					name = alias
				}

				fmt.Fprintf(w, "\n"+`{"target": "%s", "datapoints": [`+"\n", name)

				n := 0
				for series.Next() {
					if n > 0 {
						fmt.Fprintf(w, ",")
					}
					value := series.CurrentValue()
					ts := series.CurrentTime().Unix()
					if ts > 0 {
						if math.IsNaN(value) || math.IsInf(value, 0) {
							fmt.Fprintf(w, "[null, %v]", ts)
						} else {
							fmt.Fprintf(w, "[%v, %v]", value, ts)
						}
						n++
					}
				}
				if nn < len(seriesMap)-1 || tn < len(r.Form["target"])-1 {
					fmt.Fprintf(w, "]},\n")
				} else {
					fmt.Fprintf(w, "]}")
				}
				series.Close()
				nn++
			}
		}
		fmt.Fprintf(w, "]\n")
	}
}

func parseTime(s string) (*time.Time, error) {

	if len(s) == 0 {
		return nil, nil
	}

	if s[0] == '-' { // relative
		if dur, err := misc.BetterParseDuration(s[1:len(s)]); err == nil {
			t := time.Now().Add(-dur)
			return &t, nil
		} else {
			return nil, fmt.Errorf("parseTime(): Error parsing relative time %q: %v", s, err)
		}
	} else { // absolute
		if s == "now" {
			t := time.Now()
			return &t, nil
		} else if i, err := strconv.ParseInt(s, 10, 64); err == nil {
			t := time.Unix(i, 0)
			return &t, nil
		} else {
			return nil, fmt.Errorf("parseTime(): Error parsing absolute time %q: %v", s, err)
		}
	}
}

// This is not perfect, but it's better than nothing. It seeks
// identifiers containing a dot and surrounds them with quotes - this
// prevents errors for series names parts of which begin with a digit,
// which is not valid Go syntax.
func quoteIdentifiers(target string) string {
	result := target
	// Note that commas are only allowed inside {} (aka "value expression")
	parts := regexp.MustCompile(`("?[\w*][\w\-.*]*({[\w\-.*,]*})?[\w\-.*]*[\w*]"?)`).FindAllString(target, -1)
	for _, part := range parts {
		if strings.Contains(part, ".") && !strings.HasPrefix(part, "\"") {
			result = quoteIdentifiers(strings.Replace(result, part, fmt.Sprintf("%q", part), -1))
			break
		}
	}
	return result
}

func processTarget(rcache dsl.NamedDSFetcher, target string, from, to, maxPoints int64) (dsl.SeriesMap, error) {
	target = quoteIdentifiers(target)
	// In our DSL everything must be a function call, so we wrap everything in group()
	query := fmt.Sprintf("group(%s)", target)
	return dsl.ParseDsl(rcache, query, time.Unix(from, 0), time.Unix(to, 0), maxPoints)
}
<|endoftext|>


<commit_before>package main

import (
	builder "github.com/modcloth/bob"
	"github.com/modcloth/bob/config"
	"github.com/modcloth/bob/log"
	"github.com/modcloth/bob/parser"
	"github.com/modcloth/bob/version"
)

import (
	"github.com/benmanns/goworker"
	"github.com/onsi/gocleanup"
	"github.com/wsxiaoys/terminal/color"
)

import (
	"errors"
	"flag"
	"fmt"
	"os"
)

var runWorker = func() {
	logger = log.Initialize(false)
	flag.Parse()
	goworker.Register("DockerBuild", workerFunc)

	if err := goworker.Work(); err != nil {
		logger.Println(
			color.Sprintf("@{r!}Alas, something went wrong :'(@{|}\n----> %+v", err),
		)
	}
}

var workerFunc = func(queue string, args ...interface{}) (fake error) {
	if queue == "docker-build" {
		first := args[0].(map[string]interface{})
		pwd := first["pwd"].(string)
		build := fmt.Sprintf("%s/%s", pwd, first["build"].(string))

		os.Setenv("PWD", pwd)

		par, err := parser.NewParser(build, logger)
		if err != nil {
			logger.Println(
				color.Sprintf("@{r!}Alas, could not generate parser@{|}\n----> %+v", err),
			)
			gocleanup.Exit(73)
		}

		commandSequence, err := par.Parse()
		if err != nil {
			runtime.Println(color.Sprintf("@{r!}Alas, could not parse@{|}\n----> %+v", err))
			gocleanup.Exit(23)
		}

		bob, err := builder.NewBuilder(logger, true)
		if err != nil {
			logger.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %+v",
					err,
				),
			)
			gocleanup.Exit(61)
		}
		bob.Builderfile = build

		if err = bob.Build(commandSequence); err != nil {
			logger.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %+v",
					err,
				),
			)
			gocleanup.Exit(29)
		}
		return
	}

	return errors.New("invalid attempt to use as a goworker")
}

var allTheThings = func() {
	runtime = config.NewRuntime()
	ver = version.NewVersion()

	// if user requests version/branch/rev
	if runtime.Version {
		runtime.Println(ver.Version)
	} else if runtime.VersionFull {
		runtime.Println(ver.VersionFull)
	} else if runtime.Branch {
		runtime.Println(ver.Branch)
	} else if runtime.Rev {
		runtime.Println(ver.Rev)
	} else if runtime.Lintfile != "" {
		// lint
		par, _ = parser.NewParser(runtime.Lintfile, runtime)
		par.AssertLint()
	} else {
		if runtime.Builderfile == "" {
			runtime.Builderfile = "bob.toml"
		}
		// otherwise, build
		par, err := parser.NewParser(runtime.Builderfile, runtime)
		if err != nil {
			runtime.Println(
				color.Sprintf("@{r!}Alas, could not generate parser@{|}\n----> %+v", err),
			)
			gocleanup.Exit(73)
		}

		commandSequence, err := par.Parse()
		if err != nil {
			runtime.Println(color.Sprintf("@{r!}Alas, could not parse@{|}\n----> %+v", err))
			gocleanup.Exit(23)
		}

		bob, err := builder.NewBuilder(runtime, true)
		if err != nil {
			runtime.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %+v",
					err,
				),
			)
			gocleanup.Exit(61)
		}

		bob.Builderfile = runtime.Builderfile

		if err = bob.Build(commandSequence); err != nil {
			runtime.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %+v",
					err,
				),
			)
			gocleanup.Exit(29)
		}
	}

	gocleanup.Exit(0)
}
<commit_msg>Making sure we print error messages more informatively<commit_after>package main

import (
	builder "github.com/modcloth/bob"
	"github.com/modcloth/bob/config"
	"github.com/modcloth/bob/log"
	"github.com/modcloth/bob/parser"
	"github.com/modcloth/bob/version"
)

import (
	"github.com/benmanns/goworker"
	"github.com/onsi/gocleanup"
	"github.com/wsxiaoys/terminal/color"
)

import (
	"errors"
	"flag"
	"fmt"
	"os"
)

var runWorker = func() {
	logger = log.Initialize(false)
	flag.Parse()
	goworker.Register("DockerBuild", workerFunc)

	if err := goworker.Work(); err != nil {
		logger.Println(
			color.Sprintf("@{r!}Alas, something went wrong :'(@{|}\n----> %q", err),
		)
	}
}

var workerFunc = func(queue string, args ...interface{}) (fake error) {
	if queue == "docker-build" {
		first := args[0].(map[string]interface{})
		pwd := first["pwd"].(string)
		build := fmt.Sprintf("%s/%s", pwd, first["build"].(string))

		os.Setenv("PWD", pwd)

		par, err := parser.NewParser(build, logger)
		if err != nil {
			logger.Println(
				color.Sprintf("@{r!}Alas, could not generate parser@{|}\n----> %q", err),
			)
			gocleanup.Exit(73)
		}

		commandSequence, err := par.Parse()
		if err != nil {
			runtime.Println(color.Sprintf("@{r!}Alas, could not parse@{|}\n----> %q", err))
			gocleanup.Exit(23)
		}

		bob, err := builder.NewBuilder(logger, true)
		if err != nil {
			logger.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %q",
					err,
				),
			)
			gocleanup.Exit(61)
		}
		bob.Builderfile = build

		if err = bob.Build(commandSequence); err != nil {
			logger.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %q",
					err,
				),
			)
			gocleanup.Exit(29)
		}
		return
	}

	return errors.New("invalid attempt to use as a goworker")
}

var allTheThings = func() {
	runtime = config.NewRuntime()
	ver = version.NewVersion()

	// if user requests version/branch/rev
	if runtime.Version {
		runtime.Println(ver.Version)
	} else if runtime.VersionFull {
		runtime.Println(ver.VersionFull)
	} else if runtime.Branch {
		runtime.Println(ver.Branch)
	} else if runtime.Rev {
		runtime.Println(ver.Rev)
	} else if runtime.Lintfile != "" {
		// lint
		par, _ = parser.NewParser(runtime.Lintfile, runtime)
		par.AssertLint()
	} else {
		if runtime.Builderfile == "" {
			runtime.Builderfile = "bob.toml"
		}
		// otherwise, build
		par, err := parser.NewParser(runtime.Builderfile, runtime)
		if err != nil {
			runtime.Println(
				color.Sprintf("@{r!}Alas, could not generate parser@{|}\n----> %q", err),
			)
			gocleanup.Exit(73)
		}

		commandSequence, err := par.Parse()
		if err != nil {
			runtime.Println(color.Sprintf("@{r!}Alas, could not parse@{|}\n----> %q", err))
			gocleanup.Exit(23)
		}

		bob, err := builder.NewBuilder(runtime, true)
		if err != nil {
			runtime.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %q",
					err,
				),
			)
			gocleanup.Exit(61)
		}

		bob.Builderfile = runtime.Builderfile

		if err = bob.Build(commandSequence); err != nil {
			runtime.Println(
				color.Sprintf(
					"@{r!}Alas, I am unable to complete my assigned build because of...@{|}\n----> %q",
					err,
				),
			)
			gocleanup.Exit(29)
		}
	}

	gocleanup.Exit(0)
}
<|endoftext|>


<commit_before>package resolves

import (
	"strconv"
	"strings"
	"time"

	"github.com/hysios/apiai-go"
	"github.com/wanliu/brain_data/database"
	"github.com/wanliu/flow/builtin/ai"

	. "github.com/wanliu/flow/context"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OrderResolve struct {
	AiParams  ai.AiOrder
	Products  ItemsResolve
	Gifts     ItemsResolve
	Address   string
	Customer  string
	Time      time.Time
	DefTime   string
	Current   Resolve
	Note      string
	UpdatedAt time.Time
	Editing   bool
	Canceled  bool

	User *database.User
}

func NewOrderResolve(ctx Context) *OrderResolve {
	resolve := new(OrderResolve)
	resolve.Touch()

	aiResult := ctx.Value("Result").(apiai.Result)

	resolve.AiParams = ai.ApiAiOrder{AiResult: aiResult}
	resolve.ExtractFromParams()

	if viewer := ctx.Value("Viewer"); viewer != nil {
		user := viewer.(*database.User)
		resolve.User = user
	}

	return resolve
}

func (r *OrderResolve) Solve(aiResult apiai.Result) string {
	return r.Answer()
}

func (r *OrderResolve) Touch() {
	r.UpdatedAt = time.Now()
}

func (r OrderResolve) Modifable(expireMin int) bool {
	return !r.Expired(expireMin) || r.Submited()
}

// TODO
func (r OrderResolve) Cancelable() bool {
	return true
}

// TODO
func (r *OrderResolve) Cancel() bool {
	r.Canceled = true
	return true
}

func (r OrderResolve) Fulfiled() bool {
	return len(r.Products.Products) > 0 && (r.Address != "" || r.Customer != "")
}

func (r OrderResolve) Expired(expireMin int) bool {
	return r.UpdatedAt.Add(time.Duration(expireMin)*time.Minute).UnixNano() < time.Now().UnixNano()
}

// TODO
func (r OrderResolve) Submited() bool {
	return false
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OrderResolve) ExtractFromParams() {
	r.ExtractItems()
	r.ExtractGiftItems()
	r.ExtractAddress()
	r.ExtractCustomer()
	r.ExtractTime()
	r.ExtractNote()
}

func (r *OrderResolve) ExtractItems() {
	for _, i := range r.AiParams.Items() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Products.Products = append(r.Products.Products, item)
	}
}

func (r *OrderResolve) ExtractGiftItems() {
	for _, i := range r.AiParams.GiftItems() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Gifts.Products = append(r.Gifts.Products, item)
	}
}

func (r *OrderResolve) ExtractAddress() {
	r.Address = r.AiParams.Address()
}

func (r *OrderResolve) ExtractCustomer() {
	r.Customer = r.AiParams.Customer()
}

func (r *OrderResolve) ExtractTime() {
	r.Time = r.AiParams.Time()
}

func (r *OrderResolve) ExtractNote() {
	r.Note = r.AiParams.Note()
}

func (r *OrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OrderResolve) Answer() string {
	if r.Fulfiled() {
		return r.PostOrderAndAnswer()
	} else {
		return r.AnswerHead() + r.AnswerFooter("", "")
	}
}

func (r *OrderResolve) PostOrderAndAnswer() string {
	items := make([]database.OrderItem, 0, 0)
	gifts := make([]database.GiftItem, 0, 0)

	for _, pr := range r.Products.Products {
		item, err := database.NewOrderItem("", pr.Product, uint(pr.Quantity), pr.Price)
		if err != nil {
			return err.Error()
		}
		items = append(items, *item)
	}

	for _, pr := range r.Gifts.Products {
		gift, err := database.NewGiftItem("", pr.Product, uint(pr.Quantity))
		if err != nil {
			return err.Error()
		}

		gifts = append(gifts, *gift)
	}

	order, err := r.User.CreateSaledOrder(r.Address, r.Note, r.Time, 0, items, gifts)

	if err != nil {
		return err.Error()
	} else {
		return r.AnswerHead() + r.AnswerBody() + r.AnswerFooter(order.ID, order.GlobelId())
	}
}

func (r OrderResolve) AddressInfo() string {
	if r.Address != "" && r.Customer != "" {
		return "地址:" + r.Address + r.Customer + "\n"
	} else if r.Address != "" {
		return "地址:" + r.Address + "\n"
	} else if r.Customer != "" {
		return "客户:" + r.Customer + "\n"
	} else {
		return ""
	}
}

func (r OrderResolve) AnswerHead() string {
	desc := "订单正在处理, 已经添加" + CnNum(len(r.Products.Products)) + "种产品"

	if r.Fulfiled() {
		desc = "订单已经生成, 共" + CnNum(len(r.Products.Products)) + "种产品"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + ", " + CnNum(len(r.Gifts.Products)) + "种赠品" + "\n"
	} else {
		desc = desc + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerBody() string {
	desc := ""

	for _, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + "申请的赠品:\n"

		for _, g := range r.Gifts.Products {
			desc = desc + g.Product + " " + strconv.Itoa(g.Quantity) + "件\n"
		}
	}

	desc = desc + "时间:" + r.Time.Format("2006年01月02日") + "\n"

	if r.Note != "" {
		desc = desc + "备注：" + r.Note + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerFooter(no, id interface{}) string {
	desc := ""

	if r.Fulfiled() {
		desc = desc + r.AddressInfo()
		desc = desc + "订单已经生成，订单号为：" + fmt.Sprint(no) + "\n"
		desc = desc + "订单入口: http://wanliu.biz/orders/" + fmt.Sprint(id)
	} else {
		desc = desc + "还缺少收货地址或客户信息\n"
	}

	return desc
}

func CnNum(num int) string {
	switch num {
	case 1:
		return "一"
	case 2:
		return "两"
	case 3:
		return "三"
	case 4:
		return "四"
	case 5:
		return "五"
	case 6:
		return "六"
	case 7:
		return "七"
	case 8:
		return "八"
	case 9:
		return "九"
	case 10:
		return "十"
	default:
		return strconv.Itoa(num)
	}
}
<commit_msg>create order for user<commit_after>package resolves

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/hysios/apiai-go"
	"github.com/wanliu/brain_data/database"
	"github.com/wanliu/flow/builtin/ai"

	. "github.com/wanliu/flow/context"
)

// 处理开单的逻辑结构, 不需要是组件
// 作为context的一个部分，或者存在一个Value中
type OrderResolve struct {
	AiParams  ai.AiOrder
	Products  ItemsResolve
	Gifts     ItemsResolve
	Address   string
	Customer  string
	Time      time.Time
	DefTime   string
	Current   Resolve
	Note      string
	UpdatedAt time.Time
	Editing   bool
	Canceled  bool

	User *database.User
}

func NewOrderResolve(ctx Context) *OrderResolve {
	resolve := new(OrderResolve)
	resolve.Touch()

	aiResult := ctx.Value("Result").(apiai.Result)

	resolve.AiParams = ai.ApiAiOrder{AiResult: aiResult}
	resolve.ExtractFromParams()

	if viewer := ctx.Value("Viewer"); viewer != nil {
		user := viewer.(*database.User)
		resolve.User = user
	}

	return resolve
}

func (r *OrderResolve) Solve(aiResult apiai.Result) string {
	return r.Answer()
}

func (r *OrderResolve) Touch() {
	r.UpdatedAt = time.Now()
}

func (r OrderResolve) Modifable(expireMin int) bool {
	return !r.Expired(expireMin) || r.Submited()
}

// TODO
func (r OrderResolve) Cancelable() bool {
	return true
}

// TODO
func (r *OrderResolve) Cancel() bool {
	r.Canceled = true
	return true
}

func (r OrderResolve) Fulfiled() bool {
	return len(r.Products.Products) > 0 && (r.Address != "" || r.Customer != "")
}

func (r OrderResolve) Expired(expireMin int) bool {
	return r.UpdatedAt.Add(time.Duration(expireMin)*time.Minute).UnixNano() < time.Now().UnixNano()
}

// TODO
func (r OrderResolve) Submited() bool {
	return false
}

// 从ｌｕｉｓ数据构造结构数据
func (r *OrderResolve) ExtractFromParams() {
	r.ExtractItems()
	r.ExtractGiftItems()
	r.ExtractAddress()
	r.ExtractCustomer()
	r.ExtractTime()
	r.ExtractNote()
}

func (r *OrderResolve) ExtractItems() {
	for _, i := range r.AiParams.Items() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Products.Products = append(r.Products.Products, item)
	}
}

func (r *OrderResolve) ExtractGiftItems() {
	for _, i := range r.AiParams.GiftItems() {
		name := strings.Replace(i.Product, "%", "%%", -1)
		item := &ItemResolve{
			Resolved: true,
			Name:     name,
			Price:    i.Price,
			Quantity: i.Quantity,
			Product:  name,
		}

		r.Gifts.Products = append(r.Gifts.Products, item)
	}
}

func (r *OrderResolve) ExtractAddress() {
	r.Address = r.AiParams.Address()
}

func (r *OrderResolve) ExtractCustomer() {
	r.Customer = r.AiParams.Customer()
}

func (r *OrderResolve) ExtractTime() {
	r.Time = r.AiParams.Time()
}

func (r *OrderResolve) ExtractNote() {
	r.Note = r.AiParams.Note()
}

func (r *OrderResolve) SetDefTime(t string) {
	r.DefTime = t

	if r.Time.IsZero() && r.DefTime != "" {
		r.SetTimeByDef()
	}
}

func (r *OrderResolve) SetTimeByDef() {
	if r.DefTime == "今天" {
		r.Time = time.Now()
	} else if r.DefTime == "明天" {
		r.Time = time.Now().Add(24 * time.Hour)
	}
}

func (r OrderResolve) EmptyProducts() bool {
	return len(r.Products.Products) == 0
}

func (r OrderResolve) Answer() string {
	if r.Fulfiled() {
		return r.PostOrderAndAnswer()
	} else {
		return r.AnswerHead() + r.AnswerFooter("", "")
	}
}

func (r *OrderResolve) PostOrderAndAnswer() string {
	items := make([]database.OrderItem, 0, 0)
	gifts := make([]database.GiftItem, 0, 0)

	for _, pr := range r.Products.Products {
		item, err := database.NewOrderItem("", pr.Product, uint(pr.Quantity), pr.Price)
		if err != nil {
			return err.Error()
		}
		items = append(items, *item)
	}

	for _, pr := range r.Gifts.Products {
		gift, err := database.NewGiftItem("", pr.Product, uint(pr.Quantity))
		if err != nil {
			return err.Error()
		}

		gifts = append(gifts, *gift)
	}

	order, err := r.User.CreateSaledOrder(r.Address, r.Note, r.Time, 0, items, gifts)

	if err != nil {
		return err.Error()
	} else {
		return r.AnswerHead() + r.AnswerBody() + r.AnswerFooter(order.ID, order.GlobelId())
	}
}

func (r OrderResolve) AddressInfo() string {
	if r.Address != "" && r.Customer != "" {
		return "地址:" + r.Address + r.Customer + "\n"
	} else if r.Address != "" {
		return "地址:" + r.Address + "\n"
	} else if r.Customer != "" {
		return "客户:" + r.Customer + "\n"
	} else {
		return ""
	}
}

func (r OrderResolve) AnswerHead() string {
	desc := "订单正在处理, 已经添加" + CnNum(len(r.Products.Products)) + "种产品"

	if r.Fulfiled() {
		desc = "订单已经生成, 共" + CnNum(len(r.Products.Products)) + "种产品"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + ", " + CnNum(len(r.Gifts.Products)) + "种赠品" + "\n"
	} else {
		desc = desc + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerBody() string {
	desc := ""

	for _, p := range r.Products.Products {
		desc = desc + p.Product + " " + strconv.Itoa(p.Quantity) + "件\n"
	}

	if len(r.Gifts.Products) > 0 {
		desc = desc + "申请的赠品:\n"

		for _, g := range r.Gifts.Products {
			desc = desc + g.Product + " " + strconv.Itoa(g.Quantity) + "件\n"
		}
	}

	desc = desc + "时间:" + r.Time.Format("2006年01月02日") + "\n"

	if r.Note != "" {
		desc = desc + "备注：" + r.Note + "\n"
	}

	return desc
}

func (r OrderResolve) AnswerFooter(no, id interface{}) string {
	desc := ""

	if r.Fulfiled() {
		desc = desc + r.AddressInfo()
		desc = desc + "订单已经生成，订单号为：" + fmt.Sprint(no) + "\n"
		desc = desc + "订单入口: http://wanliu.biz/orders/" + fmt.Sprint(id)
	} else {
		desc = desc + "还缺少收货地址或客户信息\n"
	}

	return desc
}

func CnNum(num int) string {
	switch num {
	case 1:
		return "一"
	case 2:
		return "两"
	case 3:
		return "三"
	case 4:
		return "四"
	case 5:
		return "五"
	case 6:
		return "六"
	case 7:
		return "七"
	case 8:
		return "八"
	case 9:
		return "九"
	case 10:
		return "十"
	default:
		return strconv.Itoa(num)
	}
}
<|endoftext|>


<commit_before>package process

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

// BufferSize determines how much output will be read into memory before resorting to using a temporary file
const BufferSize = 1048576

// UnknownExit is used when the return/exit-code of the command is not known.
const UnknownExit = 1

func getShell() string {
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "sh"
	}
	return shell
}

// Command contains a buffered reader with the realized stdout of the process along with the exit code.
type Command struct {
	*bufio.Reader
	tmpName string
	Err     error
	cmd     string
}

func (c *Command) error() string {
	if c == nil || c.Err == nil {
		return ""
	}
	return c.Err.Error()
}

func (c *Command) String() string {
	cmd := c.cmd
	if len(c.cmd) > 100 {
		cmd = cmd[:80] + "..."
	}
	out, _ := c.Peek(20)
	return fmt.Sprintf("Command('%s', output[:20]: %s, exit-code: %d, error: %s)",
		cmd, strings.Replace(string(out), "\n", "\\n", -1), c.ExitCode(), c.error())
}

// ExitCode returns the exit code associated with a given error
func (c *Command) ExitCode() int {
	if c.Err == nil {
		return 0
	}
	if ex, ok := c.Err.(*exec.ExitError); ok {
		if st, ok := ex.Sys().(syscall.WaitStatus); ok {
			return st.ExitStatus()
		}
	}
	return UnknownExit
}

func cleanup(c *Command) {
	os.Remove(c.tmpName)
}

func newCommand(rdr *bufio.Reader, tmpName string, cmd string, err error) *Command {
	c := &Command{rdr, tmpName, err, cmd}
	if tmpName != "" {
		runtime.SetFinalizer(c, cleanup)
	}
	return c
}

// Run takes a command string, executes the command,
// Blocks until the output is finished and returns a *Command
// that is an io.Reader. If retries > 0 it will retry on a
// non-zero exit-code.
func Run(command string, retries int) *Command {
	c := oneRun(command)
	for retries > 0 && c.ExitCode() != 0 {
		retries--
		c = oneRun(command)
	}
	return c
}

func oneRun(command string) *Command {

	cmd := exec.Command(getShell(), "-c", command)

	opipe, err := cmd.StdoutPipe()
	if err != nil {
		return newCommand(nil, "", command, err)
	}
	defer opipe.Close()

	cmd.Stderr = os.Stderr

	err = cmd.Start()
	if err != nil {
		return newCommand(nil, "", command, err)
	}

	bpipe := bufio.NewReaderSize(opipe, BufferSize)
	var res []byte
	res, err = bpipe.Peek(BufferSize)

	// less than BufferSize bytes in output...
	if err == bufio.ErrBufferFull || err == io.EOF {
		err = cmd.Wait()
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	if err != nil {
		return newCommand(nil, "", command, err)
	}

	// more than BufferSize bytes in output. must use tmpfile
	var tmp *os.File
	tmp, err = ioutil.TempFile("", "gargsTmp.")
	if err != nil {
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	btmp := bufio.NewWriter(tmp)
	_, err = io.CopyBuffer(btmp, bpipe, res)
	if err != nil {
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	err = opipe.Close()
	btmp.Flush()
	_, err = tmp.Seek(0, 0)
	return newCommand(bufio.NewReader(tmp), tmp.Name(), command, err)
}

// Runner accepts commands from a channel and sends a bufio.Reader on the returned channel.
// done allows the caller to stop Runner, for example if an error occurs.
// It will parallelize according to GOMAXPROCS.
func Runner(commands <-chan string, retries int, cancel <-chan bool) chan *Command {

	stdout := make(chan *Command, runtime.GOMAXPROCS(0))

	wg := &sync.WaitGroup{}
	wg.Add(runtime.GOMAXPROCS(0))

	// Start a number of workers equal to the requested procs.
	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go func() {
			defer wg.Done()
			// workers read off the same channel of incoming commands.
			for cmdStr := range commands {
				v := Run(cmdStr, retries)
				select {
				case stdout <- v:
				// if we receive from this, we must exit.
				// receive from closed channel will continually yield false
				// so it does what we expect.
				case <-cancel:
					return
				}
			}
		}()
	}

	// wait for all the workers to finish.
	go func() {
		wg.Wait()
		close(stdout)
	}()

	return stdout
}
<commit_msg>simplify<commit_after>package process

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

// BufferSize determines how much output will be read into memory before resorting to using a temporary file
const BufferSize = 1048576

// UnknownExit is used when the return/exit-code of the command is not known.
const UnknownExit = 1

func getShell() string {
	shell := os.Getenv("SHELL")
	if shell == "" {
		shell = "sh"
	}
	return shell
}

// Command contains a buffered reader with the realized stdout of the process along with the exit code.
type Command struct {
	*bufio.Reader
	tmpName string
	Err     error
	cmd     string
}

func (c *Command) error() string {
	if c == nil || c.Err == nil {
		return ""
	}
	return c.Err.Error()
}

func (c *Command) String() string {
	cmd := c.cmd
	if len(c.cmd) > 100 {
		cmd = cmd[:80] + "..."
	}
	out, _ := c.Peek(20)
	return fmt.Sprintf("Command('%s', output[:20]: %s, exit-code: %d, error: %s)",
		cmd, strings.Replace(string(out), "\n", "\\n", -1), c.ExitCode(), c.error())
}

// ExitCode returns the exit code associated with a given error
func (c *Command) ExitCode() int {
	if c.Err == nil {
		return 0
	}
	if ex, ok := c.Err.(*exec.ExitError); ok {
		if st, ok := ex.Sys().(syscall.WaitStatus); ok {
			return st.ExitStatus()
		}
	}
	return UnknownExit
}

func cleanup(c *Command) {
	os.Remove(c.tmpName)
}

func newCommand(rdr *bufio.Reader, tmpName string, cmd string, err error) *Command {
	c := &Command{rdr, tmpName, err, cmd}
	if tmpName != "" {
		runtime.SetFinalizer(c, cleanup)
	}
	return c
}

// Run takes a command string, executes the command,
// Blocks until the output is finished and returns a *Command
// that is an io.Reader. If retries > 0 it will retry on a
// non-zero exit-code.
func Run(command string, retries int) *Command {
	c := oneRun(command)
	for retries > 0 && c.ExitCode() != 0 {
		retries--
		c = oneRun(command)
	}
	return c
}

func oneRun(command string) *Command {

	cmd := exec.Command(getShell(), "-c", command)

	opipe, err := cmd.StdoutPipe()
	if err != nil {
		return newCommand(nil, "", command, err)
	}
	defer opipe.Close()

	cmd.Stderr = os.Stderr

	err = cmd.Start()
	if err != nil {
		return newCommand(nil, "", command, err)
	}

	bpipe := bufio.NewReaderSize(opipe, BufferSize)
	var res []byte
	res, err = bpipe.Peek(BufferSize)

	// less than BufferSize bytes in output...
	if err == bufio.ErrBufferFull || err == io.EOF {
		err = cmd.Wait()
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	if err != nil {
		return newCommand(nil, "", command, err)
	}

	// more than BufferSize bytes in output. must use tmpfile
	var tmp *os.File
	tmp, err = ioutil.TempFile("", "gargsTmp.")
	if err != nil {
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	btmp := bufio.NewWriter(tmp)
	_, err = io.CopyBuffer(btmp, bpipe, res)
	if err != nil {
		return newCommand(bufio.NewReader(bytes.NewReader(res)), "", command, err)
	}
	err = opipe.Close()
	btmp.Flush()
	_, err = tmp.Seek(0, 0)
	return newCommand(bufio.NewReader(tmp), tmp.Name(), command, err)
}

// Runner accepts commands from a channel and sends a bufio.Reader on the returned channel.
// done allows the caller to stop Runner, for example if an error occurs.
// It will parallelize according to GOMAXPROCS.
func Runner(commands <-chan string, retries int, cancel <-chan bool) chan *Command {

	stdout := make(chan *Command, runtime.GOMAXPROCS(0))

	wg := &sync.WaitGroup{}
	wg.Add(runtime.GOMAXPROCS(0))

	// Start a number of workers equal to the requested procs.
	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
		go func() {
			defer wg.Done()
			// workers read off the same channel of incoming commands.
			for cmdStr := range commands {
				select {
				case stdout <- Run(cmdStr, retries):
				// if we receive from this, we must exit.
				// receive from closed channel will continually yield false
				// so it does what we expect.
				case <-cancel:
					return
				}
			}
		}()
	}

	// wait for all the workers to finish.
	go func() {
		wg.Wait()
		close(stdout)
	}()

	return stdout
}
<|endoftext|>