<commit_before>// Package parse implements parsing of the BUILD files via an embedded Python interpreter.
//
// The actual work here is done by an embedded PyPy instance. Various rules are built in to
// the binary itself using go-bindata to embed the .py files; these are always available to
// all programs which is rather nice, but it does mean that must be run before 'go run' etc
// will work as expected.
package parse

import (
	"fmt"
	"path"
	"strings"

	"gopkg.in/op/go-logging.v1"

	"github.com/thought-machine/please/src/cli"
	"github.com/thought-machine/please/src/core"
	"github.com/thought-machine/please/src/fs"
	"github.com/thought-machine/please/src/worker"
)

var log = logging.MustGetLogger("parse")

// Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package.
// It is not an error if the package has already been parsed.
//
// By default, after the package is parsed, any targets that are now needed for the build and ready
// to be built are queued, and any new packages are queued for parsing. When a specific label is requested
// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it:
// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only
// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added.
// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed
// even when we're not otherwise building targets.
func Parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) {
	if err := parse(tid, state, label, dependor, include, exclude, forSubinclude); err != nil {
		state.LogBuildError(tid, label, core.ParseFailed, err, "Failed to parse package")
	}
}

func parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) error {
	// See if something else has parsed this package first.
	pkg := state.WaitForPackage(label)
	if pkg != nil {
		// Does exist, all we need to do is toggle on this target
		return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
	}
	// If we get here then it falls to us to parse this package.
	state.LogBuildResult(tid, label, core.PackageParsing, "Parsing...")

	subrepo, err := checkSubrepo(tid, state, label, dependor)
	if err != nil {
		return err
	} else if subrepo != nil && subrepo.Target != nil {
		// We have got the definition of the subrepo but it depends on something, make sure that has been built.
		state.WaitForBuiltTarget(subrepo.Target.Label, label)
	}
	// Subrepo & nothing else means we just want to ensure that subrepo is present.
	if label.Subrepo != "" && label.PackageName == "" && label.Name == "" {
		return nil
	}
	pkg, err = parsePackage(state, label, dependor, subrepo)
	if err != nil {
		return err
	}
	state.LogBuildResult(tid, label, core.PackageParsed, "Parsed package")
	return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
}

// checkSubrepo checks whether this guy exists within a subrepo. If so we will need to make sure that's available first.
func checkSubrepo(tid int, state *core.BuildState, label, dependor core.BuildLabel) (*core.Subrepo, error) {
	if label.Subrepo == "" {
		return nil, nil
	} else if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	// We don't have the definition of it at all. Need to parse that first.
	sl := label.SubrepoLabel()
	if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, "", label); err != nil {
		return nil, err
	} else if !handled {
		if _, err := parseSubrepoPackage(tid, state, sl.PackageName, dependor.Subrepo, label); err != nil {
			return nil, err
		}
	}
	if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	} else if subrepo := checkArchSubrepo(state, label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	return nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo)
}

// parseSubrepoPackage parses a package to make sure subrepos are available.
func parseSubrepoPackage(tid int, state *core.BuildState, pkg, subrepo string, dependor core.BuildLabel) (bool, error) {
	if state.Graph.Package(pkg, subrepo) == nil {
		// Don't have it already, must parse.
		label := core.BuildLabel{Subrepo: subrepo, PackageName: pkg, Name: "all"}
		return true, parse(tid, state, label, dependor, nil, nil, true)
	}
	return false, nil
}

// checkArchSubrepo checks if a target refers to a cross-compiling subrepo.
// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to.
func checkArchSubrepo(state *core.BuildState, name string) *core.Subrepo {
	var arch cli.Arch
	if err := arch.UnmarshalFlag(name); err == nil {
		return state.Graph.MaybeAddSubrepo(core.SubrepoForArch(state, arch))
	}
	return nil
}

// activateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses.
func activateTarget(state *core.BuildState, pkg *core.Package, label, dependor core.BuildLabel, forSubinclude bool, include, exclude []string) error {
	if !label.IsAllTargets() && state.Graph.Target(label) == nil {
		if label.Subrepo == "" && label.PackageName == "" && label.Name == dependor.Subrepo {
			if subrepo := checkArchSubrepo(state, label.Name); subrepo != nil {
				return nil
			}
		}
		if state.Config.Bazel.Compatibility && forSubinclude {
			// Bazel allows some things that look like build targets but aren't - notably the syntax
			// to load(). It suits us to treat that as though it is one, but we now have to
			// implicitly make it available.
			exportFile(state, pkg, label)
		} else {
			msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name)
			if dependor != core.OriginalTarget {
				msg += fmt.Sprintf(" (depended on by %s)", dependor)
			}
			return fmt.Errorf(msg + suggestTargets(pkg, label, dependor))
		}
	}
	if state.ParsePackageOnly && !forSubinclude {
		return nil // Some kinds of query don't need a full recursive parse.
	} else if label.IsAllTargets() {
		if dependor == core.OriginalTarget {
			for _, target := range pkg.AllTargets() {
				// Don't activate targets that were added in a post-build function; that causes a race condition
				// between the post-build functions running and other things trying to activate them too early.
				if state.ShouldInclude(target) && !target.AddedPostBuild {
					// Must always do this for coverage because we need to calculate sources of
					// non-test targets later on.
					if !state.NeedTests || target.IsTest || state.NeedCoverage {
						state.QueueTarget(target.Label, dependor, false, dependor.IsAllTargets())
					}
				}
			}
		}
	} else {
		for _, l := range state.Graph.DependentTargets(dependor, label) {
			// We use :all to indicate a dependency needed for parse.
			state.QueueTarget(l, dependor, false, forSubinclude || dependor.IsAllTargets())
		}
	}
	return nil
}

// parsePackage performs the initial parse of a package.
func parsePackage(state *core.BuildState, label, dependor core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) {
	packageName := label.PackageName
	pkg := core.NewPackage(packageName)
	pkg.Subrepo = subrepo
	if subrepo != nil {
		pkg.SubrepoName = subrepo.Name
	}
	filename, dir := buildFileName(state, label.PackageName, subrepo)
	if filename == "" {
		if success, err := providePackage(state, pkg); err != nil {
			return nil, err
		} else if !success && packageName == "" && dependor.Subrepo == "pleasings" && subrepo == nil && state.Config.Parse.BuiltinPleasings {
			// Deliberate fallthrough, for the case where someone depended on the default
			// @pleasings subrepo, and there is no BUILD file at their root.
		} else if !success {
			exists := core.PathExists(dir)
			// Handle quite a few cases to provide more obvious error messages.
			if dependor != core.OriginalTarget && exists {
				return nil, fmt.Errorf("%s depends on %s, but there's no %s file in %s/", dependor, label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			} else if dependor != core.OriginalTarget {
				return nil, fmt.Errorf("%s depends on %s, but the directory %s doesn't exist", dependor, label, dir)
			} else if exists {
				return nil, fmt.Errorf("Can't build %s; there's no %s file in %s/", label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			}
			return nil, fmt.Errorf("Can't build %s; the directory %s doesn't exist", label, dir)
		}
	} else {
		pkg.Filename = filename
		if err := state.Parser.ParseFile(state, pkg, pkg.Filename); err != nil {
			return nil, err
		}
	}
	// If the config setting is on, we "magically" register a default repo called @pleasings.
	if packageName == "" && subrepo == nil && state.Config.Parse.BuiltinPleasings && pkg.Target("pleasings") == nil {
		if _, err := state.Parser.(*aspParser).asp.ParseReader(pkg, strings.NewReader(pleasings)); err != nil {
			log.Fatalf("Failed to load pleasings: %s", err) // This shouldn't happen, of course.
		}
	}
	// Verify some details of the output files in the background. Don't need to wait for this
	// since it only issues warnings sometimes.
	go pkg.VerifyOutputs()
	state.Graph.AddPackage(pkg) // Calling this means nobody else will add entries to pendingTargets for this package.
	return pkg, nil
}

// buildFileName returns the name of the BUILD file for a package, or the empty string if one
// doesn't exist. It also returns the directory that it looked in.
func buildFileName(state *core.BuildState, pkgName string, subrepo *core.Subrepo) (string, string) {
	config := state.Config
	if subrepo != nil {
		pkgName = subrepo.Dir(pkgName)
		config = subrepo.State.Config
	}
	// Bazel defines targets in its "external" package from its WORKSPACE file.
	// We will fake this by treating that as an actual package file...
	// TODO(peterebden): They may be moving away from their "external" nomenclature?
	if state.Config.Bazel.Compatibility && pkgName == "external" || pkgName == "workspace" {
		return "WORKSPACE", ""
	}
	for _, buildFileName := range config.Parse.BuildFileName {
		if filename := path.Join(core.RepoRoot, pkgName, buildFileName); fs.FileExists(filename) {
			return filename, pkgName
		}
	}
	return "", pkgName
}

func rescanDeps(state *core.BuildState, changed map[*core.BuildTarget]struct{}) {
	// Run over all the changed targets in this package and ensure that any newly added dependencies enter the build queue.
	for target := range changed {
		if !state.Graph.AllDependenciesResolved(target) {
			for _, dep := range target.DeclaredDependencies() {
				state.Graph.AddDependency(target.Label, dep)
			}
		}
		if s := target.State(); s < core.Built && s > core.Inactive {
			state.QueueTarget(target.Label, core.OriginalTarget, true, false)
		}
	}
}

// This is the builtin subrepo for pleasings.
// TODO(peterebden): Should really provide a github_archive builtin that knows how to construct
//                   the URL and strip_prefix etc.
const pleasings = `
http_archive(
    name = "pleasings",
    strip_prefix = "pleasings-master",
    urls = ["https://github.com/thought-machine/pleasings/archive/master.zip"],
)
`

// providePackage looks through all the configured BUILD file providers to see if any of them
// can handle the given package. It returns true if any of them did.
// N.B. More than one is allowed to handle a single directory.
func providePackage(state *core.BuildState, pkg *core.Package) (bool, error) {
	if len(state.Config.Provider) == 0 {
		return false, nil
	}
	success := false
	label := pkg.Label()
	for name, p := range state.Config.Provider {
		if !shouldProvide(p.Path, label) {
			continue
		}
		t := state.WaitForBuiltTarget(p.Target, label)
		outs := t.Outputs()
		if !t.IsBinary && len(outs) != 1 {
			log.Error("Cannot use %s as build provider %s, it must be a binary with exactly 1 output.", p.Target, name)
			continue
		}
		dir := pkg.SourceRoot()
		resp, err := worker.ProvideParse(state, path.Join(t.OutDir(), outs[0]), dir)
		if err != nil {
			return false, fmt.Errorf("Failed to start build provider %s: %s", name, err)
		} else if resp != "" {
			log.Debug("Received BUILD file from %s provider for %s: %s", name, dir, resp)
			if err := state.Parser.ParseReader(state, pkg, strings.NewReader(resp)); err != nil {
				return false, err
			}
			success = true
		}
	}
	return success, nil
}

// shouldProvide returns true if a provider's set of configured paths overlaps a package.
func shouldProvide(paths []core.BuildLabel, label core.BuildLabel) bool {
	for _, p := range paths {
		if p.Includes(label) {
			return true
		}
	}
	return false
}

// exportFile adds a single-file export target. This is primarily used for Bazel compat.
func exportFile(state *core.BuildState, pkg *core.Package, label core.BuildLabel) {
	t := core.NewBuildTarget(label)
	t.Subrepo = pkg.Subrepo
	t.IsFilegroup = true
	t.AddSource(core.NewFileLabel(label.Name, pkg))
	state.AddTarget(pkg, t)
}
<commit_msg>Additional fallback for finding subrepos.<commit_after>// Package parse implements parsing of the BUILD files via an embedded Python interpreter.
//
// The actual work here is done by an embedded PyPy instance. Various rules are built in to
// the binary itself using go-bindata to embed the .py files; these are always available to
// all programs which is rather nice, but it does mean that must be run before 'go run' etc
// will work as expected.
package parse

import (
	"fmt"
	"path"
	"strings"

	"gopkg.in/op/go-logging.v1"

	"github.com/thought-machine/please/src/cli"
	"github.com/thought-machine/please/src/core"
	"github.com/thought-machine/please/src/fs"
	"github.com/thought-machine/please/src/worker"
)

var log = logging.MustGetLogger("parse")

// Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package.
// It is not an error if the package has already been parsed.
//
// By default, after the package is parsed, any targets that are now needed for the build and ready
// to be built are queued, and any new packages are queued for parsing. When a specific label is requested
// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it:
// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only
// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added.
// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed
// even when we're not otherwise building targets.
func Parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) {
	if err := parse(tid, state, label, dependor, include, exclude, forSubinclude); err != nil {
		state.LogBuildError(tid, label, core.ParseFailed, err, "Failed to parse package")
	}
}

func parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) error {
	// See if something else has parsed this package first.
	pkg := state.WaitForPackage(label)
	if pkg != nil {
		// Does exist, all we need to do is toggle on this target
		return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
	}
	// If we get here then it falls to us to parse this package.
	state.LogBuildResult(tid, label, core.PackageParsing, "Parsing...")

	subrepo, err := checkSubrepo(tid, state, label, dependor)
	if err != nil {
		return err
	} else if subrepo != nil && subrepo.Target != nil {
		// We have got the definition of the subrepo but it depends on something, make sure that has been built.
		state.WaitForBuiltTarget(subrepo.Target.Label, label)
	}
	// Subrepo & nothing else means we just want to ensure that subrepo is present.
	if label.Subrepo != "" && label.PackageName == "" && label.Name == "" {
		return nil
	}
	pkg, err = parsePackage(state, label, dependor, subrepo)
	if err != nil {
		return err
	}
	state.LogBuildResult(tid, label, core.PackageParsed, "Parsed package")
	return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
}

// checkSubrepo checks whether this guy exists within a subrepo. If so we will need to make sure that's available first.
func checkSubrepo(tid int, state *core.BuildState, label, dependor core.BuildLabel) (*core.Subrepo, error) {
	if label.Subrepo == "" {
		return nil, nil
	} else if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	// We don't have the definition of it at all. Need to parse that first.
	sl := label.SubrepoLabel()
	if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, "", label); err != nil {
		return nil, err
	} else if !handled {
		if _, err := parseSubrepoPackage(tid, state, sl.PackageName, dependor.Subrepo, label); err != nil {
			return nil, err
		}
	}
	if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	} else if subrepo := checkArchSubrepo(state, label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	// Fix for #577; fallback like above, it might be defined within the subrepo.
	if handled, err := parseSubrepoPackage(tid, state, sl.PackageName, dependor.Subrepo, label); handled && err == nil {
		return state.Graph.Subrepo(label.Subrepo), nil
	}
	return nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo)
}

// parseSubrepoPackage parses a package to make sure subrepos are available.
func parseSubrepoPackage(tid int, state *core.BuildState, pkg, subrepo string, dependor core.BuildLabel) (bool, error) {
	if state.Graph.Package(pkg, subrepo) == nil {
		// Don't have it already, must parse.
		label := core.BuildLabel{Subrepo: subrepo, PackageName: pkg, Name: "all"}
		return true, parse(tid, state, label, dependor, nil, nil, true)
	}
	return false, nil
}

// checkArchSubrepo checks if a target refers to a cross-compiling subrepo.
// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to.
func checkArchSubrepo(state *core.BuildState, name string) *core.Subrepo {
	var arch cli.Arch
	if err := arch.UnmarshalFlag(name); err == nil {
		return state.Graph.MaybeAddSubrepo(core.SubrepoForArch(state, arch))
	}
	return nil
}

// activateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses.
func activateTarget(state *core.BuildState, pkg *core.Package, label, dependor core.BuildLabel, forSubinclude bool, include, exclude []string) error {
	if !label.IsAllTargets() && state.Graph.Target(label) == nil {
		if label.Subrepo == "" && label.PackageName == "" && label.Name == dependor.Subrepo {
			if subrepo := checkArchSubrepo(state, label.Name); subrepo != nil {
				return nil
			}
		}
		if state.Config.Bazel.Compatibility && forSubinclude {
			// Bazel allows some things that look like build targets but aren't - notably the syntax
			// to load(). It suits us to treat that as though it is one, but we now have to
			// implicitly make it available.
			exportFile(state, pkg, label)
		} else {
			msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name)
			if dependor != core.OriginalTarget {
				msg += fmt.Sprintf(" (depended on by %s)", dependor)
			}
			return fmt.Errorf(msg + suggestTargets(pkg, label, dependor))
		}
	}
	if state.ParsePackageOnly && !forSubinclude {
		return nil // Some kinds of query don't need a full recursive parse.
	} else if label.IsAllTargets() {
		if dependor == core.OriginalTarget {
			for _, target := range pkg.AllTargets() {
				// Don't activate targets that were added in a post-build function; that causes a race condition
				// between the post-build functions running and other things trying to activate them too early.
				if state.ShouldInclude(target) && !target.AddedPostBuild {
					// Must always do this for coverage because we need to calculate sources of
					// non-test targets later on.
					if !state.NeedTests || target.IsTest || state.NeedCoverage {
						state.QueueTarget(target.Label, dependor, false, dependor.IsAllTargets())
					}
				}
			}
		}
	} else {
		for _, l := range state.Graph.DependentTargets(dependor, label) {
			// We use :all to indicate a dependency needed for parse.
			state.QueueTarget(l, dependor, false, forSubinclude || dependor.IsAllTargets())
		}
	}
	return nil
}

// parsePackage performs the initial parse of a package.
func parsePackage(state *core.BuildState, label, dependor core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) {
	packageName := label.PackageName
	pkg := core.NewPackage(packageName)
	pkg.Subrepo = subrepo
	if subrepo != nil {
		pkg.SubrepoName = subrepo.Name
	}
	filename, dir := buildFileName(state, label.PackageName, subrepo)
	if filename == "" {
		if success, err := providePackage(state, pkg); err != nil {
			return nil, err
		} else if !success && packageName == "" && dependor.Subrepo == "pleasings" && subrepo == nil && state.Config.Parse.BuiltinPleasings {
			// Deliberate fallthrough, for the case where someone depended on the default
			// @pleasings subrepo, and there is no BUILD file at their root.
		} else if !success {
			exists := core.PathExists(dir)
			// Handle quite a few cases to provide more obvious error messages.
			if dependor != core.OriginalTarget && exists {
				return nil, fmt.Errorf("%s depends on %s, but there's no %s file in %s/", dependor, label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			} else if dependor != core.OriginalTarget {
				return nil, fmt.Errorf("%s depends on %s, but the directory %s doesn't exist", dependor, label, dir)
			} else if exists {
				return nil, fmt.Errorf("Can't build %s; there's no %s file in %s/", label, buildFileNames(state.Config.Parse.BuildFileName), dir)
			}
			return nil, fmt.Errorf("Can't build %s; the directory %s doesn't exist", label, dir)
		}
	} else {
		pkg.Filename = filename
		if err := state.Parser.ParseFile(state, pkg, pkg.Filename); err != nil {
			return nil, err
		}
	}
	// If the config setting is on, we "magically" register a default repo called @pleasings.
	if packageName == "" && subrepo == nil && state.Config.Parse.BuiltinPleasings && pkg.Target("pleasings") == nil {
		if _, err := state.Parser.(*aspParser).asp.ParseReader(pkg, strings.NewReader(pleasings)); err != nil {
			log.Fatalf("Failed to load pleasings: %s", err) // This shouldn't happen, of course.
		}
	}
	// Verify some details of the output files in the background. Don't need to wait for this
	// since it only issues warnings sometimes.
	go pkg.VerifyOutputs()
	state.Graph.AddPackage(pkg) // Calling this means nobody else will add entries to pendingTargets for this package.
	return pkg, nil
}

// buildFileName returns the name of the BUILD file for a package, or the empty string if one
// doesn't exist. It also returns the directory that it looked in.
func buildFileName(state *core.BuildState, pkgName string, subrepo *core.Subrepo) (string, string) {
	config := state.Config
	if subrepo != nil {
		pkgName = subrepo.Dir(pkgName)
		config = subrepo.State.Config
	}
	// Bazel defines targets in its "external" package from its WORKSPACE file.
	// We will fake this by treating that as an actual package file...
	// TODO(peterebden): They may be moving away from their "external" nomenclature?
	if state.Config.Bazel.Compatibility && pkgName == "external" || pkgName == "workspace" {
		return "WORKSPACE", ""
	}
	for _, buildFileName := range config.Parse.BuildFileName {
		if filename := path.Join(core.RepoRoot, pkgName, buildFileName); fs.FileExists(filename) {
			return filename, pkgName
		}
	}
	return "", pkgName
}

func rescanDeps(state *core.BuildState, changed map[*core.BuildTarget]struct{}) {
	// Run over all the changed targets in this package and ensure that any newly added dependencies enter the build queue.
	for target := range changed {
		if !state.Graph.AllDependenciesResolved(target) {
			for _, dep := range target.DeclaredDependencies() {
				state.Graph.AddDependency(target.Label, dep)
			}
		}
		if s := target.State(); s < core.Built && s > core.Inactive {
			state.QueueTarget(target.Label, core.OriginalTarget, true, false)
		}
	}
}

// This is the builtin subrepo for pleasings.
// TODO(peterebden): Should really provide a github_archive builtin that knows how to construct
//                   the URL and strip_prefix etc.
const pleasings = `
http_archive(
    name = "pleasings",
    strip_prefix = "pleasings-master",
    urls = ["https://github.com/thought-machine/pleasings/archive/master.zip"],
)
`

// providePackage looks through all the configured BUILD file providers to see if any of them
// can handle the given package. It returns true if any of them did.
// N.B. More than one is allowed to handle a single directory.
func providePackage(state *core.BuildState, pkg *core.Package) (bool, error) {
	if len(state.Config.Provider) == 0 {
		return false, nil
	}
	success := false
	label := pkg.Label()
	for name, p := range state.Config.Provider {
		if !shouldProvide(p.Path, label) {
			continue
		}
		t := state.WaitForBuiltTarget(p.Target, label)
		outs := t.Outputs()
		if !t.IsBinary && len(outs) != 1 {
			log.Error("Cannot use %s as build provider %s, it must be a binary with exactly 1 output.", p.Target, name)
			continue
		}
		dir := pkg.SourceRoot()
		resp, err := worker.ProvideParse(state, path.Join(t.OutDir(), outs[0]), dir)
		if err != nil {
			return false, fmt.Errorf("Failed to start build provider %s: %s", name, err)
		} else if resp != "" {
			log.Debug("Received BUILD file from %s provider for %s: %s", name, dir, resp)
			if err := state.Parser.ParseReader(state, pkg, strings.NewReader(resp)); err != nil {
				return false, err
			}
			success = true
		}
	}
	return success, nil
}

// shouldProvide returns true if a provider's set of configured paths overlaps a package.
func shouldProvide(paths []core.BuildLabel, label core.BuildLabel) bool {
	for _, p := range paths {
		if p.Includes(label) {
			return true
		}
	}
	return false
}

// exportFile adds a single-file export target. This is primarily used for Bazel compat.
func exportFile(state *core.BuildState, pkg *core.Package, label core.BuildLabel) {
	t := core.NewBuildTarget(label)
	t.Subrepo = pkg.Subrepo
	t.IsFilegroup = true
	t.AddSource(core.NewFileLabel(label.Name, pkg))
	state.AddTarget(pkg, t)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"regexp";
	"testing";
)

type DialErrorTest struct {
	Net string;
	Laddr string;
	Raddr string;
	Pattern string;
}

var dialErrorTests = []DialErrorTest {
	DialErrorTest{
		"datakit", "", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	DialErrorTest{
		"tcp", "", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no such host",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\..*\.( on .*)?: no such host`,
	},
	DialErrorTest{
		"tcp", "", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	DialErrorTest{
		"unix", "", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	DialErrorTest{
		"unix", "", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket)",
	},
}

func TestDialError(t *testing.T) {
	for i, tt := range dialErrorTests {
		c, e := Dial(tt.Net, tt.Laddr, tt.Raddr);
		if c != nil {
			c.Close();
		}
		if e == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern);
			continue;
		}
		s := e.String();
		match, _ := regexp.MatchString(tt.Pattern, s);
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern);
		}
	}
}
<commit_msg>two more regexp tweaks<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"regexp";
	"testing";
)

type DialErrorTest struct {
	Net string;
	Laddr string;
	Raddr string;
	Pattern string;
}

var dialErrorTests = []DialErrorTest {
	DialErrorTest{
		"datakit", "", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	DialErrorTest{
		"tcp", "", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	DialErrorTest{
		"tcp", "", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\..*\.( on .*)?: no (.*)`,
	},
	DialErrorTest{
		"tcp", "", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	DialErrorTest{
		"unix", "", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	DialErrorTest{
		"unix", "", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket)",
	},
}

func TestDialError(t *testing.T) {
	for i, tt := range dialErrorTests {
		c, e := Dial(tt.Net, tt.Laddr, tt.Raddr);
		if c != nil {
			c.Close();
		}
		if e == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern);
			continue;
		}
		s := e.String();
		match, _ := regexp.MatchString(tt.Pattern, s);
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern);
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO It would be nice to use a mock DNS server, to eliminate
// external dependencies.

package net

import (
	"testing"
)

func TestGoogleSRV(t *testing.T) {
	_, addrs, err := LookupSRV("xmpp-server", "tcp", "google.com")
	if err != nil {
		t.Errorf("failed: %s", err)
	}
	if len(addrs) == 0 {
		t.Errorf("no results")
	}
}
<commit_msg>net: disable one more external network test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO It would be nice to use a mock DNS server, to eliminate
// external dependencies.

package net

import (
	"testing"
)

func TestGoogleSRV(t *testing.T) {
	if testing.Short() {
		return
	}
	_, addrs, err := LookupSRV("xmpp-server", "tcp", "google.com")
	if err != nil {
		t.Errorf("failed: %s", err)
	}
	if len(addrs) == 0 {
		t.Errorf("no results")
	}
}
<|endoftext|>


<commit_before>package rest

import (
	"github.com/ant0ine/go-json-rest/rest/test"
	"testing"
)

func TestApiNoAppNoMiddleware(t *testing.T) {

	api := NewApi()
	if api == nil {
		t.Fatal("Api object must be instantiated")
	}

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must be have been create")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
}

func TestApiSimpleAppNoMiddleware(t *testing.T) {

	api := NewApi()
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must be have been create")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(`{"Id":"123"}`)
}

func TestDevStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultDevStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must be have been create")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs("{\n  \"Id\": \"123\"\n}")
}

func TestProdStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultProdStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must be have been create")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.ContentEncodingIsGzip()
}

func TestCommonStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultCommonStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must be have been create")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(`{"Id":"123"}`)
}
<commit_msg>Fix old typo<commit_after>package rest

import (
	"github.com/ant0ine/go-json-rest/rest/test"
	"testing"
)

func TestApiNoAppNoMiddleware(t *testing.T) {

	api := NewApi()
	if api == nil {
		t.Fatal("Api object must be instantiated")
	}

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must have been created")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
}

func TestApiSimpleAppNoMiddleware(t *testing.T) {

	api := NewApi()
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must have been created")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(`{"Id":"123"}`)
}

func TestDevStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultDevStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must have been created")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs("{\n  \"Id\": \"123\"\n}")
}

func TestProdStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultProdStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must have been created")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.ContentEncodingIsGzip()
}

func TestCommonStack(t *testing.T) {

	api := NewApi()
	api.Use(DefaultCommonStack...)
	api.SetApp(AppSimple(func(w ResponseWriter, r *Request) {
		w.WriteJson(map[string]string{"Id": "123"})
	}))

	handler := api.MakeHandler()
	if handler == nil {
		t.Fatal("the http.Handler must have been created")
	}

	recorded := test.RunRequest(t, handler, test.MakeSimpleRequest("GET", "http://localhost/", nil))
	recorded.CodeIs(200)
	recorded.ContentTypeIsJson()
	recorded.BodyIs(`{"Id":"123"}`)
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"
	"io/ioutil"
	"os"

	kv "gopkg.in/Clever/kayvee-go.v6"
)

var teamName string

func init() {
	teamName = os.Getenv("_TEAM_OWNER")
	if teamName == "" {
		teamName = "UNSET"
	}
}

func setDefaults(output map[string]interface{}) map[string]interface{} {
	otype, ok := output["type"].(string)
	if !ok {
		return output
	}

	switch otype {
	case "metrics":
		fallthrough
	case "alerts":
		if _, ok := output["value_field"]; !ok {
			output["value_field"] = "value"
		}
	}

	return output
}

// Route returns routing metadata for the log line `msg`. The outputs (with
// variable substitutions performed) for each rule matched are placed under the
// "routes" key.
func (r *RuleRouter) Route(msg map[string]interface{}) map[string]interface{} {
	outputs := []map[string]interface{}{}
	for _, rule := range r.rules {
		if rule.Matches(msg) {
			outputs = append(outputs, rule.OutputFor(msg))
		}
	}
	return map[string]interface{}{
		"team":        teamName,
		"kv_version":  kv.Version,
		"kv_language": "go",
		"routes":      outputs,
	}
}

// NewFromConfig constructs a Router using the configuration specified as yaml
// in `filename`. The routing rules should be placed under the "routes" key on
// the root-level map in the file. Validation is performed as described in
// parse.go.
func NewFromConfig(filename string) (Router, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fileBytes, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	router, err := NewFromConfigBytes(fileBytes)
	if err != nil {
		return nil, fmt.Errorf(
			"Error initializing kayvee log router from file '%s':\n%s",
			filename, err.Error(),
		)
	}
	return router, nil
}

// NewFromConfigBytes constructs a Router using the configuration specified
// as bytes typically read from a binary file. This allows us to
// package kv routing yaml files into binaries like gearcmd.
// The routing rules should be placed under the "routes" key on
// the root-level map in the file. Validation is performed as described in
// parse.go.
func NewFromConfigBytes(fileBytes []byte) (Router, error) {
	routes, err := parse(fileBytes)
	if err != nil {
		return &RuleRouter{}, err
	}

	return NewFromRoutes(routes)
}

// NewFromRoutes constructs a RuleRouter using the provided map of route names
// to Rules.
func NewFromRoutes(routes map[string]Rule) (Router, error) {
	router := &RuleRouter{}
	for name, rule := range routes {
		output, err := substituteEnvVars(rule.Output)
		if err != nil {
			return router, err
		}
		output = setDefaults(output)

		rule.Name = name
		rule.Output = output
		router.rules = append(router.rules, rule)
	}

	return router, nil
}
<commit_msg>Added _TEAM_OWNER fallback.  This for lambdas<commit_after>package router

import (
	"fmt"
	"io/ioutil"
	"os"

	kv "gopkg.in/Clever/kayvee-go.v6"
)

var teamName string

func init() {
	teamName = os.Getenv("_TEAM_OWNER")
	if teamName == "" {
		teamName = os.Getenv("TEAM_OWNER")
		if teamName == "" {
			teamName = "UNSET"
		}
	}
}

func setDefaults(output map[string]interface{}) map[string]interface{} {
	otype, ok := output["type"].(string)
	if !ok {
		return output
	}

	switch otype {
	case "metrics":
		fallthrough
	case "alerts":
		if _, ok := output["value_field"]; !ok {
			output["value_field"] = "value"
		}
	}

	return output
}

// Route returns routing metadata for the log line `msg`. The outputs (with
// variable substitutions performed) for each rule matched are placed under the
// "routes" key.
func (r *RuleRouter) Route(msg map[string]interface{}) map[string]interface{} {
	outputs := []map[string]interface{}{}
	for _, rule := range r.rules {
		if rule.Matches(msg) {
			outputs = append(outputs, rule.OutputFor(msg))
		}
	}
	return map[string]interface{}{
		"team":        teamName,
		"kv_version":  kv.Version,
		"kv_language": "go",
		"routes":      outputs,
	}
}

// NewFromConfig constructs a Router using the configuration specified as yaml
// in `filename`. The routing rules should be placed under the "routes" key on
// the root-level map in the file. Validation is performed as described in
// parse.go.
func NewFromConfig(filename string) (Router, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fileBytes, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	router, err := NewFromConfigBytes(fileBytes)
	if err != nil {
		return nil, fmt.Errorf(
			"Error initializing kayvee log router from file '%s':\n%s",
			filename, err.Error(),
		)
	}
	return router, nil
}

// NewFromConfigBytes constructs a Router using the configuration specified
// as bytes typically read from a binary file. This allows us to
// package kv routing yaml files into binaries like gearcmd.
// The routing rules should be placed under the "routes" key on
// the root-level map in the file. Validation is performed as described in
// parse.go.
func NewFromConfigBytes(fileBytes []byte) (Router, error) {
	routes, err := parse(fileBytes)
	if err != nil {
		return &RuleRouter{}, err
	}

	return NewFromRoutes(routes)
}

// NewFromRoutes constructs a RuleRouter using the provided map of route names
// to Rules.
func NewFromRoutes(routes map[string]Rule) (Router, error) {
	router := &RuleRouter{}
	for name, rule := range routes {
		output, err := substituteEnvVars(rule.Output)
		if err != nil {
			return router, err
		}
		output = setDefaults(output)

		rule.Name = name
		rule.Output = output
		router.rules = append(router.rules, rule)
	}

	return router, nil
}
<|endoftext|>


<commit_before>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

/*
{
	"dest_dir"		:	"/Users/jemy/Backup",
	"bucket"		:	"test-bucket",
	"domain"		:	"<Your bucket domain>",
	"access_key"	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"is_private"	:	false,
	"prefix"		:	"demo/"
}
*/
type DownloadConfig struct {
	DestDir   string `json:"dest_dir"`
	Bucket    string `json:"bucket"`
	Domain    string `json:"domain"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	IsPrivate bool   `json:"is_private"`
	Prefix    string `json:"prefix,omitempty"`
}

func QiniuDownload(threadCount int, downloadConfigFile string) {
	cnfFp, err := os.Open(downloadConfigFile)
	if err != nil {
		log.Error("Open download config file", downloadConfigFile, "failed,", err)
		return
	}
	defer cnfFp.Close()
	cnfData, err := ioutil.ReadAll(cnfFp)
	if err != nil {
		log.Error("Read download config file error", err)
		return
	}
	downConfig := DownloadConfig{}
	cnfErr := json.Unmarshal(cnfData, &downConfig)
	if cnfErr != nil {
		log.Error("Parse download config error", err)
		return
	}
	cnfJson, _ := json.Marshal(&downConfig)
	jobId := fmt.Sprintf("%x", md5.Sum(cnfJson))
	jobListName := fmt.Sprintf("%s.list.txt", jobId)
	acct := Account{
		AccessKey: downConfig.AccessKey,
		SecretKey: downConfig.SecretKey,
	}
	bLister := ListBucket{
		Account: acct,
	}
	log.Debug("List bucket...")
	listErr := bLister.List(downConfig.Bucket, downConfig.Prefix, jobListName)
	if listErr != nil {
		log.Error("List bucket error", listErr)
		return
	}
	listFp, openErr := os.Open(jobListName)
	if openErr != nil {
		log.Error("Open list file error", openErr)
		return
	}
	defer listFp.Close()
	listScanner := bufio.NewScanner(listFp)
	listScanner.Split(bufio.ScanLines)
	downWorkGroup := sync.WaitGroup{}
	downCounter := 0
	if threadCount < 0 || threadCount > 10 {
		threadCount = 5
	}
	for listScanner.Scan() {
		downCounter += 1
		if downCounter%threadCount == 0 {
			downWorkGroup.Wait()
		}
		line := strings.TrimSpace(listScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 2 {
			fileKey := items[0]
			fileSize, _ := strconv.ParseInt(items[1], 10, 64)
			//not backup yet
			if !checkLocalDuplicate(downConfig.DestDir, fileKey, fileSize) {
				downWorkGroup.Add(1)
				go func() {
					defer downWorkGroup.Done()
					downloadFile(downConfig, fileKey)
				}()
			}
		}
	}
	downWorkGroup.Wait()
	fmt.Println("All downloaded!")
}

func checkLocalDuplicate(destDir string, fileKey string, fileSize int64) bool {
	dup := false
	filePath := filepath.Join(destDir, fileKey)
	fStat, statErr := os.Stat(filePath)
	if statErr == nil {
		//exist, check file size
		localFileSize := fStat.Size()
		if localFileSize == fileSize {
			dup = true
		}
	}
	return dup
}

func downloadFile(downConfig DownloadConfig, fileKey string) {
	localFilePath := filepath.Join(downConfig.DestDir, fileKey)
	ldx := strings.LastIndex(localFilePath, string(os.PathSeparator))
	if ldx != -1 {
		localFileDir := localFilePath[:ldx]
		err := os.MkdirAll(localFileDir, 0775)
		if err != nil {
			log.Error("MkdirAll failed for", localFileDir)
			return
		}
	}
	fmt.Println("Downloading", fileKey, "=>", localFilePath, "...")
	downUrl := strings.Join([]string{downConfig.Domain, fileKey}, "/")
	if downConfig.IsPrivate {
		now := time.Now().Add(time.Second * 3600 * 24)
		downUrl = fmt.Sprintf("%s?e=%d", downUrl, now.Unix())
		mac := digest.Mac{downConfig.AccessKey, []byte(downConfig.SecretKey)}
		token := digest.Sign(&mac, []byte(downUrl))
		downUrl = fmt.Sprintf("%s&token=", downUrl, token)
	}

	resp, respErr := rpc.DefaultClient.Get(nil, downUrl)
	if respErr != nil {
		log.Error("Download", fileKey, "failed by url", downUrl)
		return
	}
	defer resp.Body.Close()
	localFp, openErr := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY, 0666)
	if openErr != nil {
		log.Error("Open local file", localFilePath, "failed")
		return
	}
	defer localFp.Close()
	_, err := io.Copy(localFp, resp.Body)
	if err != nil {
		log.Error("Download", fileKey, "failed", err)
	}
}
<commit_msg>Add download status code check and fix token bug.<commit_after>package qshell

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/log"
	"github.com/qiniu/rpc"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

/*
{
	"dest_dir"		:	"/Users/jemy/Backup",
	"bucket"		:	"test-bucket",
	"domain"		:	"<Your bucket domain>",
	"access_key"	:	"<Your AccessKey>",
	"secret_key"	:	"<Your SecretKey>",
	"is_private"	:	false,
	"prefix"		:	"demo/"
}
*/
type DownloadConfig struct {
	DestDir   string `json:"dest_dir"`
	Bucket    string `json:"bucket"`
	Domain    string `json:"domain"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	IsPrivate bool   `json:"is_private"`
	Prefix    string `json:"prefix,omitempty"`
}

func QiniuDownload(threadCount int, downloadConfigFile string) {
	cnfFp, err := os.Open(downloadConfigFile)
	if err != nil {
		log.Error("Open download config file", downloadConfigFile, "failed,", err)
		return
	}
	defer cnfFp.Close()
	cnfData, err := ioutil.ReadAll(cnfFp)
	if err != nil {
		log.Error("Read download config file error", err)
		return
	}
	downConfig := DownloadConfig{}
	cnfErr := json.Unmarshal(cnfData, &downConfig)
	if cnfErr != nil {
		log.Error("Parse download config error", err)
		return
	}
	cnfJson, _ := json.Marshal(&downConfig)
	jobId := fmt.Sprintf("%x", md5.Sum(cnfJson))
	jobListName := fmt.Sprintf("%s.list.txt", jobId)
	acct := Account{
		AccessKey: downConfig.AccessKey,
		SecretKey: downConfig.SecretKey,
	}
	bLister := ListBucket{
		Account: acct,
	}
	log.Debug("List bucket...")
	listErr := bLister.List(downConfig.Bucket, downConfig.Prefix, jobListName)
	if listErr != nil {
		log.Error("List bucket error", listErr)
		return
	}
	listFp, openErr := os.Open(jobListName)
	if openErr != nil {
		log.Error("Open list file error", openErr)
		return
	}
	defer listFp.Close()
	listScanner := bufio.NewScanner(listFp)
	listScanner.Split(bufio.ScanLines)
	downWorkGroup := sync.WaitGroup{}
	downCounter := 0
	if threadCount < 0 || threadCount > 10 {
		threadCount = 5
	}
	for listScanner.Scan() {
		downCounter += 1
		if downCounter%threadCount == 0 {
			downWorkGroup.Wait()
		}
		line := strings.TrimSpace(listScanner.Text())
		items := strings.Split(line, "\t")
		if len(items) > 2 {
			fileKey := items[0]
			fileSize, _ := strconv.ParseInt(items[1], 10, 64)
			//not backup yet
			if !checkLocalDuplicate(downConfig.DestDir, fileKey, fileSize) {
				downWorkGroup.Add(1)
				go func() {
					defer downWorkGroup.Done()
					downloadFile(downConfig, fileKey)
				}()
			}
		}
	}
	downWorkGroup.Wait()
	fmt.Println("All downloaded!")
}

func checkLocalDuplicate(destDir string, fileKey string, fileSize int64) bool {
	dup := false
	filePath := filepath.Join(destDir, fileKey)
	fStat, statErr := os.Stat(filePath)
	if statErr == nil {
		//exist, check file size
		localFileSize := fStat.Size()
		if localFileSize == fileSize {
			dup = true
		}
	}
	return dup
}

func downloadFile(downConfig DownloadConfig, fileKey string) {
	localFilePath := filepath.Join(downConfig.DestDir, fileKey)
	ldx := strings.LastIndex(localFilePath, string(os.PathSeparator))
	if ldx != -1 {
		localFileDir := localFilePath[:ldx]
		err := os.MkdirAll(localFileDir, 0775)
		if err != nil {
			log.Error("MkdirAll failed for", localFileDir)
			return
		}
	}
	fmt.Println("Downloading", fileKey, "=>", localFilePath, "...")
	downUrl := strings.Join([]string{downConfig.Domain, fileKey}, "/")
	if downConfig.IsPrivate {
		now := time.Now().Add(time.Second * 3600 * 24)
		downUrl = fmt.Sprintf("%s?e=%d", downUrl, now.Unix())
		mac := digest.Mac{downConfig.AccessKey, []byte(downConfig.SecretKey)}
		token := digest.Sign(&mac, []byte(downUrl))
		downUrl = fmt.Sprintf("%s&token=%s", downUrl, token)
	}
	resp, respErr := rpc.DefaultClient.Get(nil, downUrl)
	if respErr != nil {
		log.Error("Download", fileKey, "failed by url", downUrl)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode == 200 {
		localFp, openErr := os.OpenFile(localFilePath, os.O_CREATE|os.O_WRONLY, 0666)
		if openErr != nil {
			log.Error("Open local file", localFilePath, "failed")
			return
		}
		defer localFp.Close()
		_, err := io.Copy(localFp, resp.Body)
		if err != nil {
			log.Error("Download", fileKey, "failed", err)
		}
	} else {
		log.Error("Download", fileKey, "failed by url", downUrl)
	}
}
<|endoftext|>


<commit_before>package dao

import (
	"bytes"
	"fmt"

	_D "github.com/deze333/diag"
)

//------------------------------------------------------------
// Collection indexes
//------------------------------------------------------------

// Ensures multiple indexes. Error is suppressed yet reported.
func (dao *DAO) EnsureIndexes() {
	if dao.indexes.isIndexed {
		return
	}

	for _, index := range dao.indexes.indexes {
		err := dao.Coll.EnsureIndex(index)
		if err != nil {
			_D.SOS("db", "Collection set index error", "db", dao.dbname, "coll", dao.collname, "error", err)
		}
	}

	// All indexed
	dao.indexes.isIndexed = true

	// Output indexes
	dao.DebugIndexes()
}

// Sets collection to not indexed. Used in situation of collection drop.
func (dao *DAO) expireIndexes() {
	dao.indexes.isIndexed = false
}

// Debug method that prints active indexes for given collection.
func (dao *DAO) DebugIndexes() {
	indexes, err := dao.Coll.Indexes()
	if err != nil {
		panic(fmt.Sprintf("MongoDB cannot read indexes for '%v' due to error: %v", dao.collname, err.Error()))
		return
	}

	var buf bytes.Buffer
	for _, index := range indexes {
		buf.WriteString(fmt.Sprint(index.Key))
		buf.WriteString(", ")
	}

	// Ideally, indexing only happens once per deployment, hence WARNING
	_D.WARNING("DAO: Indexes present", dao.collname, "idx", buf.String())
}
<commit_msg>change to index debug<commit_after>package dao

import (
	"bytes"
	"fmt"

	_D "github.com/deze333/diag"
)

//------------------------------------------------------------
// Collection indexes
//------------------------------------------------------------

// Ensures multiple indexes. Error is suppressed yet reported.
func (dao *DAO) EnsureIndexes() {
	if dao.indexes.isIndexed {
		return
	}

	for _, index := range dao.indexes.indexes {
		err := dao.Coll.EnsureIndex(index)
		if err != nil {
			_D.SOS("db", "Collection set index error", "db", dao.dbname, "coll", dao.collname, "error", err)
		}
	}

	// All indexed
	dao.indexes.isIndexed = true

	// Output indexes
	dao.DebugIndexes()
}

// Sets collection to not indexed. Used in situation of collection drop.
func (dao *DAO) expireIndexes() {
	dao.indexes.isIndexed = false
}

// Debug method that prints active indexes for given collection.
func (dao *DAO) DebugIndexes() {
	indexes, err := dao.Coll.Indexes()
	if err != nil {
		panic(fmt.Sprintf("MongoDB cannot read indexes for '%v' due to error: %v", dao.collname, err.Error()))
		return
	}

	var buf bytes.Buffer
	for _, index := range indexes {
		buf.WriteString(fmt.Sprint(index.Key))
		buf.WriteString(", ")
	}

	// Ideally, indexing only happens once per deployment, hence this NOTE
	_D.NOTE2("DAO "+dao.collname, "idx", buf.String())
}
<|endoftext|>


<commit_before>package processor_test

import (
	. "ci.guzzler.io/guzzler/corcel/processor"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("ListRingIterator", func() {

	It("Loops round", func() {
		peopleKey := "People"
		data := map[string][]map[string]interface{}{}
		data[peopleKey] = []map[string]interface{}{}

		bob := map[string]interface{}{
			"name": "bob",
			"age":  30,
		}
		carol := map[string]interface{}{
			"name": "carol",
			"age":  31,
		}
		alice := map[string]interface{}{
			"name": "alice",
			"age":  32,
		}

		data[peopleKey] = append(data[peopleKey], bob)
		data[peopleKey] = append(data[peopleKey], carol)
		data[peopleKey] = append(data[peopleKey], alice)

		iterator := NewListRingIterator(data)

		values1 := iterator.Values()
		Expect(values1["$People.name"]).To(Equal("bob"))
		Expect(values1["$People.age"]).To(Equal(30))

		values2 := iterator.Values()
		Expect(values2["$People.name"]).To(Equal("carol"))
		Expect(values2["$People.age"]).To(Equal(31))

		values3 := iterator.Values()
		Expect(values3["$People.name"]).To(Equal("alice"))
		Expect(values3["$People.age"]).To(Equal(32))
	})

})
<commit_msg>Tested uneven lists<commit_after>package processor_test

import (
	. "ci.guzzler.io/guzzler/corcel/processor"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func getPeopleCollection() []map[string]interface{} {
	return []map[string]interface{}{
		map[string]interface{}{
			"name": "bob",
			"age":  30,
		},
		map[string]interface{}{
			"name": "carol",
			"age":  31,
		},
		map[string]interface{}{
			"name": "alice",
			"age":  32,
		},
	}
}

func getProductsCollection() []map[string]interface{} {
	return []map[string]interface{}{
		map[string]interface{}{
			"name": "toaster",
			"sku":  "1234",
		},
		map[string]interface{}{
			"name": "grinder",
			"sku":  "5678",
		},
	}
}

var _ = Describe("ListRingIterator", func() {

	It("Loops round", func() {
		data := map[string][]map[string]interface{}{}
		data["People"] = getPeopleCollection()

		iterator := NewListRingIterator(data)

		values1 := iterator.Values()
		Expect(values1["$People.name"]).To(Equal("bob"))
		Expect(values1["$People.age"]).To(Equal(30))

		values2 := iterator.Values()
		Expect(values2["$People.name"]).To(Equal("carol"))
		Expect(values2["$People.age"]).To(Equal(31))

		values3 := iterator.Values()
		Expect(values3["$People.name"]).To(Equal("alice"))
		Expect(values3["$People.age"]).To(Equal(32))
	})

	It("Loops around uneven lists", func() {
		data := map[string][]map[string]interface{}{}
		data["People"] = getPeopleCollection()
		data["Products"] = getProductsCollection()

		iterator := NewListRingIterator(data)

		values1 := iterator.Values()
		Expect(values1["$People.name"]).To(Equal("bob"))
		Expect(values1["$People.age"]).To(Equal(30))
		Expect(values1["$Products.name"]).To(Equal("toaster"))
		Expect(values1["$Products.sku"]).To(Equal("1234"))

		values2 := iterator.Values()
		Expect(values2["$People.name"]).To(Equal("carol"))
		Expect(values2["$People.age"]).To(Equal(31))
		Expect(values2["$Products.name"]).To(Equal("grinder"))
		Expect(values2["$Products.sku"]).To(Equal("5678"))

		values3 := iterator.Values()
		Expect(values3["$People.name"]).To(Equal("alice"))
		Expect(values3["$People.age"]).To(Equal(32))
		Expect(values3["$Products.name"]).To(Equal("toaster"))
		Expect(values3["$Products.sku"]).To(Equal("1234"))
	})

})
<|endoftext|>


<commit_before>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/satori/go.uuid"
)

const loginPage = `
<!DOCTYPE html>

    <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
    <meta charset="utf-8">
    <title>FCS LPC Testbench</title>

    <link rel="import" href="bower_components/paper-styles/paper-styles-classes.html">
    <link rel="import" href="fcs-lpc-motor.html">

			<fcs-lpc-motor-login></fcs-lpc-motor-login>
			%s

`

func (srv *server) handleLogin(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		fmt.Fprintf(w, loginPage, "")
		return
	}
	client, cookie, err := srv.checkCredentials(w, r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = r.ParseForm()
	if err != nil {
		fmt.Fprintf(w, loginPage, "
ERROR: Parsing form ("+err.Error()+")")
		return
	}

	user := r.FormValue("username")
	pass := r.FormValue("password")
	if !srv.authenticate(user, pass) {
		fmt.Fprintf(w, loginPage, "
ERROR: Wrong username/pass
")
		return
	}

	client.auth = true
	client.name = user
	srv.session.set(cookie, client)

	r.SetBasicAuth(user, pass)
	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func (srv *server) handleLogout(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		fmt.Fprintf(w, loginPage, "")
		return
	}
	cookie, err := r.Cookie("FCS_TOKEN")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	srv.session.del(cookie.Value)
	// delete cookie now.
	cookie.MaxAge = -1

	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func (srv *server) authenticate(user, pass string) bool {
	v, ok := srv.session.password(user)
	if !ok {
		return false
	}

	return subtle.ConstantTimeCompare([]byte(pass), []byte(v)) == 1
}

func (srv *server) checkCredentials(w http.ResponseWriter, r *http.Request) (webClient, string, error) {
	var (
		ok     = false
		client webClient
	)
	cookie, err := r.Cookie("FCS_TOKEN")
	if err != nil {
		if err != http.ErrNoCookie {
			return client, "", err
		}
		err = nil
	}

	if cookie != nil {
		client, ok = srv.session.get(cookie.Value)
	}

	if !ok {
		cookie = &http.Cookie{
			Name:  "FCS_TOKEN",
			Value: uuid.NewV4().String(),
		}
		client = webClient{auth: false, token: cookie.Value}
		srv.session.set(cookie.Value, client)
	}

	http.SetCookie(w, cookie)
	return client, cookie.Value, nil
}

type webClient struct {
	name  string
	token string
	auth  bool
}

type authRegistry struct {
	store map[string]webClient
	mu    sync.RWMutex
	db    map[string]string
}

func newAuthRegistry() *authRegistry {
	db := make(map[string]string)
	f, err := os.Open("passwd.json")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	defer f.Close()
	err = json.NewDecoder(f).Decode(&db)
	if err != nil {
		log.Fatalf("error decoding JSON db: %v\n", err)
	}

	if *mockFlag {
		db["faux-fcs"] = "faux-fcs"
	}

	return &authRegistry{
		store: make(map[string]webClient),
		db:    db,
	}
}

func (reg *authRegistry) password(user string) (string, bool) {
	v, ok := reg.db[user]
	return v, ok
}

func (reg *authRegistry) get(cookie string) (webClient, bool) {
	reg.mu.RLock()
	client, ok := reg.store[cookie]
	reg.mu.RUnlock()
	return client, ok
}

func (reg *authRegistry) set(cookie string, client webClient) {
	reg.mu.Lock()
	reg.store[cookie] = client
	reg.mu.Unlock()
}

func (reg *authRegistry) del(cookie string) {
	reg.mu.Lock()
	delete(reg.store, cookie)
	reg.mu.Unlock()
}
<commit_msg>login: update for new uuid API<commit_after>// Copyright ©2016 The go-lsst Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"crypto/subtle"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/satori/uuid"
)

const loginPage = `
<!DOCTYPE html>

    <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
    <meta charset="utf-8">
    <title>FCS LPC Testbench</title>

    <link rel="import" href="bower_components/paper-styles/paper-styles-classes.html">
    <link rel="import" href="fcs-lpc-motor.html">

			<fcs-lpc-motor-login></fcs-lpc-motor-login>
			%s

`

func (srv *server) handleLogin(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		fmt.Fprintf(w, loginPage, "")
		return
	}
	client, cookie, err := srv.checkCredentials(w, r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = r.ParseForm()
	if err != nil {
		fmt.Fprintf(w, loginPage, "
ERROR: Parsing form ("+err.Error()+")")
		return
	}

	user := r.FormValue("username")
	pass := r.FormValue("password")
	if !srv.authenticate(user, pass) {
		fmt.Fprintf(w, loginPage, "
ERROR: Wrong username/pass
")
		return
	}

	client.auth = true
	client.name = user
	srv.session.set(cookie, client)

	r.SetBasicAuth(user, pass)
	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func (srv *server) handleLogout(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		fmt.Fprintf(w, loginPage, "")
		return
	}
	cookie, err := r.Cookie("FCS_TOKEN")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	srv.session.del(cookie.Value)
	// delete cookie now.
	cookie.MaxAge = -1

	http.Redirect(w, r, "/", http.StatusSeeOther)
}

func (srv *server) authenticate(user, pass string) bool {
	v, ok := srv.session.password(user)
	if !ok {
		return false
	}

	return subtle.ConstantTimeCompare([]byte(pass), []byte(v)) == 1
}

func (srv *server) checkCredentials(w http.ResponseWriter, r *http.Request) (webClient, string, error) {
	var (
		ok     = false
		client webClient
	)
	cookie, err := r.Cookie("FCS_TOKEN")
	if err != nil {
		if err != http.ErrNoCookie {
			return client, "", err
		}
		err = nil
	}

	if cookie != nil {
		client, ok = srv.session.get(cookie.Value)
	}

	if !ok {
		cookie = &http.Cookie{
			Name:  "FCS_TOKEN",
			Value: uuid.Must(uuid.NewV4()).String(),
		}
		client = webClient{auth: false, token: cookie.Value}
		srv.session.set(cookie.Value, client)
	}

	http.SetCookie(w, cookie)
	return client, cookie.Value, nil
}

type webClient struct {
	name  string
	token string
	auth  bool
}

type authRegistry struct {
	store map[string]webClient
	mu    sync.RWMutex
	db    map[string]string
}

func newAuthRegistry() *authRegistry {
	db := make(map[string]string)
	f, err := os.Open("passwd.json")
	if err != nil {
		log.Fatalf("error: %v\n", err)
	}
	defer f.Close()
	err = json.NewDecoder(f).Decode(&db)
	if err != nil {
		log.Fatalf("error decoding JSON db: %v\n", err)
	}

	if *mockFlag {
		db["faux-fcs"] = "faux-fcs"
	}

	return &authRegistry{
		store: make(map[string]webClient),
		db:    db,
	}
}

func (reg *authRegistry) password(user string) (string, bool) {
	v, ok := reg.db[user]
	return v, ok
}

func (reg *authRegistry) get(cookie string) (webClient, bool) {
	reg.mu.RLock()
	client, ok := reg.store[cookie]
	reg.mu.RUnlock()
	return client, ok
}

func (reg *authRegistry) set(cookie string, client webClient) {
	reg.mu.Lock()
	reg.store[cookie] = client
	reg.mu.Unlock()
}

func (reg *authRegistry) del(cookie string) {
	reg.mu.Lock()
	delete(reg.store, cookie)
	reg.mu.Unlock()
}
<|endoftext|>


<commit_before>package file

import (
	"encoding/csv"
	"errors"
	"github.com/3onyc/hipdate/shared"
	"github.com/3onyc/hipdate/sources"
	"gopkg.in/fsnotify.v1"
	"log"
	"os"
	"path"
	"sync"
)

var (
	MissingFilepathError = errors.New("path not specified")
)

type FileSource struct {
	cce chan *shared.ChangeEvent
	wg  *sync.WaitGroup
	sc  chan bool
	p   string
	lf  [][]string
	w   *fsnotify.Watcher
}

func NewFileSource(
	opt shared.OptionMap,
	cce chan *shared.ChangeEvent,
	wg *sync.WaitGroup,
	sc chan bool,
) (
	sources.Source,
	error,
) {
	p, ok := opt["path"]
	if !ok {
		return nil, MissingFilepathError
	}

	return &FileSource{
		cce: cce,
		wg:  wg,
		sc:  sc,
		p:   p,
	}, nil
}

func (fs *FileSource) eventHandler(
	cfe chan fsnotify.Event,
	ce chan error,
) {
	for {
		select {
		case e := <-ce:
			log.Println("ERROR [source:file] Watcher:", e)
		case fe := <-cfe:
			if fe.Name != fs.p {
				continue
			}

			if fe.Op&fsnotify.Write == fsnotify.Write && fe.Op&fsnotify.Chmod != fsnotify.Chmod {
				r, err := fs.processFile()
				if err != nil {
					log.Println("CRITICAL [source:file]", err)
				}

				fs.processRecords("remove", fs.lf)
				fs.processRecords("add", r)
				fs.lf = r
			}
		case <-fs.sc:
			fs.Stop()
			return
		}
	}
}

func (fs *FileSource) Stop() {
	log.Println("INFO [source:file] Stopping watcher ...")
	if err := fs.w.Remove(path.Dir(fs.p)); err != nil {
		log.Println("ERROR [source:file] watcher:", err)
	}

	fs.wg.Done()
}

func (fs *FileSource) Start() {
	fs.wg.Add(1)

	log.Println("INFO [source:file] Loading file source...")
	if err := fs.Initialise(); err != nil {
		log.Println("ERROR [source:file]", err)
	}

	log.Println("INFO [source:file] Starting watcher ...")
	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Println("ERROR [source:file]", err)
		return
	}

	fs.w = w
	fs.w.Add(path.Dir(fs.p))

	fs.eventHandler(fs.w.Events, fs.w.Errors)
}

func (fs *FileSource) Initialise() error {
	r, err := fs.processFile()
	if err != nil {
		return err
	}
	fs.lf = r
	fs.processRecords("add", r)

	return nil
}

func (fs *FileSource) processFile() ([][]string, error) {
	f, err := os.Open(fs.p)
	if err != nil {
		return nil, err
	}

	r := csv.NewReader(f)
	r.FieldsPerRecord = -1
	rec, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	return rec, nil
}

func (fs *FileSource) processRecords(e string, r [][]string) {
	for _, l := range r {
		h := shared.Host(l[0])
		for _, u := range l[1:] {
			ep, err := shared.NewEndpointFromUrl(u)
			if err != nil {
				log.Printf("WARN [source:file] Couldn't parse URL %s, skipping", ep, err)
				continue
			}

			ce := shared.NewChangeEvent(e, h, *ep)
			fs.cce <- ce
		}
	}
}

func init() {
	sources.SourceMap["file"] = NewFileSource
}
<commit_msg>Fix log.Printf call<commit_after>package file

import (
	"encoding/csv"
	"errors"
	"github.com/3onyc/hipdate/shared"
	"github.com/3onyc/hipdate/sources"
	"gopkg.in/fsnotify.v1"
	"log"
	"os"
	"path"
	"sync"
)

var (
	MissingFilepathError = errors.New("path not specified")
)

type FileSource struct {
	cce chan *shared.ChangeEvent
	wg  *sync.WaitGroup
	sc  chan bool
	p   string
	lf  [][]string
	w   *fsnotify.Watcher
}

func NewFileSource(
	opt shared.OptionMap,
	cce chan *shared.ChangeEvent,
	wg *sync.WaitGroup,
	sc chan bool,
) (
	sources.Source,
	error,
) {
	p, ok := opt["path"]
	if !ok {
		return nil, MissingFilepathError
	}

	return &FileSource{
		cce: cce,
		wg:  wg,
		sc:  sc,
		p:   p,
	}, nil
}

func (fs *FileSource) eventHandler(
	cfe chan fsnotify.Event,
	ce chan error,
) {
	for {
		select {
		case e := <-ce:
			log.Println("ERROR [source:file] Watcher:", e)
		case fe := <-cfe:
			if fe.Name != fs.p {
				continue
			}

			if fe.Op&fsnotify.Write == fsnotify.Write && fe.Op&fsnotify.Chmod != fsnotify.Chmod {
				r, err := fs.processFile()
				if err != nil {
					log.Println("CRITICAL [source:file]", err)
				}

				fs.processRecords("remove", fs.lf)
				fs.processRecords("add", r)
				fs.lf = r
			}
		case <-fs.sc:
			fs.Stop()
			return
		}
	}
}

func (fs *FileSource) Stop() {
	log.Println("INFO [source:file] Stopping watcher ...")
	if err := fs.w.Remove(path.Dir(fs.p)); err != nil {
		log.Println("ERROR [source:file] watcher:", err)
	}

	fs.wg.Done()
}

func (fs *FileSource) Start() {
	fs.wg.Add(1)

	log.Println("INFO [source:file] Loading file source...")
	if err := fs.Initialise(); err != nil {
		log.Println("ERROR [source:file]", err)
	}

	log.Println("INFO [source:file] Starting watcher ...")
	w, err := fsnotify.NewWatcher()
	if err != nil {
		log.Println("ERROR [source:file]", err)
		return
	}

	fs.w = w
	fs.w.Add(path.Dir(fs.p))

	fs.eventHandler(fs.w.Events, fs.w.Errors)
}

func (fs *FileSource) Initialise() error {
	r, err := fs.processFile()
	if err != nil {
		return err
	}
	fs.lf = r
	fs.processRecords("add", r)

	return nil
}

func (fs *FileSource) processFile() ([][]string, error) {
	f, err := os.Open(fs.p)
	if err != nil {
		return nil, err
	}

	r := csv.NewReader(f)
	r.FieldsPerRecord = -1
	rec, err := r.ReadAll()
	if err != nil {
		return nil, err
	}

	return rec, nil
}

func (fs *FileSource) processRecords(e string, r [][]string) {
	for _, l := range r {
		h := shared.Host(l[0])
		for _, u := range l[1:] {
			ep, err := shared.NewEndpointFromUrl(u)
			if err != nil {
				log.Printf("WARN [source:file] Couldn't parse URL %s, skipping (%s)", ep, err)
				continue
			}

			ce := shared.NewChangeEvent(e, h, *ep)
			fs.cce <- ce
		}
	}
}

func init() {
	sources.SourceMap["file"] = NewFileSource
}
<|endoftext|>


<commit_before>package guage

import (
	"fmt"
	ui "github.com/gizak/termui"
	DB "github.com/vrecan/FluxDash/influx"
	"github.com/vrecan/FluxDash/merge"
	"github.com/vrecan/FluxDash/query"
	TS "github.com/vrecan/FluxDash/timeselect"
)

type GaugeInfo struct {
}

type Gauge struct {
	From                    string       `json:"from"`
	Where                   string       `json:"where"`
	BorderLabel             string       `json:"borderlabel"`
	Border                  bool         `json:"border"`
	BorderFg                ui.Attribute `json:"borderfg"`
	BorderBg                ui.Attribute `json:"borderbg"`
	BorderLeft              bool         `json:borderleft"`
	BorderRight             bool         `json:"borderright"`
	BorderTop               bool         `json:"bordertop"`
	BorderBottom            bool         `json:"borderbottom"`
	BorderLabelFg           ui.Attribute `json:"borderlabelfg"`
	BorderLabelBg           ui.Attribute `json:"borderlabelbg"`
	Display                 bool         `json:"display"`
	Bg                      ui.Attribute `json:"bg"`
	Width                   int          `json:"width"`
	Height                  int          `json:"height"`
	PaddingTop              int          `json:"paddingtop"`
	PaddingBottom           int          `json:"paddingbottom"`
	PaddingLeft             int          `json:"paddingleft"`
	PaddingRight            int          `json:"paddingright"`
	BarColor                ui.Attribute `json:"barcolor"`
	PercentColor            ui.Attribute `json:"percentcolor"`
	PercentColorHighlighted ui.Attribute `json:"percentcolorhighlighted"`
	Label                   string       `json:"label"`
	LabelAlign              ui.Align     `json:"labelalign"`
	G                       *ui.Gauge    `json:"-"`
	db                      *DB.Influx   `json:"-"`
}

func NewGauge(db *DB.Influx, g *Gauge) *Gauge {
	g.db = db
	g.G = ui.NewGauge()
	merge.Merge(g, g.G, "G", "db")
	return g
}

func (s *Gauge) Gauges() *ui.Gauge {
	return s.G
}

func (s *Gauge) Update(time TS.TimeSelect) {
	t, _, _ := time.CurTime()
	s.SetData(t)
	s.SetTitle(t)
}

func (s *Gauge) SetData(time string) {
	meanTotal := query.GetIntData(s.db, query.Build("mean(value)", s.From, s.Where, time, ""))
	s.G.Percent = meanTotal[0]
}

func (s *Gauge) SetTitle(time string) {
	meanTotal := query.GetIntData(s.db, query.Build("mean(value)", s.From, s.Where, time, ""))
	if len(meanTotal) > 0 {
		s.G.Percent = meanTotal[0]
	} else {
		s.G.Percent = 0
	}
	maxTotal := query.GetIntData(s.db, query.Build("max(value)", s.From, s.Where, time, ""))
	s.G.Label = fmt.Sprintf("%s mean:%v%% max:%v%%", s.Label, s.G.Percent, maxTotal[0])
}
<commit_msg>improve docs<commit_after>package guage

import (
	"fmt"
	ui "github.com/gizak/termui"
	DB "github.com/vrecan/FluxDash/influx"
	"github.com/vrecan/FluxDash/merge"
	"github.com/vrecan/FluxDash/query"
	TS "github.com/vrecan/FluxDash/timeselect"
)

//Gauge is a simple percentage gauge of a statistic.
type Gauge struct {
	From                    string       `json:"from"`
	Where                   string       `json:"where"`
	BorderLabel             string       `json:"borderlabel"`
	Border                  bool         `json:"border"`
	BorderFg                ui.Attribute `json:"borderfg"`
	BorderBg                ui.Attribute `json:"borderbg"`
	BorderLeft              bool         `json:borderleft"`
	BorderRight             bool         `json:"borderright"`
	BorderTop               bool         `json:"bordertop"`
	BorderBottom            bool         `json:"borderbottom"`
	BorderLabelFg           ui.Attribute `json:"borderlabelfg"`
	BorderLabelBg           ui.Attribute `json:"borderlabelbg"`
	Display                 bool         `json:"display"`
	Bg                      ui.Attribute `json:"bg"`
	Width                   int          `json:"width"`
	Height                  int          `json:"height"`
	PaddingTop              int          `json:"paddingtop"`
	PaddingBottom           int          `json:"paddingbottom"`
	PaddingLeft             int          `json:"paddingleft"`
	PaddingRight            int          `json:"paddingright"`
	BarColor                ui.Attribute `json:"barcolor"`
	PercentColor            ui.Attribute `json:"percentcolor"`
	PercentColorHighlighted ui.Attribute `json:"percentcolorhighlighted"`
	Label                   string       `json:"label"`
	LabelAlign              ui.Align     `json:"labelalign"`
	G                       *ui.Gauge    `json:"-"`
	db                      *DB.Influx   `json:"-"`
}

//NewGauge will create a gauge from a partial gauge generated from a json dashboard.
func NewGauge(db *DB.Influx, g *Gauge) *Gauge {
	g.db = db
	g.G = ui.NewGauge()
	merge.Merge(g, g.G, "G", "db")
	return g
}

//Update the gauge data from influxdb queries.
func (s *Gauge) Update(time TS.TimeSelect) {
	t, _, _ := time.CurTime()
	s.SetData(t)
	s.SetTitle(t)
}

//SetData will set the data for the bar.
func (s *Gauge) SetData(time string) {
	meanTotal := query.GetIntData(s.db, query.Build("mean(value)", s.From, s.Where, time, ""))
	s.G.Percent = meanTotal[0]
}

//SetTitle will set the label of the gauge.
func (s *Gauge) SetTitle(time string) {
	meanTotal := query.GetIntData(s.db, query.Build("mean(value)", s.From, s.Where, time, ""))
	if len(meanTotal) > 0 {
		s.G.Percent = meanTotal[0]
	} else {
		s.G.Percent = 0
	}
	maxTotal := query.GetIntData(s.db, query.Build("max(value)", s.From, s.Where, time, ""))
	s.G.Label = fmt.Sprintf("%s mean:%v%% max:%v%%", s.Label, s.G.Percent, maxTotal[0])
}
<|endoftext|>


<commit_before>// Copyright 2021 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package readerdriver

import (
	"io"
	"runtime"
	"sync"
	"time"

	"github.com/hajimehoshi/ebiten/v2/audio/internal/oboe"
)

func IsAvailable() bool {
	return true
}

type context struct {
	sampleRate      int
	channelNum      int
	bitDepthInBytes int
}

func NewContext(sampleRate int, channelNum int, bitDepthInBytes int) (Context, chan struct{}, error) {
	ready := make(chan struct{})
	close(ready)

	c := &context{
		sampleRate:      sampleRate,
		channelNum:      channelNum,
		bitDepthInBytes: bitDepthInBytes,
	}
	return c, ready, nil
}

func (c *context) NewPlayer(src io.Reader) Player {
	p := &player{
		context: c,
		src:     src,
		cond:    sync.NewCond(&sync.Mutex{}),
		volume:  1,
	}
	runtime.SetFinalizer(p, (*player).Close)
	return p
}

func (c *context) Suspend() error {
	return oboe.Suspend()
}

func (c *context) Resume() error {
	return oboe.Resume()
}

type player struct {
	context *context
	p       *oboe.Player
	src     io.Reader
	err     error
	cond    *sync.Cond
	closed  bool
	volume  float64
}

func (p *player) Pause() {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.err != nil {
		return
	}
	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	if err := p.p.Pause(); err != nil {
		p.setErrorImpl(err)
		return
	}
	p.cond.Signal()
}

func (p *player) Play() {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.err != nil {
		return
	}
	if p.p != nil && p.p.IsPlaying() {
		return
	}
	defer p.cond.Signal()
	var runLoop bool
	if p.p == nil {
		p.p = oboe.NewPlayer(p.context.sampleRate, p.context.channelNum, p.context.bitDepthInBytes, p.volume, func() {
			p.cond.Signal()
		})
		runLoop = true
	}

	buf := make([]byte, p.context.maxBufferSize())
	for p.p.UnplayedBufferSize() < p.context.maxBufferSize() {
		n, err := p.src.Read(buf)
		if err != nil && err != io.EOF {
			p.setErrorImpl(err)
			return
		}
		p.p.AppendBuffer(buf[:n])
		if err == io.EOF {
			break
		}
	}

	if err := p.p.Play(); err != nil {
		p.setErrorImpl(err)
		return
	}
	if runLoop {
		go p.loop()
	}
}

func (p *player) IsPlaying() bool {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	if p.p == nil {
		return false
	}
	return p.p.IsPlaying()
}

func (p *player) Reset() {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.err != nil {
		return
	}
	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	if err := p.p.Close(); err != nil {
		p.setErrorImpl(err)
		return
	}
	p.p = nil
	p.cond.Signal()
}

func (p *player) Volume() float64 {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.volume
}

func (p *player) SetVolume(volume float64) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	p.volume = volume
	if p.p == nil {
		return
	}
	p.p.SetVolume(volume)
}

func (p *player) UnplayedBufferSize() int {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	if p.p == nil {
		return 0
	}
	return p.p.UnplayedBufferSize()
}

func (p *player) Err() error {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.err
}

func (p *player) Close() error {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.closeImpl()
}

func (p *player) closeImpl() error {
	defer p.cond.Signal()

	runtime.SetFinalizer(p, nil)
	p.closed = true
	if p.p == nil {
		return p.err
	}
	if err := p.p.Close(); err != nil && p.err == nil {
		// Do not call setErrorImpl, or this can cause infinite recursive.
		p.err = err
		return p.err
	}
	p.p = nil
	return p.err
}

func (p *player) setError(err error) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	p.setErrorImpl(err)
}

func (p *player) setErrorImpl(err error) {
	p.err = err
	p.closeImpl()
}

func (p *player) shouldWait() bool {
	if p.closed {
		return false
	}
	if p.p == nil {
		return false
	}
	if p.p.IsPlaying() {
		return p.p.UnplayedBufferSize() >= p.context.maxBufferSize()
	}
	return true
}

func (p *player) wait() bool {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	for p.shouldWait() {
		p.cond.Wait()
	}
	return p.p != nil && p.p.IsPlaying()
}

func (p *player) write(buf []byte) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	p.p.AppendBuffer(buf)
}

func (p *player) loop() {
	buf := make([]byte, 4096)
	for {
		if !p.wait() {
			return
		}

		n, err := p.src.Read(buf)
		if err != nil && err != io.EOF {
			p.setError(err)
			return
		}
		p.write(buf[:n])

		// Now p.p.Reset() doesn't close the stream gracefully. Then buffer size check is necessary here.
		if err == io.EOF && p.UnplayedBufferSize() == 0 {
			// Even when the unplayed buffer size is 0, the audio data in the hardware might not be played yet (#1632).
			// Just wait for a while.
			time.Sleep(100 * time.Millisecond)
			p.Reset()
			return
		}
	}
}
<commit_msg>audio/internal/readerdriver: Make Play async on Android<commit_after>// Copyright 2021 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package readerdriver

import (
	"io"
	"runtime"
	"sync"
	"time"

	"github.com/hajimehoshi/ebiten/v2/audio/internal/oboe"
)

func IsAvailable() bool {
	return true
}

type context struct {
	sampleRate      int
	channelNum      int
	bitDepthInBytes int
}

func NewContext(sampleRate int, channelNum int, bitDepthInBytes int) (Context, chan struct{}, error) {
	ready := make(chan struct{})
	close(ready)

	c := &context{
		sampleRate:      sampleRate,
		channelNum:      channelNum,
		bitDepthInBytes: bitDepthInBytes,
	}
	return c, ready, nil
}

func (c *context) NewPlayer(src io.Reader) Player {
	p := &player{
		context: c,
		src:     src,
		cond:    sync.NewCond(&sync.Mutex{}),
		volume:  1,
	}
	runtime.SetFinalizer(p, (*player).Close)
	return p
}

func (c *context) Suspend() error {
	return oboe.Suspend()
}

func (c *context) Resume() error {
	return oboe.Resume()
}

type player struct {
	context *context
	p       *oboe.Player
	src     io.Reader
	err     error
	cond    *sync.Cond
	closed  bool
	volume  float64
}

func (p *player) Pause() {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.err != nil {
		return
	}
	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	if err := p.p.Pause(); err != nil {
		p.setErrorImpl(err)
		return
	}
	p.cond.Signal()
}

func (p *player) Play() {
	// Call Play asynchronously since Oboe's Play might take long.
	ch := make(chan struct{})
	go func() {
		p.cond.L.Lock()
		defer p.cond.L.Unlock()
		close(ch)
		p.playImpl()
	}()

	// Wait until the mutex is locked in the above goroutine.
	<-ch
}

func (p *player) playImpl() {
	if p.err != nil {
		return
	}
	if p.p != nil && p.p.IsPlaying() {
		return
	}
	defer p.cond.Signal()
	var runLoop bool
	if p.p == nil {
		p.p = oboe.NewPlayer(p.context.sampleRate, p.context.channelNum, p.context.bitDepthInBytes, p.volume, func() {
			p.cond.Signal()
		})
		runLoop = true
	}

	buf := make([]byte, p.context.maxBufferSize())
	for p.p.UnplayedBufferSize() < p.context.maxBufferSize() {
		n, err := p.src.Read(buf)
		if err != nil && err != io.EOF {
			p.setErrorImpl(err)
			return
		}
		p.p.AppendBuffer(buf[:n])
		if err == io.EOF {
			break
		}
	}

	if err := p.p.Play(); err != nil {
		p.setErrorImpl(err)
		return
	}
	if runLoop {
		go p.loop()
	}
}

func (p *player) IsPlaying() bool {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	if p.p == nil {
		return false
	}
	return p.p.IsPlaying()
}

func (p *player) Reset() {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.err != nil {
		return
	}
	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	if err := p.p.Close(); err != nil {
		p.setErrorImpl(err)
		return
	}
	p.p = nil
	p.cond.Signal()
}

func (p *player) Volume() float64 {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.volume
}

func (p *player) SetVolume(volume float64) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	p.volume = volume
	if p.p == nil {
		return
	}
	p.p.SetVolume(volume)
}

func (p *player) UnplayedBufferSize() int {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	if p.p == nil {
		return 0
	}
	return p.p.UnplayedBufferSize()
}

func (p *player) Err() error {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.err
}

func (p *player) Close() error {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	return p.closeImpl()
}

func (p *player) closeImpl() error {
	defer p.cond.Signal()

	runtime.SetFinalizer(p, nil)
	p.closed = true
	if p.p == nil {
		return p.err
	}
	if err := p.p.Close(); err != nil && p.err == nil {
		// Do not call setErrorImpl, or this can cause infinite recursive.
		p.err = err
		return p.err
	}
	p.p = nil
	return p.err
}

func (p *player) setError(err error) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()
	p.setErrorImpl(err)
}

func (p *player) setErrorImpl(err error) {
	p.err = err
	p.closeImpl()
}

func (p *player) shouldWait() bool {
	if p.closed {
		return false
	}
	if p.p == nil {
		return false
	}
	if p.p.IsPlaying() {
		return p.p.UnplayedBufferSize() >= p.context.maxBufferSize()
	}
	return true
}

func (p *player) wait() bool {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	for p.shouldWait() {
		p.cond.Wait()
	}
	return p.p != nil && p.p.IsPlaying()
}

func (p *player) write(buf []byte) {
	p.cond.L.Lock()
	defer p.cond.L.Unlock()

	if p.closed {
		return
	}
	if p.p == nil {
		return
	}
	p.p.AppendBuffer(buf)
}

func (p *player) loop() {
	buf := make([]byte, 4096)
	for {
		if !p.wait() {
			return
		}

		n, err := p.src.Read(buf)
		if err != nil && err != io.EOF {
			p.setError(err)
			return
		}
		p.write(buf[:n])

		// Now p.p.Reset() doesn't close the stream gracefully. Then buffer size check is necessary here.
		if err == io.EOF && p.UnplayedBufferSize() == 0 {
			// Even when the unplayed buffer size is 0, the audio data in the hardware might not be played yet (#1632).
			// Just wait for a while.
			time.Sleep(100 * time.Millisecond)
			p.Reset()
			return
		}
	}
}
<|endoftext|>