<commit_before>// +build mage

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/magefile/mage/mg" // mg contains helpful utility functions, like Deps
	"github.com/magefile/mage/sh" // mg contains helpful utility functions, like Deps
)

const localWorkDir = "./sparta"

var header = strings.Repeat("-", 80)

var ignoreSubdirectoryPaths = []string{
	".vendor",
	".sparta",
	".vscode",
	"/resources/describe",
}

// Default target to run when none is specified
// If not set, running mage will list available targets
// var Default = Build

func mageScript(commands [][]string) error {
	for _, eachCommand := range commands {
		var commandErr error
		if len(eachCommand) <= 1 {
			commandErr = sh.Run(eachCommand[0])
		} else {
			commandErr = sh.Run(eachCommand[0], eachCommand[1:]...)
		}
		if commandErr != nil {
			return commandErr
		}
	}
	return nil
}
func mageLog(formatSpecifier string, args ...interface{}) {
	if mg.Verbose() {
		if len(args) != 0 {
			log.Printf(formatSpecifier, args...)
		} else {
			log.Printf(formatSpecifier)
		}
	}
}

func sourceFilesOfType(extension string) ([]string, error) {
	testExtension := strings.TrimPrefix(extension, ".")
	testExtension = fmt.Sprintf(".%s", testExtension)

	files := make([]string, 0)
	walker := func(path string, info os.FileInfo, err error) error {
		contains := false
		for _, eachComponent := range ignoreSubdirectoryPaths {
			contains = strings.Contains(path, eachComponent)
			if contains {
				break
			}
		}
		if !contains && (filepath.Ext(path) == testExtension) {
			files = append(files, path)
		}
		return nil
	}
	goSourceFilesErr := filepath.Walk(".", walker)
	return files, goSourceFilesErr
}

func applyToSource(fileExtension string, commandParts ...string) error {
	if len(commandParts) <= 0 {
		return errors.New("applyToSource requires a command to apply to source files")
	}
	eligibleSourceFiles, eligibleSourceFilesErr := sourceFilesOfType(fileExtension)
	if eligibleSourceFilesErr != nil {
		return eligibleSourceFilesErr
	}

	mageLog(header)
	mageLog("Applying `%s` to %d `*.%s` source files", commandParts[0], len(eligibleSourceFiles), fileExtension)
	mageLog(header)

	commandArgs := []string{}
	if len(commandParts) > 1 {
		for _, eachPart := range commandParts[1:] {
			commandArgs = append(commandArgs, eachPart)
		}
	}
	for _, eachFile := range eligibleSourceFiles {
		applyArgs := append(commandArgs, eachFile)
		applyErr := sh.Run(commandParts[0], applyArgs...)
		if applyErr != nil {
			return applyErr
		}
	}
	return nil
}

func markdownSourceApply(commandParts ...string) error {
	return applyToSource("md", commandParts...)
}
func goSourceApply(commandParts ...string) error {
	return applyToSource("go", commandParts...)
}

func EnsureCleanTree() error {
	cleanTreeScript := [][]string{
		// No dirty trees
		{"git", "diff", "--exit-code"},
	}
	return mageScript(cleanTreeScript)
}

// GenerateBuildInfo creates the automatic buildinfo.go file so that we can
// stamp the SHA into the binaries we build...
func GenerateBuildInfo() error {
	mg.SerialDeps(EnsureCleanTree)

	// The first thing we need is the `git` SHA
	cmd := exec.Command("git", "rev-parse", "HEAD")
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		return err
	}
	stdOutResult := strings.TrimSpace(string(stdout.Bytes()))

	// Super = update the buildinfo data
	buildInfoTemplate := `package sparta

// THIS FILE IS AUTOMATICALLY GENERATED
// DO NOT EDIT
// CREATED: %s

// SpartaGitHash is the commit hash of this Sparta library
const SpartaGitHash = "%s"
`
	updatedInfo := fmt.Sprintf(buildInfoTemplate, time.Now().UTC(), stdOutResult)
	// Write it to the output location...
	writeErr := ioutil.WriteFile("./buildinfo.go", []byte(updatedInfo), os.ModePerm)

	if writeErr != nil {
		return writeErr
	}
	commitGenerateCommands := [][]string{
		{"git", "diff"},
		{"git", "commit", "-a", "-m", `"Autogenerated build info"`},
	}
	return mageScript(commitGenerateCommands)

}

// GenerateConstants runs the set of commands that update the embedded CONSTANTS
// for both local and AWS Lambda execution
func GenerateConstants() error {
	generateCommands := [][]string{
		// Create the embedded version
		{"go", "run", "$GOPATH/src/github.com/mjibson/esc/main.go", "-o", "./CONSTANTS.go", "-private", "-pkg", "sparta", "./resources"},
		//Create a secondary CONSTANTS_AWSBINARY.go file with empty content.
		{"go", "run", "$GOPATH/src/github.com/mjibson/esc/main.go", "-o", "./CONSTANTS_AWSBINARY.go", "-private", "-pkg", "sparta", "./resources/awsbinary/README.md"},
		//The next step will insert the
		// build tags at the head of each file so that they are mutually exclusive
		{"go", "run", "./cmd/insertTags/main.go", "./CONSTANTS", "!lambdabinary"},
		{"go", "run", "./cmd/insertTags/main.go", "./CONSTANTS_AWSBINARY", "lambdabinary"},
		{"git", "commit", "-a", "-m", "Autogenerated constants"},
	}
	return mageScript(generateCommands)
}

// InstallBuildRequirements installs or updates the dependent
// packages that aren't referenced by the source, but are needed
// to build the Sparta source
func InstallBuildRequirements() error {
	mageLog("`go get` update flags (env.GO_GET_FLAG): %s", os.Getenv("GO_GET_FLAG"))

	requirements := []string{
		"github.com/golang/dep/...",
		"honnef.co/go/tools/cmd/megacheck",
		"honnef.co/go/tools/cmd/gosimple",
		"honnef.co/go/tools/cmd/unused",
		"honnef.co/go/tools/cmd/staticcheck",
		"golang.org/x/tools/cmd/goimports",
		"github.com/fzipp/gocyclo",
		"github.com/golang/lint/golint",
		"github.com/mjibson/esc",
		"github.com/securego/gosec/cmd/gosec/...",
		"github.com/client9/misspell/cmd/misspell",
	}
	for _, eachDep := range requirements {
		cmdErr := sh.Run("go",
			"get",
			os.Getenv("GO_GET_FLAG"),
			eachDep)

		// cmdErr := exec.Command(.Run()
		if cmdErr != nil {
			return cmdErr
		}
	}
	return nil
}

// EnsureSpelling ensures that there are no misspellings in the source
func EnsureSpelling() error {
	goSpelling := func() error {
		return goSourceApply("misspell", "-error")
	}
	mdSpelling := func() error {
		return markdownSourceApply("misspell", "-error")
	}
	mg.SerialDeps(
		goSpelling,
		mdSpelling)
	return nil
}

// EnsureVet ensures that the source has been `go vet`ted
func EnsureVet() error {
	return goSourceApply("go", "tool", "vet")
}

// EnsureLint ensures that the source is `golint`ed
func EnsureLint() error {
	return goSourceApply("golint")
}

// EnsureFormatted ensures that the source code is formatted with goimports
func EnsureFormatted() error {
	return goSourceApply("goimports", "-d")
}

// EnsureStaticChecks ensures that the source code passes static code checks
func EnsureStaticChecks() error {
	// Megacheck
	megacheckErr := sh.Run("megacheck",
		"-ignore",
		"github.com/mweagle/Sparta/CONSTANTS.go:*")
	if megacheckErr != nil {
		return megacheckErr
	}
	// Gosec
	return sh.Run("gosec",
		"-exclude=G204,G505,G401",
		"./...")
}

// EnsureAllPreconditions ensures that the source passes *ALL* static `ensure*`
// precondition steps
func EnsureAllPreconditions() error {
	mg.SerialDeps(
		InstallBuildRequirements,
		EnsureVet,
		EnsureLint,
		EnsureFormatted,
		EnsureStaticChecks,
		EnsureSpelling,
	)
	return nil
}

// EnsureTravisBuildEnvironment is the command that sets up the Travis
// environment to run the build.
func EnsureTravisBuildEnvironment() error {
	mg.SerialDeps(InstallBuildRequirements)

	// Super run some commands
	travisComands := [][]string{
		[]string{"dep", "version"},
		[]string{"dep", "ensure", "-v"},
		[]string{"rsync", "-a", "--quiet", "--remove-source-files", "./vendor/", "$GOPATH/src"},
	}
	return mageScript(travisComands)
}

// Build the application
func Build() error {
	mg.Deps(EnsureAllPreconditions)
	return sh.Run("go", "build", ".")
}

// Clean the working directory
func Clean() error {
	cleanCommands := [][]string{
		[]string{"go", "clean", "."},
		[]string{"rm", "-rf", "./graph.html"},
		[]string{"rsync", "-a", "--quiet", "--remove-source-files", "./vendor/", "$GOPATH/src"},
	}
	return mageScript(cleanCommands)
}

// Describe runs the `TestDescribe` test to generate a describe HTML output
// file at graph.html
func Describe() error {
	describeCommands := [][]string{
		[]string{"rm", "-rf", "./graph.html"},
		[]string{"go", "test", "-v", "-run", "TestDescribe"},
	}
	return mageScript(describeCommands)
}

// Publish the latest source
func Publish() error {
	mg.SerialDeps(GenerateBuildInfo)

	describeCommands := [][]string{
		[]string{"git", "push", "origin"},
	}
	return mageScript(describeCommands)
}

// Test runs the Sparta tests
func Test() {
	testCommand := func() error {
		return sh.Run("go",
			"test",
			"-cover",
			"-race",
			"./...")
	}
	mg.SerialDeps(
		EnsureAllPreconditions,
		testCommand,
	)
}

// TestCover runs the test and opens up the resulting report
func TestCover() error {
	// mg.SerialDeps(
	// 	EnsureAllPreconditions,
	// )
	coverageReport := fmt.Sprintf("%s/cover.out", localWorkDir)
	testCoverCommands := [][]string{
		[]string{"go", "test", fmt.Sprintf("-coverprofile=%s", coverageReport), "."},
		[]string{"go", "tool", "cover", fmt.Sprintf("-html=%s", coverageReport)},
		[]string{"rm", coverageReport},
		[]string{"open", fmt.Sprintf("%s/cover.html", localWorkDir)},
	}
	return mageScript(testCoverCommands)
}

// TravisBuild is the task to build in the context of a Travis CI pipeline
func TravisBuild() error {
	mg.SerialDeps(EnsureTravisBuildEnvironment,
		Build,
		Test)
	return nil
}
<commit_msg>Use proper working dir<commit_after>// +build mage

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/magefile/mage/mg" // mg contains helpful utility functions, like Deps
	"github.com/magefile/mage/sh" // mg contains helpful utility functions, like Deps
)

const localWorkDir = "./.sparta"

var header = strings.Repeat("-", 80)

var ignoreSubdirectoryPaths = []string{
	".vendor",
	".sparta",
	".vscode",
	"/resources/describe",
}

// Default target to run when none is specified
// If not set, running mage will list available targets
// var Default = Build

func mageScript(commands [][]string) error {
	for _, eachCommand := range commands {
		var commandErr error
		if len(eachCommand) <= 1 {
			commandErr = sh.Run(eachCommand[0])
		} else {
			commandErr = sh.Run(eachCommand[0], eachCommand[1:]...)
		}
		if commandErr != nil {
			return commandErr
		}
	}
	return nil
}
func mageLog(formatSpecifier string, args ...interface{}) {
	if mg.Verbose() {
		if len(args) != 0 {
			log.Printf(formatSpecifier, args...)
		} else {
			log.Printf(formatSpecifier)
		}
	}
}

func sourceFilesOfType(extension string) ([]string, error) {
	testExtension := strings.TrimPrefix(extension, ".")
	testExtension = fmt.Sprintf(".%s", testExtension)

	files := make([]string, 0)
	walker := func(path string, info os.FileInfo, err error) error {
		contains := false
		for _, eachComponent := range ignoreSubdirectoryPaths {
			contains = strings.Contains(path, eachComponent)
			if contains {
				break
			}
		}
		if !contains && (filepath.Ext(path) == testExtension) {
			files = append(files, path)
		}
		return nil
	}
	goSourceFilesErr := filepath.Walk(".", walker)
	return files, goSourceFilesErr
}

func applyToSource(fileExtension string, commandParts ...string) error {
	if len(commandParts) <= 0 {
		return errors.New("applyToSource requires a command to apply to source files")
	}
	eligibleSourceFiles, eligibleSourceFilesErr := sourceFilesOfType(fileExtension)
	if eligibleSourceFilesErr != nil {
		return eligibleSourceFilesErr
	}

	mageLog(header)
	mageLog("Applying `%s` to %d `*.%s` source files", commandParts[0], len(eligibleSourceFiles), fileExtension)
	mageLog(header)

	commandArgs := []string{}
	if len(commandParts) > 1 {
		for _, eachPart := range commandParts[1:] {
			commandArgs = append(commandArgs, eachPart)
		}
	}
	for _, eachFile := range eligibleSourceFiles {
		applyArgs := append(commandArgs, eachFile)
		applyErr := sh.Run(commandParts[0], applyArgs...)
		if applyErr != nil {
			return applyErr
		}
	}
	return nil
}

func markdownSourceApply(commandParts ...string) error {
	return applyToSource("md", commandParts...)
}
func goSourceApply(commandParts ...string) error {
	return applyToSource("go", commandParts...)
}

func EnsureCleanTree() error {
	cleanTreeScript := [][]string{
		// No dirty trees
		{"git", "diff", "--exit-code"},
	}
	return mageScript(cleanTreeScript)
}

// GenerateBuildInfo creates the automatic buildinfo.go file so that we can
// stamp the SHA into the binaries we build...
func GenerateBuildInfo() error {
	mg.SerialDeps(EnsureCleanTree)

	// The first thing we need is the `git` SHA
	cmd := exec.Command("git", "rev-parse", "HEAD")
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if err != nil {
		return err
	}
	stdOutResult := strings.TrimSpace(string(stdout.Bytes()))

	// Super = update the buildinfo data
	buildInfoTemplate := `package sparta

// THIS FILE IS AUTOMATICALLY GENERATED
// DO NOT EDIT
// CREATED: %s

// SpartaGitHash is the commit hash of this Sparta library
const SpartaGitHash = "%s"
`
	updatedInfo := fmt.Sprintf(buildInfoTemplate, time.Now().UTC(), stdOutResult)
	// Write it to the output location...
	writeErr := ioutil.WriteFile("./buildinfo.go", []byte(updatedInfo), os.ModePerm)

	if writeErr != nil {
		return writeErr
	}
	commitGenerateCommands := [][]string{
		{"git", "diff"},
		{"git", "commit", "-a", "-m", `"Autogenerated build info"`},
	}
	return mageScript(commitGenerateCommands)

}

// GenerateConstants runs the set of commands that update the embedded CONSTANTS
// for both local and AWS Lambda execution
func GenerateConstants() error {
	generateCommands := [][]string{
		// Create the embedded version
		{"go", "run", "$GOPATH/src/github.com/mjibson/esc/main.go", "-o", "./CONSTANTS.go", "-private", "-pkg", "sparta", "./resources"},
		//Create a secondary CONSTANTS_AWSBINARY.go file with empty content.
		{"go", "run", "$GOPATH/src/github.com/mjibson/esc/main.go", "-o", "./CONSTANTS_AWSBINARY.go", "-private", "-pkg", "sparta", "./resources/awsbinary/README.md"},
		//The next step will insert the
		// build tags at the head of each file so that they are mutually exclusive
		{"go", "run", "./cmd/insertTags/main.go", "./CONSTANTS", "!lambdabinary"},
		{"go", "run", "./cmd/insertTags/main.go", "./CONSTANTS_AWSBINARY", "lambdabinary"},
		{"git", "commit", "-a", "-m", "Autogenerated constants"},
	}
	return mageScript(generateCommands)
}

// InstallBuildRequirements installs or updates the dependent
// packages that aren't referenced by the source, but are needed
// to build the Sparta source
func InstallBuildRequirements() error {
	mageLog("`go get` update flags (env.GO_GET_FLAG): %s", os.Getenv("GO_GET_FLAG"))

	requirements := []string{
		"github.com/golang/dep/...",
		"honnef.co/go/tools/cmd/megacheck",
		"honnef.co/go/tools/cmd/gosimple",
		"honnef.co/go/tools/cmd/unused",
		"honnef.co/go/tools/cmd/staticcheck",
		"golang.org/x/tools/cmd/goimports",
		"github.com/fzipp/gocyclo",
		"github.com/golang/lint/golint",
		"github.com/mjibson/esc",
		"github.com/securego/gosec/cmd/gosec/...",
		"github.com/client9/misspell/cmd/misspell",
	}
	for _, eachDep := range requirements {
		cmdErr := sh.Run("go",
			"get",
			os.Getenv("GO_GET_FLAG"),
			eachDep)

		// cmdErr := exec.Command(.Run()
		if cmdErr != nil {
			return cmdErr
		}
	}
	return nil
}

// EnsureSpelling ensures that there are no misspellings in the source
func EnsureSpelling() error {
	goSpelling := func() error {
		return goSourceApply("misspell", "-error")
	}
	mdSpelling := func() error {
		return markdownSourceApply("misspell", "-error")
	}
	mg.SerialDeps(
		goSpelling,
		mdSpelling)
	return nil
}

// EnsureVet ensures that the source has been `go vet`ted
func EnsureVet() error {
	return goSourceApply("go", "tool", "vet")
}

// EnsureLint ensures that the source is `golint`ed
func EnsureLint() error {
	return goSourceApply("golint")
}

// EnsureFormatted ensures that the source code is formatted with goimports
func EnsureFormatted() error {
	return goSourceApply("goimports", "-d")
}

// EnsureStaticChecks ensures that the source code passes static code checks
func EnsureStaticChecks() error {
	// Megacheck
	megacheckErr := sh.Run("megacheck",
		"-ignore",
		"github.com/mweagle/Sparta/CONSTANTS.go:*")
	if megacheckErr != nil {
		return megacheckErr
	}
	// Gosec
	return sh.Run("gosec",
		"-exclude=G204,G505,G401",
		"./...")
}

// EnsureAllPreconditions ensures that the source passes *ALL* static `ensure*`
// precondition steps
func EnsureAllPreconditions() error {
	mg.SerialDeps(
		InstallBuildRequirements,
		EnsureVet,
		EnsureLint,
		EnsureFormatted,
		EnsureStaticChecks,
		EnsureSpelling,
	)
	return nil
}

// EnsureTravisBuildEnvironment is the command that sets up the Travis
// environment to run the build.
func EnsureTravisBuildEnvironment() error {
	mg.SerialDeps(InstallBuildRequirements)

	// Super run some commands
	travisComands := [][]string{
		[]string{"dep", "version"},
		[]string{"dep", "ensure", "-v"},
		[]string{"rsync", "-a", "--quiet", "--remove-source-files", "./vendor/", "$GOPATH/src"},
	}
	return mageScript(travisComands)
}

// Build the application
func Build() error {
	mg.Deps(EnsureAllPreconditions)
	return sh.Run("go", "build", ".")
}

// Clean the working directory
func Clean() error {
	cleanCommands := [][]string{
		[]string{"go", "clean", "."},
		[]string{"rm", "-rf", "./graph.html"},
		[]string{"rsync", "-a", "--quiet", "--remove-source-files", "./vendor/", "$GOPATH/src"},
	}
	return mageScript(cleanCommands)
}

// Describe runs the `TestDescribe` test to generate a describe HTML output
// file at graph.html
func Describe() error {
	describeCommands := [][]string{
		[]string{"rm", "-rf", "./graph.html"},
		[]string{"go", "test", "-v", "-run", "TestDescribe"},
	}
	return mageScript(describeCommands)
}

// Publish the latest source
func Publish() error {
	mg.SerialDeps(GenerateBuildInfo)

	describeCommands := [][]string{
		[]string{"git", "push", "origin"},
	}
	return mageScript(describeCommands)
}

// Test runs the Sparta tests
func Test() {
	testCommand := func() error {
		return sh.Run("go",
			"test",
			"-cover",
			"-race",
			"./...")
	}
	mg.SerialDeps(
		EnsureAllPreconditions,
		testCommand,
	)
}

// TestCover runs the test and opens up the resulting report
func TestCover() error {
	// mg.SerialDeps(
	// 	EnsureAllPreconditions,
	// )
	coverageReport := fmt.Sprintf("%s/cover.out", localWorkDir)
	testCoverCommands := [][]string{
		[]string{"go", "test", fmt.Sprintf("-coverprofile=%s", coverageReport), "."},
		[]string{"go", "tool", "cover", fmt.Sprintf("-html=%s", coverageReport)},
		[]string{"rm", coverageReport},
		[]string{"open", fmt.Sprintf("%s/cover.html", localWorkDir)},
	}
	return mageScript(testCoverCommands)
}

// TravisBuild is the task to build in the context of a Travis CI pipeline
func TravisBuild() error {
	mg.SerialDeps(EnsureTravisBuildEnvironment,
		Build,
		Test)
	return nil
}
<|endoftext|>


<commit_before>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package language

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/admpub/i18n"
	"github.com/admpub/log"
	"github.com/webx-top/com"
)

var defaultInstance *I18n

type I18n struct {
	*i18n.TranslatorFactory
	Translators map[string]*i18n.Translator
	config      *Config
}

func NewI18n(c *Config) *I18n {
	f, errs := i18n.NewTranslatorFactory(c.RulesPath, c.MessagesPath, c.Fallback, c.FSFunc())
	if len(errs) > 0 {
		var errMsg string
		for idx, err := range errs {
			if idx > 0 {
				errMsg += "\n"
			}
			errMsg += err.Error()
		}
		if len(errMsg) > 0 {
			panic("== i18n error: " + errMsg + "\n")
		}
	}
	defaultInstance = &I18n{
		TranslatorFactory: f,
		Translators:       make(map[string]*i18n.Translator),
		config:            c,
	}
	defaultInstance.GetAndCache(c.Default)

	return defaultInstance
}

func (a *I18n) Monitor() *I18n {
	onchange := func(file string) {
		log.Info("reload language: ", file)
		defaultInstance.Reload(file)
	}
	callback := &com.MonitorEvent{
		Modify: onchange,
		Delete: onchange,
		Rename: onchange,
	}
	callback.Watch(func(f string) bool {
		log.Info("changed language: ", f)
		return strings.HasSuffix(f, `.yaml`)
	})
	for _, mp := range a.config.MessagesPath {
		if len(mp) == 0 {
			continue
		}
		if err := callback.AddDir(mp); err != nil {
			log.Error(err)
		}
	}
	return a
}

func (a *I18n) GetAndCache(langCode string) *i18n.Translator {
	var (
		t    *i18n.Translator
		errs []error
	)
	t, errs = a.TranslatorFactory.GetTranslator(langCode)
	if len(errs) > 0 {
		if a.config.Default != langCode {
			t, errs = a.TranslatorFactory.GetTranslator(a.config.Default)
		}
	}
	if len(errs) > 0 {
		var errMsg string
		for idx, err := range errs {
			if idx > 0 {
				errMsg += "\n"
			}
			errMsg += err.Error()
		}
		if len(errMsg) > 0 {
			panic("== i18n error: " + errMsg + "\n")
		}
	}
	a.Translators[langCode] = t
	return t
}

func (a *I18n) Reload(langCode string) {
	if strings.HasSuffix(langCode, `.yaml`) {
		langCode = strings.TrimSuffix(langCode, `.yaml`)
		langCode = filepath.Base(langCode)
	}
	a.TranslatorFactory.Reload(langCode)
	if _, ok := a.Translators[langCode]; ok {
		delete(a.Translators, langCode)
	}
}

func (a *I18n) Get(langCode string) *i18n.Translator {
	t, ok := a.Translators[langCode]
	if !ok {
		t = a.GetAndCache(langCode)
	}
	return t
}

func (a *I18n) Translate(langCode, key string, args map[string]string) string {
	t := a.Get(langCode)
	translation, err := t.Translate(key, args)
	if err != nil {
		return key
	}
	return translation
}

func (a *I18n) T(langCode, key string, args ...interface{}) (t string) {
	if len(args) > 0 {
		if v, ok := args[0].(map[string]string); ok {
			t = a.Translate(langCode, key, v)
			return
		}
		t = a.Translate(langCode, key, map[string]string{})
		t = fmt.Sprintf(t, args...)
		return
	}
	t = a.Translate(langCode, key, map[string]string{})
	return
}

//T 多语言翻译
func T(langCode, key string, args ...interface{}) (t string) {
	if defaultInstance == nil {
		t = key
		if len(args) > 0 {
			t = fmt.Sprintf(t, args...)
		}
		return
	}
	t = defaultInstance.T(langCode, key, args...)
	return
}
<commit_msg>update<commit_after>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package language

import (
	"fmt"
	"path/filepath"
	"strings"
	"sync"

	"github.com/admpub/i18n"
	"github.com/admpub/log"
	"github.com/webx-top/com"
)

var defaultInstance *I18n

type I18n struct {
	*i18n.TranslatorFactory
	lock        sync.RWMutex
	translators map[string]*i18n.Translator
	config      *Config
}

func NewI18n(c *Config) *I18n {
	f, errs := i18n.NewTranslatorFactory(c.RulesPath, c.MessagesPath, c.Fallback, c.FSFunc())
	if len(errs) > 0 {
		var errMsg string
		for idx, err := range errs {
			if idx > 0 {
				errMsg += "\n"
			}
			errMsg += err.Error()
		}
		if len(errMsg) > 0 {
			panic("== i18n error: " + errMsg + "\n")
		}
	}
	defaultInstance = &I18n{
		TranslatorFactory: f,
		translators:       make(map[string]*i18n.Translator),
		config:            c,
	}
	defaultInstance.GetAndCache(c.Default)

	return defaultInstance
}

func (a *I18n) Monitor() *I18n {
	onchange := func(file string) {
		log.Info("reload language: ", file)
		defaultInstance.Reload(file)
	}
	callback := &com.MonitorEvent{
		Modify: onchange,
		Delete: onchange,
		Rename: onchange,
	}
	callback.Watch(func(f string) bool {
		log.Info("changed language: ", f)
		return strings.HasSuffix(f, `.yaml`)
	})
	for _, mp := range a.config.MessagesPath {
		if len(mp) == 0 {
			continue
		}
		if err := callback.AddDir(mp); err != nil {
			log.Error(err)
		}
	}
	return a
}

func (a *I18n) GetAndCache(langCode string) *i18n.Translator {
	var (
		t    *i18n.Translator
		errs []error
	)
	t, errs = a.TranslatorFactory.GetTranslator(langCode)
	if len(errs) > 0 {
		if a.config.Default != langCode {
			t, errs = a.TranslatorFactory.GetTranslator(a.config.Default)
		}
	}
	if len(errs) > 0 {
		var errMsg string
		for idx, err := range errs {
			if idx > 0 {
				errMsg += "\n"
			}
			errMsg += err.Error()
		}
		if len(errMsg) > 0 {
			panic("== i18n error: " + errMsg + "\n")
		}
	}
	a.lock.Lock()
	a.translators[langCode] = t
	a.lock.Unlock()
	return t
}

func (a *I18n) Reload(langCode string) {
	if strings.HasSuffix(langCode, `.yaml`) {
		langCode = strings.TrimSuffix(langCode, `.yaml`)
		langCode = filepath.Base(langCode)
	}
	a.TranslatorFactory.Reload(langCode)

	a.lock.Lock()
	delete(a.translators, langCode)
	a.lock.Unlock()
}

func (a *I18n) Get(langCode string) *i18n.Translator {
	a.lock.RLock()
	t, ok := a.translators[langCode]
	a.lock.RUnlock()
	if !ok {
		t = a.GetAndCache(langCode)
	}
	return t
}

func (a *I18n) Translate(langCode, key string, args map[string]string) string {
	t := a.Get(langCode)
	translation, err := t.Translate(key, args)
	if err != nil {
		return key
	}
	return translation
}

func (a *I18n) T(langCode, key string, args ...interface{}) (t string) {
	if len(args) > 0 {
		if v, ok := args[0].(map[string]string); ok {
			t = a.Translate(langCode, key, v)
			return
		}
		t = a.Translate(langCode, key, map[string]string{})
		t = fmt.Sprintf(t, args...)
		return
	}
	t = a.Translate(langCode, key, map[string]string{})
	return
}

//T 多语言翻译
func T(langCode, key string, args ...interface{}) (t string) {
	if defaultInstance == nil {
		t = key
		if len(args) > 0 {
			t = fmt.Sprintf(t, args...)
		}
		return
	}
	t = defaultInstance.T(langCode, key, args...)
	return
}
<|endoftext|>


<commit_before>package utils

import (
	"github.com/cosmos/cosmos-sdk/client"
	clienttx "github.com/cosmos/cosmos-sdk/client/tx"
	sdk "github.com/cosmos/cosmos-sdk/types"
)

// GenTx generates a signed mock transaction.
func GenTx(txf clienttx.Factory, gen client.TxConfig, msgs []sdk.Msg, signer string) (sdk.Tx, error) {
	tx, err := txf.BuildUnsignedTx(msgs...)
	if err != nil {
		return nil, err
	}

	err = clienttx.Sign(txf, signer, tx, true)
	if err != nil {
		return nil, err
	}

	return tx.GetTx(), nil
}
<commit_msg>Fix build errors<commit_after>package utils

import (
	"github.com/cosmos/cosmos-sdk/client"
	clienttx "github.com/cosmos/cosmos-sdk/client/tx"
	sdk "github.com/cosmos/cosmos-sdk/types"
)

// GenTx generates a signed mock transaction.
func GenTx(txf clienttx.Factory, gen client.TxConfig, msgs []sdk.Msg, signer string) (sdk.Tx, error) {
	// tx, err := txf.BuildUnsignedTx(msgs...)
	tx, err := clienttx.BuildUnsignedTx(txf, msgs...)
	if err != nil {
		return nil, err
	}

	err = clienttx.Sign(txf, signer, tx, true)
	if err != nil {
		return nil, err
	}

	return tx.GetTx(), nil
}
<|endoftext|>


<commit_before>package gensnippets

import (
	"fmt"
	"sort"
	"strings"

	"cloud.google.com/go/internal/gapicgen/gensnippets/metadata"
)

type apiInfo struct {
	// protoPkg is the proto namespace for the API package.
	protoPkg string
	// libPkg is the gapic import path.
	libPkg string
	// protoServices is a map of gapic client short names to service structs.
	protoServices map[string]*service
	// version is the Go module version for the gapic client.
	version string
	// shortName for the service.
	shortName string
}

// RegionTags gets the region tags keyed by client name and method name.
func (ai *apiInfo) RegionTags() map[string]map[string]string {
	regionTags := map[string]map[string]string{}
	for svcName, svc := range ai.protoServices {
		regionTags[svcName] = map[string]string{}
		for mName, m := range svc.methods {
			regionTags[svcName][mName] = m.regionTag
		}
	}
	return regionTags
}

// RegionTags gets the region tags keyed by client name and method name.
func (ai *apiInfo) ToSnippetMetadata() *metadata.Index {
	index := &metadata.Index{
		ClientLibrary: &metadata.ClientLibrary{
			Name:     ai.libPkg,
			Version:  ai.version,
			Language: metadata.Language_GO,
			Apis: []*metadata.Api{
				{
					Id:      ai.protoPkg,
					Version: ai.protoVersion(),
				},
			},
		},
	}

	// Sorting keys to stabilize output
	var svcKeys []string
	for k := range ai.protoServices {
		svcKeys = append(svcKeys, k)
	}
	sort.StringSlice(svcKeys).Sort()
	for _, clientShortName := range svcKeys {
		service := ai.protoServices[clientShortName]
		var methodKeys []string
		for k := range service.methods {
			methodKeys = append(methodKeys, k)
		}
		sort.StringSlice(methodKeys).Sort()
		for _, methodShortName := range methodKeys {
			method := service.methods[methodShortName]
			snip := &metadata.Snippet{
				RegionTag:   method.regionTag,
				Title:       fmt.Sprintf("%s %s Sample", ai.shortName, methodShortName),
				Description: strings.TrimSpace(method.doc),
				File:        fmt.Sprintf("%s/%s/main.go", clientShortName, methodShortName),
				Language:    metadata.Language_GO,
				Canonical:   false,
				Origin:      *metadata.Snippet_API_DEFINITION.Enum(),
				ClientMethod: &metadata.ClientMethod{
					ShortName:  methodShortName,
					FullName:   fmt.Sprintf("%s.%s.%s", ai.protoPkg, clientShortName, methodShortName),
					Async:      false,
					ResultType: method.result,
					Client: &metadata.ServiceClient{
						ShortName: clientShortName,
						FullName:  fmt.Sprintf("%s.%s", ai.protoPkg, clientShortName),
					},
					Method: &metadata.Method{
						ShortName: methodShortName,
						FullName:  fmt.Sprintf("%s.%s.%s", ai.protoPkg, service.protoName, methodShortName),
						Service: &metadata.Service{
							ShortName: service.protoName,
							FullName:  fmt.Sprintf("%s.%s", ai.protoPkg, service.protoName),
						},
					},
				},
			}
			segment := &metadata.Snippet_Segment{
				Start: int32(method.regionTagStart + 1),
				End:   int32(method.regionTagEnd - 1),
				Type:  metadata.Snippet_Segment_FULL,
			}
			snip.Segments = append(snip.Segments, segment)
			for _, param := range method.params {
				methParam := &metadata.ClientMethod_Parameter{
					Type: param.pType,
					Name: param.name,
				}
				snip.ClientMethod.Parameters = append(snip.ClientMethod.Parameters, methParam)
			}
			index.Snippets = append(index.Snippets, snip)
		}
	}
	return index
}

func (ai *apiInfo) protoVersion() string {
	ss := strings.Split(ai.protoPkg, ".")
	return ss[len(ss)-1]
}

// service associates a proto service from gapic metadata with gapic client and its methods
type service struct {
	// protoName is the name of the proto service.
	protoName string
	// methods is a map of gapic method short names to method structs.
	methods map[string]*method
}

// method associates elements of gapic client methods (docs, params and return types)
// with snippet file details such as the region tag string and line numbers.
type method struct {
	// doc is the documention for the methods.
	doc string
	// regionTag is the region tag that will be used for the generated snippet.
	regionTag string
	// regionTagStart is the line number of the START region tag in the snippet file.
	regionTagStart int
	// regionTagEnd is the line number of the END region tag in the snippet file.
	regionTagEnd int
	// params are the input parameters for the gapic method.
	params []*param
	// result is the return value for the method.
	result string
}

type param struct {
	// name of the parameter.
	name string
	// pType is the Go type for the parameter.
	pType string
}
<commit_msg>test(internal/gapicgen): add missing header (#5887)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gensnippets

import (
	"fmt"
	"sort"
	"strings"

	"cloud.google.com/go/internal/gapicgen/gensnippets/metadata"
)

type apiInfo struct {
	// protoPkg is the proto namespace for the API package.
	protoPkg string
	// libPkg is the gapic import path.
	libPkg string
	// protoServices is a map of gapic client short names to service structs.
	protoServices map[string]*service
	// version is the Go module version for the gapic client.
	version string
	// shortName for the service.
	shortName string
}

// RegionTags gets the region tags keyed by client name and method name.
func (ai *apiInfo) RegionTags() map[string]map[string]string {
	regionTags := map[string]map[string]string{}
	for svcName, svc := range ai.protoServices {
		regionTags[svcName] = map[string]string{}
		for mName, m := range svc.methods {
			regionTags[svcName][mName] = m.regionTag
		}
	}
	return regionTags
}

// RegionTags gets the region tags keyed by client name and method name.
func (ai *apiInfo) ToSnippetMetadata() *metadata.Index {
	index := &metadata.Index{
		ClientLibrary: &metadata.ClientLibrary{
			Name:     ai.libPkg,
			Version:  ai.version,
			Language: metadata.Language_GO,
			Apis: []*metadata.Api{
				{
					Id:      ai.protoPkg,
					Version: ai.protoVersion(),
				},
			},
		},
	}

	// Sorting keys to stabilize output
	var svcKeys []string
	for k := range ai.protoServices {
		svcKeys = append(svcKeys, k)
	}
	sort.StringSlice(svcKeys).Sort()
	for _, clientShortName := range svcKeys {
		service := ai.protoServices[clientShortName]
		var methodKeys []string
		for k := range service.methods {
			methodKeys = append(methodKeys, k)
		}
		sort.StringSlice(methodKeys).Sort()
		for _, methodShortName := range methodKeys {
			method := service.methods[methodShortName]
			snip := &metadata.Snippet{
				RegionTag:   method.regionTag,
				Title:       fmt.Sprintf("%s %s Sample", ai.shortName, methodShortName),
				Description: strings.TrimSpace(method.doc),
				File:        fmt.Sprintf("%s/%s/main.go", clientShortName, methodShortName),
				Language:    metadata.Language_GO,
				Canonical:   false,
				Origin:      *metadata.Snippet_API_DEFINITION.Enum(),
				ClientMethod: &metadata.ClientMethod{
					ShortName:  methodShortName,
					FullName:   fmt.Sprintf("%s.%s.%s", ai.protoPkg, clientShortName, methodShortName),
					Async:      false,
					ResultType: method.result,
					Client: &metadata.ServiceClient{
						ShortName: clientShortName,
						FullName:  fmt.Sprintf("%s.%s", ai.protoPkg, clientShortName),
					},
					Method: &metadata.Method{
						ShortName: methodShortName,
						FullName:  fmt.Sprintf("%s.%s.%s", ai.protoPkg, service.protoName, methodShortName),
						Service: &metadata.Service{
							ShortName: service.protoName,
							FullName:  fmt.Sprintf("%s.%s", ai.protoPkg, service.protoName),
						},
					},
				},
			}
			segment := &metadata.Snippet_Segment{
				Start: int32(method.regionTagStart + 1),
				End:   int32(method.regionTagEnd - 1),
				Type:  metadata.Snippet_Segment_FULL,
			}
			snip.Segments = append(snip.Segments, segment)
			for _, param := range method.params {
				methParam := &metadata.ClientMethod_Parameter{
					Type: param.pType,
					Name: param.name,
				}
				snip.ClientMethod.Parameters = append(snip.ClientMethod.Parameters, methParam)
			}
			index.Snippets = append(index.Snippets, snip)
		}
	}
	return index
}

func (ai *apiInfo) protoVersion() string {
	ss := strings.Split(ai.protoPkg, ".")
	return ss[len(ss)-1]
}

// service associates a proto service from gapic metadata with gapic client and its methods
type service struct {
	// protoName is the name of the proto service.
	protoName string
	// methods is a map of gapic method short names to method structs.
	methods map[string]*method
}

// method associates elements of gapic client methods (docs, params and return types)
// with snippet file details such as the region tag string and line numbers.
type method struct {
	// doc is the documention for the methods.
	doc string
	// regionTag is the region tag that will be used for the generated snippet.
	regionTag string
	// regionTagStart is the line number of the START region tag in the snippet file.
	regionTagStart int
	// regionTagEnd is the line number of the END region tag in the snippet file.
	regionTagEnd int
	// params are the input parameters for the gapic method.
	params []*param
	// result is the return value for the method.
	result string
}

type param struct {
	// name of the parameter.
	name string
	// pType is the Go type for the parameter.
	pType string
}
<|endoftext|>


<commit_before>package handlers_test

import (
	"github.com/aclindsa/moneygo/internal/handlers"
	"net/http"
	"strconv"
	"testing"
	"time"
)

func createTransaction(client *http.Client, transaction *handlers.Transaction) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := create(client, transaction, &s, "/transaction/", "transaction")
	return &s, err
}

func getTransaction(client *http.Client, transactionid int64) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := read(client, &s, "/transaction/"+strconv.FormatInt(transactionid, 10), "transaction")
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func updateTransaction(client *http.Client, transaction *handlers.Transaction) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := update(client, transaction, &s, "/transaction/"+strconv.FormatInt(transaction.TransactionId, 10), "transaction")
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func deleteTransaction(client *http.Client, s *handlers.Transaction) error {
	err := remove(client, "/transaction/"+strconv.FormatInt(s.TransactionId, 10), "transaction")
	if err != nil {
		return err
	}
	return nil
}

func ensureTransactionsMatch(t *testing.T, expected, tran *handlers.Transaction, accounts *[]handlers.Account, matchtransactionids, matchsplitids bool) {
	t.Helper()

	if tran.TransactionId == 0 {
		t.Errorf("TransactionId is 0")
	}

	if matchtransactionids && tran.TransactionId != expected.TransactionId {
		t.Errorf("TransactionId (%d) doesn't match what's expected (%d)", tran.TransactionId, expected.TransactionId)
	}
	if tran.Description != expected.Description {
		t.Errorf("Description doesn't match")
	}
	if tran.Date != expected.Date {
		t.Errorf("Date doesn't match")
	}

	if len(tran.Splits) != len(expected.Splits) {
		t.Fatalf("Expected %d splits, received %d", len(expected.Splits), len(tran.Splits))
	}

	foundIds := make(map[int64]bool)
	for j := 0; j < len(expected.Splits); j++ {
		origsplit := expected.Splits[j]

		if tran.Splits[j].TransactionId != tran.TransactionId {
			t.Fatalf("Split TransactionId doesn't match transaction's")
		}

		found := false
		for _, s := range tran.Splits {
			if s.SplitId == 0 {
				t.Errorf("Found SplitId that's 0")
			}
			accountid := origsplit.AccountId
			if accounts != nil {
				accountid = (*accounts)[accountid].AccountId
			}
			if origsplit.Status == s.Status &&
				origsplit.ImportSplitType == s.ImportSplitType &&
				s.AccountId == accountid &&
				s.SecurityId == -1 &&
				origsplit.RemoteId == origsplit.RemoteId &&
				origsplit.Number == s.Number &&
				origsplit.Memo == s.Memo &&
				origsplit.Amount == s.Amount &&
				(!matchsplitids || origsplit.SplitId == s.SplitId) {

				if _, ok := foundIds[s.SplitId]; ok {
					continue
				}
				foundIds[s.SplitId] = true
				found = true
				break
			}
		}
		if !found {
			t.Errorf("Unable to find matching split: %+v", origsplit)
		}
	}
}

func TestCreateTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i, orig := range data[0].transactions {
			transaction := d.transactions[i]

			ensureTransactionsMatch(t, &orig, &transaction, &d.accounts, false, false)
		}

		// Don't allow imbalanced transactions
		tran := handlers.Transaction{
			UserId:      d.users[0].UserId,
			Description: "Imbalanced",
			Date:        time.Date(2017, time.September, 1, 0, 00, 00, 0, time.UTC),
			Splits: []*handlers.Split{
				&handlers.Split{
					Status:     handlers.Reconciled,
					AccountId:  d.accounts[1].AccountId,
					SecurityId: -1,
					Amount:     "-39.98",
				},
				&handlers.Split{
					Status:     handlers.Entered,
					AccountId:  d.accounts[4].AccountId,
					SecurityId: -1,
					Amount:     "39.99",
				},
			},
		}
		_, err := createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating imbalanced transaction")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid requeset
				t.Fatalf("Unexpected API error creating imbalanced transaction: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating imbalanced transaction")
		}

		// Don't allow transactions with 0 splits
		tran.Splits = []*handlers.Split{}
		_, err = createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating with zero splits")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid requeset
				t.Fatalf("Unexpected API error creating with zero splits: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating zero splits")
		}

		// Don't allow creating a transaction for another user
		tran.UserId = d.users[1].UserId
		_, err = createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating transaction for another user")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid request
				t.Fatalf("Unexpected API error creating transction for another user: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating transaction for another user")
		}
	})
}

func TestGetTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i := 0; i < len(data[0].transactions); i++ {
			orig := data[0].transactions[i]
			curr := d.transactions[i]

			tran, err := getTransaction(d.clients[orig.UserId], curr.TransactionId)
			if err != nil {
				t.Fatalf("Error fetching transaction: %s\n", err)
			}

			ensureTransactionsMatch(t, &curr, tran, nil, true, true)
		}
	})
}

func TestUpdateTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i := 0; i < len(data[0].transactions); i++ {
			orig := data[0].transactions[i]
			curr := d.transactions[i]

			curr.Description = "more money"
			curr.Date = time.Date(2017, time.October, 18, 10, 41, 40, 0, time.UTC)

			tran, err := updateTransaction(d.clients[orig.UserId], &curr)
			if err != nil {
				t.Fatalf("Error updating transaction: %s\n", err)
			}

			ensureTransactionsMatch(t, &curr, tran, nil, true, true)

			tran.Splits = []*handlers.Split{}
			for _, s := range curr.Splits {
				var split handlers.Split
				split = *s
				tran.Splits = append(tran.Splits, &split)
			}

			// Don't allow updating transactions for other/invalid users
			tran.UserId = tran.UserId + 1
			tran2, err := updateTransaction(d.clients[orig.UserId], tran)
			if tran2.UserId != curr.UserId {
				t.Fatalf("Allowed updating transaction to have wrong UserId\n")
			}
			tran.UserId = curr.UserId

			// Make sure we can't create an unbalanced transaction
			tran.Splits[len(tran.Splits)-1].Amount = "42"
			_, err = updateTransaction(d.clients[orig.UserId], tran)
			if err == nil {
				t.Fatalf("Expected error updating imbalanced transaction")
			}
			if herr, ok := err.(*handlers.Error); ok {
				if herr.ErrorId != 3 { // Invalid requeset
					t.Fatalf("Unexpected API error updating imbalanced transaction: %s", herr)
				}
			} else {
				t.Fatalf("Unexpected error updating imbalanced transaction")
			}

			// Don't allow transactions with 0 splits
			tran.Splits = []*handlers.Split{}
			_, err = updateTransaction(d.clients[orig.UserId], tran)
			if err == nil {
				t.Fatalf("Expected error updating with zero splits")
			}
			if herr, ok := err.(*handlers.Error); ok {
				if herr.ErrorId != 3 { // Invalid requeset
					t.Fatalf("Unexpected API error updating with zero splits: %s", herr)
				}
			} else {
				t.Fatalf("Unexpected error updating zero splits")
			}

		}
	})
}
<commit_msg>testing: Test deleting transactions<commit_after>package handlers_test

import (
	"github.com/aclindsa/moneygo/internal/handlers"
	"net/http"
	"strconv"
	"testing"
	"time"
)

func createTransaction(client *http.Client, transaction *handlers.Transaction) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := create(client, transaction, &s, "/transaction/", "transaction")
	return &s, err
}

func getTransaction(client *http.Client, transactionid int64) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := read(client, &s, "/transaction/"+strconv.FormatInt(transactionid, 10), "transaction")
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func updateTransaction(client *http.Client, transaction *handlers.Transaction) (*handlers.Transaction, error) {
	var s handlers.Transaction
	err := update(client, transaction, &s, "/transaction/"+strconv.FormatInt(transaction.TransactionId, 10), "transaction")
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func deleteTransaction(client *http.Client, s *handlers.Transaction) error {
	err := remove(client, "/transaction/"+strconv.FormatInt(s.TransactionId, 10), "transaction")
	if err != nil {
		return err
	}
	return nil
}

func ensureTransactionsMatch(t *testing.T, expected, tran *handlers.Transaction, accounts *[]handlers.Account, matchtransactionids, matchsplitids bool) {
	t.Helper()

	if tran.TransactionId == 0 {
		t.Errorf("TransactionId is 0")
	}

	if matchtransactionids && tran.TransactionId != expected.TransactionId {
		t.Errorf("TransactionId (%d) doesn't match what's expected (%d)", tran.TransactionId, expected.TransactionId)
	}
	if tran.Description != expected.Description {
		t.Errorf("Description doesn't match")
	}
	if tran.Date != expected.Date {
		t.Errorf("Date doesn't match")
	}

	if len(tran.Splits) != len(expected.Splits) {
		t.Fatalf("Expected %d splits, received %d", len(expected.Splits), len(tran.Splits))
	}

	foundIds := make(map[int64]bool)
	for j := 0; j < len(expected.Splits); j++ {
		origsplit := expected.Splits[j]

		if tran.Splits[j].TransactionId != tran.TransactionId {
			t.Fatalf("Split TransactionId doesn't match transaction's")
		}

		found := false
		for _, s := range tran.Splits {
			if s.SplitId == 0 {
				t.Errorf("Found SplitId that's 0")
			}
			accountid := origsplit.AccountId
			if accounts != nil {
				accountid = (*accounts)[accountid].AccountId
			}
			if origsplit.Status == s.Status &&
				origsplit.ImportSplitType == s.ImportSplitType &&
				s.AccountId == accountid &&
				s.SecurityId == -1 &&
				origsplit.RemoteId == origsplit.RemoteId &&
				origsplit.Number == s.Number &&
				origsplit.Memo == s.Memo &&
				origsplit.Amount == s.Amount &&
				(!matchsplitids || origsplit.SplitId == s.SplitId) {

				if _, ok := foundIds[s.SplitId]; ok {
					continue
				}
				foundIds[s.SplitId] = true
				found = true
				break
			}
		}
		if !found {
			t.Errorf("Unable to find matching split: %+v", origsplit)
		}
	}
}

func TestCreateTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i, orig := range data[0].transactions {
			transaction := d.transactions[i]

			ensureTransactionsMatch(t, &orig, &transaction, &d.accounts, false, false)
		}

		// Don't allow imbalanced transactions
		tran := handlers.Transaction{
			UserId:      d.users[0].UserId,
			Description: "Imbalanced",
			Date:        time.Date(2017, time.September, 1, 0, 00, 00, 0, time.UTC),
			Splits: []*handlers.Split{
				&handlers.Split{
					Status:     handlers.Reconciled,
					AccountId:  d.accounts[1].AccountId,
					SecurityId: -1,
					Amount:     "-39.98",
				},
				&handlers.Split{
					Status:     handlers.Entered,
					AccountId:  d.accounts[4].AccountId,
					SecurityId: -1,
					Amount:     "39.99",
				},
			},
		}
		_, err := createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating imbalanced transaction")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid requeset
				t.Fatalf("Unexpected API error creating imbalanced transaction: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating imbalanced transaction")
		}

		// Don't allow transactions with 0 splits
		tran.Splits = []*handlers.Split{}
		_, err = createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating with zero splits")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid requeset
				t.Fatalf("Unexpected API error creating with zero splits: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating zero splits")
		}

		// Don't allow creating a transaction for another user
		tran.UserId = d.users[1].UserId
		_, err = createTransaction(d.clients[0], &tran)
		if err == nil {
			t.Fatalf("Expected error creating transaction for another user")
		}
		if herr, ok := err.(*handlers.Error); ok {
			if herr.ErrorId != 3 { // Invalid request
				t.Fatalf("Unexpected API error creating transction for another user: %s", herr)
			}
		} else {
			t.Fatalf("Unexpected error creating transaction for another user")
		}
	})
}

func TestGetTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i := 0; i < len(data[0].transactions); i++ {
			orig := data[0].transactions[i]
			curr := d.transactions[i]

			tran, err := getTransaction(d.clients[orig.UserId], curr.TransactionId)
			if err != nil {
				t.Fatalf("Error fetching transaction: %s\n", err)
			}

			ensureTransactionsMatch(t, &curr, tran, nil, true, true)
		}
	})
}

func TestUpdateTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i := 0; i < len(data[0].transactions); i++ {
			orig := data[0].transactions[i]
			curr := d.transactions[i]

			curr.Description = "more money"
			curr.Date = time.Date(2017, time.October, 18, 10, 41, 40, 0, time.UTC)

			tran, err := updateTransaction(d.clients[orig.UserId], &curr)
			if err != nil {
				t.Fatalf("Error updating transaction: %s\n", err)
			}

			ensureTransactionsMatch(t, &curr, tran, nil, true, true)

			tran.Splits = []*handlers.Split{}
			for _, s := range curr.Splits {
				var split handlers.Split
				split = *s
				tran.Splits = append(tran.Splits, &split)
			}

			// Don't allow updating transactions for other/invalid users
			tran.UserId = tran.UserId + 1
			tran2, err := updateTransaction(d.clients[orig.UserId], tran)
			if tran2.UserId != curr.UserId {
				t.Fatalf("Allowed updating transaction to have wrong UserId\n")
			}
			tran.UserId = curr.UserId

			// Make sure we can't create an unbalanced transaction
			tran.Splits[len(tran.Splits)-1].Amount = "42"
			_, err = updateTransaction(d.clients[orig.UserId], tran)
			if err == nil {
				t.Fatalf("Expected error updating imbalanced transaction")
			}
			if herr, ok := err.(*handlers.Error); ok {
				if herr.ErrorId != 3 { // Invalid requeset
					t.Fatalf("Unexpected API error updating imbalanced transaction: %s", herr)
				}
			} else {
				t.Fatalf("Unexpected error updating imbalanced transaction")
			}

			// Don't allow transactions with 0 splits
			tran.Splits = []*handlers.Split{}
			_, err = updateTransaction(d.clients[orig.UserId], tran)
			if err == nil {
				t.Fatalf("Expected error updating with zero splits")
			}
			if herr, ok := err.(*handlers.Error); ok {
				if herr.ErrorId != 3 { // Invalid requeset
					t.Fatalf("Unexpected API error updating with zero splits: %s", herr)
				}
			} else {
				t.Fatalf("Unexpected error updating zero splits")
			}
		}
	})
}

func TestDeleteTransaction(t *testing.T) {
	RunWith(t, &data[0], func(t *testing.T, d *TestData) {
		for i := 1; i < len(data[0].transactions); i++ {
			orig := data[0].transactions[i]
			curr := d.transactions[i]

			err := deleteTransaction(d.clients[orig.UserId], &curr)
			if err != nil {
				t.Fatalf("Error deleting transaction: %s\n", err)
			}

			_, err = getTransaction(d.clients[orig.UserId], curr.TransactionId)
			if err == nil {
				t.Fatalf("Expected error fetching deleted transaction")
			}
			if herr, ok := err.(*handlers.Error); ok {
				if herr.ErrorId != 3 { // Invalid requeset
					t.Fatalf("Unexpected API error fetching deleted transaction: %s", herr)
				}
			} else {
				t.Fatalf("Unexpected error fetching deleted transaction")
			}
		}
	})
}
<|endoftext|>


<commit_before>package dyndns

// Popular dynamic DNS service URLs
const (
	DNS_O_Matic = "https://update.dnsomatic.com/nic/update"
	DynDNS      = "https://members.dyndns.org/nic/update"
	No_IP       = "https://dynupdate.no-ip.com/nic/update"
)
<commit_msg>Add missing period<commit_after>package dyndns

// Popular dynamic DNS service URLs.
const (
	DNS_O_Matic = "https://update.dnsomatic.com/nic/update"
	DynDNS      = "https://members.dyndns.org/nic/update"
	No_IP       = "https://dynupdate.no-ip.com/nic/update"
)
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package direct contains the direct runner for running single-bundle
// pipelines in the current process. Useful for testing.
package direct

import (
	"context"
	"path"

	"github.com/apache/beam/sdks/v2/go/pkg/beam"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/log"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/vet"
)

func init() {
	beam.RegisterRunner("direct", Execute)
	beam.RegisterRunner("DirectRunner", Execute)
}

// Execute runs the pipeline in-process.
func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) {
	log.Info(ctx, "Executing pipeline with the direct runner.")

	if !beam.Initialized() {
		log.Warn(ctx, "Beam has not been initialized. Call beam.Init() before pipeline construction.")
	}

	log.Info(ctx, "Pipeline:")
	log.Info(ctx, p)
	ctx = metrics.SetBundleID(ctx, "direct") // Ensure a metrics.Store exists.

	if *jobopts.Strict {
		log.Info(ctx, "Strict mode enabled, applying additional validation.")
		if _, err := vet.Execute(ctx, p); err != nil {
			return nil, errors.Wrap(err, "strictness check failed")
		}
		log.Info(ctx, "Strict mode validation passed.")
	}

	edges, _, err := p.Build()
	if err != nil {
		return nil, errors.Wrap(err, "invalid pipeline")
	}
	plan, err := Compile(edges)
	if err != nil {
		return nil, errors.Wrap(err, "translation failed")
	}
	beam.PipelineOptions.LoadOptionsFromFlags(nil)
	log.Info(ctx, plan)

	if err = plan.Execute(ctx, "", exec.DataContext{}); err != nil {
		plan.Down(ctx) // ignore any teardown errors
		return nil, err
	}
	if err = plan.Down(ctx); err != nil {
		return nil, err
	}

	return newDirectPipelineResult(ctx)
}

type directPipelineResult struct {
	jobID   string
	metrics *metrics.Results
}

func newDirectPipelineResult(ctx context.Context) (*directPipelineResult, error) {
	metrics := metrics.ResultsExtractor(ctx)
	return &directPipelineResult{metrics: &metrics}, nil
}

func (pr directPipelineResult) Metrics() metrics.Results {
	return *pr.metrics
}

func (pr directPipelineResult) JobID() string {
	return pr.jobID
}

// Compile translates a pipeline to a multi-bundle execution plan.
func Compile(edges []*graph.MultiEdge) (*exec.Plan, error) {
	// (1) Preprocess graph structure to allow insertion of Multiplex,
	// Flatten and Discard.

	prev := make(map[int]int)      // nodeID -> #incoming
	succ := make(map[int][]linkID) // nodeID -> []linkID
	edgeMap := make(map[int]*graph.MultiEdge)

	for _, edge := range edges {
		edgeMap[edge.ID()] = edge
		for i, in := range edge.Input {
			from := in.From.ID()
			succ[from] = append(succ[from], linkID{edge.ID(), i})
		}
		for _, out := range edge.Output {
			to := out.To.ID()
			prev[to]++
		}
	}

	// (2) Constructs the plan units recursively.

	b := &builder{
		prev:  prev,
		succ:  succ,
		edges: edgeMap,
		nodes: make(map[int]exec.Node),
		links: make(map[linkID]exec.Node),
		idgen: &exec.GenID{},
	}

	var roots []exec.Unit

	for _, edge := range edges {
		switch edge.Op {
		case graph.Impulse:
			out, err := b.makeNode(edge.Output[0].To.ID())
			if err != nil {
				return nil, err
			}

			u := &Impulse{UID: b.idgen.New(), Value: edge.Value, Out: out}
			roots = append(roots, u)

		default:
			// skip non-roots
		}
	}

	return exec.NewPlan("plan", append(roots, b.units...))
}

// linkID represents an incoming data link to an Edge.
type linkID struct {
	to    int // graph.MultiEdge
	input int // input index. If > 0, it's a side or CoGBK input.
}

// builder is the recursive builder for non-root execution nodes.
type builder struct {
	prev  map[int]int              // nodeID -> #incoming
	succ  map[int][]linkID         // nodeID -> []linkID
	edges map[int]*graph.MultiEdge // edgeID -> Edge

	nodes map[int]exec.Node    // nodeID -> Node (cache)
	links map[linkID]exec.Node // linkID -> Node (cache)

	units []exec.Unit // result
	idgen *exec.GenID
}

func (b *builder) makeNodes(out []*graph.Outbound) ([]exec.Node, error) {
	var ret []exec.Node
	for _, o := range out {
		n, err := b.makeNode(o.To.ID())
		if err != nil {
			return nil, err
		}
		ret = append(ret, n)
	}
	return ret, nil
}

func (b *builder) makeNode(id int) (exec.Node, error) {
	if n, ok := b.nodes[id]; ok {
		return n, nil
	}

	list := b.succ[id]

	var u exec.Node
	switch len(list) {
	case 0:
		// Discard.

		u = &exec.Discard{UID: b.idgen.New()}

	case 1:
		return b.makeLink(list[0])

	default:
		// Multiplex.

		out, err := b.makeLinks(list)
		if err != nil {
			return nil, err
		}
		u = &exec.Multiplex{UID: b.idgen.New(), Out: out}
	}

	if count := b.prev[id]; count > 1 {
		// Guard node with Flatten, if needed.

		b.units = append(b.units, u)
		u = &exec.Flatten{UID: b.idgen.New(), N: count, Out: u}
	}

	b.nodes[id] = u
	b.units = append(b.units, u)
	return u, nil
}

func (b *builder) makeLinks(ids []linkID) ([]exec.Node, error) {
	var ret []exec.Node
	for _, id := range ids {
		n, err := b.makeLink(id)
		if err != nil {
			return nil, err
		}
		ret = append(ret, n)
	}
	return ret, nil
}

func (b *builder) makeLink(id linkID) (exec.Node, error) {
	if n, ok := b.links[id]; ok {
		return n, nil
	}

	// Process all incoming links for the edge and cache them. It thus doesn't matter
	// which exact link triggers the Node generation. The link caching is only needed
	// to process ParDo side inputs and CoGBK.

	edge := b.edges[id.to]

	out, err := b.makeNodes(edge.Output)
	if err != nil {
		return nil, err
	}

	var u exec.Node
	switch edge.Op {
	case graph.ParDo:
		pardo := &exec.ParDo{
			UID:     b.idgen.New(),
			Fn:      edge.DoFn,
			Inbound: edge.Input,
			Out:     out,
			PID:     path.Base(edge.DoFn.Name()),
		}
		u = pardo
		if edge.DoFn.IsSplittable() {
			u = &exec.SdfFallback{PDo: pardo}
		}
		if len(edge.Input) == 1 {
			break
		}

		// ParDo w/ side input. We need to insert buffering and wait. We also need to
		// ensure that we return the correct link node.

		b.units = append(b.units, u)

		w := &wait{UID: b.idgen.New(), need: len(edge.Input) - 1, next: u}
		b.units = append(b.units, w)
		b.links[linkID{edge.ID(), 0}] = w

		for i := 1; i < len(edge.Input); i++ {
			n := &buffer{uid: b.idgen.New(), next: w.ID(), read: pardo.ID(), notify: w.notify}
			pardo.Side = append(pardo.Side, n)

			b.units = append(b.units, n)
			b.links[linkID{edge.ID(), i}] = n
		}

		return b.links[id], nil

	case graph.Combine:
		usesKey := typex.IsKV(edge.Input[0].Type)

		u = &exec.Combine{
			UID:     b.idgen.New(),
			Fn:      edge.CombineFn,
			UsesKey: usesKey,
			Out:     out[0],
			PID:     path.Base(edge.CombineFn.Name()),
		}

	case graph.CoGBK:
		u = &CoGBK{UID: b.idgen.New(), Edge: edge, Out: out[0]}
		b.units = append(b.units, u)

		// CoGBK needs injection of each incoming index. If > 1 incoming,
		// insert Flatten as well.

		if len(edge.Input) > 1 {
			u = &exec.Flatten{UID: b.idgen.New(), N: len(edge.Input), Out: u}
			b.units = append(b.units, u)
		}

		for i := 0; i < len(edge.Input); i++ {
			n := &Inject{UID: b.idgen.New(), N: i, Out: u}

			b.units = append(b.units, n)
			b.links[linkID{edge.ID(), i}] = n
		}

		return b.links[id], nil

	case graph.Reshuffle:
		// Reshuffle is a no-op in the direct runner, as there's only a single bundle
		// on a single worker. Hoist the next node up in the cache.
		b.links[id] = out[0]
		return b.links[id], nil

	case graph.Flatten:
		u = &exec.Flatten{UID: b.idgen.New(), N: len(edge.Input), Out: out[0]}

		for i := 0; i < len(edge.Input); i++ {
			b.links[linkID{edge.ID(), i}] = u
		}

	case graph.WindowInto:
		u = &exec.WindowInto{UID: b.idgen.New(), Fn: edge.WindowFn, Out: out[0]}

	default:
		return nil, errors.Errorf("unexpected edge: %v", edge)
	}

	b.links[id] = u
	b.units = append(b.units, u)
	return u, nil
}
<commit_msg>Add clearer error message for xlang transforms on teh Go Direct Runner (#22562)<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package direct contains the direct runner for running single-bundle
// pipelines in the current process. Useful for testing.
package direct

import (
	"context"
	"path"

	"github.com/apache/beam/sdks/v2/go/pkg/beam"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/log"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/vet"
)

func init() {
	beam.RegisterRunner("direct", Execute)
	beam.RegisterRunner("DirectRunner", Execute)
}

// Execute runs the pipeline in-process.
func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) {
	log.Info(ctx, "Executing pipeline with the direct runner.")

	if !beam.Initialized() {
		log.Warn(ctx, "Beam has not been initialized. Call beam.Init() before pipeline construction.")
	}

	log.Info(ctx, "Pipeline:")
	log.Info(ctx, p)
	ctx = metrics.SetBundleID(ctx, "direct") // Ensure a metrics.Store exists.

	if *jobopts.Strict {
		log.Info(ctx, "Strict mode enabled, applying additional validation.")
		if _, err := vet.Execute(ctx, p); err != nil {
			return nil, errors.Wrap(err, "strictness check failed")
		}
		log.Info(ctx, "Strict mode validation passed.")
	}

	edges, _, err := p.Build()
	if err != nil {
		return nil, errors.Wrap(err, "invalid pipeline")
	}
	plan, err := Compile(edges)
	if err != nil {
		return nil, errors.Wrap(err, "translation failed")
	}
	beam.PipelineOptions.LoadOptionsFromFlags(nil)
	log.Info(ctx, plan)

	if err = plan.Execute(ctx, "", exec.DataContext{}); err != nil {
		plan.Down(ctx) // ignore any teardown errors
		return nil, err
	}
	if err = plan.Down(ctx); err != nil {
		return nil, err
	}

	return newDirectPipelineResult(ctx)
}

type directPipelineResult struct {
	jobID   string
	metrics *metrics.Results
}

func newDirectPipelineResult(ctx context.Context) (*directPipelineResult, error) {
	metrics := metrics.ResultsExtractor(ctx)
	return &directPipelineResult{metrics: &metrics}, nil
}

func (pr directPipelineResult) Metrics() metrics.Results {
	return *pr.metrics
}

func (pr directPipelineResult) JobID() string {
	return pr.jobID
}

// Compile translates a pipeline to a multi-bundle execution plan.
func Compile(edges []*graph.MultiEdge) (*exec.Plan, error) {
	// (1) Preprocess graph structure to allow insertion of Multiplex,
	// Flatten and Discard.

	prev := make(map[int]int)      // nodeID -> #incoming
	succ := make(map[int][]linkID) // nodeID -> []linkID
	edgeMap := make(map[int]*graph.MultiEdge)

	for _, edge := range edges {
		edgeMap[edge.ID()] = edge
		for i, in := range edge.Input {
			from := in.From.ID()
			succ[from] = append(succ[from], linkID{edge.ID(), i})
		}
		for _, out := range edge.Output {
			to := out.To.ID()
			prev[to]++
		}
	}

	// (2) Constructs the plan units recursively.

	b := &builder{
		prev:  prev,
		succ:  succ,
		edges: edgeMap,
		nodes: make(map[int]exec.Node),
		links: make(map[linkID]exec.Node),
		idgen: &exec.GenID{},
	}

	var roots []exec.Unit

	for _, edge := range edges {
		switch edge.Op {
		case graph.Impulse:
			out, err := b.makeNode(edge.Output[0].To.ID())
			if err != nil {
				return nil, err
			}

			u := &Impulse{UID: b.idgen.New(), Value: edge.Value, Out: out}
			roots = append(roots, u)

		default:
			// skip non-roots
		}
	}

	return exec.NewPlan("plan", append(roots, b.units...))
}

// linkID represents an incoming data link to an Edge.
type linkID struct {
	to    int // graph.MultiEdge
	input int // input index. If > 0, it's a side or CoGBK input.
}

// builder is the recursive builder for non-root execution nodes.
type builder struct {
	prev  map[int]int              // nodeID -> #incoming
	succ  map[int][]linkID         // nodeID -> []linkID
	edges map[int]*graph.MultiEdge // edgeID -> Edge

	nodes map[int]exec.Node    // nodeID -> Node (cache)
	links map[linkID]exec.Node // linkID -> Node (cache)

	units []exec.Unit // result
	idgen *exec.GenID
}

func (b *builder) makeNodes(out []*graph.Outbound) ([]exec.Node, error) {
	var ret []exec.Node
	for _, o := range out {
		n, err := b.makeNode(o.To.ID())
		if err != nil {
			return nil, err
		}
		ret = append(ret, n)
	}
	return ret, nil
}

func (b *builder) makeNode(id int) (exec.Node, error) {
	if n, ok := b.nodes[id]; ok {
		return n, nil
	}

	list := b.succ[id]

	var u exec.Node
	switch len(list) {
	case 0:
		// Discard.

		u = &exec.Discard{UID: b.idgen.New()}

	case 1:
		return b.makeLink(list[0])

	default:
		// Multiplex.

		out, err := b.makeLinks(list)
		if err != nil {
			return nil, err
		}
		u = &exec.Multiplex{UID: b.idgen.New(), Out: out}
	}

	if count := b.prev[id]; count > 1 {
		// Guard node with Flatten, if needed.

		b.units = append(b.units, u)
		u = &exec.Flatten{UID: b.idgen.New(), N: count, Out: u}
	}

	b.nodes[id] = u
	b.units = append(b.units, u)
	return u, nil
}

func (b *builder) makeLinks(ids []linkID) ([]exec.Node, error) {
	var ret []exec.Node
	for _, id := range ids {
		n, err := b.makeLink(id)
		if err != nil {
			return nil, err
		}
		ret = append(ret, n)
	}
	return ret, nil
}

func (b *builder) makeLink(id linkID) (exec.Node, error) {
	if n, ok := b.links[id]; ok {
		return n, nil
	}

	// Process all incoming links for the edge and cache them. It thus doesn't matter
	// which exact link triggers the Node generation. The link caching is only needed
	// to process ParDo side inputs and CoGBK.

	edge := b.edges[id.to]

	out, err := b.makeNodes(edge.Output)
	if err != nil {
		return nil, err
	}

	var u exec.Node
	switch edge.Op {
	case graph.ParDo:
		pardo := &exec.ParDo{
			UID:     b.idgen.New(),
			Fn:      edge.DoFn,
			Inbound: edge.Input,
			Out:     out,
			PID:     path.Base(edge.DoFn.Name()),
		}
		u = pardo
		if edge.DoFn.IsSplittable() {
			u = &exec.SdfFallback{PDo: pardo}
		}
		if len(edge.Input) == 1 {
			break
		}

		// ParDo w/ side input. We need to insert buffering and wait. We also need to
		// ensure that we return the correct link node.

		b.units = append(b.units, u)

		w := &wait{UID: b.idgen.New(), need: len(edge.Input) - 1, next: u}
		b.units = append(b.units, w)
		b.links[linkID{edge.ID(), 0}] = w

		for i := 1; i < len(edge.Input); i++ {
			n := &buffer{uid: b.idgen.New(), next: w.ID(), read: pardo.ID(), notify: w.notify}
			pardo.Side = append(pardo.Side, n)

			b.units = append(b.units, n)
			b.links[linkID{edge.ID(), i}] = n
		}

		return b.links[id], nil

	case graph.Combine:
		usesKey := typex.IsKV(edge.Input[0].Type)

		u = &exec.Combine{
			UID:     b.idgen.New(),
			Fn:      edge.CombineFn,
			UsesKey: usesKey,
			Out:     out[0],
			PID:     path.Base(edge.CombineFn.Name()),
		}

	case graph.CoGBK:
		u = &CoGBK{UID: b.idgen.New(), Edge: edge, Out: out[0]}
		b.units = append(b.units, u)

		// CoGBK needs injection of each incoming index. If > 1 incoming,
		// insert Flatten as well.

		if len(edge.Input) > 1 {
			u = &exec.Flatten{UID: b.idgen.New(), N: len(edge.Input), Out: u}
			b.units = append(b.units, u)
		}

		for i := 0; i < len(edge.Input); i++ {
			n := &Inject{UID: b.idgen.New(), N: i, Out: u}

			b.units = append(b.units, n)
			b.links[linkID{edge.ID(), i}] = n
		}

		return b.links[id], nil

	case graph.Reshuffle:
		// Reshuffle is a no-op in the direct runner, as there's only a single bundle
		// on a single worker. Hoist the next node up in the cache.
		b.links[id] = out[0]
		return b.links[id], nil

	case graph.Flatten:
		u = &exec.Flatten{UID: b.idgen.New(), N: len(edge.Input), Out: out[0]}

		for i := 0; i < len(edge.Input); i++ {
			b.links[linkID{edge.ID(), i}] = u
		}

	case graph.WindowInto:
		u = &exec.WindowInto{UID: b.idgen.New(), Fn: edge.WindowFn, Out: out[0]}

	case graph.External:
		return nil, errors.Errorf("external transforms like %v are not supported in the Go direct runner, please execute your pipeline on a different runner", edge)

	default:
		return nil, errors.Errorf("unexpected edge: %v", edge)
	}

	b.links[id] = u
	b.units = append(b.units, u)
	return u, nil
}
<|endoftext|>


<commit_before>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// AutoDateHistogramAggregation is a multi-bucket aggregation similar to the
// histogram except it can only be applied on date values, and the buckets num can bin pointed.
// See: https://www.elastic.co/guide/en/elasticsearch/reference/7.3/search-aggregations-bucket-autodatehistogram-aggregation.html
type AutoDateHistogramAggregation struct {
	field           string
	script          *Script
	missing         interface{}
	subAggregations map[string]Aggregation
	meta            map[string]interface{}

	buckets         int
	minDocCount     *int64
	timeZone        string
	format          string
	minimumInterval string
}

// NewAutoDateHistogramAggregation creates a new AutoDateHistogramAggregation.
func NewAutoDateHistogramAggregation() *AutoDateHistogramAggregation {
	return &AutoDateHistogramAggregation{
		subAggregations: make(map[string]Aggregation),
	}
}

// Field on which the aggregation is processed.
func (a *AutoDateHistogramAggregation) Field(field string) *AutoDateHistogramAggregation {
	a.field = field
	return a
}

// Script on which th
func (a *AutoDateHistogramAggregation) Script(script *Script) *AutoDateHistogramAggregation {
	a.script = script
	return a
}

// Missing configures the value to use when documents miss a value.
func (a *AutoDateHistogramAggregation) Missing(missing interface{}) *AutoDateHistogramAggregation {
	a.missing = missing
	return a
}

// SubAggregation sub aggregation
func (a *AutoDateHistogramAggregation) SubAggregation(name string, subAggregation Aggregation) *AutoDateHistogramAggregation {
	a.subAggregations[name] = subAggregation
	return a
}

// Meta sets the meta data to be included in the aggregation response.
func (a *AutoDateHistogramAggregation) Meta(metaData map[string]interface{}) *AutoDateHistogramAggregation {
	a.meta = metaData
	return a
}

// Buckets buckets num by which the aggregation gets processed.
func (a *AutoDateHistogramAggregation) Buckets(buckets int) *AutoDateHistogramAggregation {
	a.buckets = buckets
	return a
}

// MinDocCount sets the minimum document count per bucket.
// Buckets with less documents than this min value will not be returned.
func (a *AutoDateHistogramAggregation) MinDocCount(minDocCount int64) *AutoDateHistogramAggregation {
	a.minDocCount = &minDocCount
	return a
}

// TimeZone sets the timezone in which to translate dates before computing buckets.
func (a *AutoDateHistogramAggregation) TimeZone(timeZone string) *AutoDateHistogramAggregation {
	a.timeZone = timeZone
	return a
}

// Format sets the format to use for dates.
func (a *AutoDateHistogramAggregation) Format(format string) *AutoDateHistogramAggregation {
	a.format = format
	return a
}

// MinimumInterval accepted units for minimum_interval are: year/month/day/hour/minute/second
func (a *AutoDateHistogramAggregation) MinimumInterval(interval string) *AutoDateHistogramAggregation {
	a.minimumInterval = interval
	return a
}

// Source source for AutoDateHistogramAggregation
func (a *AutoDateHistogramAggregation) Source() (interface{}, error) {
	// Example:
	// {
	//     "aggs" : {
	//         "articles_over_time" : {
	//             "auto_date_histogram" : {
	//                 "field" : "date",
	//                 "buckets" : 10
	//             }
	//         }
	//     }
	// }
	//
	// This method returns only the { "auto_date_histogram" : { ... } } part.

	source := make(map[string]interface{})
	opts := make(map[string]interface{})
	source["auto_date_histogram"] = opts

	// ValuesSourceAggregationBuilder
	if a.field != "" {
		opts["field"] = a.field
	}
	if a.script != nil {
		src, err := a.script.Source()
		if err != nil {
			return nil, err
		}
		opts["script"] = src
	}
	if a.missing != nil {
		opts["missing"] = a.missing
	}

	if a.buckets > 0 {
		opts["buckets"] = a.buckets
	} else {
		opts["buckets"] = 10
	}

	if a.minDocCount != nil {
		opts["min_doc_count"] = *a.minDocCount
	}
	if a.timeZone != "" {
		opts["time_zone"] = a.timeZone
	}
	if a.format != "" {
		opts["format"] = a.format
	}
	if a.minimumInterval != "" {
		opts["minimum_interval"] = a.minimumInterval
	}

	// AggregationBuilder (SubAggregations)
	if len(a.subAggregations) > 0 {
		aggsMap := make(map[string]interface{})
		source["aggregations"] = aggsMap
		for name, aggregate := range a.subAggregations {
			src, err := aggregate.Source()
			if err != nil {
				return nil, err
			}
			aggsMap[name] = src
		}
	}

	// Add Meta data if available
	if len(a.meta) > 0 {
		source["meta"] = a.meta
	}

	return source, nil
}
<commit_msg>No default bucket count for auto-interval date agg<commit_after>// Copyright 2012-present Oliver Eilhard. All rights reserved.
// Use of this source code is governed by a MIT-license.
// See http://olivere.mit-license.org/license.txt for details.

package elastic

// AutoDateHistogramAggregation is a multi-bucket aggregation similar to the
// histogram except it can only be applied on date values, and the buckets num can bin pointed.
// See: https://www.elastic.co/guide/en/elasticsearch/reference/7.3/search-aggregations-bucket-autodatehistogram-aggregation.html
type AutoDateHistogramAggregation struct {
	field           string
	script          *Script
	missing         interface{}
	subAggregations map[string]Aggregation
	meta            map[string]interface{}

	buckets         int
	minDocCount     *int64
	timeZone        string
	format          string
	minimumInterval string
}

// NewAutoDateHistogramAggregation creates a new AutoDateHistogramAggregation.
func NewAutoDateHistogramAggregation() *AutoDateHistogramAggregation {
	return &AutoDateHistogramAggregation{
		subAggregations: make(map[string]Aggregation),
	}
}

// Field on which the aggregation is processed.
func (a *AutoDateHistogramAggregation) Field(field string) *AutoDateHistogramAggregation {
	a.field = field
	return a
}

// Script on which th
func (a *AutoDateHistogramAggregation) Script(script *Script) *AutoDateHistogramAggregation {
	a.script = script
	return a
}

// Missing configures the value to use when documents miss a value.
func (a *AutoDateHistogramAggregation) Missing(missing interface{}) *AutoDateHistogramAggregation {
	a.missing = missing
	return a
}

// SubAggregation sub aggregation
func (a *AutoDateHistogramAggregation) SubAggregation(name string, subAggregation Aggregation) *AutoDateHistogramAggregation {
	a.subAggregations[name] = subAggregation
	return a
}

// Meta sets the meta data to be included in the aggregation response.
func (a *AutoDateHistogramAggregation) Meta(metaData map[string]interface{}) *AutoDateHistogramAggregation {
	a.meta = metaData
	return a
}

// Buckets buckets num by which the aggregation gets processed.
func (a *AutoDateHistogramAggregation) Buckets(buckets int) *AutoDateHistogramAggregation {
	a.buckets = buckets
	return a
}

// MinDocCount sets the minimum document count per bucket.
// Buckets with less documents than this min value will not be returned.
func (a *AutoDateHistogramAggregation) MinDocCount(minDocCount int64) *AutoDateHistogramAggregation {
	a.minDocCount = &minDocCount
	return a
}

// TimeZone sets the timezone in which to translate dates before computing buckets.
func (a *AutoDateHistogramAggregation) TimeZone(timeZone string) *AutoDateHistogramAggregation {
	a.timeZone = timeZone
	return a
}

// Format sets the format to use for dates.
func (a *AutoDateHistogramAggregation) Format(format string) *AutoDateHistogramAggregation {
	a.format = format
	return a
}

// MinimumInterval accepted units for minimum_interval are: year/month/day/hour/minute/second
func (a *AutoDateHistogramAggregation) MinimumInterval(interval string) *AutoDateHistogramAggregation {
	a.minimumInterval = interval
	return a
}

// Source source for AutoDateHistogramAggregation
func (a *AutoDateHistogramAggregation) Source() (interface{}, error) {
	// Example:
	// {
	//     "aggs" : {
	//         "articles_over_time" : {
	//             "auto_date_histogram" : {
	//                 "field" : "date",
	//                 "buckets" : 10
	//             }
	//         }
	//     }
	// }
	//
	// This method returns only the { "auto_date_histogram" : { ... } } part.

	source := make(map[string]interface{})
	opts := make(map[string]interface{})
	source["auto_date_histogram"] = opts

	// ValuesSourceAggregationBuilder
	if a.field != "" {
		opts["field"] = a.field
	}
	if a.script != nil {
		src, err := a.script.Source()
		if err != nil {
			return nil, err
		}
		opts["script"] = src
	}
	if a.missing != nil {
		opts["missing"] = a.missing
	}

	if a.buckets > 0 {
		opts["buckets"] = a.buckets
	}

	if a.minDocCount != nil {
		opts["min_doc_count"] = *a.minDocCount
	}
	if a.timeZone != "" {
		opts["time_zone"] = a.timeZone
	}
	if a.format != "" {
		opts["format"] = a.format
	}
	if a.minimumInterval != "" {
		opts["minimum_interval"] = a.minimumInterval
	}

	// AggregationBuilder (SubAggregations)
	if len(a.subAggregations) > 0 {
		aggsMap := make(map[string]interface{})
		source["aggregations"] = aggsMap
		for name, aggregate := range a.subAggregations {
			src, err := aggregate.Source()
			if err != nil {
				return nil, err
			}
			aggsMap[name] = src
		}
	}

	// Add Meta data if available
	if len(a.meta) > 0 {
		source["meta"] = a.meta
	}

	return source, nil
}
<|endoftext|>


<commit_before>package nsf

import (
	"os"
	"testing"

	"github.com/mjibson/mog/output"
)

func TestNsf(t *testing.T) {
	f, err := os.Open("mm3.nsf")
	if err != nil {
		t.Fatal(err)
	}
	n, err := ReadNSF(f)
	if err != nil {
		t.Fatal(err)
	}
	if n.LoadAddr != 0x8000 || n.InitAddr != 0x8003 || n.PlayAddr != 0x8000 {
		t.Fatal("bad addresses")
	}
	n.Init(1)
	o, err := output.NewPulse(int(n.SampleRate), 1)
	if err != nil {
		t.Fatal(err)
	}
	for {
		o.Push(n.Play(1024))
	}
}
<commit_msg>Wait correctly<commit_after>package nsf

import (
	"os"
	"testing"
	"time"

	"github.com/mjibson/mog/output"
)

func TestNsf(t *testing.T) {
	f, err := os.Open("mm3.nsf")
	if err != nil {
		t.Fatal(err)
	}
	n, err := ReadNSF(f)
	if err != nil {
		t.Fatal(err)
	}
	if n.LoadAddr != 0x8000 || n.InitAddr != 0x8003 || n.PlayAddr != 0x8000 {
		t.Fatal("bad addresses")
	}
	n.Init(1)
	o, err := output.NewPulse(int(n.SampleRate), 1)
	if err != nil {
		t.Fatal(err)
	}
	const div = 10
	for _ = range time.Tick(time.Second / div) {
		o.Push(n.Play(int(n.SampleRate / div)))
	}
}
<|endoftext|>


<commit_before>package dory

import (
	"container/list"
	"log"
	"sync"
	"time"

	"github.com/dgryski/go-farm"
	prom "github.com/prometheus/client_golang/prometheus"
)

const (
	megabyte = 1024 * 1024

	maxUintptr = ^uintptr(0)
	maxMemory  = (maxUintptr >> 1)
)

var (
	cacheSize = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_size",
		Help: "Size of cache.",
	})
	cacheSizeMax = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_size_max",
		Help: "Maximum cache size.",
	})
	cacheKeys = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_keys",
		Help: "Number of keys in cache.",
	})
)

func init() {
	prom.MustRegister(cacheSize)
	prom.MustRegister(cacheSizeMax)
	prom.MustRegister(cacheKeys)
}

type Memcache struct {
	minFreeMem int64
	tableSize  int
	maxKeySize int
	maxValSize int

	// TODO: Document how this works.
	keys      map[uint64]*DiscardableTable
	tables    list.List
	maxTables int
	count     int
	lock      sync.Mutex
}

func NewMemcache(minFreeMem int64, tableSize, maxKeySize, maxValSize int) *Memcache {
	c := &Memcache{
		minFreeMem: minFreeMem,
		tableSize:  tableSize,
		maxKeySize: maxKeySize,
		maxValSize: maxValSize,
		keys:       make(map[uint64]*DiscardableTable),
	}
	go c.memWatcher()
	go c.sweepKeys()
	return c
}

func (c *Memcache) MinKeySize() int {
	return 1
}

func (c *Memcache) MinValSize() int {
	return 1
}

func (c *Memcache) MaxKeySize() int {
	return c.maxKeySize
}

func (c *Memcache) MaxValSize() int {
	return c.maxValSize
}

func (c *Memcache) memWatcher() {
	ticker := time.NewTicker(time.Second)
	for range ticker.C {
		memAvailable := getMemAvailable()

		c.lock.Lock()
		availableTableMem := int64(c.tables.Len()*c.tableSize) + memAvailable - c.minFreeMem
		if availableTableMem > int64(maxMemory) {
			availableTableMem = int64(maxMemory)
		}
		c.maxTables = int(availableTableMem / int64(c.tableSize))
		if c.maxTables < 0 {
			c.maxTables = 0
		}
		c.downsizeTables()
		numTables := c.tables.Len()
		maxTables := c.maxTables
		numKeys := len(c.keys)
		c.lock.Unlock()

		if debugLog {
			log.Printf("Mem avail: %d MB, table mem available: %d MB, tables: %d, max tables: %d",
				memAvailable/megabyte, availableTableMem/megabyte, numTables, maxTables)
		}

		cacheSize.Set(float64(numTables * c.tableSize))
		cacheSizeMax.Set(float64(maxTables * c.tableSize))
		cacheKeys.Set(float64(numKeys))
	}
}

func (c *Memcache) sweepKeys() {
	ticker := time.NewTicker(time.Minute)
	for range ticker.C {
		c.lock.Lock()
		start := time.Now()
		numKeys := len(c.keys)
		nils := 0
		for k, v := range c.keys {
			if v == nil || !v.IsAlive() {
				nils++
				c.erase(k)
			}
		}
		deleted := numKeys - len(c.keys)
		c.lock.Unlock()

		if debugLog {
			log.Printf("Swept %d keys in %0.3f sec, deleted %d, nil %d",
				numKeys, time.Since(start).Seconds(), deleted, nils)
		}
	}
}

func (c *Memcache) downsizeTables() {
	start := time.Now()
	deleted := 0
	for e := c.tables.Front(); e != nil; {
		next := e.Next()
		t := e.Value.(*DiscardableTable)
		if t.NumEntries() == 0 {
			t.Discard()
			c.tables.Remove(e)
			deleted++
		}
		e = next
	}
	if debugLog && deleted > 0 {
		log.Printf("Deleted %d empty tables in %0.3f sec", deleted, time.Since(start).Seconds())
	}

	start = time.Now()
	deleted = 0
	for c.tables.Len() > c.maxTables {
		last := c.tables.Back()
		t := last.Value.(*DiscardableTable)
		t.Discard()
		c.tables.Remove(last)
		deleted++
	}
	if debugLog && deleted > 0 {
		log.Printf("Deleted %d excess tables in %0.3f sec", deleted, time.Since(start).Seconds())
	}

	// TODO: Compact and merge underutilised tables.
}

func (c *Memcache) allocTable() *DiscardableTable {
	t := NewDiscardableTable(c.tableSize, c.count)
	c.count++
	return t
}

func (c *Memcache) erase(hash uint64) {
	_, ok := c.keys[hash+1]
	if ok {
		// TODO: Maybe simplify by using a dummy deleted element instead of nil.
		c.keys[hash] = nil
	} else {
		// No next hash, so no next element for linear probing.
		delete(c.keys, hash)
	}
}

func (c *Memcache) Has(key []byte) bool {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil {
			continue
		}

		if t.Has(key) {
			return true
		}
	}
	return false
}

func (c *Memcache) Get(key, buf []byte) []byte {
	hash := farm.Hash64(key)
	keyHash := hash

	c.lock.Lock()
	defer c.lock.Unlock()
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil {
			continue
		}

		outBuf := t.Get(key, buf)
		if outBuf != nil {
			if (c.count - t.Meta().(int)) > c.maxTables/2 {
				// Promote old keys to give LRU-like behaviour.
				c.putWithHash(key, outBuf, keyHash)
			}
			return outBuf
		}
	}
	return nil
}

func (c *Memcache) findPutTable(entrySize int) *DiscardableTable {
	var t *DiscardableTable
	i := 0
	// Search a few of the most recent tables for the smallest spot the entry will fit into.
	for e := c.tables.Front(); e != nil && i < 4; e = e.Next() {
		et := e.Value.(*DiscardableTable)
		if et.FreeSpace() >= entrySize {
			if t == nil || et.FreeSpace() < t.FreeSpace() {
				t = et
			}
		}
		i++
	}
	return t
}

func (c *Memcache) putWithHash(key, val []byte, hash uint64) {
	c.deleteWithHash(key, hash)

	if c.maxTables == 0 {
		return
	}

	if len(key) > c.maxKeySize || len(val) > c.maxValSize {
		return
	}
	entrySize := (*PackedTable)(nil).EntrySize(key, val)

	t := c.findPutTable(entrySize)
	if t == nil {
		if c.tables.Len() >= c.maxTables {
			last := c.tables.Back()
			bt := last.Value.(*DiscardableTable)
			bt.Discard()
			c.tables.Remove(last)
		}
		t = c.allocTable()
		c.tables.PushFront(t)
	}
	err := t.Put(key, val)
	if err != nil {
		panic(err)
	}
	for ; c.keys[hash] != nil; hash++ {
	}
	c.keys[hash] = t
}

func (c *Memcache) Put(key, val []byte) {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	c.putWithHash(key, val, hash)
}

func (c *Memcache) deleteWithHash(key []byte, hash uint64) {
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil || !t.IsAlive() {
			// While we're here, might as well clean out the garbage.
			c.erase(hash)
			continue
		}

		if t.Delete(key) {
			c.erase(hash)
			// Since the tables are exclusive, we can stop here.
			break
		}
	}
}

func (c *Memcache) Delete(key []byte) {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	c.deleteWithHash(key, hash)
}
<commit_msg>Minimise the sweeper blocking serving.<commit_after>package dory

import (
	"container/list"
	"log"
	"sync"
	"time"

	"github.com/dgryski/go-farm"
	prom "github.com/prometheus/client_golang/prometheus"
)

const (
	megabyte = 1024 * 1024

	maxUintptr = ^uintptr(0)
	maxMemory  = (maxUintptr >> 1)

	changedKeysSweepThreshold = 10000
)

var (
	cacheSize = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_size",
		Help: "Size of cache.",
	})
	cacheSizeMax = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_size_max",
		Help: "Maximum cache size.",
	})
	cacheKeys = prom.NewGauge(prom.GaugeOpts{
		Name: "dory_cache_keys",
		Help: "Number of keys in cache.",
	})
)

func init() {
	prom.MustRegister(cacheSize)
	prom.MustRegister(cacheSizeMax)
	prom.MustRegister(cacheKeys)
}

type KeyTable map[uint64]*DiscardableTable

// Sentinel value to indicate table entry has been deleted.
var deletedEntry = new(DiscardableTable)

type Memcache struct {
	minFreeMem int64
	tableSize  int
	maxKeySize int
	maxValSize int

	doSweepKeys chan KeyTable

	// TODO: Document how this works.
	keys        KeyTable
	changedKeys KeyTable
	tables      list.List
	maxTables   int
	count       int
	lock        sync.Mutex
}

func NewMemcache(minFreeMem int64, tableSize, maxKeySize, maxValSize int) *Memcache {
	c := &Memcache{
		minFreeMem:  minFreeMem,
		tableSize:   tableSize,
		maxKeySize:  maxKeySize,
		maxValSize:  maxValSize,
		doSweepKeys: make(chan KeyTable, 1),
		keys:        make(KeyTable),
		changedKeys: make(KeyTable),
	}
	go c.memWatcher()
	go c.sweepKeys()
	return c
}

func (c *Memcache) MinKeySize() int {
	return 1
}

func (c *Memcache) MinValSize() int {
	return 1
}

func (c *Memcache) MaxKeySize() int {
	return c.maxKeySize
}

func (c *Memcache) MaxValSize() int {
	return c.maxValSize
}

func (c *Memcache) memWatcher() {
	ticker := time.NewTicker(time.Second)
	for range ticker.C {
		memAvailable := getMemAvailable()

		c.lock.Lock()
		availableTableMem := int64(c.tables.Len()*c.tableSize) + memAvailable - c.minFreeMem
		if availableTableMem > int64(maxMemory) {
			availableTableMem = int64(maxMemory)
		}
		c.maxTables = int(availableTableMem / int64(c.tableSize))
		if c.maxTables < 0 {
			c.maxTables = 0
		}
		c.downsizeTables()
		numTables := c.tables.Len()
		maxTables := c.maxTables
		numKeys := len(c.keys)
		c.lock.Unlock()

		if debugLog {
			log.Printf("Mem avail: %d MB, table mem available: %d MB, tables: %d, max tables: %d",
				memAvailable/megabyte, availableTableMem/megabyte, numTables, maxTables)
		}

		cacheSize.Set(float64(numTables * c.tableSize))
		cacheSizeMax.Set(float64(maxTables * c.tableSize))
		cacheKeys.Set(float64(numKeys))
	}
}

func (c *Memcache) sweepKeys() {
	// Operate on a copy of the key map to minimise blocking.
	keysCopy := make(KeyTable)
	// Cap the amount of work done while holding c.lock per iteration.
	nils := make([]uint64, 0, 10000)

	for changed := range c.doSweepKeys {
		nils = nils[:0]

		sweepStart := time.Now()

		// Merge changes into our copy.
		for k, t := range changed {
			if t == deletedEntry {
				delete(keysCopy, k)
			} else {
				keysCopy[k] = t
			}
		}
		// Look for candidate keys to verify whether they're still valid.
		for k, t := range keysCopy {
			if t == nil || !t.IsAlive() {
				nils = append(nils, k)
			}

			if len(nils) == cap(nils) {
				break
			}
		}

		if len(nils) > 0 {
			c.lock.Lock()
			start := time.Now()
			numKeys := len(c.keys)
			for _, k := range nils {
				t, ok := c.keys[k]
				if !ok {
					delete(keysCopy, k)
				} else if t == nil || !t.IsAlive() {
					c.erase(k)
				}
			}
			deleted := numKeys - len(c.keys)

			if len(nils) == cap(nils) {
				// More work to be done.
				c.doSweep()
			}

			c.lock.Unlock()

			if debugLog {
				log.Printf("Swept %d keys in %0.3f sec, deleted %d, nils %d, total sweep time %0.3f sec",
					numKeys, time.Since(start).Seconds(), deleted, len(nils), time.Since(sweepStart).Seconds())
			}
		} else if debugLog {
			log.Printf("No nil entries to sweep, key copies %d, total sweep time %0.3f sec",
				len(keysCopy), time.Since(sweepStart).Seconds())
		}

	}
}

func (c *Memcache) doSweep() {
	select {
	case c.doSweepKeys <- c.changedKeys:
		c.changedKeys = make(KeyTable)
	default:
	}
}

func (c *Memcache) downsizeTables() {
	start := time.Now()
	deleted := 0
	for e := c.tables.Front(); e != nil; {
		next := e.Next()
		t := e.Value.(*DiscardableTable)
		if t.NumEntries() == 0 {
			t.Discard()
			c.tables.Remove(e)
			deleted++
		}
		e = next
	}
	if debugLog && deleted > 0 {
		log.Printf("Deleted %d empty tables in %0.3f sec", deleted, time.Since(start).Seconds())
	}

	start = time.Now()
	deleted = 0
	for c.tables.Len() > c.maxTables {
		last := c.tables.Back()
		t := last.Value.(*DiscardableTable)
		t.Discard()
		c.tables.Remove(last)
		deleted++
	}
	if deleted > 0 {
		// Discarding non-empty tables creates orphaned key table entries which need
		// to be swept away.
		c.doSweep()
	}
	if debugLog && deleted > 0 {
		log.Printf("Deleted %d excess tables in %0.3f sec", deleted, time.Since(start).Seconds())
	}

	// TODO: Compact and merge underutilised tables.
}

func (c *Memcache) allocTable() *DiscardableTable {
	t := NewDiscardableTable(c.tableSize, c.count)
	c.count++
	return t
}

func (c *Memcache) keyChanged(hash uint64, t *DiscardableTable) {
	c.changedKeys[hash] = t
	if len(c.changedKeys) > changedKeysSweepThreshold {
		c.doSweep()
	}
}

func (c *Memcache) erase(hash uint64) {
	_, ok := c.keys[hash+1]
	if ok {
		// TODO: Maybe simplify by using a dummy deleted element instead of nil.
		c.keys[hash] = nil
		c.keyChanged(hash, nil)
	} else {
		// No next hash, so no next element for linear probing.
		delete(c.keys, hash)
		c.keyChanged(hash, deletedEntry)
	}
}

func (c *Memcache) Has(key []byte) bool {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil {
			continue
		}

		if t.Has(key) {
			return true
		}
	}
	return false
}

func (c *Memcache) Get(key, buf []byte) []byte {
	hash := farm.Hash64(key)
	keyHash := hash

	c.lock.Lock()
	defer c.lock.Unlock()
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil {
			continue
		}

		outBuf := t.Get(key, buf)
		if outBuf != nil {
			if (c.count - t.Meta().(int)) > c.maxTables/2 {
				// Promote old keys to give LRU-like behaviour.
				c.putWithHash(key, outBuf, keyHash)
			}
			return outBuf
		}
	}
	return nil
}

func (c *Memcache) findPutTable(entrySize int) *DiscardableTable {
	var t *DiscardableTable
	i := 0
	// Search a few of the most recent tables for the smallest spot the entry will fit into.
	for e := c.tables.Front(); e != nil && i < 4; e = e.Next() {
		et := e.Value.(*DiscardableTable)
		if et.FreeSpace() >= entrySize {
			if t == nil || et.FreeSpace() < t.FreeSpace() {
				t = et
			}
		}
		i++
	}
	return t
}

func (c *Memcache) putWithHash(key, val []byte, hash uint64) {
	c.deleteWithHash(key, hash)

	if c.maxTables == 0 {
		return
	}

	if len(key) > c.maxKeySize || len(val) > c.maxValSize {
		return
	}
	entrySize := (*PackedTable)(nil).EntrySize(key, val)

	t := c.findPutTable(entrySize)
	if t == nil {
		if c.tables.Len() >= c.maxTables {
			last := c.tables.Back()
			bt := last.Value.(*DiscardableTable)
			bt.Discard()
			c.tables.Remove(last)
			c.doSweep()
		}
		t = c.allocTable()
		c.tables.PushFront(t)
	}
	err := t.Put(key, val)
	if err != nil {
		panic(err)
	}
	for ; c.keys[hash] != nil; hash++ {
	}
	c.keys[hash] = t
	c.keyChanged(hash, t)
}

func (c *Memcache) Put(key, val []byte) {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	c.putWithHash(key, val, hash)
}

func (c *Memcache) deleteWithHash(key []byte, hash uint64) {
	for ; ; hash++ {
		t, ok := c.keys[hash]
		if !ok {
			break
		} else if t == nil || !t.IsAlive() {
			// While we're here, might as well clean out the garbage.
			c.erase(hash)
			continue
		}

		if t.Delete(key) {
			c.erase(hash)
			// Since the tables are exclusive, we can stop here.
			break
		}
	}
}

func (c *Memcache) Delete(key []byte) {
	hash := farm.Hash64(key)

	c.lock.Lock()
	defer c.lock.Unlock()
	c.deleteWithHash(key, hash)
}
<|endoftext|>