<commit_before>// The Harness for a Revel program.
//
// It has a couple responsibilities:
// 1. Parse the user program, generating a main.go file that registers
//    controller classes and starts the user's server.
// 2. Build and run the user program.  Show compile errors.
// 3. Monitor the user source and re-build / restart the program when necessary.
//
// Source files are generated in the app/tmp directory.

package harness

import (
	"bytes"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"github.com/robfig/revel"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

const REGISTER_CONTROLLERS = `
// target: {{.AppName}}
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"
	{{range .ImportPaths}}
  "{{.}}"
  {{end}}
)

var (
	port *int = flag.Int("port", 0, "Port")
	importPath *string = flag.String("importPath", "", "Path to the app.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	rev.LOG.Println("Running revel server")
	flag.Parse()
	rev.Init(*importPath, "{{.RunMode}}")
	{{range $i, $c := .Controllers}}
	rev.RegisterController((*{{.PackageName}}.{{.StructName}})(nil),
		[]*rev.MethodType{
			{{range .MethodSpecs}}&rev.MethodType{
				Name: "{{.Name}}",
				Args: []*rev.MethodArg{ {{range .Args}}
					&rev.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{.TypeName}})(nil)) },{{end}}
			  },
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	rev.Run(*port)
}
`

// Reverse proxy requests to the application server.
// On each request, proxy sends (NotifyRequest = true)
// If code change has been detected in app:
// - app is rebuilt and restarted, send proxy (NotifyReady = true)
// - else, send proxy (NotifyReady = true)

type harnessProxy struct {
	proxy         *httputil.ReverseProxy
	NotifyRequest chan bool  // Strobed on every request.
	NotifyReady   chan error // Strobed when request may proceed.
}

func (hp *harnessProxy) ServeHTTP(wr http.ResponseWriter, req *http.Request) {
	// First, poll to see if there's a pending error in NotifyReady
	select {
	case err := <-hp.NotifyReady:
		if err != nil {
			serveError(wr, req, err)
		}
	default:
		// Usually do nothing.
	}

	// Notify that a request is coming through, and wait for the go-ahead.
	hp.NotifyRequest <- true
	err := <-hp.NotifyReady

	// If an error was returned, create the page and show it to the user.
	if err != nil {
		serveError(wr, req, err)
		return
	}

	// Reverse proxy the request.
	hp.proxy.ServeHTTP(wr, req)
}

func serveError(wr http.ResponseWriter, req *http.Request, err error) {
	switch e := err.(type) {
	case *rev.Error:
		rev.RenderError(wr, e)
	default:
		rev.RenderError(wr, map[string]string{
			"Title":       "Unexpected error",
			"Path":        "(unknown)",
			"Description": "An unexpected error occurred: " + err.Error(),
		})
	}
}

func startReverseProxy(port int) *harnessProxy {
	serverUrl, _ := url.ParseRequestURI(fmt.Sprintf("http://localhost:%d", port))
	reverseProxy := &harnessProxy{
		proxy:         httputil.NewSingleHostReverseProxy(serverUrl),
		NotifyRequest: make(chan bool),
		NotifyReady:   make(chan error),
	}
	go func() {
		appPort := getAppPort()
		log.Println("Listening on port", appPort)
		err := http.ListenAndServe(fmt.Sprintf(":%d", appPort), reverseProxy)
		if err != nil {
			log.Fatalln("Failed to start reverse proxy:", err)
		}
	}()
	return reverseProxy
}

func getAppPort() int {
	port, err := rev.Config.Int("http.port")
	if err != nil {
		log.Println("Parsing http.port failed:", err)
		return 9000
	}
	return port
}

var (
	// Will not watch directories with these names (or their subdirectories)
	DoNotWatch = []string{"tmp", "views"}
)

func Run(mode rev.RunMode) {

	// If we are in prod mode, just build and run the application.
	if mode == rev.PROD {
		log.Println("Building...")
		if err := rebuild(getAppPort()); err != nil {
			log.Fatalln(err)
		}
		cmd.Wait()
		return
	}

	// Get a port on which to run the application
	port := getFreePort()

	// Run a reverse proxy to it.
	proxy := startReverseProxy(port)

	// Listen for changes to the user app.
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	// Listen to all app subdirectories (except /views)
	filepath.Walk(rev.AppPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			rev.LOG.Println("error walking app:", err)
			return nil
		}
		if info.IsDir() {
			if rev.ContainsString(DoNotWatch, info.Name()) {
				return filepath.SkipDir
			}
			err = watcher.Watch(path)
			rev.LOG.Println("Watching:", path)
			if err != nil {
				rev.LOG.Println("Failed to watch", path, ":", err)
			}
		}
		return nil
	})

	// Define an exit handler that kills the revel server (since it won't die on
	// its own, if the harness exits)
	defer func() {
		if cmd != nil {
			cmd.Process.Kill()
			cmd = nil
		}
	}()

	// Start the listen / rebuild loop.
	var dirty bool = true
	for {
		err = nil

		// It spins in this loop for each inotify change, and each request.
		// If there is a request after an inotify change, it breaks out to rebuild.
		for {
			select {
			case ev := <-watcher.Event:
				// Ignore changes to dot-files.
				if !strings.HasPrefix(path.Base(ev.Name), ".") {
					log.Println(ev)
					dirty = true
				}
				continue
			case err = <-watcher.Error:
				log.Println("Inotify error:", err)
				continue
			case _ = <-proxy.NotifyRequest:
				if !dirty {
					proxy.NotifyReady <- nil
					continue
				}
			}

			break
		}

		// There has been a change to the app and a new request is pending.
		// Rebuild it and send the "ready" signal.
		log.Println("Rebuild")
		err := rebuild(port)
		if err != nil {
			log.Println(err.Error())
			proxy.NotifyReady <- err
			continue
		}
		dirty = false
		proxy.NotifyReady <- nil
	}
}

var cmd *exec.Cmd

// Rebuild the Revel application and run it on the given port.
func rebuild(port int) (compileError *rev.Error) {
	controllerSpecs, compileError := ScanControllers(path.Join(rev.AppPath, "controllers"))
	if compileError != nil {
		return compileError
	}

	tmpl := template.New("RegisterControllers")
	tmpl = template.Must(tmpl.Parse(REGISTER_CONTROLLERS))
	var registerControllerSource string = rev.ExecuteTemplate(tmpl, map[string]interface{}{
		"AppName":     rev.AppName,
		"Controllers": controllerSpecs,
		"ImportPaths": uniqueImportPaths(controllerSpecs),
		"RunMode": rev.AppMode,
	})

	// Terminate the server if it's already running.
	if cmd != nil && (cmd.ProcessState == nil || !cmd.ProcessState.Exited()) {
		log.Println("Killing revel server pid", cmd.Process.Pid)
		err := cmd.Process.Kill()
		if err != nil {
			log.Fatalln("Failed to kill revel server:", err)
		}
	}

	// Create a fresh temp dir.
	tmpPath := path.Join(rev.AppPath, "tmp")
	err := os.RemoveAll(tmpPath)
	if err != nil {
		log.Println("Failed to remove tmp dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		log.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the new file
	controllersFile, err := os.Create(path.Join(tmpPath, "main.go"))
	if err != nil {
		log.Fatalf("Failed to create main.go: %v", err)
	}
	_, err = controllersFile.WriteString(registerControllerSource)
	if err != nil {
		log.Fatalf("Failed to write to main.go: %v", err)
	}

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		log.Fatalf("Go executable not found in PATH.")
	}

	ctx := build.Default
	pkg, err := ctx.Import(rev.ImportPath, "", build.FindOnly)
	if err != nil {
		log.Fatalf("Failure importing", rev.ImportPath)
	}
	binName := path.Join(pkg.BinDir, rev.AppName)
	buildCmd := exec.Command(goPath, "build", "-o", binName, path.Join(rev.ImportPath, "app", "tmp"))
	output, err := buildCmd.CombinedOutput()

	// If we failed to build, parse the error message.
	if err != nil {
		return newCompileError(output)
	}

	// Run the server, via tmp/main.go.
	cmd = exec.Command(binName,
		fmt.Sprintf("-port=%d", port),
		fmt.Sprintf("-importPath=%s", rev.ImportPath))
	listeningWriter := startupListeningWriter{os.Stdout, make(chan bool)}
	cmd.Stdout = listeningWriter
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	if err != nil {
		log.Fatalln("Error running:", err)
	}

	<-listeningWriter.notifyReady
	return nil
}

// A io.Writer that copies to the destination, and listens for "Listening on.."
// in the stream.  (Which tells us when the revel server has finished starting up)
// This is super ghetto, but by far the simplest thing that should work.
type startupListeningWriter struct {
	dest        io.Writer
	notifyReady chan bool
}

func (w startupListeningWriter) Write(p []byte) (n int, err error) {
	if w.notifyReady != nil && bytes.Contains(p, []byte("Listening")) {
		w.notifyReady <- true
		w.notifyReady = nil
	}
	return w.dest.Write(p)
}

// Find an unused port
func getFreePort() (port int) {
	conn, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}

	port = conn.Addr().(*net.TCPAddr).Port
	err = conn.Close()
	if err != nil {
		log.Fatal(err)
	}
	return port
}

func uniqueImportPaths(specs []*ControllerSpec) (paths []string) {
	importPathMap := make(map[string]bool)
	for _, spec := range specs {
		importPathMap[spec.ImportPath] = true
		for _, methSpec := range spec.MethodSpecs {
			for _, methArg := range methSpec.Args {
				if methArg.ImportPath != "" {
					importPathMap[methArg.ImportPath] = true
				}
			}
		}
	}

	for importPath := range importPathMap {
		paths = append(paths, importPath)
	}

	return
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *rev.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		log.Println("Failed to parse build errors:\n", string(output))
		return &rev.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &rev.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := rev.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		log.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}
<commit_msg>Buffer initial watch to avoid startup deadlock on watch errors.<commit_after>// The Harness for a Revel program.
//
// It has a couple responsibilities:
// 1. Parse the user program, generating a main.go file that registers
//    controller classes and starts the user's server.
// 2. Build and run the user program.  Show compile errors.
// 3. Monitor the user source and re-build / restart the program when necessary.
//
// Source files are generated in the app/tmp directory.

package harness

import (
	"bytes"
	"fmt"
	"github.com/howeyc/fsnotify"
	"go/build"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"github.com/robfig/revel"
	"regexp"
	"strconv"
	"strings"
	"text/template"
)

const REGISTER_CONTROLLERS = `
// target: {{.AppName}}
package main

import (
	"flag"
	"reflect"
	"github.com/robfig/revel"
	{{range .ImportPaths}}
  "{{.}}"
  {{end}}
)

var (
	port *int = flag.Int("port", 0, "Port")
	importPath *string = flag.String("importPath", "", "Path to the app.")

	// So compiler won't complain if the generated code doesn't reference reflect package...
	_ = reflect.Invalid
)

func main() {
	rev.LOG.Println("Running revel server")
	flag.Parse()
	rev.Init(*importPath, "{{.RunMode}}")
	{{range $i, $c := .Controllers}}
	rev.RegisterController((*{{.PackageName}}.{{.StructName}})(nil),
		[]*rev.MethodType{
			{{range .MethodSpecs}}&rev.MethodType{
				Name: "{{.Name}}",
				Args: []*rev.MethodArg{ {{range .Args}}
					&rev.MethodArg{Name: "{{.Name}}", Type: reflect.TypeOf((*{{.TypeName}})(nil)) },{{end}}
			  },
				RenderArgNames: map[int][]string{ {{range .RenderCalls}}
					{{.Line}}: []string{ {{range .Names}}
						"{{.}}",{{end}}
					},{{end}}
				},
			},
			{{end}}
		})
	{{end}}
	rev.Run(*port)
}
`

// Reverse proxy requests to the application server.
// On each request, proxy sends (NotifyRequest = true)
// If code change has been detected in app:
// - app is rebuilt and restarted, send proxy (NotifyReady = true)
// - else, send proxy (NotifyReady = true)

type harnessProxy struct {
	proxy         *httputil.ReverseProxy
	NotifyRequest chan bool  // Strobed on every request.
	NotifyReady   chan error // Strobed when request may proceed.
}

func (hp *harnessProxy) ServeHTTP(wr http.ResponseWriter, req *http.Request) {
	// First, poll to see if there's a pending error in NotifyReady
	select {
	case err := <-hp.NotifyReady:
		if err != nil {
			serveError(wr, req, err)
		}
	default:
		// Usually do nothing.
	}

	// Notify that a request is coming through, and wait for the go-ahead.
	hp.NotifyRequest <- true
	err := <-hp.NotifyReady

	// If an error was returned, create the page and show it to the user.
	if err != nil {
		serveError(wr, req, err)
		return
	}

	// Reverse proxy the request.
	hp.proxy.ServeHTTP(wr, req)
}

func serveError(wr http.ResponseWriter, req *http.Request, err error) {
	switch e := err.(type) {
	case *rev.Error:
		rev.RenderError(wr, e)
	default:
		rev.RenderError(wr, map[string]string{
			"Title":       "Unexpected error",
			"Path":        "(unknown)",
			"Description": "An unexpected error occurred: " + err.Error(),
		})
	}
}

func startReverseProxy(port int) *harnessProxy {
	serverUrl, _ := url.ParseRequestURI(fmt.Sprintf("http://localhost:%d", port))
	reverseProxy := &harnessProxy{
		proxy:         httputil.NewSingleHostReverseProxy(serverUrl),
		NotifyRequest: make(chan bool),
		NotifyReady:   make(chan error),
	}
	go func() {
		appPort := getAppPort()
		log.Println("Listening on port", appPort)
		err := http.ListenAndServe(fmt.Sprintf(":%d", appPort), reverseProxy)
		if err != nil {
			log.Fatalln("Failed to start reverse proxy:", err)
		}
	}()
	return reverseProxy
}

func getAppPort() int {
	port, err := rev.Config.Int("http.port")
	if err != nil {
		log.Println("Parsing http.port failed:", err)
		return 9000
	}
	return port
}

var (
	// Will not watch directories with these names (or their subdirectories)
	DoNotWatch = []string{"tmp", "views"}
)

func Run(mode rev.RunMode) {

	// If we are in prod mode, just build and run the application.
	if mode == rev.PROD {
		log.Println("Building...")
		if err := rebuild(getAppPort()); err != nil {
			log.Fatalln(err)
		}
		cmd.Wait()
		return
	}

	// Get a port on which to run the application
	port := getFreePort()

	// Run a reverse proxy to it.
	proxy := startReverseProxy(port)

	// Listen for changes to the user app.
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	watcher.Event = make(chan *fsnotify.FileEvent, 10)
	watcher.Error = make(chan error, 10)

	// Listen to all app subdirectories (except /views)
	filepath.Walk(rev.AppPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			rev.LOG.Println("error walking app:", err)
			return nil
		}
		if info.IsDir() {
			if rev.ContainsString(DoNotWatch, info.Name()) {
				return filepath.SkipDir
			}
			err = watcher.Watch(path)
			rev.LOG.Println("Watching:", path)
			if err != nil {
				rev.LOG.Println("Failed to watch", path, ":", err)
			}
		}
		return nil
	})

	// Define an exit handler that kills the revel server (since it won't die on
	// its own, if the harness exits)
	defer func() {
		if cmd != nil {
			cmd.Process.Kill()
			cmd = nil
		}
	}()

	// Start the listen / rebuild loop.
	var dirty bool = true
	for {
		err = nil

		// It spins in this loop for each inotify change, and each request.
		// If there is a request after an inotify change, it breaks out to rebuild.
		for {
			select {
			case ev := <-watcher.Event:
				// Ignore changes to dot-files.
				if !strings.HasPrefix(path.Base(ev.Name), ".") {
					log.Println(ev)
					dirty = true
				}
				continue
			case err = <-watcher.Error:
				log.Println("Inotify error:", err)
				continue
			case _ = <-proxy.NotifyRequest:
				if !dirty {
					proxy.NotifyReady <- nil
					continue
				}
			}

			break
		}

		// There has been a change to the app and a new request is pending.
		// Rebuild it and send the "ready" signal.
		log.Println("Rebuild")
		err := rebuild(port)
		if err != nil {
			log.Println(err.Error())
			proxy.NotifyReady <- err
			continue
		}
		dirty = false
		proxy.NotifyReady <- nil
	}
}

var cmd *exec.Cmd

// Rebuild the Revel application and run it on the given port.
func rebuild(port int) (compileError *rev.Error) {
	controllerSpecs, compileError := ScanControllers(path.Join(rev.AppPath, "controllers"))
	if compileError != nil {
		return compileError
	}

	tmpl := template.New("RegisterControllers")
	tmpl = template.Must(tmpl.Parse(REGISTER_CONTROLLERS))
	var registerControllerSource string = rev.ExecuteTemplate(tmpl, map[string]interface{}{
		"AppName":     rev.AppName,
		"Controllers": controllerSpecs,
		"ImportPaths": uniqueImportPaths(controllerSpecs),
		"RunMode": rev.AppMode,
	})

	// Terminate the server if it's already running.
	if cmd != nil && (cmd.ProcessState == nil || !cmd.ProcessState.Exited()) {
		log.Println("Killing revel server pid", cmd.Process.Pid)
		err := cmd.Process.Kill()
		if err != nil {
			log.Fatalln("Failed to kill revel server:", err)
		}
	}

	// Create a fresh temp dir.
	tmpPath := path.Join(rev.AppPath, "tmp")
	err := os.RemoveAll(tmpPath)
	if err != nil {
		log.Println("Failed to remove tmp dir:", err)
	}
	err = os.Mkdir(tmpPath, 0777)
	if err != nil {
		log.Fatalf("Failed to make tmp directory: %v", err)
	}

	// Create the new file
	controllersFile, err := os.Create(path.Join(tmpPath, "main.go"))
	if err != nil {
		log.Fatalf("Failed to create main.go: %v", err)
	}
	_, err = controllersFile.WriteString(registerControllerSource)
	if err != nil {
		log.Fatalf("Failed to write to main.go: %v", err)
	}

	// Build the user program (all code under app).
	// It relies on the user having "go" installed.
	goPath, err := exec.LookPath("go")
	if err != nil {
		log.Fatalf("Go executable not found in PATH.")
	}

	ctx := build.Default
	pkg, err := ctx.Import(rev.ImportPath, "", build.FindOnly)
	if err != nil {
		log.Fatalf("Failure importing", rev.ImportPath)
	}
	binName := path.Join(pkg.BinDir, rev.AppName)
	buildCmd := exec.Command(goPath, "build", "-o", binName, path.Join(rev.ImportPath, "app", "tmp"))
	output, err := buildCmd.CombinedOutput()

	// If we failed to build, parse the error message.
	if err != nil {
		return newCompileError(output)
	}

	// Run the server, via tmp/main.go.
	cmd = exec.Command(binName,
		fmt.Sprintf("-port=%d", port),
		fmt.Sprintf("-importPath=%s", rev.ImportPath))
	listeningWriter := startupListeningWriter{os.Stdout, make(chan bool)}
	cmd.Stdout = listeningWriter
	cmd.Stderr = os.Stderr
	err = cmd.Start()
	if err != nil {
		log.Fatalln("Error running:", err)
	}

	<-listeningWriter.notifyReady
	return nil
}

// A io.Writer that copies to the destination, and listens for "Listening on.."
// in the stream.  (Which tells us when the revel server has finished starting up)
// This is super ghetto, but by far the simplest thing that should work.
type startupListeningWriter struct {
	dest        io.Writer
	notifyReady chan bool
}

func (w startupListeningWriter) Write(p []byte) (n int, err error) {
	if w.notifyReady != nil && bytes.Contains(p, []byte("Listening")) {
		w.notifyReady <- true
		w.notifyReady = nil
	}
	return w.dest.Write(p)
}

// Find an unused port
func getFreePort() (port int) {
	conn, err := net.Listen("tcp", ":0")
	if err != nil {
		log.Fatal(err)
	}

	port = conn.Addr().(*net.TCPAddr).Port
	err = conn.Close()
	if err != nil {
		log.Fatal(err)
	}
	return port
}

func uniqueImportPaths(specs []*ControllerSpec) (paths []string) {
	importPathMap := make(map[string]bool)
	for _, spec := range specs {
		importPathMap[spec.ImportPath] = true
		for _, methSpec := range spec.MethodSpecs {
			for _, methArg := range methSpec.Args {
				if methArg.ImportPath != "" {
					importPathMap[methArg.ImportPath] = true
				}
			}
		}
	}

	for importPath := range importPathMap {
		paths = append(paths, importPath)
	}

	return
}

// Parse the output of the "go build" command.
// Return a detailed Error.
func newCompileError(output []byte) *rev.Error {
	errorMatch := regexp.MustCompile(`(?m)^([^:#]+):(\d+):(\d+:)? (.*)$`).
		FindSubmatch(output)
	if errorMatch == nil {
		log.Println("Failed to parse build errors:\n", string(output))
		return &rev.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Description: "See console for build error.",
		}
	}

	// Read the source for the offending file.
	var (
		relFilename    = string(errorMatch[1]) // e.g. "src/revel/sample/app/controllers/app.go"
		absFilename, _ = filepath.Abs(relFilename)
		line, _        = strconv.Atoi(string(errorMatch[2]))
		description    = string(errorMatch[4])
		compileError   = &rev.Error{
			SourceType:  "Go code",
			Title:       "Go Compilation Error",
			Path:        relFilename,
			Description: description,
			Line:        line,
		}
	)

	fileStr, err := rev.ReadLines(absFilename)
	if err != nil {
		compileError.MetaError = absFilename + ": " + err.Error()
		log.Println(compileError.MetaError)
		return compileError
	}

	compileError.SourceLines = fileStr
	return compileError
}
<|endoftext|>


<commit_before>package stats

import (
	"context"
	"fmt"
	"math"
	"sync"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/shirou/gopsutil/cpu"
)

const (
	// We use this constant to override the built in 3 second timeout in gopsutil
	cpuInfoTimeout = 10 * time.Second
)

var (
	cpuMhzPerCore float64
	cpuModelName  string
	cpuNumCores   int
	cpuTotalTicks float64

	initErr error
	onceLer sync.Once
)

func Init() error {
	onceLer.Do(func() {
		var merrs *multierror.Error
		var err error
		if cpuNumCores, err = cpu.Counts(true); err != nil {
			merrs = multierror.Append(merrs, fmt.Errorf("Unable to determine the number of CPU cores available: %v", err))
		}

		var cpuInfo []cpu.InfoStat
		ctx, _ := context.WithTimeout(context.Background(), cpuInfoTimeout)

		if cpuInfo, err = cpu.InfoWithContext(ctx); err != nil {
			merrs = multierror.Append(merrs, fmt.Errorf("Unable to obtain CPU information: %v", initErr))
		}

		for _, cpu := range cpuInfo {
			cpuModelName = cpu.ModelName
			cpuMhzPerCore = cpu.Mhz
			break
		}

		// Floor all of the values such that small difference don't cause the
		// node to fall into a unique computed node class
		cpuMhzPerCore = math.Floor(cpuMhzPerCore)
		cpuTotalTicks = math.Floor(float64(cpuNumCores) * cpuMhzPerCore)

		// Set any errors that occurred
		initErr = merrs.ErrorOrNil()
	})
	return initErr
}

// CPUModelName returns the number of CPU cores available
func CPUNumCores() int {
	return cpuNumCores
}

// CPUMHzPerCore returns the MHz per CPU core
func CPUMHzPerCore() float64 {
	return cpuMhzPerCore
}

// CPUModelName returns the model name of the CPU
func CPUModelName() string {
	return cpuModelName
}

// TotalTicksAvailable calculates the total Mhz available across all cores
func TotalTicksAvailable() float64 {
	return cpuTotalTicks
}
<commit_msg>Addressed review comments<commit_after>package stats

import (
	"context"
	"fmt"
	"math"
	"sync"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/shirou/gopsutil/cpu"
)

const (
	// cpuInfoTimeout is the timeout used when gathering CPU info. This is used
	// to override the default timeout in gopsutil which has a tendency to
	// timeout on Windows.
	cpuInfoTimeout = 10 * time.Second
)

var (
	cpuMhzPerCore float64
	cpuModelName  string
	cpuNumCores   int
	cpuTotalTicks float64

	initErr error
	onceLer sync.Once
)

func Init() error {
	onceLer.Do(func() {
		var merrs *multierror.Error
		var err error
		if cpuNumCores, err = cpu.Counts(true); err != nil {
			merrs = multierror.Append(merrs, fmt.Errorf("Unable to determine the number of CPU cores available: %v", err))
		}

		var cpuInfo []cpu.InfoStat
		ctx, _ := context.WithTimeout(context.Background(), cpuInfoTimeout)
		if cpuInfo, err = cpu.InfoWithContext(ctx); err != nil {
			merrs = multierror.Append(merrs, fmt.Errorf("Unable to obtain CPU information: %v", initErr))
		}

		for _, cpu := range cpuInfo {
			cpuModelName = cpu.ModelName
			cpuMhzPerCore = cpu.Mhz
			break
		}

		// Floor all of the values such that small difference don't cause the
		// node to fall into a unique computed node class
		cpuMhzPerCore = math.Floor(cpuMhzPerCore)
		cpuTotalTicks = math.Floor(float64(cpuNumCores) * cpuMhzPerCore)

		// Set any errors that occurred
		initErr = merrs.ErrorOrNil()
	})
	return initErr
}

// CPUModelName returns the number of CPU cores available
func CPUNumCores() int {
	return cpuNumCores
}

// CPUMHzPerCore returns the MHz per CPU core
func CPUMHzPerCore() float64 {
	return cpuMhzPerCore
}

// CPUModelName returns the model name of the CPU
func CPUModelName() string {
	return cpuModelName
}

// TotalTicksAvailable calculates the total Mhz available across all cores
func TotalTicksAvailable() float64 {
	return cpuTotalTicks
}
<|endoftext|>


<commit_before>/*

   Stores response structs for API functions account.go

*/

package binance

// Result from: GET /api/v3/account
type Account struct {
	MakerCommission  int64     `json:"makerCommission"`
	TakerCommission  int64     `json:"takerCommission"`
	BuyerCommission  int64     `json:"buyerCommission"`
	SellerCommission int64     `json:"sellerCommission"`
	CanTrade         bool      `json:"canTrade"`
	CanWithdraw      bool      `json:"canWithdraw"`
	CanDeposit       bool      `json:"canDeposit"`
	Balances         []Balance `json:"balances"`
}

type Balance struct {
	Asset  string  `json:"asset"`
	Free   float64 `json:"free,string"`
	Locked float64 `json:"locked,string"`
}

// Result from: POST /api/v3/order
type PlacedOrder struct {
	Symbol        string `json:"symbol"`
	OrderId       int64  `json:"orderId"`
	ClientOrderId string `json:"clientOrderId"`
	TransactTime  int64  `json:"transactTime"`
}

// Result from: DELETE /api/v3/order
type CanceledOrder struct {
	Symbol            string `json:"symbol"`
	OrigClientOrderId string `json:"origClientOrderId"`
	OrderId           int64  `json:"orderId"`
	ClientOrderId     string `json:"clientOrderId"`
}

// Result from: GET /api/v3/order
type OrderStatus struct {
	Symbol        string  `json:"symbol"`
	OrderId       int64   `json:"orderId"`
	ClientOrderId string  `json:"clientOrderId"`
	Price         float64 `json:"price,string"`
	OrigQty       float64 `json:"origQty,string"`
	ExecutedQty   float64 `json:"executedQty,string"`
	Status        string  `json:"status"`
	TimeInForce   string  `json:"timeInForce"`
	Type          string  `json:"type"`
	Side          string  `json:"side"`
	StopPrice     float64 `json:"stopPrice,string"`
	IcebergQty    float64 `json:"icebergQty,string"`
	Time          int64   `json:"time"`
}

// Result from: GET /api/v3/myTrades
type Trade struct {
	Id              int64   `json:"id"`
	Price           float64 `json:"price,string"`
	Quantity        float64 `json:"qty,string"`
	Commission      float64 `json:"commission,string"`
	CommissionAsset string  `json:"commissionAsset"`
	Time            int64   `json:"time"`
	IsBuyer         bool    `json:"isBuyer"`
	IsMaker         bool    `json:"isMaker"`
	IsBestMatch     bool    `json:"isBestMatch"`
}

// Result from: GET /api/v3/depositHistory
type Deposit struct {
	InsertTime int64   `json:"insertTime"`
	Amount     float64 `json:"amount"`
	Asset      string  `json:"asset"`
	Address    string  `json:"address"`
	TxId       string  `json:"txId"`
	Status     int64   `json:"status"`
}

// Result from: GET /api/v3/withdrawHistory
type Withdraw struct {
	Id        string  `json:"id"`
	Amount    float64 `json:"amount"`
	Address   string  `json:"address"`
	Asset     string  `json:"asset"`
	TxId      string  `json:"txId"`
	ApplyTime int64   `json:"applyTime"`
	Status    int64   `json:"status"`
}

type WithdrawList struct {
	Withdraws []Withdraw `json:"withdrawList"`
}

type DepositList struct {
	Deposits []Deposit `json:"depositList"`
}
<commit_msg>Adds order id to trade type<commit_after>/*

   Stores response structs for API functions account.go

*/

package binance

// Result from: GET /api/v3/account
type Account struct {
	MakerCommission  int64     `json:"makerCommission"`
	TakerCommission  int64     `json:"takerCommission"`
	BuyerCommission  int64     `json:"buyerCommission"`
	SellerCommission int64     `json:"sellerCommission"`
	CanTrade         bool      `json:"canTrade"`
	CanWithdraw      bool      `json:"canWithdraw"`
	CanDeposit       bool      `json:"canDeposit"`
	Balances         []Balance `json:"balances"`
}

type Balance struct {
	Asset  string  `json:"asset"`
	Free   float64 `json:"free,string"`
	Locked float64 `json:"locked,string"`
}

// Result from: POST /api/v3/order
type PlacedOrder struct {
	Symbol        string `json:"symbol"`
	OrderId       int64  `json:"orderId"`
	ClientOrderId string `json:"clientOrderId"`
	TransactTime  int64  `json:"transactTime"`
}

// Result from: DELETE /api/v3/order
type CanceledOrder struct {
	Symbol            string `json:"symbol"`
	OrigClientOrderId string `json:"origClientOrderId"`
	OrderId           int64  `json:"orderId"`
	ClientOrderId     string `json:"clientOrderId"`
}

// Result from: GET /api/v3/order
type OrderStatus struct {
	Symbol        string  `json:"symbol"`
	OrderId       int64   `json:"orderId"`
	ClientOrderId string  `json:"clientOrderId"`
	Price         float64 `json:"price,string"`
	OrigQty       float64 `json:"origQty,string"`
	ExecutedQty   float64 `json:"executedQty,string"`
	Status        string  `json:"status"`
	TimeInForce   string  `json:"timeInForce"`
	Type          string  `json:"type"`
	Side          string  `json:"side"`
	StopPrice     float64 `json:"stopPrice,string"`
	IcebergQty    float64 `json:"icebergQty,string"`
	Time          int64   `json:"time"`
}

// Result from: GET /api/v3/myTrades
type Trade struct {
	Id              int64   `json:"id"`
    OrderId         int64   `json:"orderId"`
	Price           float64 `json:"price,string"`
	Quantity        float64 `json:"qty,string"`
	Commission      float64 `json:"commission,string"`
	CommissionAsset string  `json:"commissionAsset"`
	Time            int64   `json:"time"`
	IsBuyer         bool    `json:"isBuyer"`
	IsMaker         bool    `json:"isMaker"`
	IsBestMatch     bool    `json:"isBestMatch"`
}

// Result from: GET /api/v3/depositHistory
type Deposit struct {
	InsertTime int64   `json:"insertTime"`
	Amount     float64 `json:"amount"`
	Asset      string  `json:"asset"`
	Address    string  `json:"address"`
	TxId       string  `json:"txId"`
	Status     int64   `json:"status"`
}

// Result from: GET /api/v3/withdrawHistory
type Withdraw struct {
	Id        string  `json:"id"`
	Amount    float64 `json:"amount"`
	Address   string  `json:"address"`
	Asset     string  `json:"asset"`
	TxId      string  `json:"txId"`
	ApplyTime int64   `json:"applyTime"`
	Status    int64   `json:"status"`
}

type WithdrawList struct {
	Withdraws []Withdraw `json:"withdrawList"`
}

type DepositList struct {
	Deposits []Deposit `json:"depositList"`
}
<|endoftext|>


<commit_before>package henchman

import (
	"code.google.com/p/go.crypto/ssh"
)

type Machine struct {
	Hostname  string
	SSHConfig *ssh.ClientConfig
}

func Machines(hostnames []string, config *ssh.ClientConfig) []*Machine {
	var machines []*Machine
	for _, hostname := range hostnames {
		machine := Machine{hostname, config}
		machines = append(machines, &machine)
	}
	return machines
}
<commit_msg>Redundant statement<commit_after>package henchman

import (
	"code.google.com/p/go.crypto/ssh"
)

type Machine struct {
	Hostname  string
	SSHConfig *ssh.ClientConfig
}

func Machines(hostnames []string, config *ssh.ClientConfig) []*Machine {
	var machines []*Machine
	for _, hostname := range hostnames {
		machines = append(machines, &Machine{hostname, config})
	}
	return machines
}
<|endoftext|>


<commit_before>package bulkgetter

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/opensearch-project/opensearch-go/v2"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

// ErrHTTP represents non-404 errors in HTTP requests.
var ErrHTTP = errors.New("HTTP Error")

type bulkRequest struct {
	ctx         context.Context
	client      *opensearch.Client
	rrs         map[string]reqresp
	decodeMutex sync.Mutex
	aliases     map[string]string
}

func newBulkRequest(ctx context.Context, client *opensearch.Client, size int) *bulkRequest {
	if ctx == nil {
		panic("required context is nil")
	}
	return &bulkRequest{
		ctx:     ctx,
		client:  client,
		rrs:     make(map[string]reqresp, size),
		aliases: make(map[string]string),
	}
}

func (r *bulkRequest) sendBulkResponse(found bool, err error) {
	for _, rr := range r.rrs {
		rr.resp <- GetResponse{found, err}
		close(rr.resp)
	}
}

type responseDoc struct {
	Index  string          `json:"_index"`
	ID     string          `json:"_id"`
	Found  bool            `json:"found"`
	Source json.RawMessage `json:"_source"`
}

type aliasesResponse map[string]struct {
	Aliases map[string]struct{} `json:"aliases"`
}

func (r *bulkRequest) getAliases(indexOrAlias string) (aliasesResponse, error) {
	response := aliasesResponse{}

	falseConst := true
	req := opensearchapi.IndicesGetAliasRequest{
		Index:           []string{indexOrAlias},
		AllowNoIndices:  &falseConst,
		ExpandWildcards: "none",
	}

	res, err := req.Do(r.ctx, r.client)
	if err != nil {
		return response, fmt.Errorf("error executing request: %w", err)
	}

	defer res.Body.Close()

	if res.IsError() {
		return response, fmt.Errorf("%w: %s", ErrHTTP, res)
	}

	err = json.NewDecoder(res.Body).Decode(&response)

	return response, err
}

func (r *bulkRequest) resolveAlias(indexOrAlias string) (string, error) {
	// GET /<index_or_alias>/_alias
	// {
	// 	"<index>": {
	// 		"aliases": {
	// 			"ipfs_directories": {}
	// 		}
	// 	}
	// }

	index, ok := r.aliases[indexOrAlias]
	if ok {
		return index, nil
	}

	response, err := r.getAliases(indexOrAlias)
	if err != nil {
		return "", err
	}

	for k := range response {
		r.aliases[indexOrAlias] = k
		return k, nil
	}

	return "", fmt.Errorf("index or alias %s not found", indexOrAlias)
}

func (r *bulkRequest) keyFromResponseDoc(doc *responseDoc) string {
	return doc.Index + doc.ID
}

func (r *bulkRequest) keyFromRR(rr reqresp) (string, error) {
	indexName, err := r.resolveAlias(rr.req.Index)
	if err != nil {
		return "", err
	}
	return indexName + rr.req.DocumentID, nil
}

func (r *bulkRequest) add(rr reqresp) error {
	key, err := r.keyFromRR(rr)
	if err != nil {
		return err
	}

	r.rrs[key] = rr

	return nil
}

func (r *bulkRequest) sendResponse(key string, found bool, err error) {
	rr, keyFound := r.rrs[key]

	if !keyFound {
		panic(fmt.Sprintf("Key %s not found in reqresp %v.", key, r.rrs))
	}

	if rr.resp == nil {
		panic(fmt.Sprintf("Invalid value for response channel for reqresp %v", rr))
	}

	// log.Printf("Sending response to %v", rr.resp)
	// defer log.Printf("Done sending response")

	rr.resp <- GetResponse{found, err}
	close(rr.resp)
}

func (r *bulkRequest) getReqBody() io.Reader {
	type source struct {
		Include []string `json:"include"`
	}

	type doc struct {
		Index  string `json:"_index"`
		ID     string `json:"_id"`
		Source source `json:"_source"`
	}

	docs := make([]doc, len(r.rrs))

	i := 0
	for _, rr := range r.rrs {
		docs[i] = doc{
			Index: rr.req.Index,
			ID:    rr.req.DocumentID,
			Source: source{
				rr.req.Fields,
			},
		}

		i++
	}

	bodyStruct := struct {
		Docs []doc `json:"docs"`
	}{docs}

	var buffer bytes.Buffer

	e := json.NewEncoder(io.Writer(&buffer))
	if err := e.Encode(bodyStruct); err != nil {
		panic("Error generating MGET request body.")
	}

	return io.Reader(&buffer)
}

func (r *bulkRequest) getRequest() *opensearchapi.MgetRequest {
	body := r.getReqBody()

	trueConst := true

	req := opensearchapi.MgetRequest{
		Body:       body,
		Preference: "_local",
		Realtime:   &trueConst,
	}

	return &req
}

func decodeResponse(res *opensearchapi.Response) ([]responseDoc, error) {
	// log.Printf("Decoding response to bulk GET")
	// defer log.Printf("Done decoding response to bulk GET")

	response := struct {
		Docs []responseDoc `json:"docs"`
	}{}

	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}

	return response.Docs, nil
}

func (r *bulkRequest) decodeSource(src json.RawMessage, dst interface{}) error {
	// Wrap Unmarshall in mutex to prevent race conditions as dst may be shared!
	r.decodeMutex.Lock()
	defer r.decodeMutex.Unlock()

	return json.Unmarshal(src, dst)
}

// processResponseDoc returns found, error
func (r *bulkRequest) processResponseDoc(d *responseDoc, key string) (bool, error) {
	if d.Found {
		if err := r.decodeSource(d.Source, r.rrs[key].dst); err != nil {
			err = fmt.Errorf("error decoding source: %w", err)
			return false, err
		}

		return true, nil
	}

	return false, nil
}

func (r *bulkRequest) processResponse(res *opensearchapi.Response) error {
	// log.Printf("Processing response to bulk GET")
	// defer log.Printf("Done processing response to bulk GET")

	var err error

	if res.StatusCode == 200 {
		docs, err := decodeResponse(res)
		if err != nil {
			err = fmt.Errorf("error decoding body: %w", err)
			return err
		}

		// log.Printf("Processing %d returned documents", len(docs))

		for _, d := range docs {
			key := r.keyFromResponseDoc(&d)

			// Only decode and send response when the other side is listening.
			rr, ok := r.rrs[key]
			if !ok {
				log.Printf("%+v", r.rrs)
				panic("")
				return fmt.Errorf("unknown key '%s' in response to bulk request", key)
			}
			if rr.ctx.Err() == nil {
				found, err := r.processResponseDoc(&d, key)
				r.sendResponse(key, found, err)
			} else {
				log.Printf("Not writing response from bulk get, request context cancelled.")
				close(rr.resp)
			}
		}

		return nil
	}

	// Non-200 status codes signify an error
	if res.IsError() {
		err = fmt.Errorf("%w: %s", ErrHTTP, res)
	} else {
		err = fmt.Errorf("Unexpected HTTP return code: %d", res.StatusCode)
	}

	return err
}

func (r *bulkRequest) execute() error {
	log.Printf("Performing bulk GET, %d elements", len(r.rrs))

	res, err := r.getRequest().Do(r.ctx, r.client)
	if err != nil {
		err = fmt.Errorf("error executing request: %w", err)
		r.sendBulkResponse(false, err)
		return err
	}

	defer res.Body.Close()

	if err = r.processResponse(res); err != nil {
		r.sendBulkResponse(false, err)
		return err
	}

	return nil
}
<commit_msg>Tweak debugging/verbosity in bulk requests.<commit_after>package bulkgetter

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/opensearch-project/opensearch-go/v2"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

// ErrHTTP represents non-404 errors in HTTP requests.
var ErrHTTP = errors.New("HTTP Error")

type bulkRequest struct {
	ctx         context.Context
	client      *opensearch.Client
	rrs         map[string]reqresp
	decodeMutex sync.Mutex
	aliases     map[string]string
}

func newBulkRequest(ctx context.Context, client *opensearch.Client, size int) *bulkRequest {
	if ctx == nil {
		panic("required context is nil")
	}
	return &bulkRequest{
		ctx:     ctx,
		client:  client,
		rrs:     make(map[string]reqresp, size),
		aliases: make(map[string]string),
	}
}

func (r *bulkRequest) sendBulkResponse(found bool, err error) {
	for _, rr := range r.rrs {
		rr.resp <- GetResponse{found, err}
		close(rr.resp)
	}
}

type responseDoc struct {
	Index  string          `json:"_index"`
	ID     string          `json:"_id"`
	Found  bool            `json:"found"`
	Source json.RawMessage `json:"_source"`
}

type aliasesResponse map[string]struct {
	Aliases map[string]struct{} `json:"aliases"`
}

func (r *bulkRequest) getAliases(indexOrAlias string) (aliasesResponse, error) {
	response := aliasesResponse{}

	falseConst := true
	req := opensearchapi.IndicesGetAliasRequest{
		Index:           []string{indexOrAlias},
		AllowNoIndices:  &falseConst,
		ExpandWildcards: "none",
	}

	res, err := req.Do(r.ctx, r.client)
	if err != nil {
		return response, fmt.Errorf("error executing request: %w", err)
	}

	defer res.Body.Close()

	if res.IsError() {
		return response, fmt.Errorf("%w: %s", ErrHTTP, res)
	}

	err = json.NewDecoder(res.Body).Decode(&response)

	return response, err
}

func (r *bulkRequest) resolveAlias(indexOrAlias string) (string, error) {
	// GET /<index_or_alias>/_alias
	// {
	// 	"<index>": {
	// 		"aliases": {
	// 			"ipfs_directories": {}
	// 		}
	// 	}
	// }

	index, ok := r.aliases[indexOrAlias]
	if ok {
		return index, nil
	}

	response, err := r.getAliases(indexOrAlias)
	if err != nil {
		return "", err
	}

	for k := range response {
		r.aliases[indexOrAlias] = k
		return k, nil
	}

	return "", fmt.Errorf("index or alias %s not found", indexOrAlias)
}

func (r *bulkRequest) keyFromResponseDoc(doc *responseDoc) string {
	return doc.Index + doc.ID
}

func (r *bulkRequest) keyFromRR(rr reqresp) (string, error) {
	indexName, err := r.resolveAlias(rr.req.Index)
	if err != nil {
		return "", err
	}
	return indexName + rr.req.DocumentID, nil
}

func (r *bulkRequest) add(rr reqresp) error {
	key, err := r.keyFromRR(rr)
	if err != nil {
		return err
	}

	r.rrs[key] = rr

	return nil
}

func (r *bulkRequest) sendResponse(key string, found bool, err error) {
	rr, keyFound := r.rrs[key]

	if !keyFound {
		panic(fmt.Sprintf("Key %s not found in reqresp %v.", key, r.rrs))
	}

	if rr.resp == nil {
		panic(fmt.Sprintf("Invalid value for response channel for reqresp %v", rr))
	}

	// log.Printf("Sending response to %v", rr.resp)
	// defer log.Printf("Done sending response")

	rr.resp <- GetResponse{found, err}
	close(rr.resp)
}

func (r *bulkRequest) getReqBody() io.Reader {
	type source struct {
		Include []string `json:"include"`
	}

	type doc struct {
		Index  string `json:"_index"`
		ID     string `json:"_id"`
		Source source `json:"_source"`
	}

	docs := make([]doc, len(r.rrs))

	i := 0
	for _, rr := range r.rrs {
		docs[i] = doc{
			Index: rr.req.Index,
			ID:    rr.req.DocumentID,
			Source: source{
				rr.req.Fields,
			},
		}

		i++
	}

	bodyStruct := struct {
		Docs []doc `json:"docs"`
	}{docs}

	var buffer bytes.Buffer

	e := json.NewEncoder(io.Writer(&buffer))
	if err := e.Encode(bodyStruct); err != nil {
		panic("Error generating MGET request body.")
	}

	return io.Reader(&buffer)
}

func (r *bulkRequest) getRequest() *opensearchapi.MgetRequest {
	body := r.getReqBody()

	trueConst := true

	req := opensearchapi.MgetRequest{
		Body:       body,
		Preference: "_local",
		Realtime:   &trueConst,
	}

	return &req
}

func decodeResponse(res *opensearchapi.Response) ([]responseDoc, error) {
	// log.Printf("Decoding response to bulk GET")
	// defer log.Printf("Done decoding response to bulk GET")

	response := struct {
		Docs []responseDoc `json:"docs"`
	}{}

	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}

	return response.Docs, nil
}

func (r *bulkRequest) decodeSource(src json.RawMessage, dst interface{}) error {
	// Wrap Unmarshall in mutex to prevent race conditions as dst may be shared!
	r.decodeMutex.Lock()
	defer r.decodeMutex.Unlock()

	return json.Unmarshal(src, dst)
}

// processResponseDoc returns found, error
func (r *bulkRequest) processResponseDoc(d *responseDoc, key string) (bool, error) {
	if d.Found {
		if err := r.decodeSource(d.Source, r.rrs[key].dst); err != nil {
			err = fmt.Errorf("error decoding source: %w", err)
			return false, err
		}

		return true, nil
	}

	return false, nil
}

func (r *bulkRequest) processResponse(res *opensearchapi.Response) error {
	// log.Printf("Processing response to bulk GET")
	// defer log.Printf("Done processing response to bulk GET")

	var err error

	if res.StatusCode == 200 {
		docs, err := decodeResponse(res)
		if err != nil {
			err = fmt.Errorf("error decoding body: %w", err)
			return err
		}

		// log.Printf("Processing %d returned documents", len(docs))

		for _, d := range docs {
			key := r.keyFromResponseDoc(&d)

			// Only decode and send response when the other side is listening.
			rr, ok := r.rrs[key]
			if !ok {
				return fmt.Errorf("unknown key '%s' in response to bulk request", key)
			}
			if rr.ctx.Err() == nil {
				found, err := r.processResponseDoc(&d, key)
				r.sendResponse(key, found, err)
			} else {
				// log.Printf("Not writing response from bulk get, request context cancelled.")
				close(rr.resp)
			}
		}

		return nil
	}

	// Non-200 status codes signify an error
	if res.IsError() {
		err = fmt.Errorf("%w: %s", ErrHTTP, res)
	} else {
		err = fmt.Errorf("Unexpected HTTP return code: %d", res.StatusCode)
	}

	return err
}

func (r *bulkRequest) execute() error {
	log.Printf("Performing bulk GET, %d elements", len(r.rrs))

	res, err := r.getRequest().Do(r.ctx, r.client)
	if err != nil {
		err = fmt.Errorf("error executing request: %w", err)
		r.sendBulkResponse(false, err)
		return err
	}

	defer res.Body.Close()

	if err = r.processResponse(res); err != nil {
		r.sendBulkResponse(false, err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package bulkgetter

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/opensearch-project/opensearch-go/v2"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

// ErrHTTP represents non-404 errors in HTTP requests.
var ErrHTTP = errors.New("HTTP Error")

type bulkRequest struct {
	ctx         context.Context
	client      *opensearch.Client
	rrs         map[string]reqresp
	decodeMutex sync.Mutex
	aliases     map[string]string
}

func newBulkRequest(ctx context.Context, client *opensearch.Client, size int) *bulkRequest {
	if ctx == nil {
		panic("required context is nil")
	}
	return &bulkRequest{
		ctx:     ctx,
		client:  client,
		rrs:     make(map[string]reqresp, size),
		aliases: make(map[string]string),
	}
}

func (r *bulkRequest) sendBulkResponse(found bool, err error) {
	for _, rr := range r.rrs {
		rr.resp <- GetResponse{found, err}
		close(rr.resp)
	}
}

type responseDoc struct {
	Index  string          `json:"_index"`
	ID     string          `json:"_id"`
	Found  bool            `json:"found"`
	Source json.RawMessage `json:"_source"`
}

type aliasesResponse map[string]struct {
	Aliases map[string]struct{} `json:"aliases"`
}

func (r *bulkRequest) getAliases(indexOrAlias string) (aliasesResponse, error) {
	response := aliasesResponse{}

	falseConst := true
	req := opensearchapi.IndicesGetAliasRequest{
		Index:           []string{indexOrAlias},
		AllowNoIndices:  &falseConst,
		ExpandWildcards: "none",
	}

	res, err := req.Do(r.ctx, r.client)
	if err != nil {
		return response, fmt.Errorf("error executing request: %w", err)
	}

	defer res.Body.Close()

	if res.IsError() {
		return response, fmt.Errorf("%w: %s", ErrHTTP, res)
	}

	err = json.NewDecoder(res.Body).Decode(&response)

	return response, err
}

func (r *bulkRequest) resolveAlias(indexOrAlias string) (string, error) {
	// GET /<index_or_alias>/_alias
	// {
	// 	"<index>": {
	// 		"aliases": {
	// 			"ipfs_directories": {}
	// 		}
	// 	}
	// }

	index, ok := r.aliases[indexOrAlias]
	if ok {
		return index, nil
	}

	response, err := r.getAliases(indexOrAlias)
	if err != nil {
		return "", err
	}

	for k := range response {
		r.aliases[indexOrAlias] = k
		return k, nil
	}

	return "", fmt.Errorf("index or alias %s not found", indexOrAlias)
}

func (r *bulkRequest) keyFromResponseDoc(doc *responseDoc) string {
	return doc.Index + doc.ID
}

func (r *bulkRequest) keyFromRR(rr reqresp) (string, error) {
	indexName, err := r.resolveAlias(rr.req.Index)
	if err != nil {
		return "", err
	}
	return indexName + rr.req.DocumentID, nil
}

func (r *bulkRequest) add(rr reqresp) error {
	key, err := r.keyFromRR(rr)
	if err != nil {
		return err
	}

	r.rrs[key] = rr

	return nil
}

func (r *bulkRequest) sendResponse(key string, found bool, err error) {
	rr, keyFound := r.rrs[key]

	if !keyFound {
		panic(fmt.Sprintf("Key %s not found in reqresp %v.", key, r.rrs))
	}

	if rr.resp == nil {
		panic(fmt.Sprintf("Invalid value for response channel for reqresp %v", rr))
	}

	// log.Printf("Sending response to %v", rr.resp)
	// defer log.Printf("Done sending response")

	rr.resp <- GetResponse{found, err}
	close(rr.resp)
}

func (r *bulkRequest) getReqBody() io.Reader {
	type source struct {
		Include []string `json:"include"`
	}

	type doc struct {
		Index  string `json:"_index"`
		ID     string `json:"_id"`
		Source source `json:"_source"`
	}

	docs := make([]doc, len(r.rrs))

	i := 0
	for _, rr := range r.rrs {
		docs[i] = doc{
			Index: rr.req.Index,
			ID:    rr.req.DocumentID,
			Source: source{
				rr.req.Fields,
			},
		}

		i++
	}

	bodyStruct := struct {
		Docs []doc `json:"docs"`
	}{docs}

	var buffer bytes.Buffer

	e := json.NewEncoder(io.Writer(&buffer))
	if err := e.Encode(bodyStruct); err != nil {
		panic("Error generating MGET request body.")
	}

	return io.Reader(&buffer)
}

func (r *bulkRequest) getRequest() *opensearchapi.MgetRequest {
	body := r.getReqBody()

	req := opensearchapi.MgetRequest{
		Body: body,
	}

	return &req
}

func decodeResponse(res *opensearchapi.Response) ([]responseDoc, error) {
	// log.Printf("Decoding response to bulk GET")
	// defer log.Printf("Done decoding response to bulk GET")

	response := struct {
		Docs []responseDoc `json:"docs"`
	}{}

	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}

	return response.Docs, nil
}

func (r *bulkRequest) decodeSource(src json.RawMessage, dst interface{}) error {
	// Wrap Unmarshall in mutex to prevent race conditions as dst may be shared!
	r.decodeMutex.Lock()
	defer r.decodeMutex.Unlock()

	return json.Unmarshal(src, dst)
}

// processResponseDoc returns found, error
func (r *bulkRequest) processResponseDoc(d *responseDoc, key string) (bool, error) {
	if d.Found {
		if err := r.decodeSource(d.Source, r.rrs[key].dst); err != nil {
			err = fmt.Errorf("error decoding source: %w", err)
			return false, err
		}

		return true, nil
	}

	return false, nil
}

func (r *bulkRequest) processResponse(res *opensearchapi.Response) error {
	// log.Printf("Processing response to bulk GET")
	// defer log.Printf("Done processing response to bulk GET")

	var err error

	if res.StatusCode == 200 {
		docs, err := decodeResponse(res)
		if err != nil {
			err = fmt.Errorf("error decoding body: %w", err)
			return err
		}

		// log.Printf("Processing %d returned documents", len(docs))

		for _, d := range docs {
			key := r.keyFromResponseDoc(&d)

			// Only decode and send response when the other side is listening.
			rr, ok := r.rrs[key]
			if !ok {
				return fmt.Errorf("unknown key '%s' in response to bulk request", key)
			}
			if rr.ctx.Err() == nil {
				found, err := r.processResponseDoc(&d, key)
				r.sendResponse(key, found, err)
			} else {
				// log.Printf("Not writing response from bulk get, request context cancelled.")
				close(rr.resp)
			}
		}

		return nil
	}

	// Non-200 status codes signify an error
	if res.IsError() {
		err = fmt.Errorf("%w: %s", ErrHTTP, res)
	} else {
		err = fmt.Errorf("Unexpected HTTP return code: %d", res.StatusCode)
	}

	return err
}

func (r *bulkRequest) execute() error {
	log.Printf("Performing bulk GET, %d elements", len(r.rrs))

	res, err := r.getRequest().Do(r.ctx, r.client)
	if err != nil {
		err = fmt.Errorf("error executing request: %w", err)
		r.sendBulkResponse(false, err)
		return err
	}

	defer res.Body.Close()

	if err = r.processResponse(res); err != nil {
		r.sendBulkResponse(false, err)
		return err
	}

	return nil
}
<commit_msg>More sensible, less verbose, output bulk from getter.<commit_after>package bulkgetter

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/opensearch-project/opensearch-go/v2"
	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
)

// ErrHTTP represents non-404 errors in HTTP requests.
var ErrHTTP = errors.New("HTTP Error")

type bulkRequest struct {
	ctx         context.Context
	client      *opensearch.Client
	rrs         map[string]reqresp
	decodeMutex sync.Mutex
	aliases     map[string]string
}

func newBulkRequest(ctx context.Context, client *opensearch.Client, size int) *bulkRequest {
	if ctx == nil {
		panic("required context is nil")
	}
	return &bulkRequest{
		ctx:     ctx,
		client:  client,
		rrs:     make(map[string]reqresp, size),
		aliases: make(map[string]string),
	}
}

func (r *bulkRequest) sendBulkResponse(found bool, err error) {
	for _, rr := range r.rrs {
		rr.resp <- GetResponse{found, err}
		close(rr.resp)
	}
}

type responseDoc struct {
	Index  string          `json:"_index"`
	ID     string          `json:"_id"`
	Found  bool            `json:"found"`
	Source json.RawMessage `json:"_source"`
}

type aliasesResponse map[string]struct {
	Aliases map[string]struct{} `json:"aliases"`
}

func (r *bulkRequest) getAliases(indexOrAlias string) (aliasesResponse, error) {
	response := aliasesResponse{}

	falseConst := true
	req := opensearchapi.IndicesGetAliasRequest{
		Index:           []string{indexOrAlias},
		AllowNoIndices:  &falseConst,
		ExpandWildcards: "none",
	}

	res, err := req.Do(r.ctx, r.client)
	if err != nil {
		return response, fmt.Errorf("error executing request: %w", err)
	}

	defer res.Body.Close()

	if res.IsError() {
		return response, fmt.Errorf("%w: %s", ErrHTTP, res)
	}

	err = json.NewDecoder(res.Body).Decode(&response)

	return response, err
}

func (r *bulkRequest) resolveAlias(indexOrAlias string) (string, error) {
	// GET /<index_or_alias>/_alias
	// {
	// 	"<index>": {
	// 		"aliases": {
	// 			"ipfs_directories": {}
	// 		}
	// 	}
	// }

	index, ok := r.aliases[indexOrAlias]
	if ok {
		return index, nil
	}

	response, err := r.getAliases(indexOrAlias)
	if err != nil {
		return "", err
	}

	for k := range response {
		r.aliases[indexOrAlias] = k
		return k, nil
	}

	return "", fmt.Errorf("index or alias %s not found", indexOrAlias)
}

func (r *bulkRequest) keyFromResponseDoc(doc *responseDoc) string {
	return doc.Index + doc.ID
}

func (r *bulkRequest) keyFromRR(rr reqresp) (string, error) {
	indexName, err := r.resolveAlias(rr.req.Index)
	if err != nil {
		return "", err
	}
	return indexName + rr.req.DocumentID, nil
}

func (r *bulkRequest) add(rr reqresp) error {
	key, err := r.keyFromRR(rr)
	if err != nil {
		return err
	}

	r.rrs[key] = rr

	return nil
}

func (r *bulkRequest) sendResponse(key string, found bool, err error) {
	rr, keyFound := r.rrs[key]

	if !keyFound {
		panic(fmt.Sprintf("Key %s not found in reqresp %v.", key, r.rrs))
	}

	if rr.resp == nil {
		panic(fmt.Sprintf("Invalid value for response channel for reqresp %v", rr))
	}

	// log.Printf("Sending response to %v", rr.resp)
	// defer log.Printf("Done sending response")

	rr.resp <- GetResponse{found, err}
	close(rr.resp)
}

func (r *bulkRequest) getReqBody() io.Reader {
	type source struct {
		Include []string `json:"include"`
	}

	type doc struct {
		Index  string `json:"_index"`
		ID     string `json:"_id"`
		Source source `json:"_source"`
	}

	docs := make([]doc, len(r.rrs))

	i := 0
	for _, rr := range r.rrs {
		docs[i] = doc{
			Index: rr.req.Index,
			ID:    rr.req.DocumentID,
			Source: source{
				rr.req.Fields,
			},
		}

		i++
	}

	bodyStruct := struct {
		Docs []doc `json:"docs"`
	}{docs}

	var buffer bytes.Buffer

	e := json.NewEncoder(io.Writer(&buffer))
	if err := e.Encode(bodyStruct); err != nil {
		panic("Error generating MGET request body.")
	}

	return io.Reader(&buffer)
}

func (r *bulkRequest) getRequest() *opensearchapi.MgetRequest {
	body := r.getReqBody()

	req := opensearchapi.MgetRequest{
		Body: body,
	}

	return &req
}

func decodeResponse(res *opensearchapi.Response) ([]responseDoc, error) {
	// log.Printf("Decoding response to bulk GET")
	// defer log.Printf("Done decoding response to bulk GET")

	response := struct {
		Docs []responseDoc `json:"docs"`
	}{}

	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
		return nil, err
	}

	return response.Docs, nil
}

func (r *bulkRequest) decodeSource(src json.RawMessage, dst interface{}) error {
	// Wrap Unmarshall in mutex to prevent race conditions as dst may be shared!
	r.decodeMutex.Lock()
	defer r.decodeMutex.Unlock()

	return json.Unmarshal(src, dst)
}

// processResponseDoc returns found, error
func (r *bulkRequest) processResponseDoc(d *responseDoc, key string) (bool, error) {
	if d.Found {
		if err := r.decodeSource(d.Source, r.rrs[key].dst); err != nil {
			err = fmt.Errorf("error decoding source: %w", err)
			return false, err
		}

		return true, nil
	}

	return false, nil
}

func (r *bulkRequest) processResponse(res *opensearchapi.Response) error {
	// log.Printf("Processing response to bulk GET")
	// defer log.Printf("Done processing response to bulk GET")

	var err error

	if res.StatusCode == 200 {
		docs, err := decodeResponse(res)
		if err != nil {
			err = fmt.Errorf("error decoding body: %w", err)
			return err
		}

		// log.Printf("Processing %d returned documents", len(docs))

		for _, d := range docs {
			key := r.keyFromResponseDoc(&d)

			// Only decode and send response when the other side is listening.
			rr, ok := r.rrs[key]
			if !ok {
				// Panic, this is a proper bug.
				panic(fmt.Sprintf("unknown key '%s' in response to bulk request", key))
			}
			if rr.ctx.Err() == nil {
				found, err := r.processResponseDoc(&d, key)
				r.sendResponse(key, found, err)
			} else {
				// log.Printf("Not writing response from bulk get, request context cancelled.")
				close(rr.resp)
			}
		}

		return nil
	}

	// Non-200 status codes signify an error
	if res.IsError() {
		err = fmt.Errorf("%w: %s", ErrHTTP, res)
	} else {
		err = fmt.Errorf("Unexpected HTTP return code: %d", res.StatusCode)
	}

	return err
}

func (r *bulkRequest) execute() error {
	log.Printf("Performing bulk GET, %d elements", len(r.rrs))

	res, err := r.getRequest().Do(r.ctx, r.client)
	if err != nil {
		err = fmt.Errorf("error executing request: %w", err)
		r.sendBulkResponse(false, err)
		return err
	}

	defer res.Body.Close()

	if err = r.processResponse(res); err != nil {
		r.sendBulkResponse(false, err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"net/http"

	bleveHttp "github.com/blevesearch/bleve/http"

	"github.com/gorilla/mux"
)

func NewManagerRESTRouter(mgr *Manager, staticDir string, mr *MsgRing) (
	*mux.Router, error) {
	// create a router to serve static files
	r := staticFileRouter(staticDir, []string{
		"/indexes",
		"/monitor",
		"/manage",
		"/logs",
		"/debug",
	})

	r.Handle("/api/log", NewGetLogHandler(mr)).Methods("GET")

	r.Handle("/api/index", NewListIndexHandler(mgr)).Methods("GET")
	r.Handle("/api/index/{indexName}", NewCreateIndexHandler(mgr)).Methods("PUT")
	r.Handle("/api/index/{indexName}", NewDeleteIndexHandler(mgr)).Methods("DELETE")
	r.Handle("/api/index/{indexName}", NewGetIndexHandler(mgr)).Methods("GET")

	if mgr.tagsMap == nil || mgr.tagsMap["queryer"] {
		r.Handle("/api/index/{indexName}/count", NewCountHandler(mgr)).Methods("GET")
		r.Handle("/api/index/{indexName}/query", NewQueryHandler(mgr)).Methods("POST")
	}

	// We use standard bleveHttp handlers for the /api/pindex-bleve endpoints.
	if mgr.tagsMap == nil || mgr.tagsMap["pindex"] {
		listIndexesHandler := bleveHttp.NewListIndexesHandler()
		r.Handle("/api/pindex", listIndexesHandler).Methods("GET")
		r.Handle("/api/pindex-bleve", listIndexesHandler).Methods("GET")

		getIndexHandler := bleveHttp.NewGetIndexHandler()
		getIndexHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}",
			getIndexHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}",
			getIndexHandler).Methods("GET")

		docCountHandler := bleveHttp.NewDocCountHandler("")
		docCountHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}/count",
			docCountHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/count",
			docCountHandler).Methods("GET")

		docGetHandler := bleveHttp.NewDocGetHandler("")
		docGetHandler.IndexNameLookup = indexNameLookup
		docGetHandler.DocIDLookup = docIDLookup
		r.Handle("/api/pindex/{indexName}/doc/{docID}",
			docGetHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/doc/{docID}",
			docGetHandler).Methods("GET")

		debugDocHandler := bleveHttp.NewDebugDocumentHandler("")
		debugDocHandler.IndexNameLookup = indexNameLookup
		debugDocHandler.DocIDLookup = docIDLookup
		r.Handle("/api/pindex/{indexName}/docDebug/{docID}",
			debugDocHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/docDebug/{docID}",
			debugDocHandler).Methods("GET")

		// We have cbft purpose-built pindex query handler, instead of
		// just using bleveHttp, to handle auth and query consistency
		// across >1 pindex.
		r.Handle("/api/pindex/{indexName}/query",
			NewQueryPIndexHandler(mgr)).Methods("POST")

		searchHandler := bleveHttp.NewSearchHandler("")
		searchHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex-bleve/{indexName}/query",
			searchHandler).Methods("POST")

		listFieldsHandler := bleveHttp.NewListFieldsHandler("")
		listFieldsHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}/fields",
			listFieldsHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/fields",
			listFieldsHandler).Methods("GET")

		r.Handle("/api/feedStats",
			NewFeedStatsHandler(mgr)).Methods("GET")
	}

	r.Handle("/api/cfg", NewCfgGetHandler(mgr)).Methods("GET")
	r.Handle("/api/cfgRefresh", NewCfgRefreshHandler(mgr)).Methods("POST")

	r.Handle("/api/managerKick", NewManagerKickHandler(mgr)).Methods("POST")
	r.Handle("/api/managerMeta", NewManagerMetaHandler(mgr)).Methods("GET")

	return r, nil
}

func muxVariableLookup(req *http.Request, name string) string {
	return mux.Vars(req)[name]
}
<commit_msg>TODO comment<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package main

import (
	"net/http"

	bleveHttp "github.com/blevesearch/bleve/http"

	"github.com/gorilla/mux"
)

func NewManagerRESTRouter(mgr *Manager, staticDir string, mr *MsgRing) (
	*mux.Router, error) {
	// create a router to serve static files
	r := staticFileRouter(staticDir, []string{
		"/indexes",
		"/monitor",
		"/manage",
		"/logs",
		"/debug",
	})

	r.Handle("/api/log", NewGetLogHandler(mr)).Methods("GET")

	r.Handle("/api/index", NewListIndexHandler(mgr)).Methods("GET")
	r.Handle("/api/index/{indexName}", NewCreateIndexHandler(mgr)).Methods("PUT")
	r.Handle("/api/index/{indexName}", NewDeleteIndexHandler(mgr)).Methods("DELETE")
	r.Handle("/api/index/{indexName}", NewGetIndexHandler(mgr)).Methods("GET")

	if mgr.tagsMap == nil || mgr.tagsMap["queryer"] {
		r.Handle("/api/index/{indexName}/count", NewCountHandler(mgr)).Methods("GET")
		r.Handle("/api/index/{indexName}/query", NewQueryHandler(mgr)).Methods("POST")
	}

	// We use standard bleveHttp handlers for the /api/pindex-bleve endpoints.
	//
	// TODO: Need to cleanly separate the /api/pindex and
	// /api/pindex-bleve endpoints.
	if mgr.tagsMap == nil || mgr.tagsMap["pindex"] {
		listIndexesHandler := bleveHttp.NewListIndexesHandler()
		r.Handle("/api/pindex", listIndexesHandler).Methods("GET")
		r.Handle("/api/pindex-bleve", listIndexesHandler).Methods("GET")

		getIndexHandler := bleveHttp.NewGetIndexHandler()
		getIndexHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}",
			getIndexHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}",
			getIndexHandler).Methods("GET")

		docCountHandler := bleveHttp.NewDocCountHandler("")
		docCountHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}/count",
			docCountHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/count",
			docCountHandler).Methods("GET")

		docGetHandler := bleveHttp.NewDocGetHandler("")
		docGetHandler.IndexNameLookup = indexNameLookup
		docGetHandler.DocIDLookup = docIDLookup
		r.Handle("/api/pindex/{indexName}/doc/{docID}",
			docGetHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/doc/{docID}",
			docGetHandler).Methods("GET")

		debugDocHandler := bleveHttp.NewDebugDocumentHandler("")
		debugDocHandler.IndexNameLookup = indexNameLookup
		debugDocHandler.DocIDLookup = docIDLookup
		r.Handle("/api/pindex/{indexName}/docDebug/{docID}",
			debugDocHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/docDebug/{docID}",
			debugDocHandler).Methods("GET")

		// We have a purpose-built pindex query handler, instead of
		// just using bleveHttp, to handle auth and query consistency.
		r.Handle("/api/pindex/{indexName}/query",
			NewQueryPIndexHandler(mgr)).Methods("POST")

		searchHandler := bleveHttp.NewSearchHandler("")
		searchHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex-bleve/{indexName}/query",
			searchHandler).Methods("POST")

		listFieldsHandler := bleveHttp.NewListFieldsHandler("")
		listFieldsHandler.IndexNameLookup = indexNameLookup
		r.Handle("/api/pindex/{indexName}/fields",
			listFieldsHandler).Methods("GET")
		r.Handle("/api/pindex-bleve/{indexName}/fields",
			listFieldsHandler).Methods("GET")

		r.Handle("/api/feedStats",
			NewFeedStatsHandler(mgr)).Methods("GET")
	}

	r.Handle("/api/cfg", NewCfgGetHandler(mgr)).Methods("GET")
	r.Handle("/api/cfgRefresh", NewCfgRefreshHandler(mgr)).Methods("POST")

	r.Handle("/api/managerKick", NewManagerKickHandler(mgr)).Methods("POST")
	r.Handle("/api/managerMeta", NewManagerMetaHandler(mgr)).Methods("GET")

	return r, nil
}

func muxVariableLookup(req *http.Request, name string) string {
	return mux.Vars(req)[name]
}
<|endoftext|>


<commit_before>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package riot

import (
	"bytes"
	"fmt"
	"log"
	"os"

	"encoding/binary"
	"encoding/gob"

	"github.com/go-ego/murmur"
	"github.com/go-ego/riot/core"
	"github.com/go-ego/riot/types"
)

// New create a new engine
func New(dict ...string) *Engine {
	// func (engine *Engine) New(conf com.Config) *Engine{
	var (
		searcher = &Engine{}

		path          = "./riot-index"
		storageShards = 10
		numShards     = 10

		segmenterDict string
	)

	if len(dict) > 0 {
		segmenterDict = dict[0]
	}

	searcher.Init(types.EngineOpts{
		// Using:         using,
		StorageShards: storageShards,
		NumShards:     numShards,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		// StorageEngine: storageEngine,
		SegmenterDict: segmenterDict,
		// StopTokenFile: stopTokenFile,
	})

	// defer searcher.Close()
	os.MkdirAll(path, 0777)

	// 等待索引刷新完毕
	// searcher.Flush()
	// log.Println("recover index number: ", searcher.NumDocsIndexed())

	return searcher
}

// func (engine *Engine) IsDocExist(docId uint64) bool {
// 	return core.IsDocExist(docId)
// }

// HasDoc doc is exist return true
func (engine *Engine) HasDoc(docId uint64) bool {
	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
		engine.indexers = append(engine.indexers, core.Indexer{})

		has := engine.indexers[shard].HasDoc(docId)

		if has {
			return true
		}
	}

	return false
}

// HasDocDB doc is exist return true
func (engine *Engine) HasDocDB(docId uint64) bool {
	b := make([]byte, 10)
	length := binary.PutUvarint(b, docId)

	shard := murmur.Sum32(fmt.Sprintf("%d", docId)) %
		uint32(engine.initOptions.StorageShards)

	has, err := engine.dbs[shard].Has(b[0:length])
	if err != nil {
		log.Println("engine.dbs[shard].Has(b[0:length]): ", err)
	}

	return has
}

// GetDBAllIds get all the DocId from the storage database and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetDBAllIds() []uint64 {
	docsId := make([]uint64, 0)
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(k, v []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(k)
			docsId = append(docsId, docId)
			return nil
		})
	}

	return docsId
}

// GetDBAllDocs get the db all docs
func (engine *Engine) GetDBAllDocs() (
	docsId []uint64, docsData []types.DocIndexData) {
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(key, val []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(key)
			docsId = append(docsId, docId)

			buf := bytes.NewReader(val)
			dec := gob.NewDecoder(buf)
			var data types.DocIndexData
			err := dec.Decode(&data)
			if err != nil {
				log.Println("dec.decode: ", err)
			}

			docsData = append(docsData, data)

			return nil
		})
	}

	return docsId, docsData
}

// GetAllDocIds get all the DocId from the storage database and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetAllDocIds() []uint64 {
	return engine.GetDBAllIds()
}

// Try handler(err)
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}
<commit_msg>optimize new func allow set numshards<commit_after>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package riot

import (
	"bytes"
	"fmt"
	"log"
	"os"

	"encoding/binary"
	"encoding/gob"

	"github.com/go-ego/murmur"
	"github.com/go-ego/riot/core"
	"github.com/go-ego/riot/types"
)

// New create a new engine
func New(dict ...interface{}) *Engine {
	// func (engine *Engine) New(conf com.Config) *Engine{
	var (
		searcher = &Engine{}

		path          = "./riot-index"
		storageShards = 10
		numShards     = 10

		segmenterDict string
	)

	if len(dict) > 0 {
		segmenterDict = dict[0].(string)
	}

	if len(dict) > 1 {
		numShards = dict[1].(int)
		storageShards = dict[1].(int)
	}

	searcher.Init(types.EngineOpts{
		// Using:         using,
		StorageShards: storageShards,
		NumShards:     numShards,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		// StorageEngine: storageEngine,
		SegmenterDict: segmenterDict,
		// StopTokenFile: stopTokenFile,
	})

	// defer searcher.Close()
	os.MkdirAll(path, 0777)

	// 等待索引刷新完毕
	// searcher.Flush()
	// log.Println("recover index number: ", searcher.NumDocsIndexed())

	return searcher
}

// func (engine *Engine) IsDocExist(docId uint64) bool {
// 	return core.IsDocExist(docId)
// }

// HasDoc doc is exist return true
func (engine *Engine) HasDoc(docId uint64) bool {
	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
		engine.indexers = append(engine.indexers, core.Indexer{})

		has := engine.indexers[shard].HasDoc(docId)

		if has {
			return true
		}
	}

	return false
}

// HasDocDB doc is exist return true
func (engine *Engine) HasDocDB(docId uint64) bool {
	b := make([]byte, 10)
	length := binary.PutUvarint(b, docId)

	shard := murmur.Sum32(fmt.Sprintf("%d", docId)) %
		uint32(engine.initOptions.StorageShards)

	has, err := engine.dbs[shard].Has(b[0:length])
	if err != nil {
		log.Println("engine.dbs[shard].Has(b[0:length]): ", err)
	}

	return has
}

// GetDBAllIds get all the DocId from the storage database and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetDBAllIds() []uint64 {
	docsId := make([]uint64, 0)
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(k, v []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(k)
			docsId = append(docsId, docId)
			return nil
		})
	}

	return docsId
}

// GetDBAllDocs get the db all docs
func (engine *Engine) GetDBAllDocs() (
	docsId []uint64, docsData []types.DocIndexData) {
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(key, val []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(key)
			docsId = append(docsId, docId)

			buf := bytes.NewReader(val)
			dec := gob.NewDecoder(buf)
			var data types.DocIndexData
			err := dec.Decode(&data)
			if err != nil {
				log.Println("dec.decode: ", err)
			}

			docsData = append(docsData, data)

			return nil
		})
	}

	return docsId, docsData
}

// GetAllDocIds get all the DocId from the storage database and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetAllDocIds() []uint64 {
	return engine.GetDBAllIds()
}

// Try handler(err)
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}
<|endoftext|>


<commit_before>package webhookmodels

import (
	"fmt"
	"strings"
	"time"
)

const PaypalTimeLayout = "15:04:05 Jan 02, 2006 MST"

// Paypal is so special it deverses it's own time format.
type PaypalOwnTime struct {
	time.Time
}

func (pt *PaypalOwnTime) UnmarshalJSON(b []byte) error {
	var err error

	pt.Time, err = time.Parse(PaypalTimeLayout, strings.Replace(fmt.Sprintf("%s", b), `"`, "", -1))

	return err
}

type PaypalGenericWebhook struct {
	TransactionType string        `json:"txn_type"`
	Status          string        `json:"payment_status"`
	PayerId         string        `json:"recurring_payment_id"`
	Plan            string        `json:"product_name"`
	Amount          string        `json:"amount"`
	Currency        string        `json:"currency_code"`
	NextPaymentDate PaypalOwnTime `json:"next_payment_date"`
	PaymentDate     PaypalOwnTime `json:"payment_date"`
}
<commit_msg>paymentwebhook: check for 'N/A' in paypal webhook response time field<commit_after>package webhookmodels

import (
	"fmt"
	"strings"
	"time"
)

const PaypalTimeLayout = "15:04:05 Jan 02, 2006 MST"

// Paypal is so special it deverses it's own time format.
type PaypalOwnTime struct {
	time.Time
}

func (pt *PaypalOwnTime) UnmarshalJSON(b []byte) error {
	var err error

	str := strings.Replace(fmt.Sprintf("%s", b), `"`, "", -1)
	if str == "" || str == "N/A" {
		return nil
	}

	pt.Time, err = time.Parse(PaypalTimeLayout, str)

	return err
}

type PaypalGenericWebhook struct {
	TransactionType string        `json:"txn_type"`
	Status          string        `json:"payment_status"`
	PayerId         string        `json:"recurring_payment_id"`
	Plan            string        `json:"product_name"`
	Amount          string        `json:"amount"`
	Currency        string        `json:"currency_code"`
	NextPaymentDate PaypalOwnTime `json:"next_payment_date"`
	PaymentDate     PaypalOwnTime `json:"payment_date"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		Status:         "building",
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, "Something wrong :P")
	}

	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
	}
	ren.JSON(w, http.StatusOK, build)
}

func main() {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<commit_msg>Fix error message<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
	"github.com/unrolled/render"

	"github.com/wantedly/risu/registry"
	"github.com/wantedly/risu/schema"
)

var ren = render.New()

func create(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	defer r.Body.Close()
	var opts schema.BuildCreateOpts
	err := json.NewDecoder(r.Body).Decode(&opts)
	if err != nil {
		log.Fatal(err)
	}

	if opts.Dockerfile == "" {
		opts.Dockerfile = "Dockerfile"
	}

	build := schema.Build{
		ID:             uuid.NewUUID(),
		SourceRepo:     opts.SourceRepo,
		SourceRevision: opts.SourceRevision,
		Name:           opts.Name,
		Dockerfile:     opts.Dockerfile,
		Status:         "building",
		CreatedAt:      time.Now(),
		UpdatedAt:      time.Now(),
	}

	reg := registry.NewRegistry("localfs", "")
	reg.Set(build)

	// debug code
	builddata, err := reg.Get(build.ID)
	fmt.Fprintln(w, builddata)
}

func root(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	ren.JSON(w, http.StatusOK, map[string]string{"status": "ok"})
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	reg := registry.NewRegistry("localfs", "")
	builds, err := reg.List()
	if err != nil {
		ren.JSON(w, http.StatusInternalServerError, map[string]string{"status": "internal server error"})
	}

	ren.JSON(w, http.StatusOK, builds)
}

func show(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	id := ps.ByName("id")
	uuid := uuid.Parse(id)
	reg := registry.NewRegistry("localfs", "")
	build, err := reg.Get(uuid)
	if err != nil {
		ren.JSON(w, http.StatusNotFound, map[string]string{"status": "not found"})
	}
	ren.JSON(w, http.StatusOK, build)
}

func main() {
	router := httprouter.New()
	router.GET("/", root)
	router.GET("/builds", index)
	router.GET("/builds/:id", show)
	router.POST("/builds", create)

	n := negroni.Classic()
	n.UseHandler(router)
	n.Run(":8080")
}
<|endoftext|>


<commit_before>// Package grbac implements core of RBAC (role-based access control)
//
// https://en.wikipedia.org/wiki/Role-based_access_control
package grbac

import (
	"errors"
	"sync"
)

//Error codes returned by failures to change roles.
var (
	ErrRoleHasAlreadyPerm   = errors.New("role already has permission")
	ErrRoleNotPerm          = errors.New("role does not have permission")
	ErrRoleHasAlreadyParent = errors.New("role already has the parent ")
	ErrNoParent             = errors.New("parent does not exist")
)

// Roler represents a role in RBAC and describes minimum set of functions
// for storing, managing and checking permissions associated with the role.
type Roler interface {
	Name() string
	Permissions() map[string]bool
	AllPermissions() map[string]bool
	Permit(string) error
	IsAllowed(...string) bool
	Revoke(string) error
	Parents() map[string]Roler
	AllParents() map[string]Roler
	HasParent(string) bool
	SetParent(Roler) error
	RemoveParent(string) error
}

// Role is default implementation of Roler.
type Role struct {
	name        string
	permissions map[string]bool
	parents     map[string]Roler

	mutex sync.RWMutex
}

// NewRole creates a new instance of Role structure.
func NewRole(name string) *Role {
	return &Role{
		name:        name,
		permissions: make(map[string]bool),
		parents:     make(map[string]Roler),
		mutex:       sync.RWMutex{},
	}
}

// Name returns the name of the role.
func (r *Role) Name() string {
	return r.name
}

// Permissions returns a copy of the list of the role permissions,
// but does not include parental permissions.
func (r *Role) Permissions() map[string]bool {
	newPerms := make(map[string]bool)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.permissions {
		newPerms[k] = v
	}
	return newPerms
}

// AllPermissions returns a list of all the permissions of the role
// including parental permission.
func (r *Role) AllPermissions() map[string]bool {
	newPerms := make(map[string]bool)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.permissions {
		newPerms[k] = v
	}

	for _, p := range r.parents {
		for k, v := range p.AllPermissions() {
			newPerms[k] = v
		}
	}

	return newPerms
}

// Permit adds a permission for to the role.
//
// Returns ErrRoleHasAlreadyPerm if the role already has permission.
func (r *Role) Permit(perm string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.permissions[perm] {
		return ErrRoleHasAlreadyPerm
	}
	r.permissions[perm] = true
	return nil
}

// IsAllowed checks permissions listed in the perms.
// IsAllowed  returns true only if all permissions from perms are present
// in the role.
func (r *Role) IsAllowed(perms ...string) bool {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for _, perm := range perms {

		if _, ok := r.permissions[perm]; ok {
			continue
		}

		isFound := false
		for _, p := range r.parents {
			if p.IsAllowed(perm) {
				isFound = true
				break
			}
		}

		if !isFound {
			return false
		}
	}

	return true
}

// Revoke revokes permission from the role
// The function returns ErrRoleNotPerm if the role does not have permission
func (r *Role) Revoke(perm string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if !r.permissions[perm] {
		return ErrRoleNotPerm
	}

	delete(r.permissions, perm)
	return nil
}

// Parents returns a map to direct parents of the role.
//
// Key of the map - a name of the parent.
func (r *Role) Parents() map[string]Roler {
	newParents := make(map[string]Roler)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.parents {
		newParents[k] = v
	}
	return newParents
}

// AllParents returns a map of direct parents and subpaprents of the role.
//
// Key of the map - a name of the parent.
func (r *Role) AllParents() map[string]Roler {
	newParents := make(map[string]Roler)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.parents {
		newParents[k] = v
	}

	for _, p := range r.parents {
		for k, v := range p.AllParents() {
			newParents[k] = v
		}
	}

	return newParents
}

// HasParent checks direct parent in the role
func (r *Role) HasParent(name string) bool {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	_, ok := r.parents[name]
	return ok
}

// SetParent adds to the Role a new parent.
// Returns ErrRoleHasAlreadyParent if a parent is already available.
func (r *Role) SetParent(role Roler) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.parents[role.Name()]; ok {
		return ErrRoleHasAlreadyParent
	}

	r.parents[role.Name()] = role
	return nil
}

// RemoveParent remove parent from the role.
func (r *Role) RemoveParent(name string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.parents[name]; !ok {
		return ErrNoParent
	}

	delete(r.parents, name)
	return nil
}
<commit_msg>Remove the extra check from IsAllowed<commit_after>// Package grbac implements core of RBAC (role-based access control)
//
// https://en.wikipedia.org/wiki/Role-based_access_control
package grbac

import (
	"errors"
	"sync"
)

//Error codes returned by failures to change roles.
var (
	ErrRoleHasAlreadyPerm   = errors.New("role already has permission")
	ErrRoleNotPerm          = errors.New("role does not have permission")
	ErrRoleHasAlreadyParent = errors.New("role already has the parent ")
	ErrNoParent             = errors.New("parent does not exist")
)

// Roler represents a role in RBAC and describes minimum set of functions
// for storing, managing and checking permissions associated with the role.
type Roler interface {
	Name() string
	Permissions() map[string]bool
	AllPermissions() map[string]bool
	Permit(string) error
	IsAllowed(...string) bool
	Revoke(string) error
	Parents() map[string]Roler
	AllParents() map[string]Roler
	HasParent(string) bool
	SetParent(Roler) error
	RemoveParent(string) error
}

// Role is default implementation of Roler.
type Role struct {
	name        string
	permissions map[string]bool
	parents     map[string]Roler

	mutex sync.RWMutex
}

// NewRole creates a new instance of Role structure.
func NewRole(name string) *Role {
	return &Role{
		name:        name,
		permissions: make(map[string]bool),
		parents:     make(map[string]Roler),
		mutex:       sync.RWMutex{},
	}
}

// Name returns the name of the role.
func (r *Role) Name() string {
	return r.name
}

// Permissions returns a copy of the list of the role permissions,
// but does not include parental permissions.
func (r *Role) Permissions() map[string]bool {
	newPerms := make(map[string]bool)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.permissions {
		newPerms[k] = v
	}
	return newPerms
}

// AllPermissions returns a list of all the permissions of the role
// including parental permission.
func (r *Role) AllPermissions() map[string]bool {
	newPerms := make(map[string]bool)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.permissions {
		newPerms[k] = v
	}

	for _, p := range r.parents {
		for k, v := range p.AllPermissions() {
			newPerms[k] = v
		}
	}

	return newPerms
}

// Permit adds a permission for to the role.
//
// Returns ErrRoleHasAlreadyPerm if the role already has permission.
func (r *Role) Permit(perm string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.permissions[perm] {
		return ErrRoleHasAlreadyPerm
	}
	r.permissions[perm] = true
	return nil
}

// IsAllowed checks permissions listed in the perms.
// IsAllowed  returns true only if all permissions from perms are present
// in the role.
func (r *Role) IsAllowed(perms ...string) bool {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for _, perm := range perms {

		if r.permissions[perm] {
			continue
		}

		isFound := false
		for _, p := range r.parents {
			if p.IsAllowed(perm) {
				isFound = true
				break
			}
		}

		if !isFound {
			return false
		}
	}

	return true
}

// Revoke revokes permission from the role
// The function returns ErrRoleNotPerm if the role does not have permission
func (r *Role) Revoke(perm string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if !r.permissions[perm] {
		return ErrRoleNotPerm
	}

	delete(r.permissions, perm)
	return nil
}

// Parents returns a map to direct parents of the role.
//
// Key of the map - a name of the parent.
func (r *Role) Parents() map[string]Roler {
	newParents := make(map[string]Roler)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.parents {
		newParents[k] = v
	}
	return newParents
}

// AllParents returns a map of direct parents and subpaprents of the role.
//
// Key of the map - a name of the parent.
func (r *Role) AllParents() map[string]Roler {
	newParents := make(map[string]Roler)

	r.mutex.RLock()
	defer r.mutex.RUnlock()

	for k, v := range r.parents {
		newParents[k] = v
	}

	for _, p := range r.parents {
		for k, v := range p.AllParents() {
			newParents[k] = v
		}
	}

	return newParents
}

// HasParent checks direct parent in the role
func (r *Role) HasParent(name string) bool {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	_, ok := r.parents[name]
	return ok
}

// SetParent adds to the Role a new parent.
// Returns ErrRoleHasAlreadyParent if a parent is already available.
func (r *Role) SetParent(role Roler) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.parents[role.Name()]; ok {
		return ErrRoleHasAlreadyParent
	}

	r.parents[role.Name()] = role
	return nil
}

// RemoveParent remove parent from the role.
func (r *Role) RemoveParent(name string) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.parents[name]; !ok {
		return ErrNoParent
	}

	delete(r.parents, name)
	return nil
}
<|endoftext|>