<commit_before>// run_lua_on_workers is an application that runs the specified lua script on all
// CT workers and uploads the results to Google Storage. The requester is emailed
// when the task is done.
package main

import (
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/ct/go/ctfe/lua_scripts"
	"go.skia.org/infra/ct/go/frontend"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	emails        = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
	gaeTaskID     = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
	pagesetType   = flag.String("pageset_type", "", "The type of pagesets to use. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
	runID         = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")

	taskCompletedSuccessfully     = false
	luaScriptRemoteLink           = util.MASTER_LOGSERVER_LINK
	luaAggregatorRemoteLink       = util.MASTER_LOGSERVER_LINK
	luaOutputRemoteLink           = util.MASTER_LOGSERVER_LINK
	luaAggregatorOutputRemoteLink = util.MASTER_LOGSERVER_LINK
)

func sendEmail(recipients []string) {
	// Send completion email.
	emailSubject := fmt.Sprintf("Run lua script Cluster telemetry task has completed (%s)", *runID)
	failureHtml := ""
	if !taskCompletedSuccessfully {
		emailSubject += " with failures"
		failureHtml = util.GetFailureEmailHtml(*runID)
	}
	bodyTemplate := `
	The Cluster telemetry queued task to run lua script on %s pageset has completed.

	%s
	The output of your script is available here.

	The aggregated output of your script (if specified) is available here.

	You can schedule more runs here.

	Thanks!
	`
	emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, failureHtml, luaOutputRemoteLink, luaAggregatorOutputRemoteLink, frontend.LuaTasksWebapp)
	if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil {
		glog.Errorf("Error while sending email: %s", err)
		return
	}
}

func updateWebappTask() {
	if frontend.CtfeV2 {
		vars := lua_scripts.UpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(taskCompletedSuccessfully)
		vars.ScriptOutput = sql.NullString{String: luaOutputRemoteLink, Valid: true}
		if luaAggregatorOutputRemoteLink != "" {
			vars.AggregatedOutput = sql.NullString{String: luaAggregatorOutputRemoteLink, Valid: true}
		}
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	outputLink := luaOutputRemoteLink
	if luaAggregatorOutputRemoteLink != "" {
		// Use the aggregated output if it exists.
		outputLink = luaAggregatorOutputRemoteLink
	}
	extraData := map[string]string{
		"lua_script_link":     luaScriptRemoteLink,
		"lua_aggregator_link": luaAggregatorRemoteLink,
		"lua_output_link":     outputLink,
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateLuaTasksWebapp, extraData); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}

func main() {
	defer common.LogPanic()
	common.Init()
	frontend.MustInit()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&lua_scripts.UpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Lua script", util.GetMasterLogLink(*runID)))
	// Ensure webapp is updated and email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running Lua script on workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Upload the lua script for this run to Google storage.
	luaScriptName := *runID + ".lua"
	defer skutil.Remove(filepath.Join(os.TempDir(), luaScriptName))
	luaScriptRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "scripts")
	luaScriptRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaScriptName)
	if err := gs.UploadFile(luaScriptName, os.TempDir(), luaScriptRemoteDir); err != nil {
		glog.Errorf("Could not upload %s to %s: %s", luaScriptName, luaScriptRemoteDir, err)
		return
	}

	// Run the run_lua script on all workers.
	runLuaCmdTemplate := "DISPLAY=:0 run_lua --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --log_id={{.RunID}} --pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}};"
	runLuaTemplateParsed := template.Must(template.New("run_lua_cmd").Parse(runLuaCmdTemplate))
	luaCmdBytes := new(bytes.Buffer)
	if err := runLuaTemplateParsed.Execute(luaCmdBytes, struct {
		WorkerNum     string
		LogDir        string
		PagesetType   string
		ChromiumBuild string
		RunID         string
	}{
		WorkerNum:     util.WORKER_NUM_KEYWORD,
		LogDir:        util.GLogDir,
		PagesetType:   *pagesetType,
		ChromiumBuild: *chromiumBuild,
		RunID:         *runID,
	}); err != nil {
		glog.Errorf("Failed to execute template: %s", err)
		return
	}
	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs run_lua on all workers.
		luaCmdBytes.String(),
	}
	_, err = util.SSH(strings.Join(cmd, " "), util.Slaves, util.RUN_LUA_TIMEOUT)
	if err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}

	// Copy outputs from all slaves locally and combine it into one file.
	consolidatedFileName := "lua-output"
	consolidatedLuaOutput := filepath.Join(os.TempDir(), consolidatedFileName)
	if err := ioutil.WriteFile(consolidatedLuaOutput, []byte{}, 0660); err != nil {
		glog.Errorf("Could not create %s: %s", consolidatedLuaOutput, err)
		return
	}
	for i := 0; i < util.NUM_WORKERS; i++ {
		workerNum := i + 1
		workerRemoteOutputPath := filepath.Join(util.LuaRunsDir, *runID, fmt.Sprintf("slave%d", workerNum), "outputs", *runID+".output")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			// TODO(rmistry): Should we instead return here? We can only return
			// here if all 100 slaves reliably run without any failures which they
			// really should.
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.OpenFile(consolidatedLuaOutput, os.O_RDWR|os.O_APPEND, 0660)
		if err != nil {
			glog.Errorf("Unable to open file %s: %s", consolidatedLuaOutput, err)
			return
		}
		defer skutil.Close(out)
		defer skutil.Remove(consolidatedLuaOutput)
		if _, err = io.Copy(out, respBody); err != nil {
			glog.Errorf("Unable to write out %s to %s: %s", workerRemoteOutputPath, consolidatedLuaOutput, err)
			return
		}
	}
	// Copy the consolidated file into Google Storage.
	consolidatedOutputRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "consolidated_outputs")
	luaOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, consolidatedFileName)
	if err := gs.UploadFile(consolidatedFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
		glog.Errorf("Unable to upload %s to %s: %s", consolidatedLuaOutput, consolidatedOutputRemoteDir, err)
		return
	}

	// Upload the lua aggregator (if specified) for this run to Google storage.
	luaAggregatorName := *runID + ".aggregator"
	luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName)
	defer skutil.Remove(luaAggregatorPath)
	luaAggregatorRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaAggregatorName)
	luaAggregatorFileInfo, err := os.Stat(luaAggregatorPath)
	if !os.IsNotExist(err) && luaAggregatorFileInfo.Size() > 10 {
		if err := gs.UploadFile(luaAggregatorName, os.TempDir(), luaScriptRemoteDir); err != nil {
			glog.Errorf("Could not upload %s to %s: %s", luaAggregatorName, luaScriptRemoteDir, err)
			return
		}
		// Run the aggregator and save stdout.
		luaAggregatorOutputFileName := *runID + ".agg.output"
		luaAggregatorOutputFilePath := filepath.Join(os.TempDir(), luaAggregatorOutputFileName)
		luaAggregatorOutputFile, err := os.Create(luaAggregatorOutputFilePath)
		defer skutil.Close(luaAggregatorOutputFile)
		defer skutil.Remove(luaAggregatorOutputFilePath)
		if err != nil {
			glog.Errorf("Could not create %s: %s", luaAggregatorOutputFilePath, err)
			return
		}
		err = util.ExecuteCmd(util.BINARY_LUA, []string{luaAggregatorPath}, []string{},
			util.LUA_AGGREGATOR_TIMEOUT, luaAggregatorOutputFile, nil)
		if err != nil {
			glog.Errorf("Could not execute the lua aggregator %s: %s", luaAggregatorPath, err)
			return
		}
		// Copy the aggregator output into Google Storage.
		luaAggregatorOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, luaAggregatorOutputFileName)
		if err := gs.UploadFile(luaAggregatorOutputFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
			glog.Errorf("Unable to upload %s to %s: %s", luaAggregatorOutputFileName, consolidatedOutputRemoteDir, err)
			return
		}
	} else {
		glog.Info("A lua aggregator has not been specified.")
	}

	taskCompletedSuccessfully = true
}
<commit_msg>Fix lua output links in emails and task updates.<commit_after>// run_lua_on_workers is an application that runs the specified lua script on all
// CT workers and uploads the results to Google Storage. The requester is emailed
// when the task is done.
package main

import (
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/ct/go/ctfe/lua_scripts"
	"go.skia.org/infra/ct/go/frontend"
	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	emails        = flag.String("emails", "", "The comma separated email addresses to notify when the task is picked up and completes.")
	gaeTaskID     = flag.Int64("gae_task_id", -1, "The key of the App Engine task. This task will be updated when the task is completed.")
	pagesetType   = flag.String("pageset_type", "", "The type of pagesets to use. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
	runID         = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")

	taskCompletedSuccessfully     = false
	luaScriptRemoteLink           = util.MASTER_LOGSERVER_LINK
	luaAggregatorRemoteLink       = util.MASTER_LOGSERVER_LINK
	luaOutputRemoteLink           = ""
	luaAggregatorOutputRemoteLink = ""
)

func sendEmail(recipients []string) {
	// Send completion email.
	emailSubject := fmt.Sprintf("Run lua script Cluster telemetry task has completed (%s)", *runID)
	failureHtml := ""
	if !taskCompletedSuccessfully {
		emailSubject += " with failures"
		failureHtml = util.GetFailureEmailHtml(*runID)
	}
	scriptOutputHtml := ""
	if luaOutputRemoteLink != "" {
		scriptOutputHtml = fmt.Sprintf("The output of your script is available here.
\n", luaOutputRemoteLink)
	}
	aggregatorOutputHtml := ""
	if luaAggregatorOutputRemoteLink != "" {
		aggregatorOutputHtml = fmt.Sprintf("The aggregated output of your script is available here.
\n", luaAggregatorOutputRemoteLink)
	}
	bodyTemplate := `
	The Cluster telemetry queued task to run lua script on %s pageset has completed.

	%s
	%s
	%s
	You can schedule more runs here.

	Thanks!
	`
	emailBody := fmt.Sprintf(bodyTemplate, *pagesetType, failureHtml, scriptOutputHtml, aggregatorOutputHtml, frontend.LuaTasksWebapp)
	if err := util.SendEmail(recipients, emailSubject, emailBody); err != nil {
		glog.Errorf("Error while sending email: %s", err)
		return
	}
}

func updateWebappTask() {
	if frontend.CtfeV2 {
		vars := lua_scripts.UpdateVars{}
		vars.Id = *gaeTaskID
		vars.SetCompleted(taskCompletedSuccessfully)
		if luaOutputRemoteLink != "" {
			vars.ScriptOutput = sql.NullString{String: luaOutputRemoteLink, Valid: true}
		}
		if luaAggregatorOutputRemoteLink != "" {
			vars.AggregatedOutput = sql.NullString{String: luaAggregatorOutputRemoteLink, Valid: true}
		}
		skutil.LogErr(frontend.UpdateWebappTaskV2(&vars))
		return
	}
	outputLink := luaOutputRemoteLink
	if luaAggregatorOutputRemoteLink != "" {
		// Use the aggregated output if it exists.
		outputLink = luaAggregatorOutputRemoteLink
	}
	extraData := map[string]string{
		"lua_script_link":     luaScriptRemoteLink,
		"lua_aggregator_link": luaAggregatorRemoteLink,
		"lua_output_link":     outputLink,
	}
	if err := frontend.UpdateWebappTask(*gaeTaskID, frontend.UpdateLuaTasksWebapp, extraData); err != nil {
		glog.Errorf("Error while updating webapp task: %s", err)
		return
	}
}

func main() {
	defer common.LogPanic()
	common.Init()
	frontend.MustInit()

	// Send start email.
	emailsArr := util.ParseEmails(*emails)
	emailsArr = append(emailsArr, util.CtAdmins...)
	if len(emailsArr) == 0 {
		glog.Error("At least one email address must be specified")
		return
	}
	skutil.LogErr(frontend.UpdateWebappTaskSetStarted(&lua_scripts.UpdateVars{}, *gaeTaskID))
	skutil.LogErr(util.SendTaskStartEmail(emailsArr, "Lua script", util.GetMasterLogLink(*runID)))
	// Ensure webapp is updated and email is sent even if task fails.
	defer updateWebappTask()
	defer sendEmail(emailsArr)
	// Cleanup tmp files after the run.
	defer util.CleanTmpDir()
	// Finish with glog flush and how long the task took.
	defer util.TimeTrack(time.Now(), "Running Lua script on workers")
	defer glog.Flush()

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Upload the lua script for this run to Google storage.
	luaScriptName := *runID + ".lua"
	defer skutil.Remove(filepath.Join(os.TempDir(), luaScriptName))
	luaScriptRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "scripts")
	luaScriptRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaScriptName)
	if err := gs.UploadFile(luaScriptName, os.TempDir(), luaScriptRemoteDir); err != nil {
		glog.Errorf("Could not upload %s to %s: %s", luaScriptName, luaScriptRemoteDir, err)
		return
	}

	// Run the run_lua script on all workers.
	runLuaCmdTemplate := "DISPLAY=:0 run_lua --worker_num={{.WorkerNum}} --log_dir={{.LogDir}} --log_id={{.RunID}} --pageset_type={{.PagesetType}} --chromium_build={{.ChromiumBuild}} --run_id={{.RunID}};"
	runLuaTemplateParsed := template.Must(template.New("run_lua_cmd").Parse(runLuaCmdTemplate))
	luaCmdBytes := new(bytes.Buffer)
	if err := runLuaTemplateParsed.Execute(luaCmdBytes, struct {
		WorkerNum     string
		LogDir        string
		PagesetType   string
		ChromiumBuild string
		RunID         string
	}{
		WorkerNum:     util.WORKER_NUM_KEYWORD,
		LogDir:        util.GLogDir,
		PagesetType:   *pagesetType,
		ChromiumBuild: *chromiumBuild,
		RunID:         *runID,
	}); err != nil {
		glog.Errorf("Failed to execute template: %s", err)
		return
	}
	cmd := []string{
		fmt.Sprintf("cd %s;", util.CtTreeDir),
		"git pull;",
		"make all;",
		// The main command that runs run_lua on all workers.
		luaCmdBytes.String(),
	}
	_, err = util.SSH(strings.Join(cmd, " "), util.Slaves, util.RUN_LUA_TIMEOUT)
	if err != nil {
		glog.Errorf("Error while running cmd %s: %s", cmd, err)
		return
	}

	// Copy outputs from all slaves locally and combine it into one file.
	consolidatedFileName := "lua-output"
	consolidatedLuaOutput := filepath.Join(os.TempDir(), consolidatedFileName)
	if err := ioutil.WriteFile(consolidatedLuaOutput, []byte{}, 0660); err != nil {
		glog.Errorf("Could not create %s: %s", consolidatedLuaOutput, err)
		return
	}
	for i := 0; i < util.NUM_WORKERS; i++ {
		workerNum := i + 1
		workerRemoteOutputPath := filepath.Join(util.LuaRunsDir, *runID, fmt.Sprintf("slave%d", workerNum), "outputs", *runID+".output")
		respBody, err := gs.GetRemoteFileContents(workerRemoteOutputPath)
		if err != nil {
			glog.Errorf("Could not fetch %s: %s", workerRemoteOutputPath, err)
			// TODO(rmistry): Should we instead return here? We can only return
			// here if all 100 slaves reliably run without any failures which they
			// really should.
			continue
		}
		defer skutil.Close(respBody)
		out, err := os.OpenFile(consolidatedLuaOutput, os.O_RDWR|os.O_APPEND, 0660)
		if err != nil {
			glog.Errorf("Unable to open file %s: %s", consolidatedLuaOutput, err)
			return
		}
		defer skutil.Close(out)
		defer skutil.Remove(consolidatedLuaOutput)
		if _, err = io.Copy(out, respBody); err != nil {
			glog.Errorf("Unable to write out %s to %s: %s", workerRemoteOutputPath, consolidatedLuaOutput, err)
			return
		}
	}
	// Copy the consolidated file into Google Storage.
	consolidatedOutputRemoteDir := filepath.Join(util.LuaRunsDir, *runID, "consolidated_outputs")
	luaOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, consolidatedFileName)
	if err := gs.UploadFile(consolidatedFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
		glog.Errorf("Unable to upload %s to %s: %s", consolidatedLuaOutput, consolidatedOutputRemoteDir, err)
		return
	}

	// Upload the lua aggregator (if specified) for this run to Google storage.
	luaAggregatorName := *runID + ".aggregator"
	luaAggregatorPath := filepath.Join(os.TempDir(), luaAggregatorName)
	defer skutil.Remove(luaAggregatorPath)
	luaAggregatorRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, luaScriptRemoteDir, luaAggregatorName)
	luaAggregatorFileInfo, err := os.Stat(luaAggregatorPath)
	if !os.IsNotExist(err) && luaAggregatorFileInfo.Size() > 10 {
		if err := gs.UploadFile(luaAggregatorName, os.TempDir(), luaScriptRemoteDir); err != nil {
			glog.Errorf("Could not upload %s to %s: %s", luaAggregatorName, luaScriptRemoteDir, err)
			return
		}
		// Run the aggregator and save stdout.
		luaAggregatorOutputFileName := *runID + ".agg.output"
		luaAggregatorOutputFilePath := filepath.Join(os.TempDir(), luaAggregatorOutputFileName)
		luaAggregatorOutputFile, err := os.Create(luaAggregatorOutputFilePath)
		defer skutil.Close(luaAggregatorOutputFile)
		defer skutil.Remove(luaAggregatorOutputFilePath)
		if err != nil {
			glog.Errorf("Could not create %s: %s", luaAggregatorOutputFilePath, err)
			return
		}
		err = util.ExecuteCmd(util.BINARY_LUA, []string{luaAggregatorPath}, []string{},
			util.LUA_AGGREGATOR_TIMEOUT, luaAggregatorOutputFile, nil)
		if err != nil {
			glog.Errorf("Could not execute the lua aggregator %s: %s", luaAggregatorPath, err)
			return
		}
		// Copy the aggregator output into Google Storage.
		luaAggregatorOutputRemoteLink = util.GS_HTTP_LINK + filepath.Join(util.GS_BUCKET_NAME, consolidatedOutputRemoteDir, luaAggregatorOutputFileName)
		if err := gs.UploadFile(luaAggregatorOutputFileName, os.TempDir(), consolidatedOutputRemoteDir); err != nil {
			glog.Errorf("Unable to upload %s to %s: %s", luaAggregatorOutputFileName, consolidatedOutputRemoteDir, err)
			return
		}
	} else {
		glog.Info("A lua aggregator has not been specified.")
	}

	taskCompletedSuccessfully = true
}
<|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	jwt "github.com/dgrijalva/jwt-go"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/techjanitor/pram-get/config"
	e "github.com/techjanitor/pram-get/errors"
	u "github.com/techjanitor/pram-get/utils"
)

// checks for session cookie and handles permissions
func Auth(perms Permissions) gin.HandlerFunc {
	return func(c *gin.Context) {

		// set default anonymous user
		user := u.User{
			Id:    1,
			Group: 0,
		}

		// parse jwt token if its there
		token, err := jwt.ParseFromRequest(c.Request, func(token *jwt.Token) (interface{}, error) {

			// check alg
			_, ok := token.Method.(*jwt.SigningMethodHMAC)
			if !ok {
				return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
			}

			return []byte(config.Settings.Session.Secret), nil
		})
		// if theres no token set the anon user info
		if err == jwt.ErrNoTokenInRequest {
			err = user.Info()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err)
				return
			}
			// if theres some jwt error then return unauth
		} else if err != nil {
			c.JSON(e.ErrorMessage(e.ErrUnauthorized))
			c.Error(err)
			c.Abort()
			return
		}

		// process token
		if token != nil {
			// if the token is valid set the data
			if err == nil && token.Valid {

				uid, ok := token.Claims["user_id"].(float64)
				if !ok {
					c.JSON(e.ErrorMessage(e.ErrInternalError))
					c.Error(err)
					c.Abort()
					return
				}

				// set user id
				user.Id = uint(uid)

				// get the rest of the user info
				err = user.Info()
				if err == e.ErrNotFound {
					c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrInvalidUser.Error()})
					c.Error(err)
					return
				} else if err != nil {
					c.JSON(e.ErrorMessage(e.ErrInternalError))
					c.Error(err)
					return
				}

			} else {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err)
				return
			}

		}

		// check if user meets set permissions
		if user.Group < perms.Minimum {
			c.JSON(e.ErrorMessage(e.ErrUnauthorized))
			c.Error(e.ErrUnauthorized)
			c.Abort()
			return
		}

		// set user data
		c.Set("userdata", user)

		c.Next()

	}

}

// permissions data
type Permissions struct {
	Minimum uint
}

func SetAuthLevel() Permissions {
	return Permissions{}
}

// All users
func (p Permissions) All() Permissions {
	p.Minimum = 0
	return p
}

// registered users
func (p Permissions) Registered() Permissions {
	p.Minimum = 1
	return p
}

// moderators
func (p Permissions) Moderators() Permissions {
	p.Minimum = 2
	return p
}

// admins
func (p Permissions) Admins() Permissions {
	p.Minimum = 3
	return p
}
<commit_msg>get userdata from auth middleware<commit_after>package middleware

import (
	"fmt"
	jwt "github.com/dgrijalva/jwt-go"
	"github.com/gin-gonic/gin"
	"net/http"

	"github.com/techjanitor/pram-get/config"
	e "github.com/techjanitor/pram-get/errors"
	u "github.com/techjanitor/pram-get/utils"
)

// checks for session cookie and handles permissions
func Auth(perms Permissions) gin.HandlerFunc {
	return func(c *gin.Context) {

		// set default anonymous user
		user := u.User{
			Id:    1,
			Group: 0,
		}

		// parse jwt token if its there
		token, err := jwt.ParseFromRequest(c.Request, func(token *jwt.Token) (interface{}, error) {

			// check alg
			_, ok := token.Method.(*jwt.SigningMethodHMAC)
			if !ok {
				return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
			}

			return []byte(config.Settings.Session.Secret), nil
		})
		if err == jwt.ErrNoTokenInRequest {
			// if theres no token set the anon user info
			err = user.Info()
			if err != nil {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err)
				return
			}
		} else if err != nil {
			// if theres some jwt error then return unauth
			c.JSON(e.ErrorMessage(e.ErrUnauthorized))
			c.Error(err)
			c.Abort()
			return
		}

		// process token
		if token != nil {
			// if the token is valid set the data
			if err == nil && token.Valid {

				uid, ok := token.Claims["user_id"].(float64)
				if !ok {
					c.JSON(e.ErrorMessage(e.ErrInternalError))
					c.Error(err)
					c.Abort()
					return
				}

				// set user id
				user.Id = uint(uid)

				// get the rest of the user info
				err = user.Info()
				if err == e.ErrNotFound {
					c.JSON(http.StatusBadRequest, gin.H{"error_message": e.ErrInvalidUser.Error()})
					c.Error(err)
					return
				} else if err != nil {
					c.JSON(e.ErrorMessage(e.ErrInternalError))
					c.Error(err)
					return
				}

			} else {
				c.JSON(e.ErrorMessage(e.ErrInternalError))
				c.Error(err)
				return
			}

		}

		// check if user meets set permissions
		if user.Group < perms.Minimum {
			c.JSON(e.ErrorMessage(e.ErrUnauthorized))
			c.Error(e.ErrUnauthorized)
			c.Abort()
			return
		}

		// set user data
		c.Set("userdata", user)

		c.Next()

	}

}

// permissions data
type Permissions struct {
	Minimum uint
}

func SetAuthLevel() Permissions {
	return Permissions{}
}

// All users
func (p Permissions) All() Permissions {
	p.Minimum = 0
	return p
}

// registered users
func (p Permissions) Registered() Permissions {
	p.Minimum = 1
	return p
}

// moderators
func (p Permissions) Moderators() Permissions {
	p.Minimum = 2
	return p
}

// admins
func (p Permissions) Admins() Permissions {
	p.Minimum = 3
	return p
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

var graphdef map[string](mp.Graphs) = map[string](mp.Graphs){
	"memcached.connections": mp.Graphs{
		Label: "Memcached Connections",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "curr_connections", Label: "Connections", Diff: false},
		},
	},
	"memcached.cmd": mp.Graphs{
		Label: "Memcached Command",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "cmd_get", Label: "Get", Diff: true},
			mp.Metrics{Name: "cmd_set", Label: "Set", Diff: true},
			mp.Metrics{Name: "cmd_flush", Label: "Flush", Diff: true},
			mp.Metrics{Name: "cmd_touch", Label: "Touch", Diff: true},
		},
	},
	"memcached.hitmiss": mp.Graphs{
		Label: "Memcached Hits/Misses",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "get_hits", Label: "Get Hits", Diff: true},
			mp.Metrics{Name: "get_misses", Label: "Get Misses", Diff: true},
			mp.Metrics{Name: "delete_hits", Label: "Delete Hits", Diff: true},
			mp.Metrics{Name: "delete_misses", Label: "Delete Misses", Diff: true},
			mp.Metrics{Name: "incr_hits", Label: "Incr Hits", Diff: true},
			mp.Metrics{Name: "incr_misses", Label: "Incr Misses", Diff: true},
			mp.Metrics{Name: "cas_hits", Label: "Cas Hits", Diff: true},
			mp.Metrics{Name: "cas_misses", Label: "Cas Misses", Diff: true},
			mp.Metrics{Name: "touch_hits", Label: "Touch Hits", Diff: true},
			mp.Metrics{Name: "touch_misses", Label: "Touch Misses", Diff: true},
		},
	},
	"memcached.evictions": mp.Graphs{
		Label: "Memcached Evictions",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "evictions", Label: "Evictions", Diff: true},
		},
	},
	"memcached.unfetched": mp.Graphs{
		Label: "Memcached Unfetched",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "expired_unfetched", Label: "Expired unfetched", Diff: true},
			mp.Metrics{Name: "evicted_unfetched", Label: "Evicted unfetched", Diff: true},
		},
	},
	"memcached.rusage": mp.Graphs{
		Label: "Memcached Resouce Usage",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "rusage_user", Label: "User", Diff: true},
			mp.Metrics{Name: "rusage_system", Label: "System", Diff: true},
		},
	},
	"memcached.bytes": mp.Graphs{
		Label: "Memcached Traffics",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "bytes_read", Label: "Read", Diff: true},
			mp.Metrics{Name: "bytes_written", Label: "Write", Diff: true},
		},
	},
}

type MemcachedPlugin struct {
	Target   string
	Tempfile string
}

func (m MemcachedPlugin) FetchMetrics() (map[string]float64, error) {
	conn, err := net.Dial("tcp", m.Target)
	if err != nil {
		return nil, err
	}
	fmt.Fprintln(conn, "stats")
	scanner := bufio.NewScanner(conn)
	stat := make(map[string]float64)

	for scanner.Scan() {
		line := scanner.Text()
		s := string(line)
		if s == "END" {
			return stat, nil
		}

		res := strings.Split(s, " ")
		if res[0] == "STAT" {
			stat[res[1]], err = strconv.ParseFloat(res[2], 64)
			if err != nil {
				log.Println("FetchMetrics:", err)
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return stat, err
	}
	return nil, nil
}

func (m MemcachedPlugin) GraphDefinition() map[string](mp.Graphs) {
	return graphdef
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "11211", "Port")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var memcached MemcachedPlugin
	helper := mp.NewMackerelPlugin(memcached)

	memcached.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-memcached-%s-%s", *optHost, *optPort)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>Set memcached.Target property and exec mp.NewMackerelPlugin method needs to be swapped.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	mp "github.com/mackerelio/go-mackerel-plugin"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
)

var graphdef map[string](mp.Graphs) = map[string](mp.Graphs){
	"memcached.connections": mp.Graphs{
		Label: "Memcached Connections",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "curr_connections", Label: "Connections", Diff: false},
		},
	},
	"memcached.cmd": mp.Graphs{
		Label: "Memcached Command",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "cmd_get", Label: "Get", Diff: true},
			mp.Metrics{Name: "cmd_set", Label: "Set", Diff: true},
			mp.Metrics{Name: "cmd_flush", Label: "Flush", Diff: true},
			mp.Metrics{Name: "cmd_touch", Label: "Touch", Diff: true},
		},
	},
	"memcached.hitmiss": mp.Graphs{
		Label: "Memcached Hits/Misses",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "get_hits", Label: "Get Hits", Diff: true},
			mp.Metrics{Name: "get_misses", Label: "Get Misses", Diff: true},
			mp.Metrics{Name: "delete_hits", Label: "Delete Hits", Diff: true},
			mp.Metrics{Name: "delete_misses", Label: "Delete Misses", Diff: true},
			mp.Metrics{Name: "incr_hits", Label: "Incr Hits", Diff: true},
			mp.Metrics{Name: "incr_misses", Label: "Incr Misses", Diff: true},
			mp.Metrics{Name: "cas_hits", Label: "Cas Hits", Diff: true},
			mp.Metrics{Name: "cas_misses", Label: "Cas Misses", Diff: true},
			mp.Metrics{Name: "touch_hits", Label: "Touch Hits", Diff: true},
			mp.Metrics{Name: "touch_misses", Label: "Touch Misses", Diff: true},
		},
	},
	"memcached.evictions": mp.Graphs{
		Label: "Memcached Evictions",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "evictions", Label: "Evictions", Diff: true},
		},
	},
	"memcached.unfetched": mp.Graphs{
		Label: "Memcached Unfetched",
		Unit:  "integer",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "expired_unfetched", Label: "Expired unfetched", Diff: true},
			mp.Metrics{Name: "evicted_unfetched", Label: "Evicted unfetched", Diff: true},
		},
	},
	"memcached.rusage": mp.Graphs{
		Label: "Memcached Resouce Usage",
		Unit:  "float",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "rusage_user", Label: "User", Diff: true},
			mp.Metrics{Name: "rusage_system", Label: "System", Diff: true},
		},
	},
	"memcached.bytes": mp.Graphs{
		Label: "Memcached Traffics",
		Unit:  "bytes",
		Metrics: [](mp.Metrics){
			mp.Metrics{Name: "bytes_read", Label: "Read", Diff: true},
			mp.Metrics{Name: "bytes_written", Label: "Write", Diff: true},
		},
	},
}

type MemcachedPlugin struct {
	Target   string
	Tempfile string
}

func (m MemcachedPlugin) FetchMetrics() (map[string]float64, error) {
	conn, err := net.Dial("tcp", m.Target)
	if err != nil {
		return nil, err
	}
	fmt.Fprintln(conn, "stats")
	scanner := bufio.NewScanner(conn)
	stat := make(map[string]float64)

	for scanner.Scan() {
		line := scanner.Text()
		s := string(line)
		if s == "END" {
			return stat, nil
		}

		res := strings.Split(s, " ")
		if res[0] == "STAT" {
			stat[res[1]], err = strconv.ParseFloat(res[2], 64)
			if err != nil {
				log.Println("FetchMetrics:", err)
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return stat, err
	}
	return nil, nil
}

func (m MemcachedPlugin) GraphDefinition() map[string](mp.Graphs) {
	return graphdef
}

func main() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "11211", "Port")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var memcached MemcachedPlugin

	memcached.Target = fmt.Sprintf("%s:%s", *optHost, *optPort)
	helper := mp.NewMackerelPlugin(memcached)

	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-memcached-%s-%s", *optHost, *optPort)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>// Copyright ©2017 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.7

package c64

import (
	"fmt"
	"testing"
)

func BenchmarkDotUnitary(t *testing.B) {
	for _, tst := range []struct {
		name string
		f    func(x, y []complex64) complex64
	}{
		{"DotcUnitary", DotcUnitary},
		{"DotuUnitary", DotuUnitary},
	} {
		for _, v := range []int64{1, 2, 3, 4, 5, 10, 100, 1e3, 5e3, 1e4, 5e4} {
			t.Run(fmt.Sprintf("%s-%d", tst.name, v), func(b *testing.B) {
				x, y := x[:v], y[:v]
				b.SetBytes(128 * v)
				for i := 0; i < b.N; i++ {
					_ = tst.f(x, y)
				}
			})
		}
	}
}

func BenchmarkDotInc(t *testing.B) {
	for _, tst := range []struct {
		name string
		f    func(x, y []complex64, n, incX, incY, ix, iy uintptr) complex64
	}{
		{"DotcInc", DotcInc},
		{"DotuInc", DotuInc},
	} {
		for _, ln := range []int{1, 2, 3, 4, 5, 10, 100, 1e3, 5e3, 1e4, 5e4} {
			for _, inc := range []int{1, 2, 4, 10, -1, -2, -4, -10} {
				t.Run(fmt.Sprintf("%s-%d-inc%d", tst.name, ln, inc), func(b *testing.B) {
					b.SetBytes(int64(128 * ln))
					var idx int
					if inc < 0 {
						idx = (-ln + 1) * inc
					}
					for i := 0; i < b.N; i++ {
						_ = tst.f(x, y, uintptr(ln), uintptr(inc), uintptr(inc), uintptr(idx), uintptr(idx))
					}
				})
			}
		}
	}
}
<commit_msg>asm/c64: Adding benchmark sink variable.<commit_after>// Copyright ©2017 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build go1.7

package c64

import (
	"fmt"
	"testing"
)

var benchSink complex64

func BenchmarkDotUnitary(t *testing.B) {
	for _, tst := range []struct {
		name string
		f    func(x, y []complex64) complex64
	}{
		{"DotcUnitary", DotcUnitary},
		{"DotuUnitary", DotuUnitary},
	} {
		for _, v := range []int64{1, 2, 3, 4, 5, 10, 100, 1e3, 5e3, 1e4, 5e4} {
			t.Run(fmt.Sprintf("%s-%d", tst.name, v), func(b *testing.B) {
				x, y := x[:v], y[:v]
				b.SetBytes(128 * v)
				for i := 0; i < b.N; i++ {
					benchSink = tst.f(x, y)
				}
			})
		}
	}
}

func BenchmarkDotInc(t *testing.B) {
	for _, tst := range []struct {
		name string
		f    func(x, y []complex64, n, incX, incY, ix, iy uintptr) complex64
	}{
		{"DotcInc", DotcInc},
		{"DotuInc", DotuInc},
	} {
		for _, ln := range []int{1, 2, 3, 4, 5, 10, 100, 1e3, 5e3, 1e4, 5e4} {
			for _, inc := range []int{1, 2, 4, 10, -1, -2, -4, -10} {
				t.Run(fmt.Sprintf("%s-%d-inc%d", tst.name, ln, inc), func(b *testing.B) {
					b.SetBytes(int64(128 * ln))
					var idx int
					if inc < 0 {
						idx = (-ln + 1) * inc
					}
					for i := 0; i < b.N; i++ {
						benchSink = tst.f(x, y, uintptr(ln), uintptr(inc), uintptr(inc), uintptr(idx), uintptr(idx))
					}
				})
			}
		}
	}
}
<|endoftext|>


<commit_before>package config

import "sync"

type State int

const (
	StateStopped State = iota
	StateStarting
	StateRunning
	StateStopping
)

func (s State) String() string {
	switch s {
	case StateStopped:
		return "Stopped"
	case StateStarting:
		return "Starting"
	case StateRunning:
		return "Running"
	case StateStopping:
		return "Stopping"
	}
	return ""
}

type ServerSettings struct {
	ServerName          string
	ListenAddr          string
	DirServers, DirMaps string
}

type Server struct {
	ID   int
	Path string

	mu    sync.RWMutex
	Name  string
	Args  []string
	Map   int
	State State `json:",omitempty"`
}

type Map struct {
	ID   int
	Path string

	mu     sync.RWMutex
	Name   string
	Server int
}
<commit_msg>Mutexes now exported<commit_after>package config

import "sync"

type State int

const (
	StateStopped State = iota
	StateStarting
	StateRunning
	StateStopping
)

func (s State) String() string {
	switch s {
	case StateStopped:
		return "Stopped"
	case StateStarting:
		return "Starting"
	case StateRunning:
		return "Running"
	case StateStopping:
		return "Stopping"
	}
	return ""
}

type ServerSettings struct {
	ServerName          string
	ListenAddr          string
	DirServers, DirMaps string
}

type Server struct {
	ID   int
	Path string

	sync.RWMutex `json:"-"`
	Name         string
	Args         []string
	Map          int
	State        State `json:",omitempty"`
}

type Map struct {
	ID   int
	Path string

	sync.RWMutex `json:"-"`
	Name         string
	Server       int
}
<|endoftext|>


<commit_before>// Package ctlsock implementes the control socket interface that can be
// activated by passing "-ctlsock" on the command line.
package ctlsock

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"syscall"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// Interface should be implemented by fusefrontend[_reverse]
type Interface interface {
	EncryptPath(string) (string, error)
	DecryptPath(string) (string, error)
}

// RequestStruct is sent by a client
type RequestStruct struct {
	EncryptPath string
	DecryptPath string
}

// ResponseStruct is sent by us as response to a request
type ResponseStruct struct {
	// Result is the resulting decrypted or encrypted path. Empty on error.
	Result string
	// ErrNo is the error number as defined in errno.h.
	// 0 means success and -1 means that the error number is not known
	// (look at ErrText in this case).
	ErrNo int32
	// ErrText is a detailed error message.
	ErrText string
	// WarnText contains warnings that may have been encountered while
	// processing the message.
	WarnText string
}

type ctlSockHandler struct {
	fs     Interface
	socket *net.UnixListener
}

// Serve serves incoming connections on "sock". This call blocks so you
// probably want to run it in a new goroutine.
func Serve(sock net.Listener, fs Interface) {
	handler := ctlSockHandler{
		fs:     fs,
		socket: sock.(*net.UnixListener),
	}
	handler.acceptLoop()
}

func (ch *ctlSockHandler) acceptLoop() {
	for {
		conn, err := ch.socket.Accept()
		if err != nil {
			tlog.Warn.Printf("ctlsock: Accept error: %v", err)
			break
		}
		go ch.handleConnection(conn.(*net.UnixConn))
	}
}

func (ch *ctlSockHandler) handleConnection(conn *net.UnixConn) {
	// 2*PATH_MAX is definitely big enough for requests to decrypt or
	// encrypt paths.
	buf := make([]byte, 2*syscall.PathMax)
	for {
		n, err := conn.Read(buf)
		if err == io.EOF {
			conn.Close()
			return
		} else if err != nil {
			tlog.Warn.Printf("ctlsock: Read error: %#v", err)
			conn.Close()
			return
		}
		buf = buf[:n]
		var in RequestStruct
		err = json.Unmarshal(buf, &in)
		if err != nil {
			tlog.Warn.Printf("ctlsock: Unmarshal error: %#v", err)
			errorMsg := ResponseStruct{
				ErrNo:   int32(syscall.EINVAL),
				ErrText: err.Error(),
			}
			sendResponse(&errorMsg, conn)
		}
		ch.handleRequest(&in, conn)
		// Restore original size.
		buf = buf[:cap(buf)]
	}
}

func (ch *ctlSockHandler) handleRequest(in *RequestStruct, conn *net.UnixConn) {
	var err error
	var out ResponseStruct
	var inPath, clean string
	if in.DecryptPath != "" && in.EncryptPath != "" {
		err = errors.New("Ambigous")
	} else if in.DecryptPath == "" && in.EncryptPath == "" {
		err = errors.New("No operation")
	} else if in.DecryptPath != "" {
		inPath = in.DecryptPath
		clean = SanitizePath(inPath)
		out.Result, err = ch.fs.DecryptPath(clean)
	} else if in.EncryptPath != "" {
		inPath = in.EncryptPath
		clean = SanitizePath(inPath)
		out.Result, err = ch.fs.EncryptPath(clean)
	}
	if err != nil {
		out.ErrText = err.Error()
		out.ErrNo = -1
		// Try to extract the actual error number
		if pe, ok := err.(*os.PathError); ok {
			if se, ok := pe.Err.(syscall.Errno); ok {
				out.ErrNo = int32(se)
			}
		}
	}
	if inPath != clean {
		out.WarnText = fmt.Sprintf("Non-canonical input path %q has been interpreted as %q", inPath, clean)
	}
	sendResponse(&out, conn)
}

func sendResponse(msg *ResponseStruct, conn *net.UnixConn) {
	jsonMsg, err := json.Marshal(msg)
	if err != nil {
		tlog.Warn.Printf("ctlsock: Marshal failed: %v", err)
		return
	}
	// For convenience for the user, add a newline at the end.
	jsonMsg = append(jsonMsg, '\n')
	_, err = conn.Write(jsonMsg)
	if err != nil {
		tlog.Warn.Printf("ctlsock: Write failed: %v", err)
	}
}
<commit_msg>ctlsock: add a note about Accept() throwing errors on Close()<commit_after>// Package ctlsock implementes the control socket interface that can be
// activated by passing "-ctlsock" on the command line.
package ctlsock

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"syscall"

	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// Interface should be implemented by fusefrontend[_reverse]
type Interface interface {
	EncryptPath(string) (string, error)
	DecryptPath(string) (string, error)
}

// RequestStruct is sent by a client
type RequestStruct struct {
	EncryptPath string
	DecryptPath string
}

// ResponseStruct is sent by us as response to a request
type ResponseStruct struct {
	// Result is the resulting decrypted or encrypted path. Empty on error.
	Result string
	// ErrNo is the error number as defined in errno.h.
	// 0 means success and -1 means that the error number is not known
	// (look at ErrText in this case).
	ErrNo int32
	// ErrText is a detailed error message.
	ErrText string
	// WarnText contains warnings that may have been encountered while
	// processing the message.
	WarnText string
}

type ctlSockHandler struct {
	fs     Interface
	socket *net.UnixListener
}

// Serve serves incoming connections on "sock". This call blocks so you
// probably want to run it in a new goroutine.
func Serve(sock net.Listener, fs Interface) {
	handler := ctlSockHandler{
		fs:     fs,
		socket: sock.(*net.UnixListener),
	}
	handler.acceptLoop()
}

func (ch *ctlSockHandler) acceptLoop() {
	for {
		conn, err := ch.socket.Accept()
		if err != nil {
			// TODO Can this warning trigger when the socket it closed on
			// program exit? I have never observed it, but the documentation
			// says that Close() unblocks Accept().
			tlog.Warn.Printf("ctlsock: Accept error: %v", err)
			break
		}
		go ch.handleConnection(conn.(*net.UnixConn))
	}
}

func (ch *ctlSockHandler) handleConnection(conn *net.UnixConn) {
	// 2*PATH_MAX is definitely big enough for requests to decrypt or
	// encrypt paths.
	buf := make([]byte, 2*syscall.PathMax)
	for {
		n, err := conn.Read(buf)
		if err == io.EOF {
			conn.Close()
			return
		} else if err != nil {
			tlog.Warn.Printf("ctlsock: Read error: %#v", err)
			conn.Close()
			return
		}
		buf = buf[:n]
		var in RequestStruct
		err = json.Unmarshal(buf, &in)
		if err != nil {
			tlog.Warn.Printf("ctlsock: Unmarshal error: %#v", err)
			errorMsg := ResponseStruct{
				ErrNo:   int32(syscall.EINVAL),
				ErrText: err.Error(),
			}
			sendResponse(&errorMsg, conn)
		}
		ch.handleRequest(&in, conn)
		// Restore original size.
		buf = buf[:cap(buf)]
	}
}

func (ch *ctlSockHandler) handleRequest(in *RequestStruct, conn *net.UnixConn) {
	var err error
	var out ResponseStruct
	var inPath, clean string
	if in.DecryptPath != "" && in.EncryptPath != "" {
		err = errors.New("Ambigous")
	} else if in.DecryptPath == "" && in.EncryptPath == "" {
		err = errors.New("No operation")
	} else if in.DecryptPath != "" {
		inPath = in.DecryptPath
		clean = SanitizePath(inPath)
		out.Result, err = ch.fs.DecryptPath(clean)
	} else if in.EncryptPath != "" {
		inPath = in.EncryptPath
		clean = SanitizePath(inPath)
		out.Result, err = ch.fs.EncryptPath(clean)
	}
	if err != nil {
		out.ErrText = err.Error()
		out.ErrNo = -1
		// Try to extract the actual error number
		if pe, ok := err.(*os.PathError); ok {
			if se, ok := pe.Err.(syscall.Errno); ok {
				out.ErrNo = int32(se)
			}
		}
	}
	if inPath != clean {
		out.WarnText = fmt.Sprintf("Non-canonical input path %q has been interpreted as %q", inPath, clean)
	}
	sendResponse(&out, conn)
}

func sendResponse(msg *ResponseStruct, conn *net.UnixConn) {
	jsonMsg, err := json.Marshal(msg)
	if err != nil {
		tlog.Warn.Printf("ctlsock: Marshal failed: %v", err)
		return
	}
	// For convenience for the user, add a newline at the end.
	jsonMsg = append(jsonMsg, '\n')
	_, err = conn.Write(jsonMsg)
	if err != nil {
		tlog.Warn.Printf("ctlsock: Write failed: %v", err)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"time"

	"github.com/coreos/agro"
	"github.com/coreos/agro/models"
	"github.com/coreos/agro/ring"
	"github.com/coreos/pkg/capnslog"
)

type RebalanceStrategy int32

type Rebalancer interface {
	Leader(chans [2]chan *models.RebalanceStatus)
	AdvanceState(s *models.RebalanceStatus) (*models.RebalanceStatus, bool, error)
	OnError(error) *models.RebalanceStatus
	Timeout()
}

const (
	Error   RebalanceStrategy = iota
	Replace                   = 1
	Full                      = 2
)

type makeRebalanceFunc func(d *distributor, newring agro.Ring) Rebalancer

var (
	rebalanceTimeout   = 30 * time.Second
	rebalancerRegistry = make(map[RebalanceStrategy]makeRebalanceFunc)
	rlog               = capnslog.NewPackageLogger("github.com/coreos/agro", "rebalancer")
)

// Goroutine which watches for new rings and kicks off
// the rebalance dance.
func (d *distributor) rebalanceWatcher(closer chan struct{}) {
	ch := make(chan agro.Ring)
	d.srv.mds.SubscribeNewRings(ch)
exit:
	for {
		select {
		case <-closer:
			d.srv.mds.UnsubscribeNewRings(ch)
			close(ch)
			break exit
		case newring, ok := <-ch:
			if ok {
				if newring.Version() == d.ring.Version() {
					// No problem. We're seeing the same ring.
					continue
				}
				if newring.Version() != d.ring.Version()+1 {
					panic("replacing old ring with ring in the far future!")
				}
				d.Rebalance(newring)
			} else {
				break exit
			}
		}
	}
}

func (d *distributor) Rebalance(newring agro.Ring) {
	d.srv.updatePeerMap()
	isMember := d.ring.Members().Union(newring.Members()).Has(d.UUID())
	if !isMember {
		clog.Infof("rebalance detected, but not a member")
	}
	// TODO(barakmich): Rebalancing is tricky. But here's the entry point.
	clog.Infof("rebalancing beginning: new ring version %d for %s", newring.Version(), d.UUID())
	chans, leader, err := d.srv.mds.OpenRebalanceChannels()
	if err != nil {
		clog.Error(err)
		return
	}
	if leader {
		clog.Infof("elected as leader: %s", d.UUID())
		d.rebalanceLeader(chans, newring)
	} else {
		clog.Infof("elected to follow")
		d.rebalanceFollower(chans, newring)
	}
	d.mut.Lock()
	defer d.mut.Unlock()
	d.rebalancer = nil
}

func (d *distributor) rebalanceLeader(chans [2]chan *models.RebalanceStatus, newring agro.Ring) {
	var re Rebalancer
	switch d.ring.Type() {
	case ring.Empty:
		// We can always replace the empty ring.
		clog.Infof("replacing empty ring")
		re = rebalancerRegistry[Replace](d, newring)
	default:
		re = rebalancerRegistry[Full](d, newring)
	}
	d.mut.Lock()
	d.rebalancer = re
	d.mut.Unlock()
	re.Leader(chans)
	d.srv.mut.Lock()
	defer d.srv.mut.Unlock()
	clog.Info("leader: success, setting new ring")
	d.ring = newring
	d.srv.mds.SetRing(newring, true)
	close(chans[1])
}

func (d *distributor) rebalanceFollower(inOut [2]chan *models.RebalanceStatus, newring agro.Ring) {
	in, out := inOut[0], inOut[1]
	for {
		s := <-in
		if !s.FromLeader {
			panic("got a message not from leader")
		}
		if d.rebalancer == nil {
			d.mut.Lock()
			rlog.Debugf("creating rebalancer %d", s.RebalanceType)
			d.rebalancer = rebalancerRegistry[RebalanceStrategy(s.RebalanceType)](d, newring)
			d.mut.Unlock()
		}
		news, done, err := d.rebalancer.AdvanceState(s)
		if err != nil {
			clog.Error(err)
			stat := d.rebalancer.OnError(err)
			if stat != nil {
				out <- stat
			}
			close(out)
			return
		}
		news.UUID = d.UUID()
		out <- news
		if done {
			close(out)
			d.srv.mut.Lock()
			defer d.srv.mut.Unlock()
			clog.Info("follower: success, setting new ring")
			d.ring = newring
			return
		}
	}
}

func waitAll(c chan *models.RebalanceStatus, peerList agro.PeerList, phase int32) error {
	for len(peerList) > 0 {

		// TODO(barakmich) Check if the status is an error, such as the TTL of
		// the key being lost in etcd (thus a machine has timed out and we're in
		// trouble). LEASES.
		stat, ok := <-c
		if !ok {
			clog.Error("close before end of rebalance")
			return agro.ErrClosed
		}
		if stat.Phase == phase {
			for i, m := range peerList {
				if m == stat.UUID {
					clog.Debugf("got response from %s", stat.UUID)
					peerList = append(peerList[:i], peerList[i+1:]...)
					break
				}
			}
		}
	}
	clog.Debugf("finished waiting for members")
	return nil
}
<commit_msg>no, really, remove rebalancer. Always replace the ring.<commit_after>package server

import (
	"time"

	"github.com/coreos/agro"
)

var (
	rebalanceTimeout = 30 * time.Second
)

// Goroutine which watches for new rings and kicks off
// the rebalance dance.
func (d *distributor) rebalanceWatcher(closer chan struct{}) {
	ch := make(chan agro.Ring)
	d.srv.mds.SubscribeNewRings(ch)
exit:
	for {
		select {
		case <-closer:
			d.srv.mds.UnsubscribeNewRings(ch)
			close(ch)
			break exit
		case newring, ok := <-ch:
			if ok {
				if newring.Version() == d.ring.Version() {
					// No problem. We're seeing the same ring.
					continue
				}
				if newring.Version() != d.ring.Version()+1 {
					panic("replacing old ring with ring in the far future!")
				}
				d.mut.Lock()
				d.ring = newring
				d.mut.Unlock()
			} else {
				break exit
			}
		}
	}
}
<|endoftext|>


<commit_before>//go:generate struct-markdown

package vmx

import (
	"fmt"
	"os"

	vmwcommon "github.com/hashicorp/packer/builder/vmware/common"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/common/bootcommand"
	"github.com/hashicorp/packer/common/shutdowncommand"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

// Config is the configuration structure for the builder.
type Config struct {
	common.PackerConfig            `mapstructure:",squash"`
	common.HTTPConfig              `mapstructure:",squash"`
	common.FloppyConfig            `mapstructure:",squash"`
	bootcommand.VNCConfig          `mapstructure:",squash"`
	vmwcommon.DriverConfig         `mapstructure:",squash"`
	vmwcommon.OutputConfig         `mapstructure:",squash"`
	vmwcommon.RunConfig            `mapstructure:",squash"`
	shutdowncommand.ShutdownConfig `mapstructure:",squash"`
	vmwcommon.SSHConfig            `mapstructure:",squash"`
	vmwcommon.ToolsConfig          `mapstructure:",squash"`
	vmwcommon.VMXConfig            `mapstructure:",squash"`
	vmwcommon.ExportConfig         `mapstructure:",squash"`
	// By default Packer creates a 'full' clone of the virtual machine
	// specified in source_path. The resultant virtual machine is fully
	// independant from the parent it was cloned from.
	//
	// Setting linked to true instead causes Packer to create the virtual
	// machine as a 'linked' clone. Linked clones use and require ongoing
	// access to the disks of the parent virtual machine. The benefit of a
	// linked clone is that the clones virtual disk is typically very much
	// smaller than would be the case for a full clone. Additionally, the
	// cloned virtual machine can also be created much faster. Creating a
	// linked clone will typically only be of benefit in some advanced build
	// scenarios. Most users will wish to create a full clone instead. Defaults
	// to false.
	Linked bool `mapstructure:"linked" required:"false"`
	// The type of remote machine that will be used to
	// build this VM rather than a local desktop product. The only value accepted
	// for this currently is esx5. If this is not set, a desktop product will
	// be used. By default, this is not set.
	RemoteType string `mapstructure:"remote_type" required:"false"`
	// Path to the source VMX file to clone. If
	// remote_type is enabled then this specifies a path on the remote_host.
	SourcePath string `mapstructure:"source_path" required:"true"`
	// This is the name of the VMX file for the new virtual
	// machine, without the file extension. By default this is packer-BUILDNAME,
	// where "BUILDNAME" is the name of the build.
	VMName string `mapstructure:"vm_name" required:"false"`

	ctx interpolate.Context
}

func NewConfig(raws ...interface{}) (*Config, []string, error) {
	c := new(Config)
	err := config.Decode(c, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &c.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"boot_command",
				"tools_upload_path",
			},
		},
	}, raws...)
	if err != nil {
		return nil, nil, err
	}

	// Defaults
	if c.VMName == "" {
		c.VMName = fmt.Sprintf(
			"packer-%s-%d", c.PackerBuildName, interpolate.InitTime.Unix())
	}

	// Prepare the errors
	var errs *packer.MultiError
	errs = packer.MultiErrorAppend(errs, c.DriverConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.HTTPConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.OutputConfig.Prepare(&c.ctx, &c.PackerConfig)...)
	errs = packer.MultiErrorAppend(errs, c.RunConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ShutdownConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.SSHConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ToolsConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.VMXConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.FloppyConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.VNCConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ExportConfig.Prepare(&c.ctx)...)

	if c.RemoteType == "" {
		if c.SourcePath == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("source_path is blank, but is required"))
		} else {
			if _, err := os.Stat(c.SourcePath); err != nil {
				errs = packer.MultiErrorAppend(errs,
					fmt.Errorf("source_path is invalid: %s", err))
			}
		}
	} else {
		// Remote configuration validation
		if c.RemoteHost == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("remote_host must be specified"))
		}

		if c.RemoteType != "esx5" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Only 'esx5' value is accepted for remote_type"))
		}
	}

	err = c.DriverConfig.Validate(c.SkipExport)
	if err != nil {
		errs = packer.MultiErrorAppend(errs, err)
	}

	if c.Format != "" {
		if c.RemoteType != "esx5" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("format is only valid when remote_type=esx5"))
		}
	} else {
		c.Format = "ovf"
	}

	if !(c.Format == "ova" || c.Format == "ovf" || c.Format == "vmx") {
		errs = packer.MultiErrorAppend(errs,
			fmt.Errorf("format must be one of ova, ovf, or vmx"))
	}

	// Warnings
	var warnings []string
	if c.ShutdownCommand == "" {
		warnings = append(warnings,
			"A shutdown_command was not specified. Without a shutdown command, Packer\n"+
				"will forcibly halt the virtual machine, which may result in data loss.")
	}

	if c.Headless && c.DisableVNC {
		warnings = append(warnings,
			"Headless mode uses VNC to retrieve output. Since VNC has been disabled,\n"+
				"you won't be able to see any output.")
	}

	// Check for any errors.
	if errs != nil && len(errs.Errors) > 0 {
		return nil, warnings, errs
	}

	return c, warnings, nil
}
<commit_msg>builder.vmware.vmx: remove duplicate RemoteType field.<commit_after>//go:generate struct-markdown

package vmx

import (
	"fmt"
	"os"

	vmwcommon "github.com/hashicorp/packer/builder/vmware/common"
	"github.com/hashicorp/packer/common"
	"github.com/hashicorp/packer/common/bootcommand"
	"github.com/hashicorp/packer/common/shutdowncommand"
	"github.com/hashicorp/packer/helper/config"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template/interpolate"
)

// Config is the configuration structure for the builder.
type Config struct {
	common.PackerConfig            `mapstructure:",squash"`
	common.HTTPConfig              `mapstructure:",squash"`
	common.FloppyConfig            `mapstructure:",squash"`
	bootcommand.VNCConfig          `mapstructure:",squash"`
	vmwcommon.DriverConfig         `mapstructure:",squash"`
	vmwcommon.OutputConfig         `mapstructure:",squash"`
	vmwcommon.RunConfig            `mapstructure:",squash"`
	shutdowncommand.ShutdownConfig `mapstructure:",squash"`
	vmwcommon.SSHConfig            `mapstructure:",squash"`
	vmwcommon.ToolsConfig          `mapstructure:",squash"`
	vmwcommon.VMXConfig            `mapstructure:",squash"`
	vmwcommon.ExportConfig         `mapstructure:",squash"`
	// By default Packer creates a 'full' clone of the virtual machine
	// specified in source_path. The resultant virtual machine is fully
	// independant from the parent it was cloned from.
	//
	// Setting linked to true instead causes Packer to create the virtual
	// machine as a 'linked' clone. Linked clones use and require ongoing
	// access to the disks of the parent virtual machine. The benefit of a
	// linked clone is that the clones virtual disk is typically very much
	// smaller than would be the case for a full clone. Additionally, the
	// cloned virtual machine can also be created much faster. Creating a
	// linked clone will typically only be of benefit in some advanced build
	// scenarios. Most users will wish to create a full clone instead. Defaults
	// to false.
	Linked bool `mapstructure:"linked" required:"false"`
	// Path to the source VMX file to clone. If
	// remote_type is enabled then this specifies a path on the remote_host.
	SourcePath string `mapstructure:"source_path" required:"true"`
	// This is the name of the VMX file for the new virtual
	// machine, without the file extension. By default this is packer-BUILDNAME,
	// where "BUILDNAME" is the name of the build.
	VMName string `mapstructure:"vm_name" required:"false"`

	ctx interpolate.Context
}

func NewConfig(raws ...interface{}) (*Config, []string, error) {
	c := new(Config)
	err := config.Decode(c, &config.DecodeOpts{
		Interpolate:        true,
		InterpolateContext: &c.ctx,
		InterpolateFilter: &interpolate.RenderFilter{
			Exclude: []string{
				"boot_command",
				"tools_upload_path",
			},
		},
	}, raws...)
	if err != nil {
		return nil, nil, err
	}

	// Defaults
	if c.VMName == "" {
		c.VMName = fmt.Sprintf(
			"packer-%s-%d", c.PackerBuildName, interpolate.InitTime.Unix())
	}

	// Prepare the errors
	var errs *packer.MultiError
	errs = packer.MultiErrorAppend(errs, c.DriverConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.HTTPConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.OutputConfig.Prepare(&c.ctx, &c.PackerConfig)...)
	errs = packer.MultiErrorAppend(errs, c.RunConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ShutdownConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.SSHConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ToolsConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.VMXConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.FloppyConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.VNCConfig.Prepare(&c.ctx)...)
	errs = packer.MultiErrorAppend(errs, c.ExportConfig.Prepare(&c.ctx)...)

	if c.RemoteType == "" {
		if c.SourcePath == "" {
			errs = packer.MultiErrorAppend(errs, fmt.Errorf("source_path is blank, but is required"))
		} else {
			if _, err := os.Stat(c.SourcePath); err != nil {
				errs = packer.MultiErrorAppend(errs,
					fmt.Errorf("source_path is invalid: %s", err))
			}
		}
	} else {
		// Remote configuration validation
		if c.RemoteHost == "" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("remote_host must be specified"))
		}

		if c.RemoteType != "esx5" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("Only 'esx5' value is accepted for remote_type"))
		}
	}

	err = c.DriverConfig.Validate(c.SkipExport)
	if err != nil {
		errs = packer.MultiErrorAppend(errs, err)
	}

	if c.Format != "" {
		if c.RemoteType != "esx5" {
			errs = packer.MultiErrorAppend(errs,
				fmt.Errorf("format is only valid when remote_type=esx5"))
		}
	} else {
		c.Format = "ovf"
	}

	if !(c.Format == "ova" || c.Format == "ovf" || c.Format == "vmx") {
		errs = packer.MultiErrorAppend(errs,
			fmt.Errorf("format must be one of ova, ovf, or vmx"))
	}

	// Warnings
	var warnings []string
	if c.ShutdownCommand == "" {
		warnings = append(warnings,
			"A shutdown_command was not specified. Without a shutdown command, Packer\n"+
				"will forcibly halt the virtual machine, which may result in data loss.")
	}

	if c.Headless && c.DisableVNC {
		warnings = append(warnings,
			"Headless mode uses VNC to retrieve output. Since VNC has been disabled,\n"+
				"you won't be able to see any output.")
	}

	// Check for any errors.
	if errs != nil && len(errs.Errors) > 0 {
		return nil, warnings, errs
	}

	return c, warnings, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"sort"
	"strings"

	"github.com/lib/pq"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/database"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/postgres/symbolsearch"
	"golang.org/x/sync/errgroup"
)

func upsertSymbolSearchDocuments(ctx context.Context, tx *database.DB,
	modulePath, v string) (err error) {
	defer derrors.Wrap(&err, "upsertSymbolSearchDocuments(ctx, ddb, %q, %q)", modulePath, v)

	if !experiment.IsActive(ctx, internal.ExperimentInsertSymbolSearchDocuments) {
		return nil
	}

	// If a user is looking for the symbol "DB.Begin", from package
	// database/sql, we want them to be able to find this by searching for
	// "DB.Begin" and "sql.DB.Begin". Searching for "sql.DB", "DB", "Begin" or
	// "sql.DB" will not return "DB.Begin".
	// If a user is looking for the symbol "DB.Begin", from package
	// database/sql, we want them to be able to find this by searching for
	// "DB.Begin", "Begin", and "sql.DB.Begin". Searching for "sql.DB" or
	// "DB" will not return "DB.Begin".
	q := `
		INSERT INTO symbol_search_documents (
			package_path_id,
			symbol_name_id,
			unit_id,
			package_symbol_id,
			goos,
			goarch,
			package_name,
			package_path,
			imported_by_count
		)
		SELECT DISTINCT ON (sd.package_path_id, ps.symbol_name_id)
			sd.package_path_id,
			ps.symbol_name_id,
			sd.unit_id,
			ps.id AS package_symbol_id,
			d.goos,
			d.goarch,
			sd.name,
			sd.package_path,
			sd.imported_by_count
		FROM search_documents sd
		INNER JOIN units u ON sd.unit_id = u.id
		INNER JOIN documentation d ON d.unit_id = sd.unit_id
		INNER JOIN documentation_symbols ds ON d.id = ds.documentation_id
		INNER JOIN package_symbols ps ON ps.id = ds.package_symbol_id
		WHERE
			sd.module_path = $1 AND sd.version = $2
			AND u.name != 'main' -- do not insert data for commands
			AND sd.redistributable
		ORDER BY
			sd.package_path_id,
			ps.symbol_name_id,
			-- Order should match internal.BuildContexts.
			CASE WHEN d.goos = 'all' THEN 0
			WHEN d.goos = 'linux' THEN 1
			WHEN d.goos = 'windows' THEN 2
			WHEN d.goos = 'darwin' THEN 3
			WHEN d.goos = 'js' THEN 4
			END
		ON CONFLICT (package_path_id, symbol_name_id)
		DO UPDATE SET
			unit_id = excluded.unit_id,
			package_symbol_id = excluded.package_symbol_id,
			goos = excluded.goos,
			goarch = excluded.goarch,
			package_name = excluded.package_name,
			package_path = excluded.package_path,
			imported_by_count = excluded.imported_by_count;`
	_, err = tx.Exec(ctx, q, modulePath, v)
	return err
}

// symbolSearch searches all symbols in the symbol_search_documents table for
// the query.
//
// TODO(https://golang.org/issue/44142): factor out common code between
// symbolSearch and deepSearch.
func (db *DB) symbolSearch(ctx context.Context, q string, limit, offset, maxResultCount int) searchResponse {
	var (
		results []*SearchResult
		err     error
	)
	sr := searchResponse{source: "symbol"}
	it := symbolsearch.ParseInputType(q)
	switch it {
	case symbolsearch.InputTypeOneDot:
		results, err = runSymbolSearchOneDot(ctx, db.db, q, limit)
	case symbolsearch.InputTypeMultiWord:
		results, err = runSymbolSearchMultiWord(ctx, db.db, q, limit)
	case symbolsearch.InputTypeNoDot:
		results, err = runSymbolSearch(ctx, db.db, symbolsearch.SearchTypeSymbol, q, limit)
	case symbolsearch.InputTypeTwoDots:
		results, err = runSymbolSearch(ctx, db.db, symbolsearch.SearchTypePackageDotSymbol, q, limit, q)
	default:
		// There is no supported situation where we will get results for one
		// element containing more than 2 dots.
		return sr
	}

	if len(results) == 0 {
		if err != nil && !errors.Is(err, derrors.NotFound) {
			sr.err = err
		}
		return sr
	}
	sort.Slice(results, func(i, j int) bool {
		if results[i].NumImportedBy == results[j].NumImportedBy {
			return results[i].SymbolName < results[j].SymbolName
		}
		return results[i].NumImportedBy > results[j].NumImportedBy
	})
	if len(results) > limit {
		results = results[0:limit]
	}
	for _, r := range results {
		r.NumResults = uint64(len(results))
	}
	sr.results = results
	return sr
}

// runSymbolSearchMultiWord executes a symbol search for SearchTypeMultiWord.
func runSymbolSearchMultiWord(ctx context.Context, ddb *database.DB, q string, limit int) (_ []*SearchResult, err error) {
	defer derrors.Wrap(&err, "runSymbolSearchMultiWord(ctx, ddb, query, %q, %d)", q, limit)
	symbolToPathTokens := multiwordSearchCombinations(q)
	if len(symbolToPathTokens) == 0 {
		// There are no words in the query that could be a symbol name.
		return nil, derrors.NotFound
	}
	group, searchCtx := errgroup.WithContext(ctx)
	resultsArray := make([][]*SearchResult, len(symbolToPathTokens))
	count := 0
	for symbol, pathTokens := range symbolToPathTokens {
		symbol := symbol
		pathTokens := pathTokens
		i := count
		count += 1
		group.Go(func() error {
			st := symbolsearch.SearchTypeMultiWordExact
			if strings.Contains(q, "|") {
				st = symbolsearch.SearchTypeMultiWordOr
			}
			ids, err := fetchMatchingSymbolIDs(searchCtx, ddb, st, symbol)
			if err != nil {
				if !errors.Is(err, derrors.NotFound) {
					return err
				}
				return nil
			}
			r, err := fetchSymbolSearchResults(ctx, ddb, st, ids, limit, pathTokens)
			if err != nil {
				return err
			}
			resultsArray[i] = r
			return nil
		})
	}
	if err := group.Wait(); err != nil {
		return nil, err
	}
	return mergedResults(resultsArray, limit), nil
}

func mergedResults(resultsArray [][]*SearchResult, limit int) []*SearchResult {
	var results []*SearchResult
	for _, r := range resultsArray {
		results = append(results, r...)
	}
	sort.Slice(results, func(i, j int) bool { return results[i].NumImportedBy > results[j].NumImportedBy })
	if len(results) > limit {
		results = results[0:limit]
	}
	return results
}

// multiwordSearchCombinations returns a map of symbol name to path_tokens to
// be used for possible search combinations.
//
// For each word, check if there is an invalid symbol character or if it
// matches a common hostname. If so, the search on tsv_path_tokens must match
// that search.
//
// It is assumed that the symbol name is always 1 word. For example, if the
// user wants sql.DB.Begin, "sql DB.Begin", "sql Begin", or "sql DB" will all
// be return the relevant result, but "sql DB Begin" will not.
func multiwordSearchCombinations(q string) map[string]string {
	words := strings.Fields(q)
	symbolToPathTokens := map[string]string{}
	for i, w := range words {
		// Is this word a possible symbol name? If not, continue.
		if strings.Contains(w, "/") || strings.Contains(w, "-") || commonHostnames[w] {
			continue
		}
		// If it is, try search for this word assuming it is the symbol name
		// and everything else is a path element.
		symbolToPathTokens[w] = strings.Join(append(append([]string{}, words[0:i]...), words[i+1:]...), " & ")
	}
	if len(symbolToPathTokens) > 2 {
		// There are more than 2 possible searches that can be performed, so
		// just perform an OR query.
		orQuery := strings.Join(strings.Fields(q), " | ")
		return map[string]string{orQuery: orQuery}
	}
	return symbolToPathTokens
}

// runSymbolSearchOneDot is used when q contains only 1 dot, so the search must
// either be for <package>.<symbol> or <type>.<methodOrFieldName>.
//
// This search is split into two parallel queries, since the query is very slow
// when using an OR in the WHERE clause.
func runSymbolSearchOneDot(ctx context.Context, ddb *database.DB, q string, limit int) (_ []*SearchResult, err error) {
	group, searchCtx := errgroup.WithContext(ctx)
	resultsArray := make([][]*SearchResult, 2)
	for i, st := range []symbolsearch.SearchType{
		symbolsearch.SearchTypeSymbol,
		symbolsearch.SearchTypePackageDotSymbol,
	} {
		i := i
		st := st
		group.Go(func() error {
			var args []interface{}
			if st == symbolsearch.SearchTypePackageDotSymbol {
				args = append(args, q)
			}
			results, err := runSymbolSearch(searchCtx, ddb, st, q, limit, args...)
			if err != nil {
				return err
			}
			resultsArray[i] = results
			return nil
		})
	}
	if err := group.Wait(); err != nil {
		return nil, err
	}
	return mergedResults(resultsArray, limit), nil
}

func runSymbolSearch(ctx context.Context, ddb *database.DB,
	st symbolsearch.SearchType, q string, limit int, args ...interface{}) (_ []*SearchResult, err error) {
	defer derrors.Wrap(&err, "runSymbolSearch(ctx, ddb, query, %q, %d)", q, limit)
	ids, err := fetchMatchingSymbolIDs(ctx, ddb, st, q)
	if err != nil {
		if errors.Is(err, derrors.NotFound) {
			return nil, nil
		}
		return nil, err
	}
	return fetchSymbolSearchResults(ctx, ddb, st, ids, limit, args...)
}

// fetchMatchingSymbolIDs fetches the symbol ids to be used for a given
// symbolsearch.SearchType. It runs the query returned by
// symbolsearch.MatchingSymbolIDsQuery. The ids returned will be used by in
// runSymbolSearch.
func fetchMatchingSymbolIDs(ctx context.Context, ddb *database.DB, st symbolsearch.SearchType, q string) (_ []int, err error) {
	defer derrors.Wrap(&err, "fetchMatchingSymbolIDs(ctx, ddb, %d, %q)", st, q)
	var ids []int
	collect := func(rows *sql.Rows) error {
		var id int
		if err := rows.Scan(&id); err != nil {
			return err
		}
		ids = append(ids, id)
		return nil
	}
	query := symbolsearch.MatchingSymbolIDsQuery(st)
	if err := ddb.RunQuery(ctx, query, collect, q); err != nil {
		return nil, err
	}
	if len(ids) == 0 {
		return nil, derrors.NotFound
	}
	return ids, nil
}

// fetchSymbolSearchResults executes a symbol search for the given
// symbolsearch.SearchType and args.
func fetchSymbolSearchResults(ctx context.Context, ddb *database.DB,
	st symbolsearch.SearchType, ids []int, limit int, args ...interface{}) (results []*SearchResult, err error) {
	defer derrors.Wrap(&err, "fetchSymbolSearchResults(ctx, ddb, st: %d, ids: %v, limit:  %d, args: %v)", st, ids, limit, args)
	collect := func(rows *sql.Rows) error {
		var r SearchResult
		if err := rows.Scan(
			&r.SymbolName,
			&r.PackagePath,
			&r.ModulePath,
			&r.Version,
			&r.Name,
			&r.Synopsis,
			pq.Array(&r.Licenses),
			&r.CommitTime,
			&r.NumImportedBy,
			&r.SymbolGOOS,
			&r.SymbolGOARCH,
			&r.SymbolKind,
			&r.SymbolSynopsis); err != nil {
			return fmt.Errorf("symbolSearch: rows.Scan(): %v", err)
		}
		results = append(results, &r)
		return nil
	}
	query := symbolsearch.Query(st)
	args = append([]interface{}{pq.Array(ids), limit}, args...)
	if err := ddb.RunQuery(ctx, query, collect, args...); err != nil {
		return nil, err
	}
	return results, nil
}
<commit_msg>internal/postgres: change symbolsearch results sort<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"sort"
	"strings"

	"github.com/lib/pq"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/database"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/experiment"
	"golang.org/x/pkgsite/internal/postgres/symbolsearch"
	"golang.org/x/sync/errgroup"
)

func upsertSymbolSearchDocuments(ctx context.Context, tx *database.DB,
	modulePath, v string) (err error) {
	defer derrors.Wrap(&err, "upsertSymbolSearchDocuments(ctx, ddb, %q, %q)", modulePath, v)

	if !experiment.IsActive(ctx, internal.ExperimentInsertSymbolSearchDocuments) {
		return nil
	}

	// If a user is looking for the symbol "DB.Begin", from package
	// database/sql, we want them to be able to find this by searching for
	// "DB.Begin" and "sql.DB.Begin". Searching for "sql.DB", "DB", "Begin" or
	// "sql.DB" will not return "DB.Begin".
	// If a user is looking for the symbol "DB.Begin", from package
	// database/sql, we want them to be able to find this by searching for
	// "DB.Begin", "Begin", and "sql.DB.Begin". Searching for "sql.DB" or
	// "DB" will not return "DB.Begin".
	q := `
		INSERT INTO symbol_search_documents (
			package_path_id,
			symbol_name_id,
			unit_id,
			package_symbol_id,
			goos,
			goarch,
			package_name,
			package_path,
			imported_by_count
		)
		SELECT DISTINCT ON (sd.package_path_id, ps.symbol_name_id)
			sd.package_path_id,
			ps.symbol_name_id,
			sd.unit_id,
			ps.id AS package_symbol_id,
			d.goos,
			d.goarch,
			sd.name,
			sd.package_path,
			sd.imported_by_count
		FROM search_documents sd
		INNER JOIN units u ON sd.unit_id = u.id
		INNER JOIN documentation d ON d.unit_id = sd.unit_id
		INNER JOIN documentation_symbols ds ON d.id = ds.documentation_id
		INNER JOIN package_symbols ps ON ps.id = ds.package_symbol_id
		WHERE
			sd.module_path = $1 AND sd.version = $2
			AND u.name != 'main' -- do not insert data for commands
			AND sd.redistributable
		ORDER BY
			sd.package_path_id,
			ps.symbol_name_id,
			-- Order should match internal.BuildContexts.
			CASE WHEN d.goos = 'all' THEN 0
			WHEN d.goos = 'linux' THEN 1
			WHEN d.goos = 'windows' THEN 2
			WHEN d.goos = 'darwin' THEN 3
			WHEN d.goos = 'js' THEN 4
			END
		ON CONFLICT (package_path_id, symbol_name_id)
		DO UPDATE SET
			unit_id = excluded.unit_id,
			package_symbol_id = excluded.package_symbol_id,
			goos = excluded.goos,
			goarch = excluded.goarch,
			package_name = excluded.package_name,
			package_path = excluded.package_path,
			imported_by_count = excluded.imported_by_count;`
	_, err = tx.Exec(ctx, q, modulePath, v)
	return err
}

// symbolSearch searches all symbols in the symbol_search_documents table for
// the query.
//
// TODO(https://golang.org/issue/44142): factor out common code between
// symbolSearch and deepSearch.
func (db *DB) symbolSearch(ctx context.Context, q string, limit, offset, maxResultCount int) searchResponse {
	var (
		results []*SearchResult
		err     error
	)
	sr := searchResponse{source: "symbol"}
	it := symbolsearch.ParseInputType(q)
	switch it {
	case symbolsearch.InputTypeOneDot:
		results, err = runSymbolSearchOneDot(ctx, db.db, q, limit)
	case symbolsearch.InputTypeMultiWord:
		results, err = runSymbolSearchMultiWord(ctx, db.db, q, limit)
	case symbolsearch.InputTypeNoDot:
		results, err = runSymbolSearch(ctx, db.db, symbolsearch.SearchTypeSymbol, q, limit)
	case symbolsearch.InputTypeTwoDots:
		results, err = runSymbolSearch(ctx, db.db, symbolsearch.SearchTypePackageDotSymbol, q, limit, q)
	default:
		// There is no supported situation where we will get results for one
		// element containing more than 2 dots.
		return sr
	}

	if len(results) == 0 {
		if err != nil && !errors.Is(err, derrors.NotFound) {
			sr.err = err
		}
		return sr
	}
	sort.Slice(results, func(i, j int) bool {
		if results[i].NumImportedBy != results[j].NumImportedBy {
			return results[i].NumImportedBy > results[j].NumImportedBy
		}

		// If two packages have the same imported by count, return them in
		// alphabetical order by packge path.
		if results[i].PackagePath != results[j].PackagePath {
			return results[i].PackagePath < results[j].PackagePath
		}

		// If one package has multiple matching symbols, return them by
		// alphabetical order of symbol name.
		return results[i].SymbolName < results[j].SymbolName
	})
	if len(results) > limit {
		results = results[0:limit]
	}
	for _, r := range results {
		r.NumResults = uint64(len(results))
	}
	sr.results = results
	return sr
}

// runSymbolSearchMultiWord executes a symbol search for SearchTypeMultiWord.
func runSymbolSearchMultiWord(ctx context.Context, ddb *database.DB, q string, limit int) (_ []*SearchResult, err error) {
	defer derrors.Wrap(&err, "runSymbolSearchMultiWord(ctx, ddb, query, %q, %d)", q, limit)
	symbolToPathTokens := multiwordSearchCombinations(q)
	if len(symbolToPathTokens) == 0 {
		// There are no words in the query that could be a symbol name.
		return nil, derrors.NotFound
	}
	group, searchCtx := errgroup.WithContext(ctx)
	resultsArray := make([][]*SearchResult, len(symbolToPathTokens))
	count := 0
	for symbol, pathTokens := range symbolToPathTokens {
		symbol := symbol
		pathTokens := pathTokens
		i := count
		count += 1
		group.Go(func() error {
			st := symbolsearch.SearchTypeMultiWordExact
			if strings.Contains(q, "|") {
				st = symbolsearch.SearchTypeMultiWordOr
			}
			ids, err := fetchMatchingSymbolIDs(searchCtx, ddb, st, symbol)
			if err != nil {
				if !errors.Is(err, derrors.NotFound) {
					return err
				}
				return nil
			}
			r, err := fetchSymbolSearchResults(ctx, ddb, st, ids, limit, pathTokens)
			if err != nil {
				return err
			}
			resultsArray[i] = r
			return nil
		})
	}
	if err := group.Wait(); err != nil {
		return nil, err
	}
	return mergedResults(resultsArray, limit), nil
}

func mergedResults(resultsArray [][]*SearchResult, limit int) []*SearchResult {
	var results []*SearchResult
	for _, r := range resultsArray {
		results = append(results, r...)
	}
	sort.Slice(results, func(i, j int) bool { return results[i].NumImportedBy > results[j].NumImportedBy })
	if len(results) > limit {
		results = results[0:limit]
	}
	return results
}

// multiwordSearchCombinations returns a map of symbol name to path_tokens to
// be used for possible search combinations.
//
// For each word, check if there is an invalid symbol character or if it
// matches a common hostname. If so, the search on tsv_path_tokens must match
// that search.
//
// It is assumed that the symbol name is always 1 word. For example, if the
// user wants sql.DB.Begin, "sql DB.Begin", "sql Begin", or "sql DB" will all
// be return the relevant result, but "sql DB Begin" will not.
func multiwordSearchCombinations(q string) map[string]string {
	words := strings.Fields(q)
	symbolToPathTokens := map[string]string{}
	for i, w := range words {
		// Is this word a possible symbol name? If not, continue.
		if strings.Contains(w, "/") || strings.Contains(w, "-") || commonHostnames[w] {
			continue
		}
		// If it is, try search for this word assuming it is the symbol name
		// and everything else is a path element.
		symbolToPathTokens[w] = strings.Join(append(append([]string{}, words[0:i]...), words[i+1:]...), " & ")
	}
	if len(symbolToPathTokens) > 2 {
		// There are more than 2 possible searches that can be performed, so
		// just perform an OR query.
		orQuery := strings.Join(strings.Fields(q), " | ")
		return map[string]string{orQuery: orQuery}
	}
	return symbolToPathTokens
}

// runSymbolSearchOneDot is used when q contains only 1 dot, so the search must
// either be for <package>.<symbol> or <type>.<methodOrFieldName>.
//
// This search is split into two parallel queries, since the query is very slow
// when using an OR in the WHERE clause.
func runSymbolSearchOneDot(ctx context.Context, ddb *database.DB, q string, limit int) (_ []*SearchResult, err error) {
	group, searchCtx := errgroup.WithContext(ctx)
	resultsArray := make([][]*SearchResult, 2)
	for i, st := range []symbolsearch.SearchType{
		symbolsearch.SearchTypeSymbol,
		symbolsearch.SearchTypePackageDotSymbol,
	} {
		i := i
		st := st
		group.Go(func() error {
			var args []interface{}
			if st == symbolsearch.SearchTypePackageDotSymbol {
				args = append(args, q)
			}
			results, err := runSymbolSearch(searchCtx, ddb, st, q, limit, args...)
			if err != nil {
				return err
			}
			resultsArray[i] = results
			return nil
		})
	}
	if err := group.Wait(); err != nil {
		return nil, err
	}
	return mergedResults(resultsArray, limit), nil
}

func runSymbolSearch(ctx context.Context, ddb *database.DB,
	st symbolsearch.SearchType, q string, limit int, args ...interface{}) (_ []*SearchResult, err error) {
	defer derrors.Wrap(&err, "runSymbolSearch(ctx, ddb, query, %q, %d)", q, limit)
	ids, err := fetchMatchingSymbolIDs(ctx, ddb, st, q)
	if err != nil {
		if errors.Is(err, derrors.NotFound) {
			return nil, nil
		}
		return nil, err
	}
	return fetchSymbolSearchResults(ctx, ddb, st, ids, limit, args...)
}

// fetchMatchingSymbolIDs fetches the symbol ids to be used for a given
// symbolsearch.SearchType. It runs the query returned by
// symbolsearch.MatchingSymbolIDsQuery. The ids returned will be used by in
// runSymbolSearch.
func fetchMatchingSymbolIDs(ctx context.Context, ddb *database.DB, st symbolsearch.SearchType, q string) (_ []int, err error) {
	defer derrors.Wrap(&err, "fetchMatchingSymbolIDs(ctx, ddb, %d, %q)", st, q)
	var ids []int
	collect := func(rows *sql.Rows) error {
		var id int
		if err := rows.Scan(&id); err != nil {
			return err
		}
		ids = append(ids, id)
		return nil
	}
	query := symbolsearch.MatchingSymbolIDsQuery(st)
	if err := ddb.RunQuery(ctx, query, collect, q); err != nil {
		return nil, err
	}
	if len(ids) == 0 {
		return nil, derrors.NotFound
	}
	return ids, nil
}

// fetchSymbolSearchResults executes a symbol search for the given
// symbolsearch.SearchType and args.
func fetchSymbolSearchResults(ctx context.Context, ddb *database.DB,
	st symbolsearch.SearchType, ids []int, limit int, args ...interface{}) (results []*SearchResult, err error) {
	defer derrors.Wrap(&err, "fetchSymbolSearchResults(ctx, ddb, st: %d, ids: %v, limit:  %d, args: %v)", st, ids, limit, args)
	collect := func(rows *sql.Rows) error {
		var r SearchResult
		if err := rows.Scan(
			&r.SymbolName,
			&r.PackagePath,
			&r.ModulePath,
			&r.Version,
			&r.Name,
			&r.Synopsis,
			pq.Array(&r.Licenses),
			&r.CommitTime,
			&r.NumImportedBy,
			&r.SymbolGOOS,
			&r.SymbolGOARCH,
			&r.SymbolKind,
			&r.SymbolSynopsis); err != nil {
			return fmt.Errorf("symbolSearch: rows.Scan(): %v", err)
		}
		results = append(results, &r)
		return nil
	}
	query := symbolsearch.Query(st)
	args = append([]interface{}{pq.Array(ids), limit}, args...)
	if err := ddb.RunQuery(ctx, query, collect, args...); err != nil {
		return nil, err
	}
	return results, nil
}
<|endoftext|>


<commit_before>package web

import (
	"errors"
	"net"
	"net/http"
	"regexp"
	"strings"

	res "github.com/micro/go-micro/v2/api/resolver"
	"github.com/micro/go-micro/v2/client/selector"
	"github.com/micro/micro/v2/internal/namespace"
	"golang.org/x/net/publicsuffix"
)

var (
	re               = regexp.MustCompile("^[a-zA-Z0-9]+([a-zA-Z0-9-]*[a-zA-Z0-9]*)?$")
	defaultNamespace = namespace.DefaultNamespace + ".web"
)

type Resolver struct {
	// Type of resolver e.g path, domain
	Type string
	// a function which returns the namespace of the request
	Namespace func(*http.Request) string
	// selector to find services
	Selector selector.Selector
}

func reverse(s []string) {
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
}

func (r *Resolver) String() string {
	return "web/resolver"
}

// Info checks whether this is a web request.
// It returns host, namespace and whether its internal
func (r *Resolver) Info(req *http.Request) (string, string, bool) {
	// set to host
	host := req.URL.Hostname()

	// set as req.Host if blank
	if len(host) == 0 {
		host = req.Host
	}

	// split out ip
	if h, _, err := net.SplitHostPort(host); err == nil {
		host = h
	}

	// determine the namespace of the request
	namespace := r.Namespace(req)

	// overide host if the namespace is go.micro.web, since
	// this will also catch localhost & 127.0.0.1, resulting
	// in a more consistent dev experience
	if host == "localhost" || host == "127.0.0.1" {
		host = "web.micro.mu"
	}

	// if the type is path, always resolve using the path
	if r.Type == "path" {
		return host, namespace, true
	}

	// if the namespace is not the default (go.micro.web),
	// we always resolve using path
	if namespace != defaultNamespace {
		return host, namespace, true
	}

	// check for micro subdomains, we want to do subdomain routing
	// on these if the subdomoain routing has been specified
	if r.Type == "subdomain" && host != "web.micro.mu" && strings.HasSuffix(host, ".micro.mu") {
		return host, namespace, false
	}

	// Check if the request is a top level path
	isWeb := strings.Count(req.URL.Path, "/") == 1
	return host, namespace, isWeb
}

// Resolve replaces the values of Host, Path, Scheme to calla backend service
// It accounts for subdomains for service names based on namespace
func (r *Resolver) Resolve(req *http.Request) (*res.Endpoint, error) {
	// get host, namespace and if its an internal request
	host, _, _ := r.Info(req)

	// check for micro web
	if r.Type == "path" || host == "web.micro.mu" {
		return r.resolveWithPath(req)
	}

	domain, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		return nil, err
	}

	// get and reverse the subdomain
	subdomain := strings.TrimSuffix(host, "."+domain)
	parts := strings.Split(subdomain, ".")
	reverse(parts)

	// turn it into an alias
	alias := strings.Join(parts, ".")
	if len(alias) == 0 {
		return nil, errors.New("unknown host")
	}

	var name string
	if strings.HasSuffix(host, ".micro.mu") {
		// for micro.mu subdomains, we route foo.micro.mu/bar to
		// go.micro.web.bar
		name = defaultNamespace + "." + alias
	} else if comps := strings.Split(req.URL.Path, "/"); len(comps) > 0 {
		// for non micro.mu subdomains, we route foo.m3o.app/bar to
		// foo.web.bar
		name = alias + ".web." + comps[1]
	}

	// find the service using the selector
	next, err := r.Selector.Select(name)
	if err == selector.ErrNotFound {
		// fallback to path based
		return r.resolveWithPath(req)
	} else if err != nil {
		return nil, err
	}

	// TODO: better retry strategy
	s, err := next()
	if err != nil {
		return nil, err
	}

	// we're done
	return &res.Endpoint{
		Name:   alias,
		Method: req.Method,
		Host:   s.Address,
		Path:   req.URL.Path,
	}, nil
}

func (r *Resolver) resolveWithPath(req *http.Request) (*res.Endpoint, error) {
	parts := strings.Split(req.URL.Path, "/")
	if len(parts) < 2 {
		return nil, errors.New("unknown service")
	}

	if !re.MatchString(parts[1]) {
		return nil, res.ErrInvalidPath
	}

	_, namespace, _ := r.Info(req)
	next, err := r.Selector.Select(namespace + "." + parts[1])
	if err == selector.ErrNotFound {
		return nil, res.ErrNotFound
	} else if err != nil {
		return nil, err
	}

	// TODO: better retry strategy
	s, err := next()
	if err != nil {
		return nil, err
	}

	// we're done
	return &res.Endpoint{
		Name:   parts[1],
		Method: req.Method,
		Host:   s.Address,
		Path:   "/" + strings.Join(parts[2:], "/"),
	}, nil
}
<commit_msg>Fix /services/* path (#790)<commit_after>package web

import (
	"errors"
	"net"
	"net/http"
	"regexp"
	"strings"

	res "github.com/micro/go-micro/v2/api/resolver"
	"github.com/micro/go-micro/v2/client/selector"
	"github.com/micro/micro/v2/internal/namespace"
	"golang.org/x/net/publicsuffix"
)

var (
	re               = regexp.MustCompile("^[a-zA-Z0-9]+([a-zA-Z0-9-]*[a-zA-Z0-9]*)?$")
	defaultNamespace = namespace.DefaultNamespace + ".web"
)

type Resolver struct {
	// Type of resolver e.g path, domain
	Type string
	// a function which returns the namespace of the request
	Namespace func(*http.Request) string
	// selector to find services
	Selector selector.Selector
}

func reverse(s []string) {
	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
		s[i], s[j] = s[j], s[i]
	}
}

func (r *Resolver) String() string {
	return "web/resolver"
}

// Info checks whether this is a web request.
// It returns host, namespace and whether its internal
func (r *Resolver) Info(req *http.Request) (string, string, bool) {
	// set to host
	host := req.URL.Hostname()

	// set as req.Host if blank
	if len(host) == 0 {
		host = req.Host
	}

	// split out ip
	if h, _, err := net.SplitHostPort(host); err == nil {
		host = h
	}

	// determine the namespace of the request
	namespace := r.Namespace(req)

	// overide host if the namespace is go.micro.web, since
	// this will also catch localhost & 127.0.0.1, resulting
	// in a more consistent dev experience
	if host == "localhost" || host == "127.0.0.1" {
		host = "web.micro.mu"
	}

	// if the type is path, always resolve using the path
	if r.Type == "path" {
		return host, namespace, true
	}

	// if the namespace is not the default (go.micro.web),
	// we always resolve using path
	if namespace != defaultNamespace {
		return host, namespace, true
	}

	// check for micro subdomains, we want to do subdomain routing
	// on these if the subdomoain routing has been specified
	if r.Type == "subdomain" && host != "web.micro.mu" && strings.HasSuffix(host, ".micro.mu") {
		return host, namespace, false
	}

	// Check for services info path, also handled by micro web but
	// not a top level path. TODO: Find a better way of detecting and
	// handling the non-proxied paths.
	if strings.HasPrefix(req.URL.Path, "/service/") {
		return host, namespace, true
	}

	// Check if the request is a top level path
	isWeb := strings.Count(req.URL.Path, "/") == 1
	return host, namespace, isWeb
}

// Resolve replaces the values of Host, Path, Scheme to calla backend service
// It accounts for subdomains for service names based on namespace
func (r *Resolver) Resolve(req *http.Request) (*res.Endpoint, error) {
	// get host, namespace and if its an internal request
	host, _, _ := r.Info(req)

	// check for micro web
	if r.Type == "path" || host == "web.micro.mu" {
		return r.resolveWithPath(req)
	}

	domain, err := publicsuffix.EffectiveTLDPlusOne(host)
	if err != nil {
		return nil, err
	}

	// get and reverse the subdomain
	subdomain := strings.TrimSuffix(host, "."+domain)
	parts := strings.Split(subdomain, ".")
	reverse(parts)

	// turn it into an alias
	alias := strings.Join(parts, ".")
	if len(alias) == 0 {
		return nil, errors.New("unknown host")
	}

	var name string
	if strings.HasSuffix(host, ".micro.mu") {
		// for micro.mu subdomains, we route foo.micro.mu/bar to
		// go.micro.web.bar
		name = defaultNamespace + "." + alias
	} else if comps := strings.Split(req.URL.Path, "/"); len(comps) > 0 {
		// for non micro.mu subdomains, we route foo.m3o.app/bar to
		// foo.web.bar
		name = alias + ".web." + comps[1]
	}

	// find the service using the selector
	next, err := r.Selector.Select(name)
	if err == selector.ErrNotFound {
		// fallback to path based
		return r.resolveWithPath(req)
	} else if err != nil {
		return nil, err
	}

	// TODO: better retry strategy
	s, err := next()
	if err != nil {
		return nil, err
	}

	// we're done
	return &res.Endpoint{
		Name:   alias,
		Method: req.Method,
		Host:   s.Address,
		Path:   req.URL.Path,
	}, nil
}

func (r *Resolver) resolveWithPath(req *http.Request) (*res.Endpoint, error) {
	parts := strings.Split(req.URL.Path, "/")
	if len(parts) < 2 {
		return nil, errors.New("unknown service")
	}

	if !re.MatchString(parts[1]) {
		return nil, res.ErrInvalidPath
	}

	_, namespace, _ := r.Info(req)
	next, err := r.Selector.Select(namespace + "." + parts[1])
	if err == selector.ErrNotFound {
		return nil, res.ErrNotFound
	} else if err != nil {
		return nil, err
	}

	// TODO: better retry strategy
	s, err := next()
	if err != nil {
		return nil, err
	}

	// we're done
	return &res.Endpoint{
		Name:   parts[1],
		Method: req.Method,
		Host:   s.Address,
		Path:   "/" + strings.Join(parts[2:], "/"),
	}, nil
}
<|endoftext|>


<commit_before>package builtin

import (
	"fmt"
	"strings"

	"github.com/wanliu/flow/builtin/ai"
	"github.com/wanliu/flow/builtin/config"
	"github.com/wanliu/flow/builtin/resolves"
	"github.com/wanliu/flow/context"

	flow "github.com/wanliu/goflow"
)

type OrderItemDelete struct {
	flow.Component

	Ctx <-chan context.Request
	Out chan<- context.Request
}

func NewOrderItemDelete() interface{} {
	return new(OrderItemDelete)
}

func (c *OrderItemDelete) OnCtx(req context.Request) {
	ctx := req.Ctx
	currentOrder := ctx.CtxValue(config.CtxKeyOrder)

	if nil != currentOrder {

		cOrder := currentOrder.(resolves.OrderResolve)

		if cOrder.Expired(config.SesssionExpiredMinutes) {
			req.Res = context.Response{"会话已经过时，当前没有正在进行中的订单", ctx, nil}
			c.Out <- req
			return
		}

		cmd := req.Command
		if cmd != nil {
			// delete by command
			data := cmd.Data
			if itemName, ok := data["itemName"].(string); ok {
				itemsResolve := cOrder.Products

				removed := itemsResolve.Remove(itemName)
				if removed {
					cOrder.Products = itemsResolve
					ctx.SetCtxValue(config.CtxKeyOrder, cOrder)

					_, d := cOrder.Answer(ctx)

					data := map[string]interface{}{
						"type":   "info",
						"on":     "order",
						"action": "update",
						"data":   d,
					}

					reply := fmt.Sprintf("已经删除%v", itemName)
					req.Res = context.Response{reply, ctx, data}
					c.Out <- req
				} else {
					reply := fmt.Sprintf("无效的操作，%v不存在", itemName)
					req.Res = context.Response{reply, ctx, nil}
					c.Out <- req
				}
			} else {
				req.Res = context.Response{"无效的删除操作", ctx, nil}
				c.Out <- req
			}
		} else {
			// delete by intent
			aiResult := req.ApiAiResult

			aiExtract := ai.ApiAiOrder{AiResult: aiResult}
			deletedItems := []string{}
			products := []string{}
			itemsResolve := cOrder.Products

			for _, product := range aiExtract.Products() {
				name := product.Product
				products = append(products, name)

				removed := itemsResolve.Remove(name)
				if removed {
					deletedItems = append(deletedItems, name)
				}
			}

			if len(deletedItems) > 0 {
				cOrder.Products = itemsResolve
				_, d := cOrder.Answer(ctx)

				data := map[string]interface{}{
					"type":   "info",
					"on":     "order",
					"action": "update",
					"data":   d,
				}
				ctx.SetCtxValue(config.CtxKeyOrder, cOrder)

				reply := fmt.Sprintf("已经删除%v", strings.Join(deletedItems, ","))
				req.Res = context.Response{reply, ctx, data}
				c.Out <- req
			} else {
				_, d := cOrder.Answer(ctx)

				data := map[string]interface{}{
					"type":   "info",
					"on":     "order",
					"action": "update",
					"data":   d,
				}

				reply := fmt.Sprintf("无效的操作，%v不存在", strings.Join(products, ","))
				req.Res = context.Response{reply, ctx, data}
				c.Out <- req
			}
		}
	} else {
		req.Res = context.Response{"无效的操作，当前没有正在进行中的订单", ctx, nil}
		c.Out <- req
	}
}
<commit_msg>empty products notify<commit_after>package builtin

import (
	"fmt"
	"strings"

	"github.com/wanliu/flow/builtin/ai"
	"github.com/wanliu/flow/builtin/config"
	"github.com/wanliu/flow/builtin/resolves"
	"github.com/wanliu/flow/context"

	flow "github.com/wanliu/goflow"
)

type OrderItemDelete struct {
	flow.Component

	Ctx <-chan context.Request
	Out chan<- context.Request
}

func NewOrderItemDelete() interface{} {
	return new(OrderItemDelete)
}

func (c *OrderItemDelete) OnCtx(req context.Request) {
	ctx := req.Ctx
	currentOrder := ctx.CtxValue(config.CtxKeyOrder)

	if nil != currentOrder {

		cOrder := currentOrder.(resolves.OrderResolve)

		if cOrder.Expired(config.SesssionExpiredMinutes) {
			req.Res = context.Response{"会话已经过时，当前没有正在进行中的订单", ctx, nil}
			c.Out <- req
			return
		}

		cmd := req.Command
		if cmd != nil {
			// delete by command
			data := cmd.Data
			if itemName, ok := data["itemName"].(string); ok {
				itemsResolve := cOrder.Products

				removed := itemsResolve.Remove(itemName)
				if removed {
					cOrder.Products = itemsResolve
					ctx.SetCtxValue(config.CtxKeyOrder, cOrder)

					answer, d := cOrder.Answer(ctx)

					data := map[string]interface{}{
						"type":   "info",
						"on":     "order",
						"action": "update",
						"data":   d,
					}

					reply := fmt.Sprintf("已经删除%v, %v", itemName, answer)
					req.Res = context.Response{reply, ctx, data}
					c.Out <- req
				} else {
					reply := fmt.Sprintf("无效的操作，%v不存在", itemName)
					req.Res = context.Response{reply, ctx, nil}
					c.Out <- req
				}
			} else {
				req.Res = context.Response{"无效的删除操作", ctx, nil}
				c.Out <- req
			}
		} else {
			// delete by intent
			aiResult := req.ApiAiResult

			aiExtract := ai.ApiAiOrder{AiResult: aiResult}
			deletedItems := []string{}
			products := []string{}
			itemsResolve := cOrder.Products

			for _, product := range aiExtract.Products() {
				name := product.Product
				products = append(products, name)

				removed := itemsResolve.Remove(name)
				if removed {
					deletedItems = append(deletedItems, name)
				}
			}

			if len(deletedItems) > 0 {
				cOrder.Products = itemsResolve
				answer, d := cOrder.Answer(ctx)

				data := map[string]interface{}{
					"type":   "info",
					"on":     "order",
					"action": "update",
					"data":   d,
				}
				ctx.SetCtxValue(config.CtxKeyOrder, cOrder)

				reply := fmt.Sprintf("已经删除%v, %v", strings.Join(deletedItems, ","), answer)
				req.Res = context.Response{reply, ctx, data}
				c.Out <- req
			} else {
				_, d := cOrder.Answer(ctx)

				data := map[string]interface{}{
					"type":   "info",
					"on":     "order",
					"action": "update",
					"data":   d,
				}

				reply := fmt.Sprintf("无效的操作，%v不存在", strings.Join(products, ","))
				req.Res = context.Response{reply, ctx, data}
				c.Out <- req
			}
		}
	} else {
		req.Res = context.Response{"无效的操作，当前没有正在进行中的订单", ctx, nil}
		c.Out <- req
	}
}
<|endoftext|>