<commit_before>// +build !lambdabinary

package sparta

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"strings"
	"text/template"

	spartaCF "github.com/mweagle/Sparta/aws/cloudformation"
	"github.com/sirupsen/logrus"
)

const (
	nodeColorService     = "#720502"
	nodeColorEventSource = "#BF2803"
	nodeColorLambda      = "#F35B05"
	nodeColorAPIGateway  = "#06B5F5"
	nodeNameAPIGateway   = "API Gateway"
)

type cytoscapeData struct {
	ID               string `json:"id"`
	Image            string `json:"image"`
	BackgroundColor  string `json:"backgroundColor,omitempty"`
	Source           string `json:"source,omitempty"`
	Target           string `json:"target,omitempty"`
	Label            string `json:"label,omitempty"`
	DegreeCentrality int    `json:"degreeCentrality"`
}
type cytoscapeNode struct {
	Data    cytoscapeData `json:"data"`
	Classes string        `json:"classes,omitempty"`
}
type templateResource struct {
	KeyName string
	Data    string
}

func cytoscapeNodeID(rawData interface{}) (string, error) {
	bytes, bytesErr := json.Marshal(rawData)
	if bytesErr != nil {
		return "", bytesErr
	}
	hash := sha1.New()
	_, writeErr := hash.Write(bytes)
	if writeErr != nil {
		return "", writeErr
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

func writeNode(nodes *[]*cytoscapeNode,
	nodeName string,
	nodeColor string,
	nodeImage string,
	logger *logrus.Logger) {
	nodeID, _ := cytoscapeNodeID(nodeName)
	appendNode := &cytoscapeNode{
		Data: cytoscapeData{
			ID:    nodeID,
			Label: strings.Trim(nodeName, "\""),
		},
	}
	if nodeImage != "" {
		resourceItem := templateResourceForKey(nodeImage, logger)
		if resourceItem != nil {
			appendNode.Data.Image = fmt.Sprintf("data:image/svg+xml;base64,%s",
				base64.StdEncoding.EncodeToString([]byte(resourceItem.Data)))
		}
	}
	*nodes = append(*nodes, appendNode)

}

func writeLink(nodes *[]*cytoscapeNode, fromNode string, toNode string, label string) {
	nodeSource, _ := cytoscapeNodeID(fromNode)
	nodeTarget, _ := cytoscapeNodeID(toNode)

	*nodes = append(*nodes, &cytoscapeNode{
		Data: cytoscapeData{
			ID:     fmt.Sprintf("%d", rand.Uint64()),
			Source: nodeSource,
			Target: nodeTarget,
			Label:  label,
		},
	})
}
func templateResourceForKey(resourceKeyName string, logger *logrus.Logger) *templateResource {
	var resource *templateResource
	resourcePath := fmt.Sprintf("/resources/describe/%s",
		strings.TrimLeft(resourceKeyName, "/"))
	data, dataErr := _escFSString(false, resourcePath)
	if dataErr == nil {
		keyParts := strings.Split(resourcePath, "/")
		keyName := keyParts[len(keyParts)-1]
		resource = &templateResource{
			KeyName: keyName,
			Data:    data,
		}
		logger.WithFields(logrus.Fields{
			"Path":    resourcePath,
			"KeyName": keyName,
		}).Debug("Embedded resource")

	} else {
		logger.WithFields(logrus.Fields{
			"Path": resourcePath,
		}).Warn("Failed to embed resource")
	}
	return resource
}
func templateResourcesForKeys(resourceKeyNames []string, logger *logrus.Logger) []*templateResource {
	resources := make([]*templateResource, 0)

	for _, eachKey := range resourceKeyNames {
		loadedResource := templateResourceForKey(eachKey, logger)
		if loadedResource != nil {
			resources = append(resources, loadedResource)
		}
	}
	return resources
}

func templateCSSFiles(logger *logrus.Logger) []*templateResource {
	cssFiles := []string{"bootstrap-4.0.0/dist/css/bootstrap.min.css",
		"highlight.js/styles/xcode.css",
	}
	return templateResourcesForKeys(cssFiles, logger)
}

func templateJSFiles(logger *logrus.Logger) []*templateResource {
	jsFiles := []string{"jquery/jquery-3.3.1.min.js",
		"popper/popper.min.js",
		"bootstrap-4.0.0/dist/js/bootstrap.min.js",
		"highlight.js/highlight.pack.js",
		"dagre-0.8.2/dagre-0.8.2/dist/dagre.js",
		"cytoscapejs/cytoscape.js",
		"cytoscape.js-dagre-2.2.1/cytoscape.js-dagre-2.2.1/cytoscape-dagre.js",
		"sparta.js",
	}
	return templateResourcesForKeys(jsFiles, logger)
}

func templateImageMap(logger *logrus.Logger) map[string]string {
	images := []string{"SpartaHelmet256.png",
		"AWSIcons/Compute/Compute_AWSLambda_LambdaFunction.svg",
		"AWSIcons/Management Tools/ManagementTools_AWSCloudFormation.svg",
	}
	resources := templateResourcesForKeys(images, logger)
	imageMap := make(map[string]string)
	for _, eachResource := range resources {
		imageMap[eachResource.KeyName] = base64.StdEncoding.EncodeToString([]byte(eachResource.Data))
	}
	return imageMap
}

func iconForAWSResource(rawEmitter interface{}) string {
	jsonBytes, _ := json.Marshal(rawEmitter)
	canonicalRaw := strings.ToLower(string(jsonBytes))
	if strings.Contains(canonicalRaw, "dynamo") {
		return "AWSIcons/Database/Database_AmazonDynamoDB.svg"
	}
	if strings.Contains(canonicalRaw, "sqs") {
		return "AWSIcons/Messaging/Messaging_AmazonSQS.svg"
	}
	if strings.Contains(canonicalRaw, "sns") {
		return "AWSIcons/Messaging/Messaging_AmazonSNS_topic.svg"
	}
	if strings.Contains(canonicalRaw, "cloudwatch") {
		return "AWSIcons/Management Tools/ManagementTools_AmazonCloudWatch.svg"
	}
	if strings.Contains(canonicalRaw, "kinesis") {
		return "AWSIcons/Analytics/Analytics_AmazonKinesis.svg"
	}
	if strings.Contains(canonicalRaw, "s3") {
		return "AWSIcons/Storage/Storage_AmazonS3_bucket.svg"
	}
	return "AWSIcons/General/General_AWScloud.svg"
}

// Describe produces a graphical representation of a service's Lambda and data sources.  Typically
// automatically called as part of a compiled golang binary via the `describe` command
// line option.
func Describe(serviceName string,
	serviceDescription string,
	lambdaAWSInfos []*LambdaAWSInfo,
	api *API,
	s3Site *S3Site,
	s3BucketName string,
	buildTags string,
	linkFlags string,
	outputWriter io.Writer,
	workflowHooks *WorkflowHooks,
	logger *logrus.Logger) error {

	validationErr := validateSpartaPreconditions(lambdaAWSInfos, logger)
	if validationErr != nil {
		return validationErr
	}

	var cloudFormationTemplate bytes.Buffer
	err := Provision(true,
		serviceName,
		serviceDescription,
		lambdaAWSInfos,
		api,
		s3Site,
		s3BucketName,
		false,
		false,
		"N/A",
		"",
		buildTags,
		linkFlags,
		&cloudFormationTemplate,
		workflowHooks,
		logger)
	if nil != err {
		return err
	}

	tmpl, err := template.New("description").Parse(_escFSMustString(false, "/resources/describe/template.html"))
	if err != nil {
		return errors.New(err.Error())
	}

	cytoscapeElements := make([]*cytoscapeNode, 0)

	// Instead of inline mermaid stuff, we're going to stuff raw
	// json through. We can also include AWS images in the icon
	// using base64/encoded:
	// Example: https://cytoscape.github.io/cytoscape.js-tutorial-demo/datasets/social.json
	// Use the "fancy" CSS:
	// https://github.com/cytoscape/cytoscape.js-tutorial-demo/blob/gh-pages/stylesheets/fancy.json
	// Which is dynamically updated at: https://cytoscape.github.io/cytoscape.js-tutorial-demo/

	// Setup the root object
	writeNode(&cytoscapeElements,
		serviceName,
		nodeColorService,
		"AWSIcons/Management Tools/ManagementTools_AWSCloudFormation_stack.svg",
		logger)

	for _, eachLambda := range lambdaAWSInfos {
		// Other cytoscape nodes
		// Create the node...
		writeNode(&cytoscapeElements,
			eachLambda.lambdaFunctionName(),
			nodeColorLambda,
			"AWSIcons/Compute/Compute_AWSLambda.svg",
			logger)
		writeLink(&cytoscapeElements,
			eachLambda.lambdaFunctionName(),
			serviceName,
			"")

		// Create permission & event mappings
		// functions declared in this
		for _, eachPermission := range eachLambda.Permissions {
			nodes, err := eachPermission.descriptionInfo()
			if nil != err {
				return err
			}

			for _, eachNode := range nodes {
				name := strings.TrimSpace(eachNode.Name)
				link := strings.TrimSpace(eachNode.Relation)
				// Style it to have the Amazon color
				nodeColor := eachNode.Color
				if "" == nodeColor {
					nodeColor = nodeColorEventSource
				}

				writeNode(&cytoscapeElements,
					name,
					nodeColor,
					iconForAWSResource(eachNode.Name),
					logger)
				writeLink(&cytoscapeElements,
					name,
					eachLambda.lambdaFunctionName(),
					link)
			}
		}
		for index, eachEventSourceMapping := range eachLambda.EventSourceMappings {
			dynamicArn := spartaCF.DynamicValueToStringExpr(eachEventSourceMapping.EventSourceArn)
			jsonBytes, jsonBytesErr := json.Marshal(dynamicArn)
			if jsonBytesErr != nil {
				jsonBytes = []byte(fmt.Sprintf("%s-EventSourceMapping[%d]",
					eachLambda.lambdaFunctionName(),
					index))
			}
			nodeName := string(jsonBytes)
			writeNode(&cytoscapeElements,
				nodeName,
				nodeColorEventSource,
				iconForAWSResource(dynamicArn),
				logger)
			writeLink(&cytoscapeElements,
				nodeName,
				eachLambda.lambdaFunctionName(),
				"")
		}
	}

	// API?
	if nil != api {
		// Create the APIGateway virtual node && connect it to the application
		writeNode(&cytoscapeElements,
			nodeNameAPIGateway,
			nodeColorAPIGateway,
			"AWSIcons/Application Services/ApplicationServices_AmazonAPIGateway.svg",
			logger)
		for _, eachResource := range api.resources {
			for eachMethod := range eachResource.Methods {
				// Create the PATH node
				var nodeName = fmt.Sprintf("%s - %s", eachMethod, eachResource.pathPart)
				writeNode(&cytoscapeElements,
					nodeName,
					nodeColorAPIGateway,
					"AWSIcons/General/General_Internet.svg",
					logger)
				writeLink(&cytoscapeElements,
					nodeNameAPIGateway,
					nodeName,
					"")
				writeLink(&cytoscapeElements,
					nodeName,
					eachResource.parentLambda.lambdaFunctionName(),
					"")
			}
		}
	}
	cytoscapeBytes, _ := json.MarshalIndent(cytoscapeElements, "", " ")
	params := struct {
		SpartaVersion          string
		ServiceName            string
		ServiceDescription     string
		CloudFormationTemplate string
		CSSFiles               []*templateResource
		JSFiles                []*templateResource
		ImageMap               map[string]string
		CytoscapeData          interface{}
	}{
		SpartaGitHash[0:8],
		serviceName,
		serviceDescription,
		cloudFormationTemplate.String(),
		templateCSSFiles(logger),
		templateJSFiles(logger),
		templateImageMap(logger),
		string(cytoscapeBytes),
	}
	return tmpl.Execute(outputWriter, params)
}
<commit_msg>Minor tweak for image lookup<commit_after>// +build !lambdabinary

package sparta

import (
	"bytes"
	"crypto/sha1"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"strings"
	"text/template"

	spartaCF "github.com/mweagle/Sparta/aws/cloudformation"
	"github.com/sirupsen/logrus"
)

const (
	nodeColorService     = "#720502"
	nodeColorEventSource = "#BF2803"
	nodeColorLambda      = "#F35B05"
	nodeColorAPIGateway  = "#06B5F5"
	nodeNameAPIGateway   = "API Gateway"
)

type cytoscapeData struct {
	ID               string `json:"id"`
	Image            string `json:"image"`
	BackgroundColor  string `json:"backgroundColor,omitempty"`
	Source           string `json:"source,omitempty"`
	Target           string `json:"target,omitempty"`
	Label            string `json:"label,omitempty"`
	DegreeCentrality int    `json:"degreeCentrality"`
}
type cytoscapeNode struct {
	Data    cytoscapeData `json:"data"`
	Classes string        `json:"classes,omitempty"`
}
type templateResource struct {
	KeyName string
	Data    string
}

func cytoscapeNodeID(rawData interface{}) (string, error) {
	bytes, bytesErr := json.Marshal(rawData)
	if bytesErr != nil {
		return "", bytesErr
	}
	hash := sha1.New()
	_, writeErr := hash.Write(bytes)
	if writeErr != nil {
		return "", writeErr
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

func writeNode(nodes *[]*cytoscapeNode,
	nodeName string,
	nodeColor string,
	nodeImage string,
	logger *logrus.Logger) {
	nodeID, _ := cytoscapeNodeID(nodeName)
	appendNode := &cytoscapeNode{
		Data: cytoscapeData{
			ID:    nodeID,
			Label: strings.Trim(nodeName, "\""),
		},
	}
	if nodeImage != "" {
		resourceItem := templateResourceForKey(nodeImage, logger)
		if resourceItem != nil {
			appendNode.Data.Image = fmt.Sprintf("data:image/svg+xml;base64,%s",
				base64.StdEncoding.EncodeToString([]byte(resourceItem.Data)))
		}
	}
	*nodes = append(*nodes, appendNode)

}

func writeLink(nodes *[]*cytoscapeNode, fromNode string, toNode string, label string) {
	nodeSource, _ := cytoscapeNodeID(fromNode)
	nodeTarget, _ := cytoscapeNodeID(toNode)

	*nodes = append(*nodes, &cytoscapeNode{
		Data: cytoscapeData{
			ID:     fmt.Sprintf("%d", rand.Uint64()),
			Source: nodeSource,
			Target: nodeTarget,
			Label:  label,
		},
	})
}
func templateResourceForKey(resourceKeyName string, logger *logrus.Logger) *templateResource {
	var resource *templateResource
	resourcePath := fmt.Sprintf("/resources/describe/%s",
		strings.TrimLeft(resourceKeyName, "/"))
	data, dataErr := _escFSString(false, resourcePath)
	if dataErr == nil {
		keyParts := strings.Split(resourcePath, "/")
		keyName := keyParts[len(keyParts)-1]
		resource = &templateResource{
			KeyName: keyName,
			Data:    data,
		}
		logger.WithFields(logrus.Fields{
			"Path":    resourcePath,
			"KeyName": keyName,
		}).Debug("Embedded resource")

	} else {
		logger.WithFields(logrus.Fields{
			"Path": resourcePath,
		}).Warn("Failed to embed resource")
	}
	return resource
}
func templateResourcesForKeys(resourceKeyNames []string, logger *logrus.Logger) []*templateResource {
	resources := make([]*templateResource, 0)

	for _, eachKey := range resourceKeyNames {
		loadedResource := templateResourceForKey(eachKey, logger)
		if loadedResource != nil {
			resources = append(resources, loadedResource)
		}
	}
	return resources
}

func templateCSSFiles(logger *logrus.Logger) []*templateResource {
	cssFiles := []string{"bootstrap-4.0.0/dist/css/bootstrap.min.css",
		"highlight.js/styles/xcode.css",
	}
	return templateResourcesForKeys(cssFiles, logger)
}

func templateJSFiles(logger *logrus.Logger) []*templateResource {
	jsFiles := []string{"jquery/jquery-3.3.1.min.js",
		"popper/popper.min.js",
		"bootstrap-4.0.0/dist/js/bootstrap.min.js",
		"highlight.js/highlight.pack.js",
		"dagre-0.8.2/dagre-0.8.2/dist/dagre.js",
		"cytoscapejs/cytoscape.js",
		"cytoscape.js-dagre-2.2.1/cytoscape.js-dagre-2.2.1/cytoscape-dagre.js",
		"sparta.js",
	}
	return templateResourcesForKeys(jsFiles, logger)
}

func templateImageMap(logger *logrus.Logger) map[string]string {
	images := []string{"SpartaHelmet256.png",
		"AWSIcons/Compute/Compute_AWSLambda_LambdaFunction.svg",
		"AWSIcons/Management Tools/ManagementTools_AWSCloudFormation.svg",
	}
	resources := templateResourcesForKeys(images, logger)
	imageMap := make(map[string]string)
	for _, eachResource := range resources {
		imageMap[eachResource.KeyName] = base64.StdEncoding.EncodeToString([]byte(eachResource.Data))
	}
	return imageMap
}

// TODO - this should really be smarter, including
// looking at the referred resource to understand it's
// type
func iconForAWSResource(rawEmitter interface{}) string {
	jsonBytes, _ := json.Marshal(rawEmitter)
	canonicalRaw := strings.ToLower(string(jsonBytes))
	if strings.Contains(canonicalRaw, "dynamodb") {
		return "AWSIcons/Database/Database_AmazonDynamoDB.svg"
	}
	if strings.Contains(canonicalRaw, "sqs") {
		return "AWSIcons/Messaging/Messaging_AmazonSQS.svg"
	}
	if strings.Contains(canonicalRaw, "sns") {
		return "AWSIcons/Messaging/Messaging_AmazonSNS_topic.svg"
	}
	if strings.Contains(canonicalRaw, "cloudwatch") {
		return "AWSIcons/Management Tools/ManagementTools_AmazonCloudWatch.svg"
	}
	if strings.Contains(canonicalRaw, "kinesis") {
		return "AWSIcons/Analytics/Analytics_AmazonKinesis.svg"
	}
	if strings.Contains(canonicalRaw, "s3") {
		return "AWSIcons/Storage/Storage_AmazonS3_bucket.svg"
	}
	return "AWSIcons/General/General_AWScloud.svg"
}

// Describe produces a graphical representation of a service's Lambda and data sources.  Typically
// automatically called as part of a compiled golang binary via the `describe` command
// line option.
func Describe(serviceName string,
	serviceDescription string,
	lambdaAWSInfos []*LambdaAWSInfo,
	api *API,
	s3Site *S3Site,
	s3BucketName string,
	buildTags string,
	linkFlags string,
	outputWriter io.Writer,
	workflowHooks *WorkflowHooks,
	logger *logrus.Logger) error {

	validationErr := validateSpartaPreconditions(lambdaAWSInfos, logger)
	if validationErr != nil {
		return validationErr
	}

	var cloudFormationTemplate bytes.Buffer
	err := Provision(true,
		serviceName,
		serviceDescription,
		lambdaAWSInfos,
		api,
		s3Site,
		s3BucketName,
		false,
		false,
		"N/A",
		"",
		buildTags,
		linkFlags,
		&cloudFormationTemplate,
		workflowHooks,
		logger)
	if nil != err {
		return err
	}

	tmpl, err := template.New("description").Parse(_escFSMustString(false, "/resources/describe/template.html"))
	if err != nil {
		return errors.New(err.Error())
	}

	cytoscapeElements := make([]*cytoscapeNode, 0)

	// Instead of inline mermaid stuff, we're going to stuff raw
	// json through. We can also include AWS images in the icon
	// using base64/encoded:
	// Example: https://cytoscape.github.io/cytoscape.js-tutorial-demo/datasets/social.json
	// Use the "fancy" CSS:
	// https://github.com/cytoscape/cytoscape.js-tutorial-demo/blob/gh-pages/stylesheets/fancy.json
	// Which is dynamically updated at: https://cytoscape.github.io/cytoscape.js-tutorial-demo/

	// Setup the root object
	writeNode(&cytoscapeElements,
		serviceName,
		nodeColorService,
		"AWSIcons/Management Tools/ManagementTools_AWSCloudFormation_stack.svg",
		logger)

	for _, eachLambda := range lambdaAWSInfos {
		// Other cytoscape nodes
		// Create the node...
		writeNode(&cytoscapeElements,
			eachLambda.lambdaFunctionName(),
			nodeColorLambda,
			"AWSIcons/Compute/Compute_AWSLambda.svg",
			logger)
		writeLink(&cytoscapeElements,
			eachLambda.lambdaFunctionName(),
			serviceName,
			"")

		// Create permission & event mappings
		// functions declared in this
		for _, eachPermission := range eachLambda.Permissions {
			nodes, err := eachPermission.descriptionInfo()
			if nil != err {
				return err
			}

			for _, eachNode := range nodes {
				name := strings.TrimSpace(eachNode.Name)
				link := strings.TrimSpace(eachNode.Relation)
				// Style it to have the Amazon color
				nodeColor := eachNode.Color
				if "" == nodeColor {
					nodeColor = nodeColorEventSource
				}

				writeNode(&cytoscapeElements,
					name,
					nodeColor,
					iconForAWSResource(eachNode.Name),
					logger)
				writeLink(&cytoscapeElements,
					name,
					eachLambda.lambdaFunctionName(),
					link)
			}
		}
		for index, eachEventSourceMapping := range eachLambda.EventSourceMappings {
			dynamicArn := spartaCF.DynamicValueToStringExpr(eachEventSourceMapping.EventSourceArn)
			jsonBytes, jsonBytesErr := json.Marshal(dynamicArn)
			if jsonBytesErr != nil {
				jsonBytes = []byte(fmt.Sprintf("%s-EventSourceMapping[%d]",
					eachLambda.lambdaFunctionName(),
					index))
			}
			nodeName := string(jsonBytes)
			writeNode(&cytoscapeElements,
				nodeName,
				nodeColorEventSource,
				iconForAWSResource(dynamicArn),
				logger)
			writeLink(&cytoscapeElements,
				nodeName,
				eachLambda.lambdaFunctionName(),
				"")
		}
	}

	// API?
	if nil != api {
		// Create the APIGateway virtual node && connect it to the application
		writeNode(&cytoscapeElements,
			nodeNameAPIGateway,
			nodeColorAPIGateway,
			"AWSIcons/Application Services/ApplicationServices_AmazonAPIGateway.svg",
			logger)
		for _, eachResource := range api.resources {
			for eachMethod := range eachResource.Methods {
				// Create the PATH node
				var nodeName = fmt.Sprintf("%s - %s", eachMethod, eachResource.pathPart)
				writeNode(&cytoscapeElements,
					nodeName,
					nodeColorAPIGateway,
					"AWSIcons/General/General_Internet.svg",
					logger)
				writeLink(&cytoscapeElements,
					nodeNameAPIGateway,
					nodeName,
					"")
				writeLink(&cytoscapeElements,
					nodeName,
					eachResource.parentLambda.lambdaFunctionName(),
					"")
			}
		}
	}
	cytoscapeBytes, _ := json.MarshalIndent(cytoscapeElements, "", " ")
	params := struct {
		SpartaVersion          string
		ServiceName            string
		ServiceDescription     string
		CloudFormationTemplate string
		CSSFiles               []*templateResource
		JSFiles                []*templateResource
		ImageMap               map[string]string
		CytoscapeData          interface{}
	}{
		SpartaGitHash[0:8],
		serviceName,
		serviceDescription,
		cloudFormationTemplate.String(),
		templateCSSFiles(logger),
		templateJSFiles(logger),
		templateImageMap(logger),
		string(cytoscapeBytes),
	}
	return tmpl.Execute(outputWriter, params)
}
<|endoftext|>


<commit_before>package commandmodule

import (
	//"bytes"
	"camssh"
	"cliargs"
	"errors"
	"fmt"
	"io"
	"modules"
	"os"
	"output"
)

type CommandModule modules.TCamerataModule

func init() {
	modules.Register("command", &CommandModule{},
		"Executes --args command line on target hosts.")
}

func (me *CommandModule) Setup(args *cliargs.Arguments, stdout *output.StdoutManager, stderr *output.StderrManager) {
	me.Args = args
	me.Stdout = stdout
	me.Stderr = stderr
}

func (me *CommandModule) Prepare(host string, sshconn *camssh.SshConnection) error {
	me.Host = host
	me.Sshconn = sshconn

	if len(me.Args.MArguments) == 0 {
		return errors.New("CommandModule: Arguments cannot be empty")
	}

	return nil
}

func (me *CommandModule) Run() error {

	commandargs := me.Args.MArguments

	me.Stdout.Print(">>> CommandModule >>> Executing ", commandargs, " @", me.Host)
	if me.Args.Sudo {
		me.Stdout.Print(" as sudo")
	}
	me.Stdout.Println("")

	commandline := commandargs

	session, err := me.Sshconn.Client.NewSession()
	if err != nil {
		panic("Failed to create session: " + err.Error())
	}
	defer session.Close()

	if me.Args.Sudo {
		commandline = fmt.Sprintf("sudo -S \"%s\"", commandargs)

		go func() {
			w, _ := session.StdinPipe()
			defer w.Close()
			if me.Args.Sudo && !me.Args.SudoNoPass {
				fmt.Fprintln(w, me.Args.Pass)
			}
		}()
	}

	//var b bytes.Buffer
	//session.Stdout = &b
	go func() {
		var br int64
		r, _ := session.StdoutPipe()
		br, _ = io.Copy(os.Stdout, r)
		for br > 0 {
			br, _ = io.Copy(os.Stdout, r)
		}
	}()
	go func() {
		var br int64
		r, _ := session.StderrPipe()
		br, _ = io.Copy(os.Stderr, r)
		for br > 0 {
			br, _ = io.Copy(os.Stderr, r)
		}
	}()

	if err := session.Run(commandline); err != nil {
		me.Stderr.Println("Failed to run: ", err)
	}
	//me.Stdout.Print(">>> CommandModule >>>", b.String())
	//fmt.Println(b.String())

	return nil

}
<commit_msg>command module stdin mapped to remote stdin. dev commit<commit_after>package commandmodule

import (
	//"bytes"
	"camssh"
	"cliargs"
	"errors"
	"fmt"
	"io"
	"modules"
	"os"
	"output"
)

type CommandModule modules.TCamerataModule

func init() {
	modules.Register("command", &CommandModule{},
		"Executes --args command line on target hosts.")
}

func (me *CommandModule) Setup(args *cliargs.Arguments, stdout *output.StdoutManager, stderr *output.StderrManager) {
	me.Args = args
	me.Stdout = stdout
	me.Stderr = stderr
}

func (me *CommandModule) Prepare(host string, sshconn *camssh.SshConnection) error {
	me.Host = host
	me.Sshconn = sshconn

	if len(me.Args.MArguments) == 0 {
		return errors.New("CommandModule: Arguments cannot be empty")
	}

	return nil
}

func (me *CommandModule) Run() error {

	commandargs := me.Args.MArguments

	me.Stdout.Print(">>> CommandModule >>> Executing ", commandargs, " @", me.Host)
	if me.Args.Sudo {
		me.Stdout.Print(" as sudo")
	}
	me.Stdout.Println("")

	commandline := commandargs

	session, err := me.Sshconn.Client.NewSession()
	if err != nil {
		panic("Failed to create session: " + err.Error())
	}
	defer session.Close()

	if me.Args.Sudo {
		commandline = fmt.Sprintf("sudo -S \"%s\"", commandargs)

		go func() {
			w, _ := session.StdinPipe()
			defer w.Close()
			if me.Args.Sudo && !me.Args.SudoNoPass {
				fmt.Fprintln(w, me.Args.Pass)
			}
		}()
	}

	//var b bytes.Buffer
	//session.Stdout = &b
	go func() {
		var br int64
		r, _ := session.StdoutPipe()
		br, _ = io.Copy(os.Stdout, r)
		for br > 0 {
			br, _ = io.Copy(os.Stdout, r)
		}
	}()
	go func() {
		var br int64
		r, _ := session.StderrPipe()
		br, _ = io.Copy(os.Stderr, r)
		for br > 0 {
			br, _ = io.Copy(os.Stderr, r)
		}
	}()

	session.Stdin = os.Stdin

	if err := session.Run(commandline); err != nil {
		me.Stderr.Println("Failed to run: ", err)
	}
	//me.Stdout.Print(">>> CommandModule >>>", b.String())
	//fmt.Println(b.String())

	return nil

}
<|endoftext|>


<commit_before>package cache

import (
	"context"
	"io"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"
)

// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
	restic.Backend
	*Cache
}

// ensure cachedBackend implements restic.Backend
var _ restic.Backend = &Backend{}

// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("cache Remove(%v)", h)
	err := b.Backend.Remove(ctx, h)
	if err != nil {
		return err
	}

	return b.Cache.Remove(h)
}

type teeReader struct {
	rd  io.Reader
	wr  io.Writer
	err error
}

func (t *teeReader) Read(p []byte) (n int, err error) {
	n, err = t.rd.Read(p)
	if t.err == nil && n > 0 {
		_, t.err = t.wr.Write(p[:n])
	}

	return n, err
}

var autoCacheTypes = map[restic.FileType]struct{}{
	restic.IndexFile:    struct{}{},
	restic.SnapshotFile: struct{}{},
}

// Save stores a new file is the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	if _, ok := autoCacheTypes[h.Type]; !ok {
		return b.Backend.Save(ctx, h, rd)
	}

	debug.Log("Save(%v): auto-store in the cache", h)
	wr, err := b.Cache.SaveWriter(h)
	if err != nil {
		debug.Log("unable to save %v to cache: %v", h, err)
		return b.Backend.Save(ctx, h, rd)
	}

	tr := &teeReader{rd: rd, wr: wr}
	err = b.Backend.Save(ctx, h, tr)
	if err != nil {
		wr.Close()
		b.Cache.Remove(h)
		return err
	}

	err = wr.Close()
	if err != nil {
		debug.Log("cache writer returned error: %v", err)
		_ = b.Cache.Remove(h)
	}
	return nil
}

var autoCacheFiles = map[restic.FileType]bool{
	restic.IndexFile:    true,
	restic.SnapshotFile: true,
}

// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	if b.Cache.Has(h) {
		debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
		return b.Cache.Load(h, length, offset)
	}

	// partial file requested
	if offset != 0 || length != 0 {
		debug.Log("Load(%v, %v, %v): partial file requested, delegating to backend", h, length, offset)
		return b.Backend.Load(ctx, h, length, offset)
	}

	rd, err := b.Backend.Load(ctx, h, length, offset)
	if err != nil {
		if b.Backend.IsNotExist(err) {
			// try to remove from the cache, ignore errors
			_ = b.Cache.Remove(h)
		}

		return nil, err
	}

	if _, ok := autoCacheFiles[h.Type]; !ok {
		return rd, nil
	}

	debug.Log("auto-store %v in the cache", h)

	// cache the file, then return cached copy
	if err = b.Cache.Save(h, rd); err != nil {
		return nil, err
	}

	if err = rd.Close(); err != nil {
		// try to remove from the cache, ignore errors
		_ = b.Cache.Remove(h)
		return nil, err
	}

	// load from the cache and save in the backend
	return b.Cache.Load(h, 0, 0)
}

// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
	debug.Log("cache Stat(%v)", h)

	fi, err := b.Backend.Stat(ctx, h)
	if err != nil {
		if b.Backend.IsNotExist(err) {
			// try to remove from the cache, ignore errors
			_ = b.Cache.Remove(h)
		}

		return fi, err
	}

	return fi, err
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (b *Backend) IsNotExist(err error) bool {
	return b.Backend.IsNotExist(err)
}
<commit_msg>cache: Allow proactive caching of tree packs<commit_after>package cache

import (
	"context"
	"io"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/restic"
)

// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
	restic.Backend
	*Cache
}

// ensure cachedBackend implements restic.Backend
var _ restic.Backend = &Backend{}

// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("cache Remove(%v)", h)
	err := b.Backend.Remove(ctx, h)
	if err != nil {
		return err
	}

	return b.Cache.Remove(h)
}

type teeReader struct {
	rd  io.Reader
	wr  io.Writer
	err error
}

func (t *teeReader) Read(p []byte) (n int, err error) {
	n, err = t.rd.Read(p)
	if t.err == nil && n > 0 {
		_, t.err = t.wr.Write(p[:n])
	}

	return n, err
}

var autoCacheTypes = map[restic.FileType]struct{}{
	restic.IndexFile:    struct{}{},
	restic.SnapshotFile: struct{}{},
}

// Save stores a new file is the backend and the cache.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	if _, ok := autoCacheTypes[h.Type]; !ok {
		return b.Backend.Save(ctx, h, rd)
	}

	debug.Log("Save(%v): auto-store in the cache", h)
	wr, err := b.Cache.SaveWriter(h)
	if err != nil {
		debug.Log("unable to save %v to cache: %v", h, err)
		return b.Backend.Save(ctx, h, rd)
	}

	tr := &teeReader{rd: rd, wr: wr}
	err = b.Backend.Save(ctx, h, tr)
	if err != nil {
		wr.Close()
		b.Cache.Remove(h)
		return err
	}

	err = wr.Close()
	if err != nil {
		debug.Log("cache writer returned error: %v", err)
		_ = b.Cache.Remove(h)
	}
	return nil
}

var autoCacheFiles = map[restic.FileType]bool{
	restic.IndexFile:    true,
	restic.SnapshotFile: true,
}

func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
	rd, err := b.Backend.Load(ctx, h, 0, 0)
	if err != nil {
		return err
	}

	if err = b.Cache.Save(h, rd); err != nil {
		return err
	}

	if err = rd.Close(); err != nil {
		// try to remove from the cache, ignore errors
		_ = b.Cache.Remove(h)
		return err
	}

	return nil
}

// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	if b.Cache.Has(h) {
		debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
		return b.Cache.Load(h, length, offset)
	}

	// partial file requested
	if offset != 0 || length != 0 {
		if b.Cache.PerformReadahead(h) {
			debug.Log("performing readahead for %v", h)
			err := b.cacheFile(ctx, h)
			if err == nil {
				return b.Cache.Load(h, length, offset)
			}

			debug.Log("error caching %v: %v", h, err)
		}

		debug.Log("Load(%v, %v, %v): partial file requested, delegating to backend", h, length, offset)
		return b.Backend.Load(ctx, h, length, offset)
	}

	// if we don't automatically cache this file type, fall back to the backend
	if _, ok := autoCacheFiles[h.Type]; !ok {
		debug.Log("Load(%v, %v, %v): delegating to backend", h, length, offset)
		return b.Backend.Load(ctx, h, length, offset)
	}

	debug.Log("auto-store %v in the cache", h)
	err := b.cacheFile(ctx, h)

	if err == nil {
		// load the cached version
		return b.Cache.Load(h, 0, 0)
	}

	debug.Log("error caching %v: %v, falling back to backend", h, err)
	return b.Backend.Load(ctx, h, length, offset)
}

// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
	debug.Log("cache Stat(%v)", h)

	fi, err := b.Backend.Stat(ctx, h)
	if err != nil {
		if b.Backend.IsNotExist(err) {
			// try to remove from the cache, ignore errors
			_ = b.Cache.Remove(h)
		}

		return fi, err
	}

	return fi, err
}

// IsNotExist returns true if the error is caused by a non-existing file.
func (b *Backend) IsNotExist(err error) bool {
	return b.Backend.IsNotExist(err)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package comebackfs

import (
	"log"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/syncutil"
)

// Create a read-only file system for browsing the backup rooted by the
// supplied score. All inodes will be owned by the supplied UID/GID pair.
func NewFileSystem(
	uid uint32,
	gid uint32,
	rootScore blob.Score,
	blobStore blob.Store) (fs fuseutil.FileSystem, err error) {
	// Create the file system.
	typed := &fileSystem{
		blobStore:   blobStore,
		nextInodeID: fuseops.RootInodeID,
		inodes:      make(map[fuseops.InodeID]*inodeRecord),
	}

	fs = typed
	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	typed.Lock()
	defer typed.Unlock()

	// Set up the root inode.
	root := newDirInode(
		fuseops.InodeAttributes{
			Nlink: 1,
			Mode:  0500,
			Uid:   uid,
			Gid:   gid,
		},
		rootScore,
		blobStore)

	typed.registerInode(root)

	return
}

////////////////////////////////////////////////////////////////////////
// Internal
////////////////////////////////////////////////////////////////////////

type fileSystem struct {
	fuseutil.NotImplementedFileSystem
	blobStore blob.Store

	/////////////////////////
	// Mutable data
	/////////////////////////

	// LOCK ORDERING:
	//
	// Let FS be the file system. Define a strict partial ordering < by:
	//
	// *   For any inode I,  I < FS.
	// *   For any handle H, H < FS.
	//
	// and follow the rule "acquire B while holding A only if A < B".
	//
	// In other words:
	//
	// *   Don't hold more than one inode or handle lock at a time.
	// *   Don't acquire an inode or handle lock before the file system lock.
	//
	// The intuition is that inode and handle locks are held for long operations,
	// but the file system lock is lightweight and must not be.
	mu syncutil.InvariantMutex

	// The next inode ID we will hand out.
	//
	// INVARIANT: nextInodeID >= fuseops.RootInodeID
	nextInodeID fuseops.InodeID

	// The inodes we currently know.
	//
	// INVARIANT: For all k, k < nextInodeID
	// INVARIANT: For all v, v.lookupCount > 0
	inodes map[fuseops.InodeID]*inodeRecord
}

// An inode and its lookup count.
type inodeRecord struct {
	lookupCount uint64
	in          inode
}

// LOCKS_REQUIRED(fs)
func (fs *fileSystem) checkInvariants() {
	// INVARIANT: nextInodeID >= fuseops.RootInodeID
	if !(fs.nextInodeID >= fuseops.RootInodeID) {
		log.Fatalf("Unexpected nextInodeID: %d", fs.nextInodeID)
	}

	// INVARIANT: For all k, k < nextInodeID
	for k, _ := range fs.inodes {
		if !(k < fs.nextInodeID) {
			log.Fatalf("ID %d not less than nextInodeID %d", k, fs.nextInodeID)
		}
	}

	// INVARIANT: For all v, v.lookupCount > 0
	for k, v := range fs.inodes {
		if !(v.lookupCount > 0) {
			log.Fatalf("Inode %d has invalid lookupCount %d", k, v.lookupCount)
		}
	}
}

// Register the supplied inode, returning its ID. Set the initial lookup count
// to one.
//
// LOCKS_REQUIRED(fs)
func (fs *fileSystem) registerInode(in inode) (id fuseops.InodeID) {
	id = fs.nextInodeID
	fs.nextInodeID++

	fs.inodes[id] = &inodeRecord{
		lookupCount: 1,
		in:          in,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs)
func (fs *fileSystem) Lock() {
	fs.mu.Lock()
}

// LOCKS_REQUIRED(fs)
func (fs *fileSystem) Unlock() {
	fs.mu.Unlock()
}

// LOCKS_EXCLUDED(fs)
func (fs *fileSystem) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp) (err error) {
	// Find the inode.
	rec := fs.inodes[op.Inode]
	if rec == nil {
		log.Fatalf("Inode %d not found", op.Inode)
	}

	// Decrement its lookup count.
	if rec.lookupCount < op.N {
		log.Fatalf(
			"Inode %d has lookup count %d, decrementing by %d",
			op.Inode,
			rec.lookupCount,
			op.N)
	}

	rec.lookupCount -= op.N
	if rec.lookupCount == 0 {
		delete(fs.inodes, op.Inode)
	}

	return
}
<commit_msg>Fixed a concurrency bug.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package comebackfs

import (
	"log"

	"golang.org/x/net/context"

	"github.com/jacobsa/comeback/internal/blob"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/syncutil"
)

// Create a read-only file system for browsing the backup rooted by the
// supplied score. All inodes will be owned by the supplied UID/GID pair.
func NewFileSystem(
	uid uint32,
	gid uint32,
	rootScore blob.Score,
	blobStore blob.Store) (fs fuseutil.FileSystem, err error) {
	// Create the file system.
	typed := &fileSystem{
		blobStore:   blobStore,
		nextInodeID: fuseops.RootInodeID,
		inodes:      make(map[fuseops.InodeID]*inodeRecord),
	}

	fs = typed
	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	typed.Lock()
	defer typed.Unlock()

	// Set up the root inode.
	root := newDirInode(
		fuseops.InodeAttributes{
			Nlink: 1,
			Mode:  0500,
			Uid:   uid,
			Gid:   gid,
		},
		rootScore,
		blobStore)

	typed.registerInode(root)

	return
}

////////////////////////////////////////////////////////////////////////
// Internal
////////////////////////////////////////////////////////////////////////

type fileSystem struct {
	fuseutil.NotImplementedFileSystem
	blobStore blob.Store

	/////////////////////////
	// Mutable data
	/////////////////////////

	// LOCK ORDERING:
	//
	// Let FS be the file system. Define a strict partial ordering < by:
	//
	// *   For any inode I,  I < FS.
	// *   For any handle H, H < FS.
	//
	// and follow the rule "acquire B while holding A only if A < B".
	//
	// In other words:
	//
	// *   Don't hold more than one inode or handle lock at a time.
	// *   Don't acquire an inode or handle lock before the file system lock.
	//
	// The intuition is that inode and handle locks are held for long operations,
	// but the file system lock is lightweight and must not be.
	mu syncutil.InvariantMutex

	// The next inode ID we will hand out.
	//
	// INVARIANT: nextInodeID >= fuseops.RootInodeID
	//
	// GUARDED_BY(mu)
	nextInodeID fuseops.InodeID

	// The inodes we currently know.
	//
	// INVARIANT: For all k, k < nextInodeID
	// INVARIANT: For all v, v.lookupCount > 0
	//
	// GUARDED_BY(mu)
	inodes map[fuseops.InodeID]*inodeRecord
}

// An inode and its lookup count.
type inodeRecord struct {
	lookupCount uint64
	in          inode
}

// LOCKS_REQUIRED(fs)
func (fs *fileSystem) checkInvariants() {
	// INVARIANT: nextInodeID >= fuseops.RootInodeID
	if !(fs.nextInodeID >= fuseops.RootInodeID) {
		log.Fatalf("Unexpected nextInodeID: %d", fs.nextInodeID)
	}

	// INVARIANT: For all k, k < nextInodeID
	for k, _ := range fs.inodes {
		if !(k < fs.nextInodeID) {
			log.Fatalf("ID %d not less than nextInodeID %d", k, fs.nextInodeID)
		}
	}

	// INVARIANT: For all v, v.lookupCount > 0
	for k, v := range fs.inodes {
		if !(v.lookupCount > 0) {
			log.Fatalf("Inode %d has invalid lookupCount %d", k, v.lookupCount)
		}
	}
}

// Register the supplied inode, returning its ID. Set the initial lookup count
// to one.
//
// LOCKS_REQUIRED(fs)
func (fs *fileSystem) registerInode(in inode) (id fuseops.InodeID) {
	id = fs.nextInodeID
	fs.nextInodeID++

	fs.inodes[id] = &inodeRecord{
		lookupCount: 1,
		in:          in,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// LOCKS_EXCLUDED(fs)
func (fs *fileSystem) Lock() {
	fs.mu.Lock()
}

// LOCKS_REQUIRED(fs)
func (fs *fileSystem) Unlock() {
	fs.mu.Unlock()
}

// LOCKS_EXCLUDED(fs)
func (fs *fileSystem) ForgetInode(
	ctx context.Context,
	op *fuseops.ForgetInodeOp) (err error) {
	fs.Lock()
	defer fs.Unlock()

	// Find the inode.
	rec := fs.inodes[op.Inode]
	if rec == nil {
		log.Fatalf("Inode %d not found", op.Inode)
	}

	// Decrement its lookup count.
	if rec.lookupCount < op.N {
		log.Fatalf(
			"Inode %d has lookup count %d, decrementing by %d",
			op.Inode,
			rec.lookupCount,
			op.N)
	}

	rec.lookupCount -= op.N
	if rec.lookupCount == 0 {
		delete(fs.inodes, op.Inode)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/google/safehtml"
	"github.com/google/safehtml/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

// UnitPage contains data needed to render the unit template.
type UnitPage struct {
	basePage
	// Unit is the unit for this page.
	Unit *internal.UnitMeta

	// Breadcrumb contains data used to render breadcrumb UI elements.
	Breadcrumb breadcrumb

	// Title is the title of the page.
	Title string

	// URLPath is the path suitable for links on the page.
	// See the unitURLPath for details.
	URLPath string

	// CanonicalURLPath is a permanent representation of the URL path for a
	// unit.
	// It uses the resolved module path and version.
	// For example, if the latest version of /my.module/pkg is version v1.5.2,
	// the canonical URL path for that unit would be /my.module@v1.5.2/pkg
	CanonicalURLPath string

	// The version string formatted for display.
	DisplayVersion string

	// LinkVersion is version string suitable for links used to compute
	// latest badges.
	LinkVersion string

	// LatestURL is a url pointing to the latest version of a unit.
	LatestURL string

	// LatestMinorClass is the CSS class that describes the current unit's minor
	// version in relationship to the latest version of the unit.
	LatestMinorClass string

	// Information about the latest major version of the module.
	LatestMajorVersion    string
	LatestMajorVersionURL string

	// PageType is the type of page (pkg, cmd, dir, std, or mod).
	PageType string

	// PageLabels are the labels that will be displayed
	// for a given page.
	PageLabels []string

	// CanShowDetails indicates whether details can be shown or must be
	// hidden due to issues like license restrictions.
	CanShowDetails bool

	// Settings contains settings for the selected tab.
	SelectedTab TabSettings

	// RedirectedFromPath is the path that redirected to the current page.
	// If non-empty, a "redirected from" banner will be displayed
	// (see static/frontend/unit/_header.tmpl).
	RedirectedFromPath string

	// Details contains data specific to the type of page being rendered.
	Details interface{}

	// Vulns holds vulnerability information.
	Vulns []Vuln

	// DepsDevURL holds the full URL to this module version on deps.dev.
	DepsDevURL string

	// IsGoProject is true if the package is from the standard library or a
	// golang.org sub-repository.
	IsGoProject bool
}

// serveUnitPage serves a unit page for a path.
func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, info *urlPathInfo) (err error) {
	defer derrors.Wrap(&err, "serveUnitPage(ctx, w, r, ds, %v)", info)
	defer middleware.ElapsedStat(ctx, "serveUnitPage")()

	tab := r.FormValue("tab")
	if tab == "" {
		// Default to details tab when there is no tab param.
		tab = tabMain
	}
	// Redirect to clean URL path when tab param is invalid.
	if _, ok := unitTabLookup[tab]; !ok {
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	um, err := ds.GetUnitMeta(ctx, info.fullPath, info.modulePath, info.requestedVersion)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			return err
		}
		return s.servePathNotFoundPage(w, r, ds, info.fullPath, info.modulePath, info.requestedVersion)
	}

	makeDepsDevURL := depsDevURLGenerator(ctx, um)

	// Use GOOS and GOARCH query parameters to create a build context, which
	// affects the documentation and synopsis. Omitting both results in an empty
	// build context, which will match the first (and preferred) build context.
	// It's also okay to provide just one (e.g. GOOS=windows), which will select
	// the first doc with that value, ignoring the other one.
	bc := internal.BuildContext{GOOS: r.FormValue("GOOS"), GOARCH: r.FormValue("GOARCH")}
	var getVulnEntries vulnEntriesFunc
	if s.vulnClient != nil {
		getVulnEntries = s.vulnClient.GetByModule
	}
	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, info.requestedVersion, bc, getVulnEntries)
	if err != nil {
		return err
	}
	if s.shouldServeJSON(r) {
		return s.serveJSONPage(w, r, d)
	}

	recordVersionTypeMetric(ctx, info.requestedVersion)
	if _, ok := internal.DefaultBranches[info.requestedVersion]; ok {
		// Since path@master is a moving target, we don't want it to be stale.
		// As a result, we enqueue every request of path@master to the frontend
		// task queue, which will initiate a fetch request depending on the
		// last time we tried to fetch this module version.
		//
		// Use a separate context here to prevent the context from being canceled
		// elsewhere before a task is enqueued.
		go func() {
			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
			defer cancel()
			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", um.ModulePath, info.requestedVersion)
			if _, err := s.queue.ScheduleFetch(ctx, um.ModulePath, info.requestedVersion, nil); err != nil {
				log.Errorf(ctx, "serveUnitPage(%q): scheduling fetch for %q@%q: %v",
					r.URL.Path, um.ModulePath, info.requestedVersion, err)
			}
		}()
	}

	if !isValidTabForUnit(tab, um) {
		// Redirect to clean URL path when tab param is invalid for the unit
		// type.
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	// If we've already called GetUnitMeta for an unknown module path and the latest version, pass
	// it to GetLatestInfo to avoid a redundant call.
	var latestUnitMeta *internal.UnitMeta
	if info.modulePath == internal.UnknownModulePath && info.requestedVersion == version.Latest {
		latestUnitMeta = um
	}
	latestInfo := s.GetLatestInfo(ctx, um.Path, um.ModulePath, latestUnitMeta)
	var redirectPath string
	redirectPath, err = cookie.Extract(w, r, cookie.AlternativeModuleFlash)
	if err != nil {
		// Don't fail, but don't display a banner either.
		log.Errorf(ctx, "extracting AlternativeModuleFlash cookie: %v", err)
	}
	title := pageTitle(um)
	basePage := s.newBasePage(r, title)
	tabSettings := unitTabLookup[tab]
	basePage.AllowWideContent = true
	if tabSettings.Name == "" {
		basePage.UseResponsiveLayout = true
	}
	lv := linkVersion(um.ModulePath, info.requestedVersion, um.Version)
	page := UnitPage{
		basePage:              basePage,
		Unit:                  um,
		Breadcrumb:            displayBreadcrumb(um, info.requestedVersion),
		Title:                 title,
		SelectedTab:           tabSettings,
		URLPath:               constructUnitURL(um.Path, um.ModulePath, info.requestedVersion),
		CanonicalURLPath:      canonicalURLPath(um.Path, um.ModulePath, info.requestedVersion, um.Version),
		DisplayVersion:        displayVersion(um.ModulePath, info.requestedVersion, um.Version),
		LinkVersion:           lv,
		LatestURL:             constructUnitURL(um.Path, um.ModulePath, version.Latest),
		LatestMinorClass:      latestMinorClass(lv, latestInfo),
		LatestMajorVersionURL: latestInfo.MajorUnitPath,
		PageLabels:            pageLabels(um),
		PageType:              pageType(um),
		RedirectedFromPath:    redirectPath,
		DepsDevURL:            makeDepsDevURL(),
		IsGoProject:           isGoProject(um.ModulePath),
	}

	// Show the banner if there was no error getting the latest major version,
	// and it is different from the major version of the current module path.
	latestMajor := internal.MajorVersionForModule(latestInfo.MajorModulePath)
	if latestMajor != "" && latestMajor != internal.MajorVersionForModule(um.ModulePath) {
		page.LatestMajorVersion = latestMajor
	}

	page.Details = d
	main, ok := d.(*MainDetails)
	if ok {
		page.MetaDescription = metaDescription(main.DocSynopsis)
	}

	// Get vulnerability information.
	if s.vulnClient != nil {
		page.Vulns = VulnsForPackage(um.ModulePath, um.Version, um.Path, s.vulnClient.GetByModule)
	}
	s.servePage(ctx, w, tabSettings.TemplateName, page)
	return nil
}

func latestMinorClass(version string, latest internal.LatestInfo) string {
	c := "DetailsHeader-badge"
	switch {
	case latest.MinorVersion == "":
		c += "--unknown"
	case latest.MinorVersion == version && !latest.UnitExistsAtMinor:
		c += "--notAtLatest"
	case latest.MinorVersion == version:
		c += "--latest"
	default:
		c += "--goToLatest"
	}
	return c
}

// metaDescription uses a safehtml escape hatch to build HTML used
// to render the <meta name="Description"> for unit pages as a
// workaround for https://github.com/google/safehtml/issues/6.
func metaDescription(synopsis string) safehtml.HTML {
	if synopsis == "" {
		return safehtml.HTML{}
	}
	return safehtml.HTMLConcat(
		uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(`<meta name="Description" content="`),
		safehtml.HTMLEscaped(synopsis),
		uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(`">`),
	)
}

// isValidTabForUnit reports whether the tab is valid for the given unit.
// It is assumed that tab is a key in unitTabLookup.
func isValidTabForUnit(tab string, um *internal.UnitMeta) bool {
	if tab == tabLicenses && !um.IsRedistributable {
		return false
	}
	if !um.IsPackage() && (tab == tabImports || tab == tabImportedBy) {
		return false
	}
	return true
}

// constructUnitURL returns a URL path that refers to the given unit at the requested
// version. If requestedVersion is "latest", then the resulting path has no
// version; otherwise, it has requestedVersion.
func constructUnitURL(fullPath, modulePath, requestedVersion string) string {
	if requestedVersion == version.Latest {
		return "/" + fullPath
	}
	v := linkVersion(modulePath, requestedVersion, requestedVersion)
	if fullPath == modulePath || modulePath == stdlib.ModulePath {
		return fmt.Sprintf("/%s@%s", fullPath, v)
	}
	return fmt.Sprintf("/%s@%s/%s", modulePath, v, strings.TrimPrefix(fullPath, modulePath+"/"))
}

// canonicalURLPath constructs a URL path to the unit that always includes the
// resolved version.
func canonicalURLPath(fullPath, modulePath, requestedVersion, resolvedVersion string) string {
	return constructUnitURL(fullPath, modulePath,
		linkVersion(modulePath, requestedVersion, resolvedVersion))
}

func isGoProject(modulePath string) bool {
	return modulePath == stdlib.ModulePath || strings.HasPrefix(modulePath, "golang.org")
}
<commit_msg>internal/frontend: serve error on nil queue<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/google/safehtml"
	"github.com/google/safehtml/uncheckedconversions"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/cookie"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/middleware"
	"golang.org/x/pkgsite/internal/stdlib"
	"golang.org/x/pkgsite/internal/version"
)

// UnitPage contains data needed to render the unit template.
type UnitPage struct {
	basePage
	// Unit is the unit for this page.
	Unit *internal.UnitMeta

	// Breadcrumb contains data used to render breadcrumb UI elements.
	Breadcrumb breadcrumb

	// Title is the title of the page.
	Title string

	// URLPath is the path suitable for links on the page.
	// See the unitURLPath for details.
	URLPath string

	// CanonicalURLPath is a permanent representation of the URL path for a
	// unit.
	// It uses the resolved module path and version.
	// For example, if the latest version of /my.module/pkg is version v1.5.2,
	// the canonical URL path for that unit would be /my.module@v1.5.2/pkg
	CanonicalURLPath string

	// The version string formatted for display.
	DisplayVersion string

	// LinkVersion is version string suitable for links used to compute
	// latest badges.
	LinkVersion string

	// LatestURL is a url pointing to the latest version of a unit.
	LatestURL string

	// LatestMinorClass is the CSS class that describes the current unit's minor
	// version in relationship to the latest version of the unit.
	LatestMinorClass string

	// Information about the latest major version of the module.
	LatestMajorVersion    string
	LatestMajorVersionURL string

	// PageType is the type of page (pkg, cmd, dir, std, or mod).
	PageType string

	// PageLabels are the labels that will be displayed
	// for a given page.
	PageLabels []string

	// CanShowDetails indicates whether details can be shown or must be
	// hidden due to issues like license restrictions.
	CanShowDetails bool

	// Settings contains settings for the selected tab.
	SelectedTab TabSettings

	// RedirectedFromPath is the path that redirected to the current page.
	// If non-empty, a "redirected from" banner will be displayed
	// (see static/frontend/unit/_header.tmpl).
	RedirectedFromPath string

	// Details contains data specific to the type of page being rendered.
	Details interface{}

	// Vulns holds vulnerability information.
	Vulns []Vuln

	// DepsDevURL holds the full URL to this module version on deps.dev.
	DepsDevURL string

	// IsGoProject is true if the package is from the standard library or a
	// golang.org sub-repository.
	IsGoProject bool
}

// serveUnitPage serves a unit page for a path.
func (s *Server) serveUnitPage(ctx context.Context, w http.ResponseWriter, r *http.Request,
	ds internal.DataSource, info *urlPathInfo) (err error) {
	defer derrors.Wrap(&err, "serveUnitPage(ctx, w, r, ds, %v)", info)
	defer middleware.ElapsedStat(ctx, "serveUnitPage")()

	tab := r.FormValue("tab")
	if tab == "" {
		// Default to details tab when there is no tab param.
		tab = tabMain
	}
	// Redirect to clean URL path when tab param is invalid.
	if _, ok := unitTabLookup[tab]; !ok {
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	um, err := ds.GetUnitMeta(ctx, info.fullPath, info.modulePath, info.requestedVersion)
	if err != nil {
		if !errors.Is(err, derrors.NotFound) {
			return err
		}
		return s.servePathNotFoundPage(w, r, ds, info.fullPath, info.modulePath, info.requestedVersion)
	}

	makeDepsDevURL := depsDevURLGenerator(ctx, um)

	// Use GOOS and GOARCH query parameters to create a build context, which
	// affects the documentation and synopsis. Omitting both results in an empty
	// build context, which will match the first (and preferred) build context.
	// It's also okay to provide just one (e.g. GOOS=windows), which will select
	// the first doc with that value, ignoring the other one.
	bc := internal.BuildContext{GOOS: r.FormValue("GOOS"), GOARCH: r.FormValue("GOARCH")}
	var getVulnEntries vulnEntriesFunc
	if s.vulnClient != nil {
		getVulnEntries = s.vulnClient.GetByModule
	}
	d, err := fetchDetailsForUnit(ctx, r, tab, ds, um, info.requestedVersion, bc, getVulnEntries)
	if err != nil {
		return err
	}
	if s.shouldServeJSON(r) {
		return s.serveJSONPage(w, r, d)
	}

	recordVersionTypeMetric(ctx, info.requestedVersion)
	if _, ok := internal.DefaultBranches[info.requestedVersion]; ok {
		// Since path@master is a moving target, we don't want it to be stale.
		// As a result, we enqueue every request of path@master to the frontend
		// task queue, which will initiate a fetch request depending on the
		// last time we tried to fetch this module version.
		//
		// Use a separate context here to prevent the context from being canceled
		// elsewhere before a task is enqueued.
		if s.queue == nil {
			return &serverError{
				status: http.StatusBadRequest,
				err:    err,
				epage: &errorPage{
					MessageData: fmt.Sprintf(`Default branches like "@%s" are not supported. Omit to get the current version.`,
						info.requestedVersion),
				},
			}
		}
		go func() {
			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
			defer cancel()
			log.Infof(ctx, "serveUnitPage: Scheduling %q@%q to be fetched", um.ModulePath, info.requestedVersion)
			if _, err := s.queue.ScheduleFetch(ctx, um.ModulePath, info.requestedVersion, nil); err != nil {
				log.Errorf(ctx, "serveUnitPage(%q): scheduling fetch for %q@%q: %v",
					r.URL.Path, um.ModulePath, info.requestedVersion, err)
			}
		}()
	}

	if !isValidTabForUnit(tab, um) {
		// Redirect to clean URL path when tab param is invalid for the unit
		// type.
		http.Redirect(w, r, r.URL.Path, http.StatusFound)
		return nil
	}

	// If we've already called GetUnitMeta for an unknown module path and the latest version, pass
	// it to GetLatestInfo to avoid a redundant call.
	var latestUnitMeta *internal.UnitMeta
	if info.modulePath == internal.UnknownModulePath && info.requestedVersion == version.Latest {
		latestUnitMeta = um
	}
	latestInfo := s.GetLatestInfo(ctx, um.Path, um.ModulePath, latestUnitMeta)
	var redirectPath string
	redirectPath, err = cookie.Extract(w, r, cookie.AlternativeModuleFlash)
	if err != nil {
		// Don't fail, but don't display a banner either.
		log.Errorf(ctx, "extracting AlternativeModuleFlash cookie: %v", err)
	}
	title := pageTitle(um)
	basePage := s.newBasePage(r, title)
	tabSettings := unitTabLookup[tab]
	basePage.AllowWideContent = true
	if tabSettings.Name == "" {
		basePage.UseResponsiveLayout = true
	}
	lv := linkVersion(um.ModulePath, info.requestedVersion, um.Version)
	page := UnitPage{
		basePage:              basePage,
		Unit:                  um,
		Breadcrumb:            displayBreadcrumb(um, info.requestedVersion),
		Title:                 title,
		SelectedTab:           tabSettings,
		URLPath:               constructUnitURL(um.Path, um.ModulePath, info.requestedVersion),
		CanonicalURLPath:      canonicalURLPath(um.Path, um.ModulePath, info.requestedVersion, um.Version),
		DisplayVersion:        displayVersion(um.ModulePath, info.requestedVersion, um.Version),
		LinkVersion:           lv,
		LatestURL:             constructUnitURL(um.Path, um.ModulePath, version.Latest),
		LatestMinorClass:      latestMinorClass(lv, latestInfo),
		LatestMajorVersionURL: latestInfo.MajorUnitPath,
		PageLabels:            pageLabels(um),
		PageType:              pageType(um),
		RedirectedFromPath:    redirectPath,
		DepsDevURL:            makeDepsDevURL(),
		IsGoProject:           isGoProject(um.ModulePath),
	}

	// Show the banner if there was no error getting the latest major version,
	// and it is different from the major version of the current module path.
	latestMajor := internal.MajorVersionForModule(latestInfo.MajorModulePath)
	if latestMajor != "" && latestMajor != internal.MajorVersionForModule(um.ModulePath) {
		page.LatestMajorVersion = latestMajor
	}

	page.Details = d
	main, ok := d.(*MainDetails)
	if ok {
		page.MetaDescription = metaDescription(main.DocSynopsis)
	}

	// Get vulnerability information.
	if s.vulnClient != nil {
		page.Vulns = VulnsForPackage(um.ModulePath, um.Version, um.Path, s.vulnClient.GetByModule)
	}
	s.servePage(ctx, w, tabSettings.TemplateName, page)
	return nil
}

func latestMinorClass(version string, latest internal.LatestInfo) string {
	c := "DetailsHeader-badge"
	switch {
	case latest.MinorVersion == "":
		c += "--unknown"
	case latest.MinorVersion == version && !latest.UnitExistsAtMinor:
		c += "--notAtLatest"
	case latest.MinorVersion == version:
		c += "--latest"
	default:
		c += "--goToLatest"
	}
	return c
}

// metaDescription uses a safehtml escape hatch to build HTML used
// to render the <meta name="Description"> for unit pages as a
// workaround for https://github.com/google/safehtml/issues/6.
func metaDescription(synopsis string) safehtml.HTML {
	if synopsis == "" {
		return safehtml.HTML{}
	}
	return safehtml.HTMLConcat(
		uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(`<meta name="Description" content="`),
		safehtml.HTMLEscaped(synopsis),
		uncheckedconversions.HTMLFromStringKnownToSatisfyTypeContract(`">`),
	)
}

// isValidTabForUnit reports whether the tab is valid for the given unit.
// It is assumed that tab is a key in unitTabLookup.
func isValidTabForUnit(tab string, um *internal.UnitMeta) bool {
	if tab == tabLicenses && !um.IsRedistributable {
		return false
	}
	if !um.IsPackage() && (tab == tabImports || tab == tabImportedBy) {
		return false
	}
	return true
}

// constructUnitURL returns a URL path that refers to the given unit at the requested
// version. If requestedVersion is "latest", then the resulting path has no
// version; otherwise, it has requestedVersion.
func constructUnitURL(fullPath, modulePath, requestedVersion string) string {
	if requestedVersion == version.Latest {
		return "/" + fullPath
	}
	v := linkVersion(modulePath, requestedVersion, requestedVersion)
	if fullPath == modulePath || modulePath == stdlib.ModulePath {
		return fmt.Sprintf("/%s@%s", fullPath, v)
	}
	return fmt.Sprintf("/%s@%s/%s", modulePath, v, strings.TrimPrefix(fullPath, modulePath+"/"))
}

// canonicalURLPath constructs a URL path to the unit that always includes the
// resolved version.
func canonicalURLPath(fullPath, modulePath, requestedVersion, resolvedVersion string) string {
	return constructUnitURL(fullPath, modulePath,
		linkVersion(modulePath, requestedVersion, resolvedVersion))
}

func isGoProject(modulePath string) bool {
	return modulePath == stdlib.ModulePath || strings.HasPrefix(modulePath, "golang.org")
}
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"strconv"
	"strings"
)

const (
	maxMatchLength = 258
	minMatchLength = 4
	hashBits       = 15
	hashSize       = 1 << hashBits
	hashMask       = (1 << hashBits) - 1
)

type compressionLevel struct {
	good, lazy, nice, chain int
}

var levels = []compressionLevel{
	{}, // 0
	{3, 0, 8, 4},
	{3, 0, 16, 8},
	{3, 0, 32, 32},
	{4, 4, 16, 16},
	{8, 16, 32, 32},
	{8, 16, 128, 128},
	{8, 32, 128, 256},
	{32, 128, 258, 1024},
	{32, 258, 258, 4096},
}

type dictator struct {
	// Pseudo deflate variables, we need those to perform deflate like matching, to identify strings that are emmited as is
	compressionLevel
	window   []byte
	hashHead [32768]int
	hashPrev [16384]int
	// Accumulate characters emmited as is
	stringBuf [16384]byte
	stringLen int
	// Count all the strings here
	table map[string]int
}

func (d *dictator) init(level int) (err error) {
	if level < 4 || level > 9 {
		return fmt.Errorf("Only supposts levels [4, 9], got %d", level)
	}

	d.compressionLevel = levels[level]
	d.stringLen = 0
	d.table = make(map[string]int)
	for i := range d.hashHead {
		d.hashHead[i] = -1
	}
	return nil
}

// Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up.
func (d *dictator) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {

	minMatchLook := maxMatchLength
	if lookahead < minMatchLook {
		minMatchLook = lookahead
	}

	win := d.window

	// We quit when we get a match that's at least nice long
	nice := len(win) - pos

	if d.nice < nice {
		nice = d.nice
	}

	// If we've got a match that's good enough, only look in 1/4 the chain.
	tries := d.chain
	length = prevLength
	if length >= d.good {
		tries >>= 2
	}

	w0 := win[pos]
	w1 := win[pos+1]
	wEnd := win[pos+length]

	for i := prevHead; tries > 0; tries-- {
		if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] {
			n := 2
			for pos+n < len(win) && win[i+n] == win[pos+n] {
				n++
			}

			if n > length && (n > 3) {
				length = n
				offset = pos - i
				ok = true
				if n >= nice {
					// The match is good enough that we don't try to find a better one.
					break
				}
				wEnd = win[pos+n]
			}
		}
		if i = d.hashPrev[i]; i < 0 {
			break
		}
	}
	return
}

func (d *dictator) findUncompressable(in []byte) {
	d.window = in
	pos := 0
	length := minMatchLength - 1

	for {
		lookahead := len(in) - pos
		if lookahead <= minMatchLength {
			break
		}

		hash := ((int(in[pos]) << 10) ^ (int(in[pos]) << 5) ^ (int(in[pos]))) & hashMask
		hashHead := d.hashHead[hash]
		d.hashPrev[pos] = hashHead
		d.hashHead[hash] = pos

		prevLength := length
		if hashHead >= 0 && prevLength < d.nice {
			if newLength, _, ok := d.findMatch(pos, hashHead, minMatchLength-1, lookahead); ok {
				length = newLength
			}
		}

		// Now deflate would output the previous match, therefore if accumulated enough uncompressed bytes, "flush" them
		if prevLength >= minMatchLength && length <= prevLength {
			if d.stringLen >= minMatchLength {
				key := string(d.stringBuf[:d.stringLen])
				d.table[key]++
				d.stringLen = 0
			}
			newPos := pos + prevLength - 1
			if newPos >= len(in) {
				break
			}
			pos++
			for pos < newPos {
				hash := ((int(in[pos]) << 10) ^ (int(in[pos]) << 5) ^ (int(in[pos]))) & hashMask
				hashHead := d.hashHead[hash]
				d.hashPrev[pos] = hashHead
				d.hashHead[hash] = pos
				pos++
			}
			length = minMatchLength - 1
			// Or the previous literal
		} else if pos > 0 {
			d.stringBuf[d.stringLen] = d.window[pos-1]
			d.stringLen++
			pos++
		} else {
			pos++
		}
	}
	if d.stringLen > minMatchLength {
		//fmt.Println(string(d.stringBuf[:d.stringLen]))
		d.table[string(d.stringBuf[:d.stringLen])]++
		d.stringLen = 0
	}
}

// An Item is something we manage in a priority queue.
type scoredString struct {
	value string // The value of the item; arbitrary.
	score int    // The priority of the item in the queue.
}

// A PriorityQueue implements heap.Interface and holds scoredStrings
type PriorityQueue []*scoredString

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].score > pq[j].score
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

func (pq *PriorityQueue) Push(x interface{}) {
	item := x.(*scoredString)
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

func main() {
	var window [16384]byte
	var d dictator
	var table map[string]int
	table = make(map[string]int)
	pq := make(PriorityQueue, 0)
	var dictionary string

	dictLen, _ := strconv.Atoi(os.Args[1])
	path := os.Args[2]

	files, _ := ioutil.ReadDir(path)

	percent := float64(0)
	for num, f := range files {
		file, err := os.Open(path + "/" + f.Name()) // For read access.
		if err != nil {
			continue
		}
		count, err := file.Read(window[:len(window)])
		d.init(4)
		// Create a table of all uncompressable strings in the fime
		d.findUncompressable(window[:count])
		// Merge with the main table
		for k, _ := range d.table {
			table[k]++
		}

		file.Close()

		if newPercent := float64(num) / float64(len(files)) * 100; (newPercent - percent) >= 1 {
			percent = math.Floor(newPercent)
			fmt.Printf("\r%.2f%% ", newPercent)
		}
	}
	fmt.Println("\r100%%")
	fmt.Println("Total uncompressible strings found: ", len(table))
	// If a string appeares in less than 1% of the files, it is probably useless
	threshold := int(math.Ceil(float64(len(files)) * 0.01))
	// Remove unique strings, score others and put into a heap
	heap.Init(&pq)
	for i, v := range table {
		if v < threshold {
			delete(table, i)
		} else {
			item := &scoredString{i, (v * (len(i) - 3)) / len(i)}
			heap.Push(&pq, item)
		}
	}
	fmt.Println("Uncompressible strings with frequency greater than ", threshold, ": ", len(table))

	// Start poping strings from the heap. We want the highest scoring closer to the end, so they are encoded with smaller distance value
	for (pq.Len() > 0) && (len(dictionary) < dictLen) {
		item := heap.Pop(&pq).(*scoredString)
		// Ignore strings that already made it to the dictionary, append others in front
		if !strings.Contains(dictionary, item.value) {
			dictionary = item.value + dictionary
		}
	}
	// Truncate
	if len(dictionary) > dictLen {
		dictionary = dictionary[len(dictionary)-dictLen:]
	}
	// Write
	ioutil.WriteFile(os.Args[3], []byte(dictionary), 0644)
}
<commit_msg>Print usage message when given wrong arguments.<commit_after>package main

import (
	"container/heap"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path"
	"strconv"
	"strings"
)

const (
	maxMatchLength = 258
	minMatchLength = 4
	hashBits       = 15
	hashSize       = 1 << hashBits
	hashMask       = (1 << hashBits) - 1
)

type compressionLevel struct {
	good, lazy, nice, chain int
}

var levels = []compressionLevel{
	{}, // 0
	{3, 0, 8, 4},
	{3, 0, 16, 8},
	{3, 0, 32, 32},
	{4, 4, 16, 16},
	{8, 16, 32, 32},
	{8, 16, 128, 128},
	{8, 32, 128, 256},
	{32, 128, 258, 1024},
	{32, 258, 258, 4096},
}

type dictator struct {
	// Pseudo deflate variables, we need those to perform deflate like matching, to identify strings that are emmited as is
	compressionLevel
	window   []byte
	hashHead [32768]int
	hashPrev [16384]int
	// Accumulate characters emmited as is
	stringBuf [16384]byte
	stringLen int
	// Count all the strings here
	table map[string]int
}

func (d *dictator) init(level int) (err error) {
	if level < 4 || level > 9 {
		return fmt.Errorf("Only supposts levels [4, 9], got %d", level)
	}

	d.compressionLevel = levels[level]
	d.stringLen = 0
	d.table = make(map[string]int)
	for i := range d.hashHead {
		d.hashHead[i] = -1
	}
	return nil
}

// Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up.
func (d *dictator) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {

	minMatchLook := maxMatchLength
	if lookahead < minMatchLook {
		minMatchLook = lookahead
	}

	win := d.window

	// We quit when we get a match that's at least nice long
	nice := len(win) - pos

	if d.nice < nice {
		nice = d.nice
	}

	// If we've got a match that's good enough, only look in 1/4 the chain.
	tries := d.chain
	length = prevLength
	if length >= d.good {
		tries >>= 2
	}

	w0 := win[pos]
	w1 := win[pos+1]
	wEnd := win[pos+length]

	for i := prevHead; tries > 0; tries-- {
		if w0 == win[i] && w1 == win[i+1] && wEnd == win[i+length] {
			n := 2
			for pos+n < len(win) && win[i+n] == win[pos+n] {
				n++
			}

			if n > length && (n > 3) {
				length = n
				offset = pos - i
				ok = true
				if n >= nice {
					// The match is good enough that we don't try to find a better one.
					break
				}
				wEnd = win[pos+n]
			}
		}
		if i = d.hashPrev[i]; i < 0 {
			break
		}
	}
	return
}

func (d *dictator) findUncompressable(in []byte) {
	d.window = in
	pos := 0
	length := minMatchLength - 1

	for {
		lookahead := len(in) - pos
		if lookahead <= minMatchLength {
			break
		}

		hash := ((int(in[pos]) << 10) ^ (int(in[pos]) << 5) ^ (int(in[pos]))) & hashMask
		hashHead := d.hashHead[hash]
		d.hashPrev[pos] = hashHead
		d.hashHead[hash] = pos

		prevLength := length
		if hashHead >= 0 && prevLength < d.nice {
			if newLength, _, ok := d.findMatch(pos, hashHead, minMatchLength-1, lookahead); ok {
				length = newLength
			}
		}

		// Now deflate would output the previous match, therefore if accumulated enough uncompressed bytes, "flush" them
		if prevLength >= minMatchLength && length <= prevLength {
			if d.stringLen >= minMatchLength {
				key := string(d.stringBuf[:d.stringLen])
				d.table[key]++
				d.stringLen = 0
			}
			newPos := pos + prevLength - 1
			if newPos >= len(in) {
				break
			}
			pos++
			for pos < newPos {
				hash := ((int(in[pos]) << 10) ^ (int(in[pos]) << 5) ^ (int(in[pos]))) & hashMask
				hashHead := d.hashHead[hash]
				d.hashPrev[pos] = hashHead
				d.hashHead[hash] = pos
				pos++
			}
			length = minMatchLength - 1
			// Or the previous literal
		} else if pos > 0 {
			d.stringBuf[d.stringLen] = d.window[pos-1]
			d.stringLen++
			pos++
		} else {
			pos++
		}
	}
	if d.stringLen > minMatchLength {
		//fmt.Println(string(d.stringBuf[:d.stringLen]))
		d.table[string(d.stringBuf[:d.stringLen])]++
		d.stringLen = 0
	}
}

// An Item is something we manage in a priority queue.
type scoredString struct {
	value string // The value of the item; arbitrary.
	score int    // The priority of the item in the queue.
}

// A PriorityQueue implements heap.Interface and holds scoredStrings
type PriorityQueue []*scoredString

func (pq PriorityQueue) Len() int { return len(pq) }

func (pq PriorityQueue) Less(i, j int) bool {
	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
	return pq[i].score > pq[j].score
}

func (pq PriorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

func (pq *PriorityQueue) Push(x interface{}) {
	item := x.(*scoredString)
	*pq = append(*pq, item)
}

func (pq *PriorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

func PrintUsage() {
	fmt.Printf("Usage: %s dictionary_size input_directory output_file\n", path.Base(os.Args[0]))
}

func main() {
	var window [16384]byte
	var d dictator
	var table map[string]int
	table = make(map[string]int)
	pq := make(PriorityQueue, 0)
	var dictionary string

	if len(os.Args) != 4 {
		PrintUsage()
		return
	}

	dictLen, _ := strconv.Atoi(os.Args[1])
	path := os.Args[2]

	files, _ := ioutil.ReadDir(path)

	percent := float64(0)
	for num, f := range files {
		file, err := os.Open(path + "/" + f.Name()) // For read access.
		if err != nil {
			continue
		}
		count, err := file.Read(window[:len(window)])
		d.init(4)
		// Create a table of all uncompressable strings in the fime
		d.findUncompressable(window[:count])
		// Merge with the main table
		for k, _ := range d.table {
			table[k]++
		}

		file.Close()

		if newPercent := float64(num) / float64(len(files)) * 100; (newPercent - percent) >= 1 {
			percent = math.Floor(newPercent)
			fmt.Printf("\r%.2f%% ", newPercent)
		}
	}
	fmt.Println("\r100%%")
	fmt.Println("Total uncompressible strings found: ", len(table))
	// If a string appeares in less than 1% of the files, it is probably useless
	threshold := int(math.Ceil(float64(len(files)) * 0.01))
	// Remove unique strings, score others and put into a heap
	heap.Init(&pq)
	for i, v := range table {
		if v < threshold {
			delete(table, i)
		} else {
			item := &scoredString{i, (v * (len(i) - 3)) / len(i)}
			heap.Push(&pq, item)
		}
	}
	fmt.Println("Uncompressible strings with frequency greater than ", threshold, ": ", len(table))

	// Start poping strings from the heap. We want the highest scoring closer to the end, so they are encoded with smaller distance value
	for (pq.Len() > 0) && (len(dictionary) < dictLen) {
		item := heap.Pop(&pq).(*scoredString)
		// Ignore strings that already made it to the dictionary, append others in front
		if !strings.Contains(dictionary, item.value) {
			dictionary = item.value + dictionary
		}
	}
	// Truncate
	if len(dictionary) > dictLen {
		dictionary = dictionary[len(dictionary)-dictLen:]
	}
	// Write
	ioutil.WriteFile(os.Args[3], []byte(dictionary), 0644)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pixels

import (
	"image"
	"image/color"

	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/graphics/opengl"
)

type drawImageHistoryItem struct {
	image    *graphics.Image
	vertices []int16
	geom     graphics.Matrix
	colorm   graphics.Matrix
	mode     opengl.CompositeMode
}

// Pixels represents pixels of an image for restoring when GL context is lost.
type Pixels struct {
	image *graphics.Image

	// basePixels and baseColor are exclusive.
	basePixels       []uint8
	baseColor        color.Color
	drawImageHistory []*drawImageHistoryItem
}

func NewPixels(image *graphics.Image) *Pixels {
	return &Pixels{
		image: image,
	}
}

func (p *Pixels) Clear() {
	p.basePixels = nil
	p.baseColor = nil
	p.drawImageHistory = nil
}

func (p *Pixels) Fill(clr color.Color) {
	p.basePixels = nil
	p.baseColor = clr
	p.drawImageHistory = nil
}

func (p *Pixels) ReplacePixels(pixels []uint8) {
	if p.basePixels == nil {
		p.basePixels = make([]uint8, len(pixels))
	}
	copy(p.basePixels, pixels)
	p.baseColor = nil
	p.drawImageHistory = nil
}

func (p *Pixels) AppendDrawImageHistory(image *graphics.Image, vertices []int16, geom graphics.Matrix, colorm graphics.Matrix, mode opengl.CompositeMode) {
	item := &drawImageHistoryItem{
		image:    image,
		vertices: vertices,
		geom:     geom,
		colorm:   colorm,
		mode:     mode,
	}
	p.drawImageHistory = append(p.drawImageHistory, item)
}

// At returns a color value at idx.
//
// Note that this must not be called until context is available.
// This means Pixels members must match with acutal state in GPU.
func (p *Pixels) At(idx int, context *opengl.Context) (color.Color, error) {
	if p.basePixels == nil || p.drawImageHistory != nil {
		var err error
		p.basePixels, err = p.image.Pixels(context)
		if err != nil {
			return nil, err
		}
		p.baseColor = nil
		p.drawImageHistory = nil
	}
	r, g, b, a := p.basePixels[idx], p.basePixels[idx+1], p.basePixels[idx+2], p.basePixels[idx+3]
	return color.RGBA{r, g, b, a}, nil
}

func (p *Pixels) DependsOn(target *graphics.Image) bool {
	for _, c := range p.drawImageHistory {
		if c.image == target {
			return true
		}
	}
	return false
}

func (p *Pixels) Reset(context *opengl.Context) error {
	var err error
	p.basePixels, err = p.image.Pixels(context)
	if err != nil {
		return err
	}
	p.baseColor = nil
	p.drawImageHistory = nil
	return nil
}

func (p *Pixels) HasDependency() bool {
	return p.drawImageHistory != nil
}

// Restore restores the pixels using its history.
//
// restore is the only function that the pixel data is not present on GPU when this is called.
func (p *Pixels) Restore(context *opengl.Context, width, height int, filter opengl.Filter) (*graphics.Image, error) {
	img := image.NewRGBA(image.Rect(0, 0, width, height))
	if p.basePixels != nil {
		for j := 0; j < height; j++ {
			copy(img.Pix[j*img.Stride:], p.basePixels[j*width*4:(j+1)*width*4])
		}
	}
	gimg, err := graphics.NewImageFromImage(img, filter)
	if err != nil {
		return nil, err
	}
	if p.baseColor != nil {
		if p.basePixels != nil {
			panic("not reach")
		}
		if err := gimg.Fill(p.baseColor); err != nil {
			return nil, err
		}
	}
	for _, c := range p.drawImageHistory {
		// c.image.impl must be already restored.
		/*if c.image.impl.hasHistory() {
			panic("not reach")
		}*/
		if err := gimg.DrawImage(c.image, c.vertices, c.geom, c.colorm, c.mode); err != nil {
			return nil, err
		}
	}
	p.basePixels, err = gimg.Pixels(context)
	if err != nil {
		return nil, err
	}
	p.baseColor = nil
	p.drawImageHistory = nil
	return gimg, nil
}
<commit_msg>pixels: Refactoring<commit_after>// Copyright 2016 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pixels

import (
	"image"
	"image/color"

	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/graphics/opengl"
)

type drawImageHistoryItem struct {
	image    *graphics.Image
	vertices []int16
	geom     graphics.Matrix
	colorm   graphics.Matrix
	mode     opengl.CompositeMode
}

// Pixels represents pixels of an image for restoring when GL context is lost.
type Pixels struct {
	image *graphics.Image

	// basePixels and baseColor are exclusive.
	basePixels       []uint8
	baseColor        color.Color
	drawImageHistory []*drawImageHistoryItem
}

func NewPixels(image *graphics.Image) *Pixels {
	return &Pixels{
		image: image,
	}
}

func (p *Pixels) Clear() {
	p.basePixels = nil
	p.baseColor = nil
	p.drawImageHistory = nil
}

func (p *Pixels) Fill(clr color.Color) {
	p.basePixels = nil
	p.baseColor = clr
	p.drawImageHistory = nil
}

func (p *Pixels) ReplacePixels(pixels []uint8) {
	if p.basePixels == nil {
		p.basePixels = make([]uint8, len(pixels))
	}
	copy(p.basePixels, pixels)
	p.baseColor = nil
	p.drawImageHistory = nil
}

func (p *Pixels) AppendDrawImageHistory(image *graphics.Image, vertices []int16, geom graphics.Matrix, colorm graphics.Matrix, mode opengl.CompositeMode) {
	item := &drawImageHistoryItem{
		image:    image,
		vertices: vertices,
		geom:     geom,
		colorm:   colorm,
		mode:     mode,
	}
	p.drawImageHistory = append(p.drawImageHistory, item)
}

// At returns a color value at idx.
//
// Note that this must not be called until context is available.
// This means Pixels members must match with acutal state in GPU.
func (p *Pixels) At(idx int, context *opengl.Context) (color.Color, error) {
	if p.basePixels == nil || p.drawImageHistory != nil {
		if err := p.Reset(context); err != nil {
			return nil, err
		}
	}
	r, g, b, a := p.basePixels[idx], p.basePixels[idx+1], p.basePixels[idx+2], p.basePixels[idx+3]
	return color.RGBA{r, g, b, a}, nil
}

func (p *Pixels) DependsOn(target *graphics.Image) bool {
	for _, c := range p.drawImageHistory {
		if c.image == target {
			return true
		}
	}
	return false
}

func (p *Pixels) Reset(context *opengl.Context) error {
	var err error
	p.basePixels, err = p.image.Pixels(context)
	if err != nil {
		return err
	}
	p.baseColor = nil
	p.drawImageHistory = nil
	return nil
}

func (p *Pixels) HasDependency() bool {
	return p.drawImageHistory != nil
}

// Restore restores the pixels using its history.
//
// Restore is the only function that the pixel data is not present on GPU when this is called.
func (p *Pixels) Restore(context *opengl.Context, width, height int, filter opengl.Filter) (*graphics.Image, error) {
	img := image.NewRGBA(image.Rect(0, 0, width, height))
	if p.basePixels != nil {
		for j := 0; j < height; j++ {
			copy(img.Pix[j*img.Stride:], p.basePixels[j*width*4:(j+1)*width*4])
		}
	}
	gimg, err := graphics.NewImageFromImage(img, filter)
	if err != nil {
		return nil, err
	}
	if p.baseColor != nil {
		if p.basePixels != nil {
			panic("not reach")
		}
		if err := gimg.Fill(p.baseColor); err != nil {
			return nil, err
		}
	}
	for _, c := range p.drawImageHistory {
		// c.image.impl must be already restored.
		/*if c.image.impl.hasHistory() {
			panic("not reach")
		}*/
		if err := gimg.DrawImage(c.image, c.vertices, c.geom, c.colorm, c.mode); err != nil {
			return nil, err
		}
	}
	p.basePixels, err = gimg.Pixels(context)
	if err != nil {
		return nil, err
	}
	p.baseColor = nil
	p.drawImageHistory = nil
	return gimg, nil
}
<|endoftext|>