<commit_before>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	latest "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha4"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/karrick/godirwalk"
	"github.com/moby/buildkit/frontend/dockerfile/command"
	"github.com/moby/buildkit/frontend/dockerfile/parser"
	"github.com/moby/buildkit/frontend/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// RetrieveImage is overridden for unit testing
var RetrieveImage = retrieveImage

func ValidateDockerfile(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		logrus.Warnf("opening file %s: %s", path, err.Error())
		return false
	}
	res, err := parser.Parse(f)
	if err != nil || res == nil || len(res.AST.Children) == 0 {
		return false
	}
	// validate each node contains valid dockerfile directive
	for _, child := range res.AST.Children {
		_, ok := command.Commands[child.Value]
		if !ok {
			return false
		}
	}

	return true
}

func expandBuildArgs(nodes []*parser.Node, buildArgs map[string]*string) {
	var key, value string

	for _, node := range nodes {
		switch node.Value {
		case command.Arg:
			// build arg's key
			keyValue := strings.Split(node.Next.Value, "=")
			key = keyValue[0]

			// build arg's value
			if buildArgs[key] != nil {
				value = *buildArgs[key]
			} else if len(keyValue) > 1 {
				value = keyValue[1]
			}
		default:
			if key != "" {
				// replace $key with value
				for curr := node; curr != nil; curr = curr.Next {
					curr.Value = util.Expand(curr.Value, key, value)
				}
			}
		}
	}
}

func readDockerfile(workspace, absDockerfilePath string, buildArgs map[string]*string) ([]string, error) {
	f, err := os.Open(absDockerfilePath)
	if err != nil {
		return nil, errors.Wrapf(err, "opening dockerfile: %s", absDockerfilePath)
	}
	defer f.Close()

	res, err := parser.Parse(f)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	expandBuildArgs(res.AST.Children, buildArgs)

	// Then process onbuilds, if present.
	onbuildsImages := [][]string{}
	stages := map[string]bool{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case command.From:
			imageName := value.Next.Value
			if _, found := stages[imageName]; found {
				continue
			}

			next := value.Next.Next
			if next != nil && strings.ToLower(next.Value) == "as" {
				if next.Next != nil {
					stages[next.Next.Value] = true
				}
			}

			onbuilds, err := processBaseImage(imageName)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
			onbuildsImages = append(onbuildsImages, onbuilds)
		}
	}

	var copied [][]string
	envs := map[string]string{}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case command.Add, command.Copy:
				files, _ := processCopy(value, envs)
				if len(files) > 0 {
					copied = append(copied, files)
				}
			case command.Env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, image := range onbuildsImages {
		for _, ob := range image {
			obRes, err := parser.Parse(strings.NewReader(ob))
			if err != nil {
				return nil, err
			}
			dispatchInstructions(obRes)
		}
	}

	dispatchInstructions(res)

	expandedPaths := make(map[string]bool)
	for _, files := range copied {
		matchesOne := false

		for _, p := range files {
			path := filepath.Join(workspace, p)
			if _, err := os.Stat(path); err == nil {
				expandedPaths[p] = true
				matchesOne = true
				continue
			}

			files, err := filepath.Glob(path)
			if err != nil {
				return nil, errors.Wrap(err, "invalid glob pattern")
			}
			if files == nil {
				continue
			}

			for _, f := range files {
				rel, err := filepath.Rel(workspace, f)
				if err != nil {
					return nil, fmt.Errorf("getting relative path of %s", f)
				}

				expandedPaths[rel] = true
			}
			matchesOne = true
		}

		if !matchesOne {
			return nil, fmt.Errorf("file pattern %s must match at least one file", files)
		}
	}

	var deps []string
	for dep := range expandedPaths {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)

	return deps, nil
}

// GetDependencies finds the sources dependencies for the given docker artifact.
// All paths are relative to the workspace.
func GetDependencies(workspace string, a *latest.DockerArtifact) ([]string, error) {
	absDockerfilePath, err := NormalizeDockerfilePath(workspace, a.DockerfilePath)
	if err != nil {
		return nil, errors.Wrap(err, "normalizing dockerfile path")
	}

	deps, err := readDockerfile(workspace, absDockerfilePath, a.BuildArgs)
	if err != nil {
		return nil, err
	}

	// Read patterns to ignore
	var excludes []string
	dockerignorePath := filepath.Join(workspace, ".dockerignore")
	if _, err := os.Stat(dockerignorePath); !os.IsNotExist(err) {
		r, err := os.Open(dockerignorePath)
		if err != nil {
			return nil, err
		}
		defer r.Close()

		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
	}

	pExclude, err := fileutils.NewPatternMatcher(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "invalid exclude patterns")
	}

	// Walk the workspace
	files := make(map[string]bool)
	for _, dep := range deps {
		dep = filepath.Clean(dep)
		absDep := filepath.Join(workspace, dep)

		fi, err := os.Stat(absDep)
		if err != nil {
			return nil, errors.Wrapf(err, "stating file %s", absDep)
		}

		switch mode := fi.Mode(); {
		case mode.IsDir():
			if err := godirwalk.Walk(absDep, &godirwalk.Options{
				Unsorted: true,
				Callback: func(fpath string, info *godirwalk.Dirent) error {
					relPath, err := filepath.Rel(workspace, fpath)
					if err != nil {
						return err
					}

					ignored, err := pExclude.Matches(relPath)
					if err != nil {
						return err
					}

					if info.IsDir() {
						if ignored {
							return filepath.SkipDir
						}
					} else if !ignored {
						files[relPath] = true
					}

					return nil
				},
			}); err != nil {
				return nil, errors.Wrapf(err, "walking folder %s", absDep)
			}
		case mode.IsRegular():
			ignored, err := pExclude.Matches(dep)
			if err != nil {
				return nil, err
			}

			if !ignored {
				files[dep] = true
			}
		}
	}

	// Always add dockerfile even if it's .dockerignored. The daemon will need it anyways.
	if !filepath.IsAbs(a.DockerfilePath) {
		files[a.DockerfilePath] = true
	} else {
		files[absDockerfilePath] = true
	}

	// Ignore .dockerignore
	delete(files, ".dockerignore")

	var dependencies []string
	for file := range files {
		dependencies = append(dependencies, file)
	}
	sort.Strings(dependencies)

	return dependencies, nil
}

func processBaseImage(baseImageName string) ([]string, error) {
	if strings.ToLower(baseImageName) == "scratch" {
		return nil, nil
	}

	logrus.Debugf("Checking base image %s for ONBUILD triggers.", baseImageName)
	img, err := RetrieveImage(baseImageName)
	if err != nil {
		return nil, err
	}

	logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, baseImageName)
	return img.Config.OnBuild, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*v1.ConfigFile, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*v1.ConfigFile), nil
	}

	client, err := NewAPIClient()
	if err != nil {
		return nil, err
	}

	cfg := &v1.ConfigFile{}
	raw, err := retrieveLocalImage(client, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = retrieveRemoteConfig(image)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client APIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteConfig(identifier string) (*v1.ConfigFile, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return nil, errors.Wrap(err, "getting image")
	}

	return img.ConfigFile()
}

func processCopy(value *parser.Node, envs map[string]string) ([]string, error) {
	var copied []string

	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return nil, errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil, nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			copied = append(copied, src)
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}

	return copied, nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<commit_msg>Cleanup docker file parsing<commit_after>/*
Copyright 2018 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	latest "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha4"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/docker/docker/builder/dockerignore"
	"github.com/docker/docker/pkg/fileutils"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/karrick/godirwalk"
	"github.com/moby/buildkit/frontend/dockerfile/command"
	"github.com/moby/buildkit/frontend/dockerfile/parser"
	"github.com/moby/buildkit/frontend/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type from struct {
	image string
	as    string
}

// RetrieveImage is overridden for unit testing
var RetrieveImage = retrieveImage

func ValidateDockerfile(path string) bool {
	f, err := os.Open(path)
	if err != nil {
		logrus.Warnf("opening file %s: %s", path, err.Error())
		return false
	}
	res, err := parser.Parse(f)
	if err != nil || res == nil || len(res.AST.Children) == 0 {
		return false
	}
	// validate each node contains valid dockerfile directive
	for _, child := range res.AST.Children {
		_, ok := command.Commands[child.Value]
		if !ok {
			return false
		}
	}

	return true
}

func expandBuildArgs(nodes []*parser.Node, buildArgs map[string]*string) {
	var key, value string

	for _, node := range nodes {
		switch node.Value {
		case command.Arg:
			// build arg's key
			keyValue := strings.Split(node.Next.Value, "=")
			key = keyValue[0]

			// build arg's value
			if buildArgs[key] != nil {
				value = *buildArgs[key]
			} else if len(keyValue) > 1 {
				value = keyValue[1]
			}
		default:
			if key != "" {
				// replace $key with value
				for curr := node; curr != nil; curr = curr.Next {
					curr.Value = util.Expand(curr.Value, key, value)
				}
			}
		}
	}
}

func fromInstructions(nodes []*parser.Node) []from {
	var list []from

	for _, node := range nodes {
		if node.Value != command.From {
			continue
		}

		list = append(list, fromInstruction(node))
	}

	return list
}

func fromInstruction(node *parser.Node) from {
	var as string
	if next := node.Next.Next; next != nil && strings.ToLower(next.Value) == "as" && next.Next != nil {
		as = next.Next.Value
	}

	return from{
		image: node.Next.Value,
		as:    as,
	}
}

func onbuildImages(nodes []*parser.Node) [][]string {
	var onbuildImages [][]string

	stages := map[string]bool{}
	for _, from := range fromInstructions(nodes) {
		if _, found := stages[from.image]; found {
			continue
		}

		if from.as != "" {
			stages[from.as] = true
		}

		if strings.ToLower(from.image) == "scratch" {
			continue
		}

		logrus.Debugf("Checking base image %s for ONBUILD triggers.", from.image)
		img, err := RetrieveImage(from.image)
		if err != nil {
			logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			continue
		}

		logrus.Debugf("Found onbuild triggers %v in image %s", img.Config.OnBuild, from.image)
		onbuildImages = append(onbuildImages, img.Config.OnBuild)
	}

	return onbuildImages
}

func readDockerfile(workspace, absDockerfilePath string, buildArgs map[string]*string) ([]string, error) {
	f, err := os.Open(absDockerfilePath)
	if err != nil {
		return nil, errors.Wrapf(err, "opening dockerfile: %s", absDockerfilePath)
	}
	defer f.Close()

	res, err := parser.Parse(f)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}

	expandBuildArgs(res.AST.Children, buildArgs)
	onbuildImages := onbuildImages(res.AST.Children)

	var copied [][]string
	envs := map[string]string{}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case command.Add, command.Copy:
				files, _ := processCopy(value, envs)
				if len(files) > 0 {
					copied = append(copied, files)
				}
			case command.Env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, image := range onbuildImages {
		for _, ob := range image {
			obRes, err := parser.Parse(strings.NewReader(ob))
			if err != nil {
				return nil, err
			}
			dispatchInstructions(obRes)
		}
	}

	dispatchInstructions(res)

	expandedPaths := make(map[string]bool)
	for _, files := range copied {
		matchesOne := false

		for _, p := range files {
			path := filepath.Join(workspace, p)
			if _, err := os.Stat(path); err == nil {
				expandedPaths[p] = true
				matchesOne = true
				continue
			}

			files, err := filepath.Glob(path)
			if err != nil {
				return nil, errors.Wrap(err, "invalid glob pattern")
			}
			if files == nil {
				continue
			}

			for _, f := range files {
				rel, err := filepath.Rel(workspace, f)
				if err != nil {
					return nil, fmt.Errorf("getting relative path of %s", f)
				}

				expandedPaths[rel] = true
			}
			matchesOne = true
		}

		if !matchesOne {
			return nil, fmt.Errorf("file pattern %s must match at least one file", files)
		}
	}

	var deps []string
	for dep := range expandedPaths {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)

	return deps, nil
}

// GetDependencies finds the sources dependencies for the given docker artifact.
// All paths are relative to the workspace.
func GetDependencies(workspace string, a *latest.DockerArtifact) ([]string, error) {
	absDockerfilePath, err := NormalizeDockerfilePath(workspace, a.DockerfilePath)
	if err != nil {
		return nil, errors.Wrap(err, "normalizing dockerfile path")
	}

	deps, err := readDockerfile(workspace, absDockerfilePath, a.BuildArgs)
	if err != nil {
		return nil, err
	}

	// Read patterns to ignore
	var excludes []string
	dockerignorePath := filepath.Join(workspace, ".dockerignore")
	if _, err := os.Stat(dockerignorePath); !os.IsNotExist(err) {
		r, err := os.Open(dockerignorePath)
		if err != nil {
			return nil, err
		}
		defer r.Close()

		excludes, err = dockerignore.ReadAll(r)
		if err != nil {
			return nil, err
		}
	}

	pExclude, err := fileutils.NewPatternMatcher(excludes)
	if err != nil {
		return nil, errors.Wrap(err, "invalid exclude patterns")
	}

	// Walk the workspace
	files := make(map[string]bool)
	for _, dep := range deps {
		dep = filepath.Clean(dep)
		absDep := filepath.Join(workspace, dep)

		fi, err := os.Stat(absDep)
		if err != nil {
			return nil, errors.Wrapf(err, "stating file %s", absDep)
		}

		switch mode := fi.Mode(); {
		case mode.IsDir():
			if err := godirwalk.Walk(absDep, &godirwalk.Options{
				Unsorted: true,
				Callback: func(fpath string, info *godirwalk.Dirent) error {
					relPath, err := filepath.Rel(workspace, fpath)
					if err != nil {
						return err
					}

					ignored, err := pExclude.Matches(relPath)
					if err != nil {
						return err
					}

					if info.IsDir() {
						if ignored {
							return filepath.SkipDir
						}
					} else if !ignored {
						files[relPath] = true
					}

					return nil
				},
			}); err != nil {
				return nil, errors.Wrapf(err, "walking folder %s", absDep)
			}
		case mode.IsRegular():
			ignored, err := pExclude.Matches(dep)
			if err != nil {
				return nil, err
			}

			if !ignored {
				files[dep] = true
			}
		}
	}

	// Always add dockerfile even if it's .dockerignored. The daemon will need it anyways.
	if !filepath.IsAbs(a.DockerfilePath) {
		files[a.DockerfilePath] = true
	} else {
		files[absDockerfilePath] = true
	}

	// Ignore .dockerignore
	delete(files, ".dockerignore")

	var dependencies []string
	for file := range files {
		dependencies = append(dependencies, file)
	}
	sort.Strings(dependencies)

	return dependencies, nil
}

var imageCache sync.Map

func retrieveImage(image string) (*v1.ConfigFile, error) {
	cachedCfg, present := imageCache.Load(image)
	if present {
		return cachedCfg.(*v1.ConfigFile), nil
	}

	client, err := NewAPIClient()
	if err != nil {
		return nil, err
	}

	cfg := &v1.ConfigFile{}
	raw, err := retrieveLocalImage(client, image)
	if err == nil {
		if err := json.Unmarshal(raw, cfg); err != nil {
			return nil, err
		}
	} else {
		cfg, err = retrieveRemoteConfig(image)
		if err != nil {
			return nil, errors.Wrap(err, "getting remote config")
		}
	}

	imageCache.Store(image, cfg)

	return cfg, nil
}

func retrieveLocalImage(client APIClient, image string) ([]byte, error) {
	_, raw, err := client.ImageInspectWithRaw(context.Background(), image)
	if err != nil {
		return nil, err
	}

	return raw, nil
}

func retrieveRemoteConfig(identifier string) (*v1.ConfigFile, error) {
	img, err := remoteImage(identifier)
	if err != nil {
		return nil, errors.Wrap(err, "getting image")
	}

	return img.ConfigFile()
}

func processCopy(value *parser.Node, envs map[string]string) ([]string, error) {
	var copied []string

	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return nil, errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil, nil
		}
		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
			copied = append(copied, src)
		} else {
			logrus.Debugf("Skipping watch on remote dependency %s", src)
		}

		value = value.Next
	}

	return copied, nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package ditaconvert

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/url"
	"path"
	"strings"

	"github.com/raintreeinc/ditaconvert/html"
)

func TODO(context *Context, dec *xml.Decoder, start xml.StartElement) error {
	context.Encoder.WriteRaw(`<div class="conversion-error">TODO ` + start.Name.Local + `</div>`)
	dec.Skip()
	return nil
}

/* TODO: unify all rules
func Rename(tag string, attrs... xml.Attr) TokenProcessor {
	return func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
		start.Name.Local = tag
		return context.EmitWithChildren(dec, start)
	}
}

func RenameWithClass(tag string, class string) TokenProcessor {
	return func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
		start.Name.Local = tag
		setAttr(&start, "class", class)
		return context.EmitWithChildren(dec, start)
	}
}*/

func NewDefaultRules() *Rules {
	return &Rules{
		Rename: map[string]Renaming{
			// conversion
			"xref": {"a", ""},
			"link": {"a", ""},

			//lists
			"choices":         {"ul", ""},
			"choice":          {"li", ""},
			"steps-unordered": {"ul", ""},
			"steps":           {"ol", ""},
			"step":            {"li", ""}, // handled with a custom rule
			"substeps":        {"ol", ""},
			"substep":         {"li", ""}, // handled with a custom rule

			"b":     {"strong", ""},
			"i":     {"em", ""},
			"lines": {"pre", ""},

			"codeblock": {"pre", "codeblock"},
			"pre":       {"pre", "pre"},

			"codeph":      {"samp", "codeph"},
			"cmdname":     {"span", "cmdname"},
			"cmd":         {"span", "cmd"},
			"shortcut":    {"span", "shortcut"},
			"wintitle":    {"span", "wintitle"},
			"filepath":    {"span", "filepath"},
			"menucascade": {"span", "menucascade"},
			"msgph":       {"span", "msgph"},
			"reportitem":  {"span", "reportitem"},
			"varname":     {"span", "varname"},

			"option":   {"span", "option"},
			"synph":    {"span", ""},
			"delim":    {"span", ""},
			"sep":      {"span", ""},
			"parmname": {"span", ""},

			"userinput": {"kbd", "userinput"},

			"image": {"img", ""},

			// ui
			"uicontrol": {"b", ""},

			// divs
			"context":    {"div", ""},
			"result":     {"div", ""},
			"stepresult": {"div", ""},
			"stepxmp":    {"div", ""},
			"info":       {"div", ""},
			"note":       {"div", ""},
			"refsyn":     {"div", ""},
			"bodydiv":    {"div", ""},
			"fig":        {"div", ""},

			"prereq":  {"div", ""},
			"postreq": {"div", ""},

			"colspec": {"colgroup", ""},

			"row":   {"tr", ""},
			"entry": {"td", ""},

			// RAINTREE SPECIFIC
			"keystroke": {"b", "key"},
			"secright":  {"span", "secright"},

			// faq
			"faq":          {"dl", ""},
			"faq-item":     {"div", ""},
			"faq-question": {"dt", "dlterm"},
			"faq-answer":   {"dd", ""},

			//UI items
			"ui-item-list": {"dl", ""},
			"ui-item":      {"div", ""},
			//"ui-item-name":        {"dt", "dlterm"},
			"ui-item-description": {"dd", ""},

			// setup options
			"setup-options":            {"dl", ""},
			"setup-option":             {"div", ""},
			"setup-option-name":        {"dt", "dlterm"},
			"setup-option-description": {"dd", ""},

			"section":    {"div", "section"},
			"example":    {"div", "example"},
			"sectiondiv": {"div", ""},
			"title":      {"h2", "sectiontitle"},

			// ??
			"dlentry": {"div", ""},
			"dt":      {"dt", "dlterm"},
		},
		Skip: map[string]bool{
			"br":            true,
			"draft-comment": true,

			// RAINTREE SPECIFIC
			"settinghead": true,
		},
		Unwrap: map[string]bool{
			"tgroup": true,
		},
		Custom: map[string]TokenProcessor{
			"a": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				var href, desc string
				var internal bool

				href = getAttr(&start, "href")
				if href != "" {
					href, _, desc, internal = context.ResolveLinkInfo(href)
					setAttr(&start, "href", href)
				}

				if desc != "" && getAttr(&start, "title") == "" {
					setAttr(&start, "title", desc)
				}

				setAttr(&start, "scope", "")
				if internal && href != "" {
					//setAttr(&start, "data-link", href)
				}

				if getAttr(&start, "format") != "" && href != "" {
					setAttr(&start, "format", "")
					ext := strings.ToLower(path.Ext(href))
					if ext == ".pdf" || ext == ".doc" || ext == ".xml" || ext == ".rtf" || ext == ".zip" || ext == ".exe" {
						setAttr(&start, "download", path.Base(href))
					} else {
						setAttr(&start, "target", "_blank")
					}
				}

				return context.EmitWithChildren(dec, start)
			},
			"img": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				href := getAttr(&start, "href")
				//setAttr(&start, "src", context.InlinedImageURL(href))
				setAttr(&start, "src", href)
				setAttr(&start, "href", "")

				placement := getAttr(&start, "placement")
				setAttr(&start, "placement", "")
				if placement == "break" {
					context.Encoder.WriteStart("p",
						xml.Attr{Name: xml.Name{Local: "class"}, Value: "image"})
				}

				err := context.EmitWithChildren(dec, start)

				if placement == "break" {
					context.Encoder.WriteEnd("p")
				}

				return err
			},
			"data": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				datatype := strings.ToLower(getAttr(&start, "datatype"))
				if datatype == "rttutorial" {
					dec.Skip()
					href := getAttr(&start, "href")

					const videof = `` +
						`<video controls>` +
						`	<source src="%s" type="video/mp4">` +
						`	<object classid="clsid:d27cdb6e-ae6d-11cf-96b8-444553540000" codebase="http://fpdownload.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=8,0,0,0" ` +
						`		<param name="SRC" value="http://ie.microsoft.com/testdrive/IEBlog/Common/player.swf?file=%s">` +
						`		<p>Video playback not supported</p>` +
						`	</object>` +
						`</video>`

					srcurl := html.NormalizeURL(href)
					urlarg := url.QueryEscape(href)

					context.Encoder.WriteRaw(fmt.Sprintf(videof, srcurl, urlarg))
					return nil
				}

				return context.EmitWithChildren(dec, start)
			},
			"imagemap": ConvertImageMap,

			"note": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				typ := getAttr(&start, "type")
				if typ == "other" {
					typ = getAttr(&start, "othertype")
				}
				if typ == "" {
					typ = "note"
				}
				setAttr(&start, "type", "")
				setAttr(&start, "othertype", "")
				setAttr(&start, "class", "note")
				mdiclass := "note-outline"
				switch typ {
				case "tip":
					mdiclass = "lightbulb-outline"
				case "caution":
					mdiclass = "alert"
				case "Extra":
					mdiclass = "key"
				case "Rev-Edition":
					mdiclass = "elevation-rise"
				case "PDF":
					mdiclass = "book-open"
				}

				context.check(context.Encoder.WriteStart("div", start.Attr...))
				context.check(context.Encoder.WriteRaw(`<i class="mdi mdi-` + mdiclass + `" title="` + typ + `"></i> `))
				context.check(context.Encoder.WriteStart("span"))
				err := context.Recurse(dec)
				context.check(context.Encoder.WriteEnd("span"))
				context.check(context.Encoder.WriteEnd("div"))
				return err
			},
			"menucascade": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "span"

				setAttr(&start, "class", "menucascade")
				// encode starting tag and attributes
				if err := context.Encoder.Encode(start); err != nil {
					return err
				}

				first := true

				// recurse on child tokens
				for {
					token, err := dec.Token()
					if err != nil {
						if err == io.EOF {
							return nil
						}
						return err
					}

					if _, ended := token.(xml.EndElement); ended {
						break
					}
					if _, starting := token.(xml.StartElement); starting {
						if !first {
							context.Encoder.WriteRaw(">")
						}
						first = false
					}
					if err := context.Handle(dec, token); err != nil {
						return err
					}
				}

				// always encode ending tag
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return nil
			},

			"ui-item-name": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "dt"

				setAttr(&start, "class", "ui-item-name")
				// encode starting tag and attributes
				if err := context.Encoder.Encode(start); err != nil {
					return err
				}

				first := true

				// recurse on child tokens
				for {
					token, err := dec.Token()
					if err != nil {
						if err == io.EOF {
							return nil
						}
						return err
					}

					if _, ended := token.(xml.EndElement); ended {
						break
					}
					if _, starting := token.(xml.StartElement); starting {
						if !first {
							context.Encoder.WriteRaw(", ")
						}
						first = false
					}
					if err := context.Handle(dec, token); err != nil {
						return err
					}
				}

				// always encode ending tag
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return nil
			},

			"simpletable": HandleSimpleTable,
			"table":       HandleTable,
			"settings":    HandleSettings,

			"step": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "li"

				context.check(context.Encoder.Encode(start))
				if getAttr(&start, "importance") == "optional" {
					context.Encoder.WriteRaw("(Optional) ")
				}
				err := context.Recurse(dec)
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return err
			},
			"substep": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "li"

				context.check(context.Encoder.Encode(start))
				if getAttr(&start, "importance") == "optional" {
					context.Encoder.WriteRaw("(Optional) ")
				}
				err := context.Recurse(dec)
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return err
			},
			"p": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {

				err := context.EmitWithChildren(dec, start)

				if getAttr(&start, "outputclass") == "no-results-warning" {
					context.Encoder.WriteRaw(`<hr class="no-results-warning">`)
				}

				return err
			},
		},
	}
}
<commit_msg>Figure elements are now tagged with the "figure" class<commit_after>package ditaconvert

import (
	"encoding/xml"
	"fmt"
	"io"
	"net/url"
	"path"
	"strings"

	"github.com/raintreeinc/ditaconvert/html"
)

func TODO(context *Context, dec *xml.Decoder, start xml.StartElement) error {
	context.Encoder.WriteRaw(`<div class="conversion-error">TODO ` + start.Name.Local + `</div>`)
	dec.Skip()
	return nil
}

/* TODO: unify all rules
func Rename(tag string, attrs... xml.Attr) TokenProcessor {
	return func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
		start.Name.Local = tag
		return context.EmitWithChildren(dec, start)
	}
}

func RenameWithClass(tag string, class string) TokenProcessor {
	return func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
		start.Name.Local = tag
		setAttr(&start, "class", class)
		return context.EmitWithChildren(dec, start)
	}
}*/

func NewDefaultRules() *Rules {
	return &Rules{
		Rename: map[string]Renaming{
			// conversion
			"xref": {"a", ""},
			"link": {"a", ""},

			//lists
			"choices":         {"ul", ""},
			"choice":          {"li", ""},
			"steps-unordered": {"ul", ""},
			"steps":           {"ol", ""},
			"step":            {"li", ""}, // handled with a custom rule
			"substeps":        {"ol", ""},
			"substep":         {"li", ""}, // handled with a custom rule

			"b":     {"strong", ""},
			"i":     {"em", ""},
			"lines": {"pre", ""},

			"codeblock": {"pre", "codeblock"},
			"pre":       {"pre", "pre"},

			"codeph":      {"samp", "codeph"},
			"cmdname":     {"span", "cmdname"},
			"cmd":         {"span", "cmd"},
			"shortcut":    {"span", "shortcut"},
			"wintitle":    {"span", "wintitle"},
			"filepath":    {"span", "filepath"},
			"menucascade": {"span", "menucascade"},
			"msgph":       {"span", "msgph"},
			"reportitem":  {"span", "reportitem"},
			"varname":     {"span", "varname"},

			"option":   {"span", "option"},
			"synph":    {"span", ""},
			"delim":    {"span", ""},
			"sep":      {"span", ""},
			"parmname": {"span", ""},

			"userinput": {"kbd", "userinput"},

			"image": {"img", ""},

			// ui
			"uicontrol": {"b", ""},

			// divs
			"context":    {"div", ""},
			"result":     {"div", ""},
			"stepresult": {"div", ""},
			"stepxmp":    {"div", ""},
			"info":       {"div", ""},
			"note":       {"div", ""},
			"refsyn":     {"div", ""},
			"bodydiv":    {"div", ""},
			"fig":        {"div", "figure"},

			"prereq":  {"div", ""},
			"postreq": {"div", ""},

			"colspec": {"colgroup", ""},

			"row":   {"tr", ""},
			"entry": {"td", ""},

			// RAINTREE SPECIFIC
			"keystroke": {"b", "key"},
			"secright":  {"span", "secright"},

			// faq
			"faq":          {"dl", ""},
			"faq-item":     {"div", ""},
			"faq-question": {"dt", "dlterm"},
			"faq-answer":   {"dd", ""},

			//UI items
			"ui-item-list": {"dl", ""},
			"ui-item":      {"div", ""},
			//"ui-item-name":        {"dt", "dlterm"},
			"ui-item-description": {"dd", ""},

			// setup options
			"setup-options":            {"dl", ""},
			"setup-option":             {"div", ""},
			"setup-option-name":        {"dt", "dlterm"},
			"setup-option-description": {"dd", ""},

			"section":    {"div", "section"},
			"example":    {"div", "example"},
			"sectiondiv": {"div", ""},
			"title":      {"h2", "sectiontitle"},

			// ??
			"dlentry": {"div", ""},
			"dt":      {"dt", "dlterm"},
		},
		Skip: map[string]bool{
			"br":            true,
			"draft-comment": true,

			// RAINTREE SPECIFIC
			"settinghead": true,
		},
		Unwrap: map[string]bool{
			"tgroup": true,
		},
		Custom: map[string]TokenProcessor{
			"a": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				var href, desc string
				var internal bool

				href = getAttr(&start, "href")
				if href != "" {
					href, _, desc, internal = context.ResolveLinkInfo(href)
					setAttr(&start, "href", href)
				}

				if desc != "" && getAttr(&start, "title") == "" {
					setAttr(&start, "title", desc)
				}

				setAttr(&start, "scope", "")
				if internal && href != "" {
					//setAttr(&start, "data-link", href)
				}

				if getAttr(&start, "format") != "" && href != "" {
					setAttr(&start, "format", "")
					ext := strings.ToLower(path.Ext(href))
					if ext == ".pdf" || ext == ".doc" || ext == ".xml" || ext == ".rtf" || ext == ".zip" || ext == ".exe" {
						setAttr(&start, "download", path.Base(href))
					} else {
						setAttr(&start, "target", "_blank")
					}
				}

				return context.EmitWithChildren(dec, start)
			},
			"img": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				href := getAttr(&start, "href")
				//setAttr(&start, "src", context.InlinedImageURL(href))
				setAttr(&start, "src", href)
				setAttr(&start, "href", "")

				placement := getAttr(&start, "placement")
				setAttr(&start, "placement", "")
				if placement == "break" {
					context.Encoder.WriteStart("p",
						xml.Attr{Name: xml.Name{Local: "class"}, Value: "image"})
				}

				err := context.EmitWithChildren(dec, start)

				if placement == "break" {
					context.Encoder.WriteEnd("p")
				}

				return err
			},
			"data": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				datatype := strings.ToLower(getAttr(&start, "datatype"))
				if datatype == "rttutorial" {
					dec.Skip()
					href := getAttr(&start, "href")

					const videof = `` +
						`<video controls>` +
						`	<source src="%s" type="video/mp4">` +
						`	<object classid="clsid:d27cdb6e-ae6d-11cf-96b8-444553540000" codebase="http://fpdownload.macromedia.com/pub/shockwave/cabs/flash/swflash.cab#version=8,0,0,0" ` +
						`		<param name="SRC" value="http://ie.microsoft.com/testdrive/IEBlog/Common/player.swf?file=%s">` +
						`		<p>Video playback not supported</p>` +
						`	</object>` +
						`</video>`

					srcurl := html.NormalizeURL(href)
					urlarg := url.QueryEscape(href)

					context.Encoder.WriteRaw(fmt.Sprintf(videof, srcurl, urlarg))
					return nil
				}

				return context.EmitWithChildren(dec, start)
			},
			"imagemap": ConvertImageMap,

			"note": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				typ := getAttr(&start, "type")
				if typ == "other" {
					typ = getAttr(&start, "othertype")
				}
				if typ == "" {
					typ = "note"
				}
				setAttr(&start, "type", "")
				setAttr(&start, "othertype", "")
				setAttr(&start, "class", "note")
				mdiclass := "note-outline"
				switch typ {
				case "tip":
					mdiclass = "lightbulb-outline"
				case "caution":
					mdiclass = "alert"
				case "Extra":
					mdiclass = "key"
				case "Rev-Edition":
					mdiclass = "elevation-rise"
				case "PDF":
					mdiclass = "book-open"
				}

				context.check(context.Encoder.WriteStart("div", start.Attr...))
				context.check(context.Encoder.WriteRaw(`<i class="mdi mdi-` + mdiclass + `" title="` + typ + `"></i> `))
				context.check(context.Encoder.WriteStart("span"))
				err := context.Recurse(dec)
				context.check(context.Encoder.WriteEnd("span"))
				context.check(context.Encoder.WriteEnd("div"))
				return err
			},
			"menucascade": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "span"

				setAttr(&start, "class", "menucascade")
				// encode starting tag and attributes
				if err := context.Encoder.Encode(start); err != nil {
					return err
				}

				first := true

				// recurse on child tokens
				for {
					token, err := dec.Token()
					if err != nil {
						if err == io.EOF {
							return nil
						}
						return err
					}

					if _, ended := token.(xml.EndElement); ended {
						break
					}
					if _, starting := token.(xml.StartElement); starting {
						if !first {
							context.Encoder.WriteRaw(">")
						}
						first = false
					}
					if err := context.Handle(dec, token); err != nil {
						return err
					}
				}

				// always encode ending tag
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return nil
			},

			"ui-item-name": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "dt"

				setAttr(&start, "class", "ui-item-name")
				// encode starting tag and attributes
				if err := context.Encoder.Encode(start); err != nil {
					return err
				}

				first := true

				// recurse on child tokens
				for {
					token, err := dec.Token()
					if err != nil {
						if err == io.EOF {
							return nil
						}
						return err
					}

					if _, ended := token.(xml.EndElement); ended {
						break
					}
					if _, starting := token.(xml.StartElement); starting {
						if !first {
							context.Encoder.WriteRaw(", ")
						}
						first = false
					}
					if err := context.Handle(dec, token); err != nil {
						return err
					}
				}

				// always encode ending tag
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return nil
			},

			"simpletable": HandleSimpleTable,
			"table":       HandleTable,
			"settings":    HandleSettings,

			"step": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "li"

				context.check(context.Encoder.Encode(start))
				if getAttr(&start, "importance") == "optional" {
					context.Encoder.WriteRaw("(Optional) ")
				}
				err := context.Recurse(dec)
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return err
			},
			"substep": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {
				start.Name.Local = "li"

				context.check(context.Encoder.Encode(start))
				if getAttr(&start, "importance") == "optional" {
					context.Encoder.WriteRaw("(Optional) ")
				}
				err := context.Recurse(dec)
				context.check(context.Encoder.Encode(xml.EndElement{start.Name}))

				return err
			},
			"p": func(context *Context, dec *xml.Decoder, start xml.StartElement) error {

				err := context.EmitWithChildren(dec, start)

				if getAttr(&start, "outputclass") == "no-results-warning" {
					context.Encoder.WriteRaw(`<hr class="no-results-warning">`)
				}

				return err
			},
		},
	}
}
<|endoftext|>


<commit_before>//
// Copyright 2017 Alsanium, SAS. or its affiliates. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package apigatewayproxy

import (
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/eawsy/aws-lambda-go-core/service/lambda/runtime"
	"github.com/eawsy/aws-lambda-go-event/service/lambda/runtime/event/apigatewayproxyevt"
)

// Handler responds to a Lambda function invocation.
type Handler func(json.RawMessage, *runtime.Context) (*Response, error)

// Server defines parameters for handling requests coming from Amazon API
// Gateway. The zero value for Server is not a valid configuration, use New
// instead.
type Server struct {
	pt string
	ts map[string]bool
}

// New returns an initialized server to handle requests from Amazon API Gateway.
// The given media types slice may be nil, if Amazon API Gateway Binary support
// is not enabled. Otherwise, it should be an array of supported media types as
// configured in Amazon API Gateway.
func New(ln net.Listener, ts []string) *Server {
	s := &Server{"http://" + ln.Addr().String(), make(map[string]bool)}
	for _, t := range ts {
		s.ts[t] = true
	}
	return s
}

// Response defines parameters for a well formed response AWS Lambda should
// return to Amazon API Gateway.
type Response struct {
	StatusCode      int               `json:"statusCode"`
	Headers         map[string]string `json:"headers,omitempty"`
	Body            string            `json:"body,omitempty"`
	IsBase64Encoded bool              `json:"isBase64Encoded"`
}

// Handle responds to an AWS Lambda proxy function invocation via Amazon API
// Gateway.
// It transforms the Amazon API Gateway Proxy event to a standard HTTP request
// suitable for the Go net/http package. Then, it submits the data to the
// network listener so that it can be consumed by HTTP handler. Finally, it
// waits for the network listener to return response from handler and transmits
// it back to Amazon API Gateway.
func (s *Server) Handle(evt json.RawMessage, ctx *runtime.Context) (gwres *Response, dummy error) {
	gwreq := new(apigatewayproxyevt.Event)
	gwres = &Response{StatusCode: http.StatusInternalServerError}

	if err := json.Unmarshal(evt, &gwreq); err != nil {
		log.Println(err)
		return
	}

	u, err := url.Parse(gwreq.Path)
	if err != nil {
		log.Println(err)
		return
	}

	q := u.Query()
	for k, v := range gwreq.QueryStringParameters {
		q.Set(k, v)
	}
	u.RawQuery = q.Encode()

	dec := gwreq.Body
	if gwreq.IsBase64Encoded {
		data, err := base64.StdEncoding.DecodeString(dec)
		if err != nil {
			log.Println(err)
			return
		}
		dec = string(data)
	}

	req, err := http.NewRequest(gwreq.HTTPMethod, s.pt+u.String(), strings.NewReader(dec))
	if err != nil {
		log.Println(err)
		return
	}

	gwreq.Body = "... truncated"

	for k, v := range gwreq.Headers {
		req.Header.Set(k, v)
	}
	if len(req.Header.Get("X-Forwarded-For")) == 0 {
		req.Header.Set("X-Forwarded-For", gwreq.RequestContext.Identity.SourceIP)
	}

	hbody, err := json.Marshal(gwreq)
	if err != nil {
		log.Println(err)
		return
	}
	req.Header.Set("X-ApiGatewayProxy-Event", string(hbody))

	hctx, err := json.Marshal(ctx)
	if err != nil {
		log.Println(err)
		return
	}
	req.Header.Set("X-ApiGatewayProxy-Context", string(hctx))

	req.Host = gwreq.Headers["Host"]

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Println(err)
		return
	}
	res.Body.Close()

	ct := res.Header.Get("Content-Type")
	if ct == "" {
		ct = http.DetectContentType(body)
		res.Header.Set("Content-Type", ct)
	}

	if _, ok := s.ts[ct]; ok {
		gwres.Body = base64.StdEncoding.EncodeToString(body)
		gwres.IsBase64Encoded = true
	} else {
		gwres.Body = string(body)
	}

	gwres.Headers = make(map[string]string)
	for k := range res.Header {
		gwres.Headers[k] = res.Header.Get(k)
	}

	gwres.StatusCode = res.StatusCode

	return
}
<commit_msg>Fix follow redirect bug<commit_after>//
// Copyright 2017 Alsanium, SAS. or its affiliates. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package apigatewayproxy

import (
	"encoding/base64"
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"

	"github.com/eawsy/aws-lambda-go-core/service/lambda/runtime"
	"github.com/eawsy/aws-lambda-go-event/service/lambda/runtime/event/apigatewayproxyevt"
)

// Handler responds to a Lambda function invocation.
type Handler func(json.RawMessage, *runtime.Context) (*Response, error)

// Server defines parameters for handling requests coming from Amazon API
// Gateway. The zero value for Server is not a valid configuration, use New
// instead.
type Server struct {
	pt string
	ts map[string]bool
}

// New returns an initialized server to handle requests from Amazon API Gateway.
// The given media types slice may be nil, if Amazon API Gateway Binary support
// is not enabled. Otherwise, it should be an array of supported media types as
// configured in Amazon API Gateway.
func New(ln net.Listener, ts []string) *Server {
	s := &Server{"http://" + ln.Addr().String(), make(map[string]bool)}
	for _, t := range ts {
		s.ts[t] = true
	}
	return s
}

// Response defines parameters for a well formed response AWS Lambda should
// return to Amazon API Gateway.
type Response struct {
	StatusCode      int               `json:"statusCode"`
	Headers         map[string]string `json:"headers,omitempty"`
	Body            string            `json:"body,omitempty"`
	IsBase64Encoded bool              `json:"isBase64Encoded"`
}

func newClient() *http.Client {
	return &http.Client{
		CheckRedirect: func(*http.Request, []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}
}

// Handle responds to an AWS Lambda proxy function invocation via Amazon API
// Gateway.
// It transforms the Amazon API Gateway Proxy event to a standard HTTP request
// suitable for the Go net/http package. Then, it submits the data to the
// network listener so that it can be consumed by HTTP handler. Finally, it
// waits for the network listener to return response from handler and transmits
// it back to Amazon API Gateway.
func (s *Server) Handle(evt json.RawMessage, ctx *runtime.Context) (gwres *Response, dummy error) {
	gwreq := new(apigatewayproxyevt.Event)
	gwres = &Response{StatusCode: http.StatusInternalServerError}

	if err := json.Unmarshal(evt, &gwreq); err != nil {
		log.Println(err)
		return
	}

	u, err := url.Parse(gwreq.Path)
	if err != nil {
		log.Println(err)
		return
	}

	q := u.Query()
	for k, v := range gwreq.QueryStringParameters {
		q.Set(k, v)
	}
	u.RawQuery = q.Encode()

	dec := gwreq.Body
	if gwreq.IsBase64Encoded {
		data, err := base64.StdEncoding.DecodeString(dec)
		if err != nil {
			log.Println(err)
			return
		}
		dec = string(data)
	}

	req, err := http.NewRequest(gwreq.HTTPMethod, s.pt+u.String(), strings.NewReader(dec))
	if err != nil {
		log.Println(err)
		return
	}

	gwreq.Body = "... truncated"

	for k, v := range gwreq.Headers {
		req.Header.Set(k, v)
	}
	if len(req.Header.Get("X-Forwarded-For")) == 0 {
		req.Header.Set("X-Forwarded-For", gwreq.RequestContext.Identity.SourceIP)
	}

	hbody, err := json.Marshal(gwreq)
	if err != nil {
		log.Println(err)
		return
	}
	req.Header.Set("X-ApiGatewayProxy-Event", string(hbody))

	hctx, err := json.Marshal(ctx)
	if err != nil {
		log.Println(err)
		return
	}
	req.Header.Set("X-ApiGatewayProxy-Context", string(hctx))

	req.Host = gwreq.Headers["Host"]

	res, err := newClient().Do(req)
	if err != nil {
		log.Println(err)
		return
	}

	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Println(err)
		return
	}
	res.Body.Close()

	ct := res.Header.Get("Content-Type")
	if ct == "" {
		ct = http.DetectContentType(body)
		res.Header.Set("Content-Type", ct)
	}

	if _, ok := s.ts[ct]; ok {
		gwres.Body = base64.StdEncoding.EncodeToString(body)
		gwres.IsBase64Encoded = true
	} else {
		gwres.Body = string(body)
	}

	gwres.Headers = make(map[string]string)
	for k := range res.Header {
		gwres.Headers[k] = res.Header.Get(k)
	}

	gwres.StatusCode = res.StatusCode

	return
}
<|endoftext|>


<commit_before>package main

import "fmt"
import least "LeastSquareCircleFit"
import "math"

func main() {
	x := []float64{0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0}
	y := []float64{0.0, 0.25, 1.0, 2.25, 4.0, 6.25, 9.0}

	xc, yc, r := least.CalcLeastSquareCircleFit(x, y)

	for i, _ := range x {
		fmt.Printf("%v %v # data\n", x[i], y[i])
	}
	fmt.Printf("\n\n%v %v # center\n\n\n", xc, yc)
	for i := 0; i < 360; i += 10 {
		xi := xc + r*math.Sin(float64(i)*math.Pi/180.0)
		yi := yc + r*math.Cos(float64(i)*math.Pi/180.0)
		fmt.Printf("%v %v #circle\n", xi, yi)
	}
	fmt.Printf("\n\n%v # radius\n\n\n", r)
}

<commit_msg>Update example.go<commit_after>package main

import "fmt"
import least "github.com/StefanSchroeder/LeastSquareCircleFit"
import "math"

func main() {
	x := []float64{0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0}
	y := []float64{0.0, 0.25, 1.0, 2.25, 4.0, 6.25, 9.0}

	xc, yc, r := least.CalcLeastSquareCircleFit(x, y)

	for i, _ := range x {
		fmt.Printf("%v %v # data\n", x[i], y[i])
	}
	fmt.Printf("\n\n%v %v # center\n\n\n", xc, yc)
	for i := 0; i < 360; i += 10 {
		xi := xc + r*math.Sin(float64(i)*math.Pi/180.0)
		yi := yc + r*math.Cos(float64(i)*math.Pi/180.0)
		fmt.Printf("%v %v #circle\n", xi, yi)
	}
	fmt.Printf("\n\n%v # radius\n\n\n", r)
}

<|endoftext|>


<commit_before>package enforcer_test

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/config"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"

	_ "github.com/go-sql-driver/mysql"
	"github.com/pivotal-cf-experimental/service-config"
)

var brokerDBName string
var c config.Config
var binaryPath string

var tempDir string
var configPath string

var adminDB *sql.DB
var adminCreds AdminCredentials
var initConfig config.Config

type AdminCredentials struct {
	User     string `yaml:"AdminUser" validate:"nonzero"`
	Password string `yaml:"AdminPassword" validate:"nonzero"`
}

func TestEnforcer(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Enforcer Suite")
}

func newDatabaseConfig(dbName string) config.Config {
	serviceConfig := service_config.New()

	var dbConfig config.Config
	err := serviceConfig.Read(&dbConfig)
	Expect(err).ToNot(HaveOccurred())

	dbConfig.DBName = dbName
	dbConfig.IgnoredUsers = append(dbConfig.IgnoredUsers, "fake-admin-user")
	dbConfig.PauseInSeconds = 1

	return dbConfig
}

func adminCredentials() AdminCredentials {
	serviceConfig := service_config.New()

	var adminCreds AdminCredentials
	err := serviceConfig.Read(&adminCreds)
	Expect(err).ToNot(HaveOccurred())

	return adminCreds
}

var _ = BeforeSuite(func() {
	initConfig = newDatabaseConfig("")

	brokerDBName = uuidWithUnderscores("db")
	c = newDatabaseConfig(brokerDBName)

	adminCreds = adminCredentials()

	adminDB, err := database.NewConnection(adminCreds.User, adminCreds.Password, initConfig.Host, initConfig.Port, initConfig.DBName)
	Expect(err).ToNot(HaveOccurred())
	defer adminDB.Close()

	_, err = adminDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", brokerDBName))
	Expect(err).ToNot(HaveOccurred())

	db, err := database.NewConnection(c.User, c.Password, c.Host, c.Port, c.DBName)
	Expect(err).ToNot(HaveOccurred())

	for _, ignoredUser := range initConfig.IgnoredUsers {
		_, err = adminDB.Exec(fmt.Sprintf("GRANT ALL PRIVILEGES ON *.* TO '%s' IDENTIFIED BY '%s' WITH GRANT OPTION",
			ignoredUser,
			"password",
		))
	}

	_, err = adminDB.Exec("FLUSH PRIVILEGES")
	Expect(err).NotTo(HaveOccurred())

	defer db.Close()

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS service_instances (
    id int(11) NOT NULL AUTO_INCREMENT,
    guid varchar(255),
    plan_guid varchar(255),
    max_storage_mb int(11) NOT NULL DEFAULT '0',
    db_name varchar(255),
    PRIMARY KEY (id)
	)`)
	Expect(err).ToNot(HaveOccurred())

	binaryPath, err = gexec.Build("github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer", "-race")
	Expect(err).ToNot(HaveOccurred())

	_, err = os.Stat(binaryPath)
	if err != nil {
		Expect(os.IsExist(err)).To(BeTrue())
	}

	tempDir, err = ioutil.TempDir(os.TempDir(), "quota-enforcer-integration-test")
	Expect(err).NotTo(HaveOccurred())

	configPath = filepath.Join(tempDir, "quotaEnforcerConfig.yml")
	writeConfig()
})

var _ = AfterSuite(func() {

	// We don't need to handle an error cleaning up the tempDir
	_ = os.RemoveAll(tempDir)

	gexec.CleanupBuildArtifacts()

	_, err := os.Stat(binaryPath)
	if err != nil {
		Expect(os.IsExist(err)).To(BeFalse())
	}

	db, err := database.NewConnection(c.User, c.Password, c.Host, c.Port, c.DBName)
	Expect(err).ToNot(HaveOccurred())
	defer db.Close()

	_, err = db.Exec("DROP TABLE IF EXISTS service_instances")
	Expect(err).ToNot(HaveOccurred())

	_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", brokerDBName))
	Expect(err).ToNot(HaveOccurred())
})

func startEnforcerWithFlags(flags ...string) *gexec.Session {

	flags = append(
		flags,
		fmt.Sprintf("-configPath=%s", configPath),
		"-logLevel=debug",
	)

	command := exec.Command(
		binaryPath,
		flags...,
	)

	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())

	return session
}

func runEnforcerContinuously(flags ...string) *gexec.Session {
	session := startEnforcerWithFlags(flags...)
	Eventually(session.Out).Should(gbytes.Say("Running continuously"))
	return session
}

func runEnforcerOnce() {
	session := startEnforcerWithFlags("-runOnce")

	Eventually(session.Out).Should(gbytes.Say("Running once"))
	// Wait for the process to finish naturally.
	// This should not take a long time
	session.Wait(5 * time.Second)
	Expect(session.ExitCode()).To(Equal(0), string(session.Err.Contents()))
}

func writeConfig() {
	fileToWrite, err := os.Create(configPath)
	Expect(err).ToNot(HaveOccurred())

	bytes, err := json.MarshalIndent(c, "", "  ")
	Expect(err).ToNot(HaveOccurred())

	_, err = fileToWrite.Write(bytes)
	Expect(err).ToNot(HaveOccurred())
}

func uuidWithUnderscores(prefix string) string {
	id, err := uuid.NewV4()
	Expect(err).ToNot(HaveOccurred())
	idString := fmt.Sprintf("%s_%s", prefix, id.String())
	return strings.Replace(idString, "-", "_", -1)
}
<commit_msg>Rename config helper<commit_after>package enforcer_test

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/nu7hatch/gouuid"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/config"
	"github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer/database"

	_ "github.com/go-sql-driver/mysql"
	"github.com/pivotal-cf-experimental/service-config"
)

var brokerDBName string
var c config.Config
var binaryPath string

var tempDir string
var configPath string

var adminDB *sql.DB
var adminCreds AdminCredentials
var initConfig config.Config

type AdminCredentials struct {
	User     string `yaml:"AdminUser" validate:"nonzero"`
	Password string `yaml:"AdminPassword" validate:"nonzero"`
}

func TestEnforcer(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Enforcer Suite")
}

func newConfig(dbName string) config.Config {
	serviceConfig := service_config.New()

	var cfg config.Config
	err := serviceConfig.Read(&cfg)
	Expect(err).ToNot(HaveOccurred())

	cfg.DBName = dbName
	cfg.IgnoredUsers = append(cfg.IgnoredUsers, "fake-admin-user")
	cfg.PauseInSeconds = 1

	return cfg
}

func adminCredentials() AdminCredentials {
	serviceConfig := service_config.New()

	var adminCreds AdminCredentials
	err := serviceConfig.Read(&adminCreds)
	Expect(err).ToNot(HaveOccurred())

	return adminCreds
}

var _ = BeforeSuite(func() {
	initConfig = newConfig("")

	brokerDBName = uuidWithUnderscores("db")
	c = newConfig(brokerDBName)

	adminCreds = adminCredentials()

	adminDB, err := database.NewConnection(adminCreds.User, adminCreds.Password, initConfig.Host, initConfig.Port, initConfig.DBName)
	Expect(err).ToNot(HaveOccurred())
	defer adminDB.Close()

	_, err = adminDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", brokerDBName))
	Expect(err).ToNot(HaveOccurred())

	db, err := database.NewConnection(c.User, c.Password, c.Host, c.Port, c.DBName)
	Expect(err).ToNot(HaveOccurred())

	for _, ignoredUser := range initConfig.IgnoredUsers {
		_, err = adminDB.Exec(fmt.Sprintf("GRANT ALL PRIVILEGES ON *.* TO '%s' IDENTIFIED BY '%s' WITH GRANT OPTION",
			ignoredUser,
			"password",
		))
	}

	_, err = adminDB.Exec("FLUSH PRIVILEGES")
	Expect(err).NotTo(HaveOccurred())

	defer db.Close()

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS service_instances (
    id int(11) NOT NULL AUTO_INCREMENT,
    guid varchar(255),
    plan_guid varchar(255),
    max_storage_mb int(11) NOT NULL DEFAULT '0',
    db_name varchar(255),
    PRIMARY KEY (id)
	)`)
	Expect(err).ToNot(HaveOccurred())

	binaryPath, err = gexec.Build("github.com/pivotal-cf-experimental/cf-mysql-quota-enforcer", "-race")
	Expect(err).ToNot(HaveOccurred())

	_, err = os.Stat(binaryPath)
	if err != nil {
		Expect(os.IsExist(err)).To(BeTrue())
	}

	tempDir, err = ioutil.TempDir(os.TempDir(), "quota-enforcer-integration-test")
	Expect(err).NotTo(HaveOccurred())

	configPath = filepath.Join(tempDir, "quotaEnforcerConfig.yml")
	writeConfig()
})

var _ = AfterSuite(func() {

	// We don't need to handle an error cleaning up the tempDir
	_ = os.RemoveAll(tempDir)

	gexec.CleanupBuildArtifacts()

	_, err := os.Stat(binaryPath)
	if err != nil {
		Expect(os.IsExist(err)).To(BeFalse())
	}

	db, err := database.NewConnection(c.User, c.Password, c.Host, c.Port, c.DBName)
	Expect(err).ToNot(HaveOccurred())
	defer db.Close()

	_, err = db.Exec("DROP TABLE IF EXISTS service_instances")
	Expect(err).ToNot(HaveOccurred())

	_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", brokerDBName))
	Expect(err).ToNot(HaveOccurred())
})

func startEnforcerWithFlags(flags ...string) *gexec.Session {

	flags = append(
		flags,
		fmt.Sprintf("-configPath=%s", configPath),
		"-logLevel=debug",
	)

	command := exec.Command(
		binaryPath,
		flags...,
	)

	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).ToNot(HaveOccurred())

	return session
}

func runEnforcerContinuously(flags ...string) *gexec.Session {
	session := startEnforcerWithFlags(flags...)
	Eventually(session.Out).Should(gbytes.Say("Running continuously"))
	return session
}

func runEnforcerOnce() {
	session := startEnforcerWithFlags("-runOnce")

	Eventually(session.Out).Should(gbytes.Say("Running once"))
	// Wait for the process to finish naturally.
	// This should not take a long time
	session.Wait(5 * time.Second)
	Expect(session.ExitCode()).To(Equal(0), string(session.Err.Contents()))
}

func writeConfig() {
	fileToWrite, err := os.Create(configPath)
	Expect(err).ToNot(HaveOccurred())

	bytes, err := json.MarshalIndent(c, "", "  ")
	Expect(err).ToNot(HaveOccurred())

	_, err = fileToWrite.Write(bytes)
	Expect(err).ToNot(HaveOccurred())
}

func uuidWithUnderscores(prefix string) string {
	id, err := uuid.NewV4()
	Expect(err).ToNot(HaveOccurred())
	idString := fmt.Sprintf("%s_%s", prefix, id.String())
	return strings.Replace(idString, "-", "_", -1)
}
<|endoftext|>


<commit_before>package integration_test

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
	"time"
)

var tmpDir string
var pathToGinkgo string

func TestIntegration(t *testing.T) {
	SetDefaultEventuallyTimeout(30 * time.Second)
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	pathToGinkgo, err := gexec.Build("github.com/onsi/ginkgo/ginkgo")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(pathToGinkgo)
}, func(computedPathToGinkgo []byte) {
	pathToGinkgo = string(computedPathToGinkgo)
})

var _ = BeforeEach(func() {
	var err error
	tmpDir, err = ioutil.TempDir("", "ginkgo-run")
	Ω(err).ShouldNot(HaveOccurred())
})

var _ = AfterEach(func() {
	err := os.RemoveAll(tmpDir)
	Ω(err).ShouldNot(HaveOccurred())
})

var _ = SynchronizedAfterSuite(func() {}, func() {
	gexec.CleanupBuildArtifacts()
})

func tmpPath(destination string) string {
	return filepath.Join(tmpDir, destination)
}

func fixturePath(name string) string {
	return filepath.Join("_fixtures", name)
}

func copyIn(sourcePath, destinationPath string, recursive bool) {
	err := os.MkdirAll(destinationPath, 0777)
	Expect(err).NotTo(HaveOccurred())

	files, err := ioutil.ReadDir(sourcePath)
	Expect(err).NotTo(HaveOccurred())
	for _, f := range files {
		srcPath := filepath.Join(sourcePath, f.Name())
		dstPath := filepath.Join(destinationPath, f.Name())
		if f.IsDir() {
			if recursive {
				copyIn(srcPath, dstPath, recursive)
			}
			continue
		}

		src, err := os.Open(srcPath)

		Expect(err).NotTo(HaveOccurred())
		defer src.Close()

		dst, err := os.Create(dstPath)
		Expect(err).NotTo(HaveOccurred())
		defer dst.Close()

		_, err = io.Copy(dst, src)
		Expect(err).NotTo(HaveOccurred())
	}
}

func sameFile(filePath, otherFilePath string) bool {
	content, readErr := ioutil.ReadFile(filePath)
	Expect(readErr).NotTo(HaveOccurred())
	otherContent, readErr := ioutil.ReadFile(otherFilePath)
	Expect(readErr).NotTo(HaveOccurred())
	Expect(string(content)).To(Equal(string(otherContent)))
	return true
}

func sameFolder(sourcePath, destinationPath string) bool {
	files, err := ioutil.ReadDir(sourcePath)
	Expect(err).NotTo(HaveOccurred())
	for _, f := range files {
		srcPath := filepath.Join(sourcePath, f.Name())
		dstPath := filepath.Join(destinationPath, f.Name())
		if f.IsDir() {
			sameFolder(srcPath, dstPath)
			continue
		}
		Expect(sameFile(srcPath, dstPath)).To(BeTrue())
	}
	return true
}

func ginkgoCommand(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(pathToGinkgo, args...)
	cmd.Dir = dir

	return cmd
}

func startGinkgo(dir string, args ...string) *gexec.Session {
	cmd := ginkgoCommand(dir, args...)
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())
	return session
}

func removeSuccessfully(path string) {
	err := os.RemoveAll(path)
	Expect(err).NotTo(HaveOccurred())
}<commit_msg>all: gofmt<commit_after>package integration_test

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"

	"testing"
	"time"
)

var tmpDir string
var pathToGinkgo string

func TestIntegration(t *testing.T) {
	SetDefaultEventuallyTimeout(30 * time.Second)
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	pathToGinkgo, err := gexec.Build("github.com/onsi/ginkgo/ginkgo")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(pathToGinkgo)
}, func(computedPathToGinkgo []byte) {
	pathToGinkgo = string(computedPathToGinkgo)
})

var _ = BeforeEach(func() {
	var err error
	tmpDir, err = ioutil.TempDir("", "ginkgo-run")
	Ω(err).ShouldNot(HaveOccurred())
})

var _ = AfterEach(func() {
	err := os.RemoveAll(tmpDir)
	Ω(err).ShouldNot(HaveOccurred())
})

var _ = SynchronizedAfterSuite(func() {}, func() {
	gexec.CleanupBuildArtifacts()
})

func tmpPath(destination string) string {
	return filepath.Join(tmpDir, destination)
}

func fixturePath(name string) string {
	return filepath.Join("_fixtures", name)
}

func copyIn(sourcePath, destinationPath string, recursive bool) {
	err := os.MkdirAll(destinationPath, 0777)
	Expect(err).NotTo(HaveOccurred())

	files, err := ioutil.ReadDir(sourcePath)
	Expect(err).NotTo(HaveOccurred())
	for _, f := range files {
		srcPath := filepath.Join(sourcePath, f.Name())
		dstPath := filepath.Join(destinationPath, f.Name())
		if f.IsDir() {
			if recursive {
				copyIn(srcPath, dstPath, recursive)
			}
			continue
		}

		src, err := os.Open(srcPath)

		Expect(err).NotTo(HaveOccurred())
		defer src.Close()

		dst, err := os.Create(dstPath)
		Expect(err).NotTo(HaveOccurred())
		defer dst.Close()

		_, err = io.Copy(dst, src)
		Expect(err).NotTo(HaveOccurred())
	}
}

func sameFile(filePath, otherFilePath string) bool {
	content, readErr := ioutil.ReadFile(filePath)
	Expect(readErr).NotTo(HaveOccurred())
	otherContent, readErr := ioutil.ReadFile(otherFilePath)
	Expect(readErr).NotTo(HaveOccurred())
	Expect(string(content)).To(Equal(string(otherContent)))
	return true
}

func sameFolder(sourcePath, destinationPath string) bool {
	files, err := ioutil.ReadDir(sourcePath)
	Expect(err).NotTo(HaveOccurred())
	for _, f := range files {
		srcPath := filepath.Join(sourcePath, f.Name())
		dstPath := filepath.Join(destinationPath, f.Name())
		if f.IsDir() {
			sameFolder(srcPath, dstPath)
			continue
		}
		Expect(sameFile(srcPath, dstPath)).To(BeTrue())
	}
	return true
}

func ginkgoCommand(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(pathToGinkgo, args...)
	cmd.Dir = dir

	return cmd
}

func startGinkgo(dir string, args ...string) *gexec.Session {
	cmd := ginkgoCommand(dir, args...)
	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())
	return session
}

func removeSuccessfully(path string) {
	err := os.RemoveAll(path)
	Expect(err).NotTo(HaveOccurred())
}
<|endoftext|>


<commit_before>package rekordo

import (
	"database/sql"
	"errors"

	"github.com/go-gorp/gorp"
	"github.com/loopfz/gadgeto/zesty"
)

// Default database settings.
const (
	maxOpenConns = 5
	maxIdleConns = 3
)

// DatabaseConfig represents the configuration used to
// register a new database.
type DatabaseConfig struct {
	Name             string
	DSN              string
	System           DBMS
	MaxOpenConns     int
	MaxIdleConns     int
	AutoCreateTables bool
}

// RegisterDatabase creates a gorp map with tables and tc and
// registers it with zesty.
func RegisterDatabase(dbcfg *DatabaseConfig, tc gorp.TypeConverter) (zesty.DB, error) {
	dbConn, err := sql.Open(dbcfg.System.DriverName(), dbcfg.DSN)
	if err != nil {
		return nil, err
	}
	// Make sure we have proper values for the database
	// settings, and replace them with default if necessary
	// before applying to the new connection.
	if dbcfg.MaxOpenConns == 0 {
		dbcfg.MaxOpenConns = maxOpenConns
	}
	dbConn.SetMaxOpenConns(dbcfg.MaxOpenConns)
	if dbcfg.MaxIdleConns == 0 {
		dbcfg.MaxIdleConns = maxIdleConns
	}
	dbConn.SetMaxIdleConns(dbcfg.MaxIdleConns)

	// Select the proper dialect used by gorp.
	var dialect gorp.Dialect
	switch dbcfg.System {
	case DatabaseMySQL:
		dialect = gorp.MySQLDialect{}
	case DatabasePostgreSQL:
		dialect = gorp.PostgresDialect{}
	case DatabaseSqlite3:
		dialect = gorp.SqliteDialect{}
	default:
		return nil, errors.New("unknown database system")
	}
	dbmap := &gorp.DbMap{
		Db:            dbConn,
		Dialect:       dialect,
		TypeConverter: tc,
	}
	modelsMu.Lock()
	tableModels := models[dbcfg.Name]
	for _, t := range tableModels {
		dbmap.AddTableWithName(t.Model, t.Name).SetKeys(t.AutoIncrement, t.Keys...)
	}
	modelsMu.Unlock()

	if dbcfg.AutoCreateTables {
		err = dbmap.CreateTablesIfNotExists()
		if err != nil {
			return nil, err
		}
	}
	db := zesty.NewDB(dbmap)
	if err := zesty.RegisterDB(db, dbcfg.Name); err != nil {
		return nil, err
	}
	return db, nil
}

// DBMS represents a database management system.
type DBMS uint8

// Database management systems.
const (
	DatabasePostgreSQL DBMS = iota ^ 42
	DatabaseMySQL
	DatabaseSqlite3
)

// DriverName returns the name of the driver for ds.
func (d DBMS) DriverName() string {
	switch d {
	case DatabasePostgreSQL:
		return "postgres"
	case DatabaseMySQL:
		return "mysql"
	case DatabaseSqlite3:
		return "sqlite3"
	}
	return ""
}
<commit_msg>rekordo: takes ConnMaxLifetime parameter in DatabaseConfig type. (#49)<commit_after>package rekordo

import (
	"database/sql"
	"errors"
	"time"

	"github.com/go-gorp/gorp"
	"github.com/loopfz/gadgeto/zesty"
)

// Default database settings.
const (
	maxOpenConns    = 5
	maxIdleConns    = 3
)

// DatabaseConfig represents the configuration used to
// register a new database.
type DatabaseConfig struct {
	Name             string
	DSN              string
	System           DBMS
	MaxOpenConns     int
	MaxIdleConns     int
	ConnMaxLifetime  time.Duration
	AutoCreateTables bool
}

// RegisterDatabase creates a gorp map with tables and tc and
// registers it with zesty.
func RegisterDatabase(dbcfg *DatabaseConfig, tc gorp.TypeConverter) (zesty.DB, error) {
	dbConn, err := sql.Open(dbcfg.System.DriverName(), dbcfg.DSN)
	if err != nil {
		return nil, err
	}
	// Make sure we have proper values for the database
	// settings, and replace them with default if necessary
	// before applying to the new connection.
	if dbcfg.MaxOpenConns == 0 {
		dbcfg.MaxOpenConns = maxOpenConns
	}
	dbConn.SetMaxOpenConns(dbcfg.MaxOpenConns)
	if dbcfg.MaxIdleConns == 0 {
		dbcfg.MaxIdleConns = maxIdleConns
	}
	dbConn.SetMaxIdleConns(dbcfg.MaxIdleConns)
	dbConn.SetConnMaxLifetime(dbcfg.ConnMaxLifetime)

	// Select the proper dialect used by gorp.
	var dialect gorp.Dialect
	switch dbcfg.System {
	case DatabaseMySQL:
		dialect = gorp.MySQLDialect{}
	case DatabasePostgreSQL:
		dialect = gorp.PostgresDialect{}
	case DatabaseSqlite3:
		dialect = gorp.SqliteDialect{}
	default:
		return nil, errors.New("unknown database system")
	}
	dbmap := &gorp.DbMap{
		Db:            dbConn,
		Dialect:       dialect,
		TypeConverter: tc,
	}
	modelsMu.Lock()
	tableModels := models[dbcfg.Name]
	for _, t := range tableModels {
		dbmap.AddTableWithName(t.Model, t.Name).SetKeys(t.AutoIncrement, t.Keys...)
	}
	modelsMu.Unlock()

	if dbcfg.AutoCreateTables {
		err = dbmap.CreateTablesIfNotExists()
		if err != nil {
			return nil, err
		}
	}
	db := zesty.NewDB(dbmap)
	if err := zesty.RegisterDB(db, dbcfg.Name); err != nil {
		return nil, err
	}
	return db, nil
}

// DBMS represents a database management system.
type DBMS uint8

// Database management systems.
const (
	DatabasePostgreSQL DBMS = iota ^ 42
	DatabaseMySQL
	DatabaseSqlite3
)

// DriverName returns the name of the driver for ds.
func (d DBMS) DriverName() string {
	switch d {
	case DatabasePostgreSQL:
		return "postgres"
	case DatabaseMySQL:
		return "mysql"
	case DatabaseSqlite3:
		return "sqlite3"
	}
	return ""
}
<|endoftext|>


<commit_before>package receiver

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
)

// Error is returned by the various receiver commands in case of error
type Error string

func (e Error) Error() string {
	return string(e)
}

// receiverOp is used to identify the different operation performed
// by the cvmfs_receiver process
type receiverOp int32

// The different operations are defined as constants. The numbering
// must match (enum receiver::Request from "cvmfs.git/cvmfs/receiver/reactor.h")
const (
	receiverQuit receiverOp = iota
	receiverEcho
	receiverGenerateToken // Unused
	receiverGetTokenID    // Unused
	receiverCheckToken    // Unused
	receiverSubmitPayload
	receiverCommit
	receiverError // Unused
)

// Receiver contains the operations that "receiver" worker processes perform
type Receiver interface {
	Quit() error
	Echo() error
	SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error
	Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error
}

// NewReceiver is the factory method for Receiver types
func NewReceiver(ctx context.Context, execPath string, mock bool) (Receiver, error) {
	if mock {
		return NewMockReceiver(ctx)
	}

	return NewCvmfsReceiver(ctx, execPath)
}

// CvmfsReceiver spawns an external cvmfs_receiver worker process
type CvmfsReceiver struct {
	worker *exec.Cmd
	stdin  io.WriteCloser
	stdout io.ReadCloser
	ctx    context.Context
}

// NewCvmfsReceiver will spawn an external cvmfs_receiver worker process and wait for a command
func NewCvmfsReceiver(ctx context.Context, execPath string) (*CvmfsReceiver, error) {
	if _, err := os.Stat(execPath); os.IsNotExist(err) {
		return nil, errors.Wrap(err, "worker process executable not found")
	}

	cmd := exec.Command(execPath, "-i", strconv.Itoa(3), "-o", strconv.Itoa(4))

	stdinRead, stdinWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create stdin pipe")
	}
	stdoutRead, stdoutWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create stdout pipe")
	}

	cmd.ExtraFiles = []*os.File{stdinRead, stdoutWrite}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "could not start worker process")
	}

	gw.LogC(ctx, "receiver", gw.LogDebug).
		Str("command", "start").
		Msg("worker process ready")

	return &CvmfsReceiver{worker: cmd, stdin: stdinWrite, stdout: stdoutRead, ctx: ctx}, nil
}

// Quit command is sent to the worker
func (r *CvmfsReceiver) Quit() error {
	defer func() {
		r.stdin.Close()
		r.stdout.Close()
	}()

	if _, err := r.call(receiverQuit, []byte{}, nil); err != nil {
		return errors.Wrap(err, "worker 'quit' call failed")
	}

	if err := r.worker.Wait(); err != nil {
		return errors.Wrap(err, "waiting for worker process failed")
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "quit").
		Msg("worker process has stopped")

	return nil
}

// Echo command is sent to the worker
func (r *CvmfsReceiver) Echo() error {
	rep, err := r.call(receiverEcho, []byte("Ping"), nil)
	if err != nil {
		return errors.Wrap(err, "worker 'echo' call failed")
	}
	reply := string(rep)

	if !strings.HasPrefix(reply, "PID: ") {
		return fmt.Errorf("invalid 'echo' reply received: %v", reply)
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "echo").
		Msgf("reply: %v", reply)

	return nil
}

// SubmitPayload command is sent to the worker
func (r *CvmfsReceiver) SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error {
	req := map[string]interface{}{"path": leasePath, "digest": digest, "header_size": headerSize}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}
	reply, err := r.call(receiverSubmitPayload, buf, payload)
	if err != nil {
		return errors.Wrap(err, "worker 'payload submission' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "submit payload").
		Msgf("result: %v", result)

	return result
}

// Commit command is sent to the worker
func (r *CvmfsReceiver) Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	req := map[string]interface{}{
		"lease_path":      leasePath,
		"old_root_hash":   oldRootHash,
		"new_root_hash":   newRootHash,
		"tag_name":        tag.Name,
		"tag_channel":     tag.Channel,
		"tag_description": tag.Description,
	}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}

	reply, err := r.call(receiverCommit, buf, nil)
	if err != nil {
		return errors.Wrap(err, "worker 'commit' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "commit").
		Msgf("result: %v", result)

	return result
}

func (r *CvmfsReceiver) call(reqID receiverOp, msg []byte, payload io.Reader) ([]byte, error) {
	if err := r.request(reqID, msg, payload); err != nil {
		return nil, err
	}
	return r.reply()
}

func (r *CvmfsReceiver) request(reqID receiverOp, msg []byte, payload io.Reader) error {
	if err := binary.Write(r.stdin, binary.LittleEndian, reqID); err != nil {
		return errors.Wrap(err, "could not write request id")
	}
	if err := binary.Write(r.stdin, binary.LittleEndian, int32(len(msg))); err != nil {
		return errors.Wrap(err, "could not write request size")
	}
	if _, err := r.stdin.Write(msg); err != nil {
		return errors.Wrap(err, "could not write request body")
	}
	if payload != nil {
		if _, err := io.Copy(r.stdin, payload); err != nil {
			return errors.Wrap(err, "could not write request payload")
		}
	}
	return nil
}

func (r *CvmfsReceiver) reply() ([]byte, error) {
	var repSize int32
	if err := binary.Read(r.stdout, binary.LittleEndian, &repSize); err != nil {
		return nil, errors.Wrap(err, "could not read reply size")
	}

	reply := make([]byte, repSize)
	reply, err := ioutil.ReadAll(io.LimitReader(r.stdout, int64(repSize)))
	if err != nil {
		return nil, errors.Wrap(err, "could not read reply body")
	}

	return reply, nil
}

func toReceiverError(reply []byte) error {
	res := make(map[string]string)
	if err := json.Unmarshal(reply, &res); err != nil {
		return errors.Wrap(err, "could not decode reply")
	}

	if status, ok := res["status"]; ok {
		if status == "ok" {
			return nil
		}

		if reason, ok := res["reason"]; ok {
			return Error(reason)
		}

		return fmt.Errorf("invalid reply")
	}

	return fmt.Errorf("invalid reply")
}
<commit_msg>Avoid reflection in IO with cvmfs_receiver<commit_after>package receiver

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"

	gw "github.com/cvmfs/gateway/internal/gateway"
	"github.com/pkg/errors"
)

// Error is returned by the various receiver commands in case of error
type Error string

func (e Error) Error() string {
	return string(e)
}

// receiverOp is used to identify the different operation performed
// by the cvmfs_receiver process
type receiverOp int32

// The different operations are defined as constants. The numbering
// must match (enum receiver::Request from "cvmfs.git/cvmfs/receiver/reactor.h")
const (
	receiverQuit receiverOp = iota
	receiverEcho
	receiverGenerateToken // Unused
	receiverGetTokenID    // Unused
	receiverCheckToken    // Unused
	receiverSubmitPayload
	receiverCommit
	receiverError // Unused
)

// Receiver contains the operations that "receiver" worker processes perform
type Receiver interface {
	Quit() error
	Echo() error
	SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error
	Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error
}

// NewReceiver is the factory method for Receiver types
func NewReceiver(ctx context.Context, execPath string, mock bool) (Receiver, error) {
	if mock {
		return NewMockReceiver(ctx)
	}

	return NewCvmfsReceiver(ctx, execPath)
}

// CvmfsReceiver spawns an external cvmfs_receiver worker process
type CvmfsReceiver struct {
	worker *exec.Cmd
	stdin  io.WriteCloser
	stdout io.ReadCloser
	ctx    context.Context
}

// NewCvmfsReceiver will spawn an external cvmfs_receiver worker process and wait for a command
func NewCvmfsReceiver(ctx context.Context, execPath string) (*CvmfsReceiver, error) {
	if _, err := os.Stat(execPath); os.IsNotExist(err) {
		return nil, errors.Wrap(err, "worker process executable not found")
	}

	cmd := exec.Command(execPath, "-i", strconv.Itoa(3), "-o", strconv.Itoa(4))

	stdinRead, stdinWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create stdin pipe")
	}
	stdoutRead, stdoutWrite, err := os.Pipe()
	if err != nil {
		return nil, errors.Wrap(err, "could not create stdout pipe")
	}

	cmd.ExtraFiles = []*os.File{stdinRead, stdoutWrite}

	if err := cmd.Start(); err != nil {
		return nil, errors.Wrap(err, "could not start worker process")
	}

	gw.LogC(ctx, "receiver", gw.LogDebug).
		Str("command", "start").
		Msg("worker process ready")

	return &CvmfsReceiver{worker: cmd, stdin: stdinWrite, stdout: stdoutRead, ctx: ctx}, nil
}

// Quit command is sent to the worker
func (r *CvmfsReceiver) Quit() error {
	defer func() {
		r.stdin.Close()
		r.stdout.Close()
	}()

	if _, err := r.call(receiverQuit, []byte{}, nil); err != nil {
		return errors.Wrap(err, "worker 'quit' call failed")
	}

	if err := r.worker.Wait(); err != nil {
		return errors.Wrap(err, "waiting for worker process failed")
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "quit").
		Msg("worker process has stopped")

	return nil
}

// Echo command is sent to the worker
func (r *CvmfsReceiver) Echo() error {
	rep, err := r.call(receiverEcho, []byte("Ping"), nil)
	if err != nil {
		return errors.Wrap(err, "worker 'echo' call failed")
	}
	reply := string(rep)

	if !strings.HasPrefix(reply, "PID: ") {
		return fmt.Errorf("invalid 'echo' reply received: %v", reply)
	}

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "echo").
		Msgf("reply: %v", reply)

	return nil
}

// SubmitPayload command is sent to the worker
func (r *CvmfsReceiver) SubmitPayload(leasePath string, payload io.Reader, digest string, headerSize int) error {
	req := map[string]interface{}{"path": leasePath, "digest": digest, "header_size": headerSize}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}
	reply, err := r.call(receiverSubmitPayload, buf, payload)
	if err != nil {
		return errors.Wrap(err, "worker 'payload submission' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "submit payload").
		Msgf("result: %v", result)

	return result
}

// Commit command is sent to the worker
func (r *CvmfsReceiver) Commit(leasePath, oldRootHash, newRootHash string, tag gw.RepositoryTag) error {
	req := map[string]interface{}{
		"lease_path":      leasePath,
		"old_root_hash":   oldRootHash,
		"new_root_hash":   newRootHash,
		"tag_name":        tag.Name,
		"tag_channel":     tag.Channel,
		"tag_description": tag.Description,
	}
	buf, err := json.Marshal(&req)
	if err != nil {
		return errors.Wrap(err, "request encoding failed")
	}

	reply, err := r.call(receiverCommit, buf, nil)
	if err != nil {
		return errors.Wrap(err, "worker 'commit' call failed")
	}

	result := toReceiverError(reply)

	gw.LogC(r.ctx, "receiver", gw.LogDebug).
		Str("command", "commit").
		Msgf("result: %v", result)

	return result
}

func (r *CvmfsReceiver) call(reqID receiverOp, msg []byte, payload io.Reader) ([]byte, error) {
	if err := r.request(reqID, msg, payload); err != nil {
		return nil, err
	}
	return r.reply()
}

func (r *CvmfsReceiver) request(reqID receiverOp, msg []byte, payload io.Reader) error {
	buf := make([]byte, 8+len(msg))
	binary.LittleEndian.PutUint32(buf[:4], uint32(reqID))
	binary.LittleEndian.PutUint32(buf[4:8], uint32(len(msg)))
	copy(buf[8:], msg)

	if _, err := r.stdin.Write(buf); err != nil {
		return errors.Wrap(err, "could not write request")
	}
	if payload != nil {
		if _, err := io.Copy(r.stdin, payload); err != nil {
			return errors.Wrap(err, "could not write request payload")
		}
	}
	return nil
}

func (r *CvmfsReceiver) reply() ([]byte, error) {
	buf := make([]byte, 4)
	if _, err := io.ReadFull(r.stdout, buf); err != nil {
		return nil, errors.Wrap(err, "could not read reply size")
	}
	repSize := int32(binary.LittleEndian.Uint32(buf))

	reply := make([]byte, repSize)
	if _, err := io.ReadFull(r.stdout, reply); err != nil {
		return nil, errors.Wrap(err, "could not read reply body")
	}

	return reply, nil
}

func toReceiverError(reply []byte) error {
	res := make(map[string]string)
	if err := json.Unmarshal(reply, &res); err != nil {
		return errors.Wrap(err, "could not decode reply")
	}

	if status, ok := res["status"]; ok {
		if status == "ok" {
			return nil
		}

		if reason, ok := res["reason"]; ok {
			return Error(reason)
		}

		return fmt.Errorf("invalid reply")
	}

	return fmt.Errorf("invalid reply")
}
<|endoftext|>