<commit_before>package runner

import (
	"archive/zip"
	"errors"
	"fmt"
	"github.com/lhchavez/quark/common"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
)

type CaseResult struct {
	Verdict  string                 `json:"verdict"`
	Name     string                 `json:"name"`
	MaxScore float64                `json:"max_score"`
	Score    float64                `json:"score"`
	Meta     map[string]RunMetadata `json:"meta"`
}

type GroupResult struct {
	Group    string       `json:"group"`
	MaxScore float64      `json:"max_score"`
	Score    float64      `json:"score"`
	Cases    []CaseResult `json:"cases"`
}

type RunResult struct {
	Verdict      string                 `json:"verdict"`
	CompileError *string                `json:"compile_error,omitempty"`
	CompileMeta  map[string]RunMetadata `json:"compile_meta"`
	Score        float64                `json:"score"`
	Time         float64                `json:"time"`
	WallTime     float64                `json:"wall_time"`
	Memory       int                    `json:"memory"`
	Groups       []GroupResult          `json:"groups"`
}

type binary struct {
	name     string
	language string
}

func Grade(
	ctx *common.Context,
	client *http.Client,
	baseURL *url.URL,
	run *common.Run,
	input common.Input,
	sandbox Sandbox,
) (*RunResult, error) {
	runResult := &RunResult{
		Verdict: "JE",
	}
	if !sandbox.Supported() {
		return runResult, errors.New("Sandbox not supported")
	}
	runRoot := path.Join(
		ctx.Config.Runner.RuntimePath,
		"grade",
		strconv.FormatUint(run.AttemptID, 10),
	)
	if !ctx.Config.Runner.PreserveFiles {
		defer os.RemoveAll(runRoot)
	}

	ctx.Log.Info("Running", "run", run)

	binaries := []binary{
		{"Main", run.Language},
	}

	generatedFiles := make([]string, 0)

	// Setup all source files.
	mainBinPath := path.Join(runRoot, "Main", "bin")
	if err := os.MkdirAll(mainBinPath, 0755); err != nil {
		return runResult, err
	}
	mainSourceFile := path.Join(mainBinPath, fmt.Sprintf("Main.%s", run.Language))
	err := ioutil.WriteFile(mainSourceFile, []byte(run.Source), 0644)
	if err != nil {
		return runResult, err
	}

	validatorBinPath := path.Join(runRoot, "validator", "bin")
	if input.Settings().Validator.Name == "custom" {
		if err := os.MkdirAll(validatorBinPath, 0755); err != nil {
			return runResult, err
		}
		validatorLang := *input.Settings().Validator.Lang
		validatorFileName := fmt.Sprintf("validator.%s", validatorLang)
		validatorSourceFile := path.Join(validatorBinPath, validatorFileName)
		err := os.Link(path.Join(input.Path(), validatorFileName), validatorSourceFile)
		if err != nil {
			return runResult, err
		}
		binaries = append(binaries, binary{"validator", validatorLang})
	}

	runResult.CompileMeta = make(map[string]RunMetadata)

	for _, b := range binaries {
		binRoot := path.Join(runRoot, b.name)
		binPath := path.Join(binRoot, "bin")
		sourceFile := path.Join(binPath, fmt.Sprintf("%s.%s", b.name, b.language))

		compileMeta, err := sandbox.Compile(
			ctx,
			b.language,
			[]string{sourceFile},
			binPath,
			path.Join(binRoot, "compile.out"),
			path.Join(binRoot, "compile.err"),
			path.Join(binRoot, "compile.meta"),
			b.name,
			[]string{},
		)
		generatedFiles = append(
			generatedFiles,
			path.Join(b.name, "compile.out"),
			path.Join(b.name, "compile.err"),
			path.Join(b.name, "compile.meta"),
		)

		if compileMeta != nil {
			runResult.CompileMeta[b.name] = *compileMeta
		}

		if err != nil || compileMeta.Verdict != "OK" {
			ctx.Log.Error("Compile error", "err", err, "compileMeta", compileMeta)
			runResult.Verdict = "CE"
			compileErrorFile := "compile.err"
			if b.language == "pas" {
				// Lazarus writes the output of the compile error in compile.out.
				compileErrorFile = "compile.out"
			} else {
				compileErrorFile = "compile.err"
			}
			compileError := getCompileError(path.Join(binRoot, compileErrorFile))
			runResult.CompileError = &compileError
			return runResult, err
		}
	}

	groupResults := make([]GroupResult, len(input.Settings().Cases))
	maxScore := run.MaxScore
	runResult.Verdict = "OK"
	wallTimeLimit := (float64)(input.Settings().Limits.OverallWallTimeLimit / 1000.0)
	for i, group := range input.Settings().Cases {
		caseResults := make([]CaseResult, len(group.Cases))
		for j, caseData := range group.Cases {
			var runMeta *RunMetadata
			if runResult.WallTime > wallTimeLimit {
				runMeta = &RunMetadata{
					Verdict: "TLE",
				}
			} else {
				runMeta, err = sandbox.Run(
					ctx,
					input,
					run.Language,
					mainBinPath,
					path.Join(input.Path(), "in", fmt.Sprintf("%s.in", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.out", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.err", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.meta", caseData.Name)),
					"Main",
					nil,
					nil,
					nil,
					[]string{},
					map[string]string{},
				)
				if err != nil {
					ctx.Log.Error("failed to run "+caseData.Name, "err", err)
				}
				generatedFiles = append(
					generatedFiles,
					fmt.Sprintf("%s.out", caseData.Name),
					fmt.Sprintf("%s.err", caseData.Name),
					fmt.Sprintf("%s.meta", caseData.Name),
				)
			}
			runResult.Verdict = worseVerdict(runResult.Verdict, runMeta.Verdict)
			runResult.Time += runMeta.Time
			runResult.WallTime += runMeta.WallTime
			runResult.Memory = max(runResult.Memory, runMeta.Memory)

			caseResults[j] = CaseResult{
				Name:     caseData.Name,
				MaxScore: maxScore * caseData.Weight,
				Verdict:  runMeta.Verdict,
				Meta: map[string]RunMetadata{
					"Main": *runMeta,
				},
			}
		}
		groupResults[i] = GroupResult{
			Group:    group.Name,
			MaxScore: maxScore * group.Weight,
			Score:    0,
			Cases:    caseResults,
		}
	}

	// Validate outputs.
	for i, group := range input.Settings().Cases {
		correct := true
		score := 0.0
		for j, caseData := range group.Cases {
			caseResults := groupResults[i].Cases[j]
			if caseResults.Verdict == "OK" {
				contestantPath := path.Join(
					runRoot, fmt.Sprintf("%s.out", caseData.Name),
				)
				if input.Settings().Validator.Name == "custom" {
					originalInputFile := path.Join(
						input.Path(),
						"in",
						fmt.Sprintf("%s.in", caseData.Name),
					)
					originalOutputFile := path.Join(
						input.Path(),
						"out",
						fmt.Sprintf("%s.out", caseData.Name),
					)
					runMetaFile := path.Join(runRoot, fmt.Sprintf("%s.meta", caseData.Name))
					validateMeta, err := sandbox.Run(
						ctx,
						input,
						*input.Settings().Validator.Lang,
						validatorBinPath,
						contestantPath,
						path.Join(runRoot, "validator", fmt.Sprintf("%s.out", caseData.Name)),
						path.Join(runRoot, "validator", fmt.Sprintf("%s.err", caseData.Name)),
						path.Join(runRoot, "validator", fmt.Sprintf("%s.meta", caseData.Name)),
						"validator",
						&originalInputFile,
						&originalOutputFile,
						&runMetaFile,
						[]string{},
						map[string]string{},
					)
					if err != nil {
						ctx.Log.Error("failed to validate "+caseData.Name, "err", err)
					}
					generatedFiles = append(
						generatedFiles,
						fmt.Sprintf("validator/%s.out", caseData.Name),
						fmt.Sprintf("validator/%s.err", caseData.Name),
						fmt.Sprintf("validator/%s.meta", caseData.Name),
					)
					if validateMeta.Verdict != "OK" {
						// If the validator did not exit cleanly, assume an empty output.
						contestantPath = "/dev/null"
					} else {
						contestantPath = path.Join(
							runRoot,
							"validator",
							fmt.Sprintf("%s.out", caseData.Name),
						)
					}
				}
				contestantFd, err := os.Open(contestantPath)
				if err != nil {
					ctx.Log.Warn("Error opening file", "path", contestantPath, "err", err)
					continue
				}
				defer contestantFd.Close()
				expectedPath := path.Join(
					input.Path(), "out", fmt.Sprintf("%s.out", caseData.Name),
				)
				expectedFd, err := os.Open(expectedPath)
				if err != nil {
					ctx.Log.Warn("Error opening file", "path", expectedPath, "err", err)
					continue
				}
				defer expectedFd.Close()
				runScore, err := CalculateScore(
					&input.Settings().Validator,
					contestantFd,
					expectedFd,
				)
				if err != nil {
					ctx.Log.Debug("error comparing values", "err", err)
				}
				caseResults.Score = maxScore * runScore * caseData.Weight
				score += runScore * caseData.Weight
				if runScore == 0 {
					correct = false
				}
				if runScore != 1 {
					runResult.Verdict = worseVerdict(runResult.Verdict, "PA")
				}
			}
		}
		if correct {
			groupResults[i].Score = maxScore * score
			runResult.Score += groupResults[i].Score
		}
	}

	runResult.Groups = groupResults

	if runResult.Verdict == "PA" && runResult.Score == 0 {
		runResult.Verdict = "WA"
	} else if runResult.Verdict == "OK" {
		runResult.Verdict = "AC"
	}

	ctx.Log.Debug("Finished running", "results", runResult)
	filesURL, err := baseURL.Parse(fmt.Sprintf("run/%d/files/", run.AttemptID))
	if err != nil {
		return runResult, err
	}
	if err := uploadFiles(
		ctx,
		client,
		filesURL.String(),
		runRoot,
		input,
		generatedFiles,
	); err != nil {
		ctx.Log.Debug("uploadFiles failed", "err", err)
		return runResult, err
	}

	return runResult, nil
}

func uploadFiles(
	ctx *common.Context,
	client *http.Client,
	uploadURL string,
	runRoot string,
	input common.Input,
	files []string,
) error {
	path, err := createZipFile(runRoot, files)
	if path != "" {
		defer os.Remove(path)
	}
	if err != nil {
		return err
	}
	fd, err := os.Open(path)
	if err != nil {
		return err
	}

	resp, err := client.Post(uploadURL, "application/zip", fd)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func createZipFile(runRoot string, files []string) (string, error) {
	zipFd, err := ioutil.TempFile(runRoot, ".results_zip")
	if err != nil {
		return "", err
	}
	zipPath := zipFd.Name()
	defer zipFd.Close()
	zip := zip.NewWriter(zipFd)
	for _, file := range files {
		f, err := os.Open(path.Join(runRoot, file))
		if err != nil {
			continue
		}
		defer f.Close()
		zf, err := zip.Create(file)
		if err != nil {
			return zipPath, err
		}
		if _, err := io.Copy(zf, f); err != nil {
			return zipPath, err
		}
	}
	return zipPath, zip.Close()
}

func getCompileError(errorFile string) string {
	fd, err := os.Open(errorFile)
	if err != nil {
		return err.Error()
	}
	defer fd.Close()
	bytes, err := ioutil.ReadAll(fd)
	if err != nil {
		return err.Error()
	}
	return string(bytes)
}

func worseVerdict(a, b string) string {
	verdictList := []string{
		"JE",
		"CE",
		"MLE",
		"RFE",
		"RTE",
		"TLE",
		"OLE",
		"WA",
		"PA",
		"AC",
		"OK",
	}
	idxA := sliceIndex(len(verdictList),
		func(i int) bool { return verdictList[i] == a })
	idxB := sliceIndex(len(verdictList),
		func(i int) bool { return verdictList[i] == b })
	return verdictList[min(idxA, idxB)]
}

func sliceIndex(limit int, predicate func(i int) bool) int {
	for i := 0; i < limit; i++ {
		if predicate(i) {
			return i
		}
	}
	return -1
}
<commit_msg>Including MaxScore in RunResult<commit_after>package runner

import (
	"archive/zip"
	"errors"
	"fmt"
	"github.com/lhchavez/quark/common"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
)

type CaseResult struct {
	Verdict  string                 `json:"verdict"`
	Name     string                 `json:"name"`
	MaxScore float64                `json:"max_score"`
	Score    float64                `json:"score"`
	Meta     map[string]RunMetadata `json:"meta"`
}

type GroupResult struct {
	Group    string       `json:"group"`
	MaxScore float64      `json:"max_score"`
	Score    float64      `json:"score"`
	Cases    []CaseResult `json:"cases"`
}

type RunResult struct {
	Verdict      string                 `json:"verdict"`
	CompileError *string                `json:"compile_error,omitempty"`
	CompileMeta  map[string]RunMetadata `json:"compile_meta"`
	Score        float64                `json:"score"`
	MaxScore     float64                `json:"max_score"`
	Time         float64                `json:"time"`
	WallTime     float64                `json:"wall_time"`
	Memory       int                    `json:"memory"`
	Groups       []GroupResult          `json:"groups"`
}

type binary struct {
	name     string
	language string
}

func Grade(
	ctx *common.Context,
	client *http.Client,
	baseURL *url.URL,
	run *common.Run,
	input common.Input,
	sandbox Sandbox,
) (*RunResult, error) {
	runResult := &RunResult{
		Verdict: "JE",
	}
	if !sandbox.Supported() {
		return runResult, errors.New("Sandbox not supported")
	}
	runRoot := path.Join(
		ctx.Config.Runner.RuntimePath,
		"grade",
		strconv.FormatUint(run.AttemptID, 10),
	)
	if !ctx.Config.Runner.PreserveFiles {
		defer os.RemoveAll(runRoot)
	}

	ctx.Log.Info("Running", "run", run)

	binaries := []binary{
		{"Main", run.Language},
	}

	generatedFiles := make([]string, 0)

	// Setup all source files.
	mainBinPath := path.Join(runRoot, "Main", "bin")
	if err := os.MkdirAll(mainBinPath, 0755); err != nil {
		return runResult, err
	}
	mainSourceFile := path.Join(mainBinPath, fmt.Sprintf("Main.%s", run.Language))
	err := ioutil.WriteFile(mainSourceFile, []byte(run.Source), 0644)
	if err != nil {
		return runResult, err
	}

	validatorBinPath := path.Join(runRoot, "validator", "bin")
	if input.Settings().Validator.Name == "custom" {
		if err := os.MkdirAll(validatorBinPath, 0755); err != nil {
			return runResult, err
		}
		validatorLang := *input.Settings().Validator.Lang
		validatorFileName := fmt.Sprintf("validator.%s", validatorLang)
		validatorSourceFile := path.Join(validatorBinPath, validatorFileName)
		err := os.Link(path.Join(input.Path(), validatorFileName), validatorSourceFile)
		if err != nil {
			return runResult, err
		}
		binaries = append(binaries, binary{"validator", validatorLang})
	}

	runResult.CompileMeta = make(map[string]RunMetadata)

	for _, b := range binaries {
		binRoot := path.Join(runRoot, b.name)
		binPath := path.Join(binRoot, "bin")
		sourceFile := path.Join(binPath, fmt.Sprintf("%s.%s", b.name, b.language))

		compileMeta, err := sandbox.Compile(
			ctx,
			b.language,
			[]string{sourceFile},
			binPath,
			path.Join(binRoot, "compile.out"),
			path.Join(binRoot, "compile.err"),
			path.Join(binRoot, "compile.meta"),
			b.name,
			[]string{},
		)
		generatedFiles = append(
			generatedFiles,
			path.Join(b.name, "compile.out"),
			path.Join(b.name, "compile.err"),
			path.Join(b.name, "compile.meta"),
		)

		if compileMeta != nil {
			runResult.CompileMeta[b.name] = *compileMeta
		}

		if err != nil || compileMeta.Verdict != "OK" {
			ctx.Log.Error("Compile error", "err", err, "compileMeta", compileMeta)
			runResult.Verdict = "CE"
			compileErrorFile := "compile.err"
			if b.language == "pas" {
				// Lazarus writes the output of the compile error in compile.out.
				compileErrorFile = "compile.out"
			} else {
				compileErrorFile = "compile.err"
			}
			compileError := getCompileError(path.Join(binRoot, compileErrorFile))
			runResult.CompileError = &compileError
			return runResult, err
		}
	}

	groupResults := make([]GroupResult, len(input.Settings().Cases))
	runResult.MaxScore = run.MaxScore
	runResult.Verdict = "OK"
	wallTimeLimit := (float64)(input.Settings().Limits.OverallWallTimeLimit / 1000.0)
	for i, group := range input.Settings().Cases {
		caseResults := make([]CaseResult, len(group.Cases))
		for j, caseData := range group.Cases {
			var runMeta *RunMetadata
			if runResult.WallTime > wallTimeLimit {
				runMeta = &RunMetadata{
					Verdict: "TLE",
				}
			} else {
				runMeta, err = sandbox.Run(
					ctx,
					input,
					run.Language,
					mainBinPath,
					path.Join(input.Path(), "in", fmt.Sprintf("%s.in", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.out", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.err", caseData.Name)),
					path.Join(runRoot, fmt.Sprintf("%s.meta", caseData.Name)),
					"Main",
					nil,
					nil,
					nil,
					[]string{},
					map[string]string{},
				)
				if err != nil {
					ctx.Log.Error("failed to run "+caseData.Name, "err", err)
				}
				generatedFiles = append(
					generatedFiles,
					fmt.Sprintf("%s.out", caseData.Name),
					fmt.Sprintf("%s.err", caseData.Name),
					fmt.Sprintf("%s.meta", caseData.Name),
				)
			}
			runResult.Verdict = worseVerdict(runResult.Verdict, runMeta.Verdict)
			runResult.Time += runMeta.Time
			runResult.WallTime += runMeta.WallTime
			runResult.Memory = max(runResult.Memory, runMeta.Memory)

			caseResults[j] = CaseResult{
				Name:     caseData.Name,
				MaxScore: runResult.MaxScore * caseData.Weight,
				Verdict:  runMeta.Verdict,
				Meta: map[string]RunMetadata{
					"Main": *runMeta,
				},
			}
		}
		groupResults[i] = GroupResult{
			Group:    group.Name,
			MaxScore: runResult.MaxScore * group.Weight,
			Score:    0,
			Cases:    caseResults,
		}
	}

	// Validate outputs.
	for i, group := range input.Settings().Cases {
		correct := true
		score := 0.0
		for j, caseData := range group.Cases {
			caseResults := groupResults[i].Cases[j]
			if caseResults.Verdict == "OK" {
				contestantPath := path.Join(
					runRoot, fmt.Sprintf("%s.out", caseData.Name),
				)
				if input.Settings().Validator.Name == "custom" {
					originalInputFile := path.Join(
						input.Path(),
						"in",
						fmt.Sprintf("%s.in", caseData.Name),
					)
					originalOutputFile := path.Join(
						input.Path(),
						"out",
						fmt.Sprintf("%s.out", caseData.Name),
					)
					runMetaFile := path.Join(runRoot, fmt.Sprintf("%s.meta", caseData.Name))
					validateMeta, err := sandbox.Run(
						ctx,
						input,
						*input.Settings().Validator.Lang,
						validatorBinPath,
						contestantPath,
						path.Join(runRoot, "validator", fmt.Sprintf("%s.out", caseData.Name)),
						path.Join(runRoot, "validator", fmt.Sprintf("%s.err", caseData.Name)),
						path.Join(runRoot, "validator", fmt.Sprintf("%s.meta", caseData.Name)),
						"validator",
						&originalInputFile,
						&originalOutputFile,
						&runMetaFile,
						[]string{},
						map[string]string{},
					)
					if err != nil {
						ctx.Log.Error("failed to validate "+caseData.Name, "err", err)
					}
					generatedFiles = append(
						generatedFiles,
						fmt.Sprintf("validator/%s.out", caseData.Name),
						fmt.Sprintf("validator/%s.err", caseData.Name),
						fmt.Sprintf("validator/%s.meta", caseData.Name),
					)
					if validateMeta.Verdict != "OK" {
						// If the validator did not exit cleanly, assume an empty output.
						contestantPath = "/dev/null"
					} else {
						contestantPath = path.Join(
							runRoot,
							"validator",
							fmt.Sprintf("%s.out", caseData.Name),
						)
					}
				}
				contestantFd, err := os.Open(contestantPath)
				if err != nil {
					ctx.Log.Warn("Error opening file", "path", contestantPath, "err", err)
					continue
				}
				defer contestantFd.Close()
				expectedPath := path.Join(
					input.Path(), "out", fmt.Sprintf("%s.out", caseData.Name),
				)
				expectedFd, err := os.Open(expectedPath)
				if err != nil {
					ctx.Log.Warn("Error opening file", "path", expectedPath, "err", err)
					continue
				}
				defer expectedFd.Close()
				runScore, err := CalculateScore(
					&input.Settings().Validator,
					contestantFd,
					expectedFd,
				)
				if err != nil {
					ctx.Log.Debug("error comparing values", "err", err)
				}
				caseResults.Score = runResult.MaxScore * runScore * caseData.Weight
				score += runScore * caseData.Weight
				if runScore == 0 {
					correct = false
				}
				if runScore != 1 {
					runResult.Verdict = worseVerdict(runResult.Verdict, "PA")
				}
			}
		}
		if correct {
			groupResults[i].Score = runResult.MaxScore * score
			runResult.Score += groupResults[i].Score
		}
	}

	runResult.Groups = groupResults

	if runResult.Verdict == "PA" && runResult.Score == 0 {
		runResult.Verdict = "WA"
	} else if runResult.Verdict == "OK" {
		runResult.Verdict = "AC"
		runResult.Score = 1
	}

	ctx.Log.Debug("Finished running", "results", runResult)
	filesURL, err := baseURL.Parse(fmt.Sprintf("run/%d/files/", run.AttemptID))
	if err != nil {
		return runResult, err
	}
	if err := uploadFiles(
		ctx,
		client,
		filesURL.String(),
		runRoot,
		input,
		generatedFiles,
	); err != nil {
		ctx.Log.Debug("uploadFiles failed", "err", err)
		return runResult, err
	}

	return runResult, nil
}

func uploadFiles(
	ctx *common.Context,
	client *http.Client,
	uploadURL string,
	runRoot string,
	input common.Input,
	files []string,
) error {
	path, err := createZipFile(runRoot, files)
	if path != "" {
		defer os.Remove(path)
	}
	if err != nil {
		return err
	}
	fd, err := os.Open(path)
	if err != nil {
		return err
	}

	resp, err := client.Post(uploadURL, "application/zip", fd)
	if err != nil {
		return err
	}
	resp.Body.Close()

	return nil
}

func createZipFile(runRoot string, files []string) (string, error) {
	zipFd, err := ioutil.TempFile(runRoot, ".results_zip")
	if err != nil {
		return "", err
	}
	zipPath := zipFd.Name()
	defer zipFd.Close()
	zip := zip.NewWriter(zipFd)
	for _, file := range files {
		f, err := os.Open(path.Join(runRoot, file))
		if err != nil {
			continue
		}
		defer f.Close()
		zf, err := zip.Create(file)
		if err != nil {
			return zipPath, err
		}
		if _, err := io.Copy(zf, f); err != nil {
			return zipPath, err
		}
	}
	return zipPath, zip.Close()
}

func getCompileError(errorFile string) string {
	fd, err := os.Open(errorFile)
	if err != nil {
		return err.Error()
	}
	defer fd.Close()
	bytes, err := ioutil.ReadAll(fd)
	if err != nil {
		return err.Error()
	}
	return string(bytes)
}

func worseVerdict(a, b string) string {
	verdictList := []string{
		"JE",
		"CE",
		"MLE",
		"RFE",
		"RTE",
		"TLE",
		"OLE",
		"WA",
		"PA",
		"AC",
		"OK",
	}
	idxA := sliceIndex(len(verdictList),
		func(i int) bool { return verdictList[i] == a })
	idxB := sliceIndex(len(verdictList),
		func(i int) bool { return verdictList[i] == b })
	return verdictList[min(idxA, idxB)]
}

func sliceIndex(limit int, predicate func(i int) bool) int {
	for i := 0; i < limit; i++ {
		if predicate(i) {
			return i
		}
	}
	return -1
}
<|endoftext|>


<commit_before>package runner

import (
	"errors"
	"fmt"
	"github.com/bivas/bitbucket-pipelines/parser"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

const (
	pipelineRunnerName       = "pipeline__runner__"
	pipelineRunnerDockerName = pipelineRunnerName + "docker"
	dockerImage              = "docker:1.8-dind"
	bootTimeout              = 30
)

//PipelineRunner : create the pipelines inner
type PipelineRunner interface {
	Setup()
	Run(commands []string) (string, error)
	Close()
}

type inner struct {
	image            string
	hostPath         string
	environment      string
	useDockerService bool
	command          *exec.Cmd
	signal           chan error
}

func (*inner) commandRun(name string, args []string) *exec.Cmd {
	log.Println("Running (commandRun)", name, args)
	return exec.Command(name, args...)
}

func (*inner) commandOutput(name string, args []string) (string, error) {
	log.Println("Running (commandOutput)", name, args)
	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	return strings.TrimSpace(string(out)), err
}

func (env *inner) docker(args ...string) (string, error) {
	return env.commandOutput("docker", args)
}

func (env *inner) pullImage() {
	ui.Info("Pulling image %s", env.image)
	out, e := env.docker("pull", env.image)
	if e != nil {
		ui.Error("Error pulling image %s %s", env.image, e)
		log.Fatalf("Error message %s\n", out)
	}
}

func (env *inner) runDockerDocker() {
	if env.useDockerService {
		log.Println("pulling", dockerImage)
		out, e := env.docker("pull", dockerImage)
		if e != nil {
			log.Fatalf("Error pulling image %s %s\n", dockerImage, e)
			log.Fatalf("Error message %s\n", out)
		}
		args := []string{"run",
			"-d",
			"--name=" + pipelineRunnerDockerName,
			"--privileged",
			dockerImage}
		dockerCommand := env.commandRun("docker", args)
		log.Println("running", dockerImage)
		go func() {
			out, err := dockerCommand.CombinedOutput()
			if err != nil {
				log.Fatalf("error combining output %s, %s", out, err)
			}
		}()
		env.waitForImage(pipelineRunnerDockerName)
	}
}

func (*inner) sendInitCommands(cmd *exec.Cmd) {
	stdin, e := cmd.StdinPipe()
	if e != nil {
		log.Fatal("error setting up stdin", e)
	}
	defer stdin.Close()
	io.WriteString(stdin, "touch /.running\n")
	io.WriteString(stdin, "while [ -e /.running ]; do sleep 1; done; exit;\n")
}

func (env *inner) runImage() {
	env.cleanup()
	ui.Info("Running image %s", env.image)
	args := []string{"run",
		"-i",
		"--rm",
		"--name=" + pipelineRunnerName,
		"--volume=" + env.hostPath + ":/wd",
		"--workdir=/wd",
		"--entrypoint=/bin/sh"}
	if env.useDockerService {
		env.runDockerDocker()
		args = append(args,
			[]string{
				"--env=DOCKER_HOST=tcp://docker:2375",
				"--link=" + pipelineRunnerDockerName + ":docker",
			}...)
	}
	if env.environment != "" {
		if isFile(env.environment) {
			args = append(args,
				[]string{
					"--env-file=" + env.environment,
				}...)
		} else {
			ui.Warning(
				"Environment file %s doesn't exist - running without environment overrides",
				env.environment)
		}
	}
	args = append(args, env.image)
	env.command = env.commandRun("docker", args)
	env.sendInitCommands(env.command)
	env.signal = make(chan error)
	go func() {
		out, err := env.command.CombinedOutput()
		if err != nil {
			env.signal <- fmt.Errorf("error combining output %s, %s", out, err)
		} else {
			env.signal <- err
		}
	}()
}

func (env *inner) stopImage() {
	env.docker("exec", "-i", pipelineRunnerName, "rm", "/.running")
	if env.useDockerService {
		env.docker("stop", pipelineRunnerDockerName)
	}
}

func (env *inner) cleanup() {
	env.docker("rm", "-f", pipelineRunnerName)
	if env.useDockerService {
		env.docker("rm", "-f", pipelineRunnerDockerName)
		os.Remove("")
	}
}

func (env *inner) Setup() {
	log.Println("Setup inner runner")
	env.pullImage()
	env.runImage()
}

func (env *inner) Close() {
	log.Println("Closing inner runner")
	env.command.Process.Kill()
	env.command.Wait()
	env.cleanup()
}

func (env *inner) waitForImage(image string) error {
	filterPs := []string{"ps", "-aq", "--filter", "name=" + image}
	id, _ := env.docker(filterPs...)
	for i := 0; ; i++ {
		if i > bootTimeout {
			return fmt.Errorf("Unable to start container after %d seconds", bootTimeout)
		}
		log.Println("Waiting for container to be available", image, id)
		if id != "" {
			return nil
		}
		time.Sleep(1 * time.Second)
		id, _ = env.docker(filterPs...)
	}
	return nil
}

func (env *inner) Run(commands []string) error {
	go func() {
		time.Sleep(5 * time.Minute)
		env.signal <- errors.New("Timeout trying to run commands")
	}()
	go func() {
		e := env.waitForImage(pipelineRunnerName)
		if e != nil {
			env.signal <- e
			return
		}
		if env.useDockerService {
			env.copyDockerBin()
		}
		for _, command := range commands {
			output, err := env.docker("exec", "-i", pipelineRunnerName, "/bin/sh", "-c", command)
			if err != nil {
				env.signal <- errors.New(fmt.Sprintln("error running", command, output, err))
			}
			ui.Info(" == Running '%s' ==>", command)
			ui.Output(output)
		}
		env.signal <- nil
	}()
	err := <-env.signal
	env.stopImage()
	return err
}

func (env *inner) copyDockerBin() {
	if out1, e1 := env.docker([]string{
		"cp",
		pipelineRunnerDockerName + ":/usr/local/bin/docker",
		"/tmp/",
	}...); e1 != nil {
		log.Fatal("copy from", out1, e1)
	}
	if out1, e1 := env.docker([]string{
		"exec",
		pipelineRunnerName,
		"mkdir",
		"-p",
		"/usr/local/bin/",
	}...); e1 != nil {
		log.Fatal("copy to", out1, e1)
	}
	if out1, e1 := env.docker([]string{
		"cp",
		"/tmp/docker",
		pipelineRunnerName + ":/usr/local/bin/",
	}...); e1 != nil {
		log.Fatal("copy to", out1, e1)
	}
}

//Run : run it!
func Run(yml string, env string) int {
	reader, e := os.Open(yml)
	if e != nil {
		log.Printf("Unable to read %s\n", yml)
		return 1
	}
	pipeline := parser.ReadPipelineDef(reader)
	path, _ := os.Getwd()
	r := &inner{
		image:            pipeline.Image,
		hostPath:         path,
		environment:      env,
		useDockerService: pipeline.Options.Docker,
	}
	defer r.Close()
	r.Setup()
	eR := r.Run(pipeline.Pipelines.Default[0].Step.Scripts)
	if eR != nil {
		log.Println("error when running", eR)
		return 1
	}
	return 0
}
<commit_msg>update to docker 17<commit_after>package runner

import (
	"errors"
	"fmt"
	"github.com/bivas/bitbucket-pipelines/parser"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

const (
	pipelineRunnerName       = "pipeline__runner__"
	pipelineRunnerDockerName = pipelineRunnerName + "docker"
	dockerImage              = "docker:17-dind"
	bootTimeout              = 30
)

//PipelineRunner : create the pipelines inner
type PipelineRunner interface {
	Setup()
	Run(commands []string) (string, error)
	Close()
}

type inner struct {
	image            string
	hostPath         string
	environment      string
	useDockerService bool
	command          *exec.Cmd
	signal           chan error
}

func (*inner) commandRun(name string, args []string) *exec.Cmd {
	log.Println("Running (commandRun)", name, args)
	return exec.Command(name, args...)
}

func (*inner) commandOutput(name string, args []string) (string, error) {
	log.Println("Running (commandOutput)", name, args)
	cmd := exec.Command(name, args...)
	out, err := cmd.CombinedOutput()
	return strings.TrimSpace(string(out)), err
}

func (env *inner) docker(args ...string) (string, error) {
	return env.commandOutput("docker", args)
}

func (env *inner) pullImage() {
	ui.Info("Pulling image %s", env.image)
	out, e := env.docker("pull", env.image)
	if e != nil {
		ui.Error("Error pulling image %s %s", env.image, e)
		log.Fatalf("Error message %s\n", out)
	}
}

func (env *inner) runDockerDocker() {
	if env.useDockerService {
		log.Println("pulling", dockerImage)
		out, e := env.docker("pull", dockerImage)
		if e != nil {
			log.Fatalf("Error pulling image %s %s\n", dockerImage, e)
			log.Fatalf("Error message %s\n", out)
		}
		args := []string{"run",
			"-d",
			"--name=" + pipelineRunnerDockerName,
			"--privileged",
			dockerImage}
		dockerCommand := env.commandRun("docker", args)
		log.Println("running", dockerImage)
		go func() {
			out, err := dockerCommand.CombinedOutput()
			if err != nil {
				log.Fatalf("error combining output %s, %s", out, err)
			}
		}()
		env.waitForImage(pipelineRunnerDockerName)
	}
}

func (*inner) sendInitCommands(cmd *exec.Cmd) {
	stdin, e := cmd.StdinPipe()
	if e != nil {
		log.Fatal("error setting up stdin", e)
	}
	defer stdin.Close()
	io.WriteString(stdin, "touch /.running\n")
	io.WriteString(stdin, "while [ -e /.running ]; do sleep 1; done; exit;\n")
}

func (env *inner) runImage() {
	env.cleanup()
	ui.Info("Running image %s", env.image)
	args := []string{"run",
		"-i",
		"--rm",
		"--name=" + pipelineRunnerName,
		"--volume=" + env.hostPath + ":/wd",
		"--workdir=/wd",
		"--entrypoint=/bin/sh"}
	if env.useDockerService {
		env.runDockerDocker()
		args = append(args,
			[]string{
				"--env=DOCKER_HOST=tcp://docker:2375",
				"--link=" + pipelineRunnerDockerName + ":docker",
			}...)
	}
	if env.environment != "" {
		if isFile(env.environment) {
			args = append(args,
				[]string{
					"--env-file=" + env.environment,
				}...)
		} else {
			ui.Warning(
				"Environment file %s doesn't exist - running without environment overrides",
				env.environment)
		}
	}
	args = append(args, env.image)
	env.command = env.commandRun("docker", args)
	env.sendInitCommands(env.command)
	env.signal = make(chan error)
	go func() {
		out, err := env.command.CombinedOutput()
		if err != nil {
			env.signal <- fmt.Errorf("error combining output %s, %s", out, err)
		} else {
			env.signal <- err
		}
	}()
}

func (env *inner) stopImage() {
	env.docker("exec", "-i", pipelineRunnerName, "rm", "/.running")
	if env.useDockerService {
		env.docker("stop", pipelineRunnerDockerName)
	}
}

func (env *inner) cleanup() {
	env.docker("rm", "-f", pipelineRunnerName)
	if env.useDockerService {
		env.docker("rm", "-f", pipelineRunnerDockerName)
		os.Remove("")
	}
}

func (env *inner) Setup() {
	log.Println("Setup inner runner")
	env.pullImage()
	env.runImage()
}

func (env *inner) Close() {
	log.Println("Closing inner runner")
	env.command.Process.Kill()
	env.command.Wait()
	env.cleanup()
}

func (env *inner) waitForImage(image string) error {
	filterPs := []string{"ps", "-aq", "--filter", "name=" + image}
	id, _ := env.docker(filterPs...)
	for i := 0; ; i++ {
		if i > bootTimeout {
			return fmt.Errorf("Unable to start container after %d seconds", bootTimeout)
		}
		log.Println("Waiting for container to be available", image, id)
		time.Sleep(1 * time.Second)
		if id != "" {
			return nil
		}
		id, _ = env.docker(filterPs...)
	}
	return nil
}

func (env *inner) Run(commands []string) error {
	go func() {
		time.Sleep(5 * time.Minute)
		env.signal <- errors.New("Timeout trying to run commands")
	}()
	go func() {
		e := env.waitForImage(pipelineRunnerName)
		if e != nil {
			env.signal <- e
			return
		}
		if env.useDockerService {
			env.copyDockerBin()
		}
		for _, command := range commands {
			output, err := env.docker("exec", "-i", pipelineRunnerName, "/bin/sh", "-c", command)
			if err != nil {
				env.signal <- errors.New(fmt.Sprintln("error running", command, output, err))
			}
			ui.Info(" == Running '%s' ==>", command)
			ui.Output(output)
		}
		env.signal <- nil
	}()
	err := <-env.signal
	env.stopImage()
	return err
}

func (env *inner) copyDockerBin() {
	if out1, e1 := env.docker([]string{
		"cp",
		pipelineRunnerDockerName + ":/usr/local/bin/docker",
		"/tmp/",
	}...); e1 != nil {
		log.Fatal("copy from", out1, e1)
	}
	if out1, e1 := env.docker([]string{
		"exec",
		pipelineRunnerName,
		"mkdir",
		"-p",
		"/usr/local/bin/",
	}...); e1 != nil {
		log.Fatal("copy to", out1, e1)
	}
	if out1, e1 := env.docker([]string{
		"cp",
		"/tmp/docker",
		pipelineRunnerName + ":/usr/local/bin/",
	}...); e1 != nil {
		log.Fatal("copy to", out1, e1)
	}
}

//Run : run it!
func Run(yml string, env string) int {
	reader, e := os.Open(yml)
	if e != nil {
		log.Printf("Unable to read %s\n", yml)
		return 1
	}
	pipeline := parser.ReadPipelineDef(reader)
	path, _ := os.Getwd()
	r := &inner{
		image:            pipeline.Image,
		hostPath:         path,
		environment:      env,
		useDockerService: pipeline.Options.Docker,
	}
	defer r.Close()
	r.Setup()
	eR := r.Run(pipeline.Pipelines.Default[0].Step.Scripts)
	if eR != nil {
		log.Println("error when running", eR)
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package scanner

const (
	TSTRING = iota
	TNUMBER
	TTRUE
	TFALSE
	TNULL
	TLBRACE
	TRBRACE
	TLBRACKET
	TRBRACKET
	TCOLON
	TCOMMA
)

var names = map[int]string{
	TSTRING:   "string",
	TNUMBER:   "number",
	TTRUE:     "true",
	TFALSE:    "false",
	TNULL:     "null",
	TLBRACE:   "left brace",
	TRBRACE:   "right brace",
	TLBRACKET: "left bracket",
	TRBRACKET: "right bracket",
	TCOLON:    "colon",
	TCOMMA:    "comma",
}

// TokenName returns a human readable version of the token.
func TokenName(tok int) string {
	return names[tok]
}
<commit_msg>Changes the integer value of TSTRING so that it can be Unscan'ed.<commit_after>package scanner

const (
	TSTRING = iota + 1
	TNUMBER
	TTRUE
	TFALSE
	TNULL
	TLBRACE
	TRBRACE
	TLBRACKET
	TRBRACKET
	TCOLON
	TCOMMA
)

var names = map[int]string{
	TSTRING:   "string",
	TNUMBER:   "number",
	TTRUE:     "true",
	TFALSE:    "false",
	TNULL:     "null",
	TLBRACE:   "left brace",
	TRBRACE:   "right brace",
	TLBRACKET: "left bracket",
	TRBRACKET: "right bracket",
	TCOLON:    "colon",
	TCOMMA:    "comma",
}

// TokenName returns a human readable version of the token.
func TokenName(tok int) string {
	return names[tok]
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux|android)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
	Android string
}{
	"windows",
	"linux",
	"darwin",
	"android"
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	// Resetting file hashes.
	fileHashMapMu.Lock()
	fileHashMap = map[string]string{}
	fileHashMapMu.Unlock()

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}

	// Only considering non-manoto versions for the latestAssetsMap
	if !buildStringContainsManoto(asset.v) {
		if g.latestAssetsMap[os][arch] == nil {
			g.latestAssetsMap[os][arch] = asset
		} else {
			// Compare against already set version.
			if asset.v.GT(g.latestAssetsMap[os][arch].v) {
				g.latestAssetsMap[os][arch] = asset
			}
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin && matches[1] != OS.Android {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<commit_msg>add comment<commit_after>package server

import (
	"fmt"
	"regexp"
	"sort"
	"sync"

	"github.com/blang/semver"
	"github.com/google/go-github/github"
)

var (
	updateAssetRe = regexp.MustCompile(`^update_(darwin|windows|linux|android)_(arm|386|amd64)\.?.*$`)

	emptyVersion semver.Version
)

// Arch holds architecture names.
var Arch = struct {
	X64 string
	X86 string
	ARM string
}{
	"amd64",
	"386",
	"arm",
}

// OS holds operating system names.
var OS = struct {
	Windows string
	Linux   string
	Darwin  string
	Android string
}{
	"windows",
	"linux",
	"darwin",
	"android",
}

// Release struct represents a single github release.
type Release struct {
	id      int
	URL     string
	Version semver.Version
	Assets  []Asset
}

type releasesByID []Release

// Asset struct represents a file included as part of a Release.
type Asset struct {
	id        int
	v         semver.Version
	Name      string
	URL       string
	LocalFile string
	Checksum  string
	Signature string
	AssetInfo
}

// AssetInfo struct holds OS and Arch information of an asset.
type AssetInfo struct {
	OS   string
	Arch string
}

// ReleaseManager struct defines a repository to pull releases from.
type ReleaseManager struct {
	client          *github.Client
	owner           string
	repo            string
	updateAssetsMap map[string]map[string]map[string]*Asset
	latestAssetsMap map[string]map[string]*Asset
	mu              *sync.RWMutex
}

func (a releasesByID) Len() int {
	return len(a)
}

func (a releasesByID) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a releasesByID) Less(i, j int) bool {
	return a[i].id < a[j].id
}

// NewReleaseManager creates a wrapper of github.Client.
func NewReleaseManager(owner string, repo string) *ReleaseManager {

	ghc := &ReleaseManager{
		client:          github.NewClient(nil),
		owner:           owner,
		repo:            repo,
		mu:              new(sync.RWMutex),
		updateAssetsMap: make(map[string]map[string]map[string]*Asset),
		latestAssetsMap: make(map[string]map[string]*Asset),
	}

	return ghc
}

// getReleases queries github for all product releases.
func (g *ReleaseManager) getReleases() ([]Release, error) {
	var releases []Release

	for page := 1; true; page++ {
		opt := &github.ListOptions{Page: page}

		rels, _, err := g.client.Repositories.ListReleases(g.owner, g.repo, opt)

		if err != nil {
			return nil, err
		}

		if len(rels) == 0 {
			break
		}

		releases = make([]Release, 0, len(rels))

		for i := range rels {
			version := *rels[i].TagName
			v, err := semver.Parse(version)
			if err != nil {
				log.Debugf("Release %q is not semantically versioned (%q). Skipping.", version, err)
				continue
			}
			rel := Release{
				id:      *rels[i].ID,
				URL:     *rels[i].ZipballURL,
				Version: v,
			}
			rel.Assets = make([]Asset, 0, len(rels[i].Assets))
			for _, asset := range rels[i].Assets {
				rel.Assets = append(rel.Assets, Asset{
					id:   *asset.ID,
					Name: *asset.Name,
					URL:  *asset.BrowserDownloadURL,
				})
			}
			log.Debugf("Release %q has %d assets...", version, len(rel.Assets))
			releases = append(releases, rel)
		}
	}

	sort.Sort(sort.Reverse(releasesByID(releases)))

	return releases, nil
}

// UpdateAssetsMap will pull published releases, scan for compatible
// update-only binaries and will add them to the updateAssetsMap.
func (g *ReleaseManager) UpdateAssetsMap() (err error) {

	var rs []Release

	log.Debugf("Getting releases...")
	if rs, err = g.getReleases(); err != nil {
		return err
	}

	// Resetting file hashes.
	fileHashMapMu.Lock()
	fileHashMap = map[string]string{}
	fileHashMapMu.Unlock()

	log.Debugf("Getting assets...")
	for i := range rs {
		log.Debugf("Getting assets for release %q...", rs[i].Version)
		for j := range rs[i].Assets {
			log.Debugf("Found %q.", rs[i].Assets[j].Name)
			// Does this asset represent a binary update?
			if isUpdateAsset(rs[i].Assets[j].Name) {
				log.Debugf("%q is an auto-update asset.", rs[i].Assets[j].Name)
				asset := rs[i].Assets[j]
				asset.v = rs[i].Version
				info, err := getAssetInfo(asset.Name)
				if err != nil {
					return fmt.Errorf("Could not get asset info: %q", err)
				}
				if err = g.pushAsset(info.OS, info.Arch, &asset); err != nil {
					return fmt.Errorf("Could not push asset: %q", err)
				}
			} else {
				log.Debugf("%q is not an auto-update asset. Skipping.", rs[i].Assets[j].Name)
			}
		}
	}

	return nil
}

func (g *ReleaseManager) getProductUpdate(os string, arch string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.latestAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.latestAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.latestAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	return g.latestAssetsMap[os][arch], nil
}

func (g *ReleaseManager) lookupAssetWithChecksum(os string, arch string, checksum string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.Checksum == checksum {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching checksum in assets list.")
}

func (g *ReleaseManager) lookupAssetWithVersion(os string, arch string, version string) (asset *Asset, err error) {
	g.mu.RLock()
	defer g.mu.RUnlock()

	if g.updateAssetsMap == nil {
		return nil, fmt.Errorf("No updates available.")
	}

	if g.updateAssetsMap[os] == nil {
		return nil, fmt.Errorf("No such OS.")
	}

	if g.updateAssetsMap[os][arch] == nil {
		return nil, fmt.Errorf("No such Arch.")
	}

	for _, a := range g.updateAssetsMap[os][arch] {
		if a.v.String() == version {
			return a, nil
		}
	}

	return nil, fmt.Errorf("Could not find a matching version in assets list.")
}

func (g *ReleaseManager) pushAsset(os string, arch string, asset *Asset) (err error) {
	g.mu.Lock()
	defer g.mu.Unlock()

	version := asset.v

	asset.OS = os
	asset.Arch = arch

	if version.EQ(emptyVersion) {
		return fmt.Errorf("Missing asset version.")
	}

	var localfile string
	if localfile, err = downloadAsset(asset.URL); err != nil {
		return err
	}

	if asset.Checksum, err = checksumForFile(localfile); err != nil {
		return err
	}

	if asset.Signature, err = signatureForFile(localfile); err != nil {
		return err
	}

	// Pushing version.
	if g.updateAssetsMap[os] == nil {
		g.updateAssetsMap[os] = make(map[string]map[string]*Asset)
	}
	if g.updateAssetsMap[os][arch] == nil {
		g.updateAssetsMap[os][arch] = make(map[string]*Asset)
	}
	g.updateAssetsMap[os][arch][version.String()] = asset

	// Setting latest version.
	if g.latestAssetsMap[os] == nil {
		g.latestAssetsMap[os] = make(map[string]*Asset)
	}

	// Only considering non-manoto versions for the latestAssetsMap
	if !buildStringContainsManoto(asset.v) {
		if g.latestAssetsMap[os][arch] == nil {
			g.latestAssetsMap[os][arch] = asset
		} else {
			// Compare against already set version.
			if asset.v.GT(g.latestAssetsMap[os][arch].v) {
				g.latestAssetsMap[os][arch] = asset
			}
		}
	}

	return nil
}

func getAssetInfo(s string) (*AssetInfo, error) {
	matches := updateAssetRe.FindStringSubmatch(s)
	if len(matches) >= 3 {
		if matches[1] != OS.Windows && matches[1] != OS.Linux && matches[1] != OS.Darwin && matches[1] != OS.Android {
			return nil, fmt.Errorf("Unknown OS: \"%s\".", matches[1])
		}
		if matches[2] != Arch.X64 && matches[2] != Arch.X86 && matches[2] != Arch.ARM {
			return nil, fmt.Errorf("Unknown architecture \"%s\".", matches[2])
		}
		info := &AssetInfo{
			OS:   matches[1],
			Arch: matches[2],
		}
		return info, nil
	}
	return nil, fmt.Errorf("Could not find asset info.")
}

func isUpdateAsset(s string) bool {
	return updateAssetRe.MatchString(s)
}
<|endoftext|>


<commit_before>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	BeforeEach(func() {
		gen = Generator{}
	})

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			adj := gen.RandomAdjective()
			// We wait for 3 millisecond because the seed we use to generate the
			// randomness has a unit of 1 nanosecond plus random test flakiness
			time.Sleep(3)
			Expect(adj).ToNot(Equal(gen.RandomAdjective()))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			noun := gen.RandomNoun()
			// We wait for 3 millisecond because the seed we use to generate the
			// randomness has a unit of 1 nanosecond plus random test flakiness
			time.Sleep(10)
			Expect(noun).ToNot(Equal(gen.RandomNoun()))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp("^\\w+-\\w+$"))
		})
	})
})
<commit_msg>make this less flaky<commit_after>package randomword_test

import (
	"time"

	. "code.cloudfoundry.org/cli/util/randomword"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Generator", func() {
	var gen Generator

	BeforeEach(func() {
		gen = Generator{}
	})

	Describe("RandomAdjective", func() {
		It("generates a random adjective each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomAdjective())
				// We wait for 3 millisecond because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3)
				setTwo = append(setTwo, gen.RandomAdjective())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("RandomNoun", func() {
		It("generates a random noun each time it is called", func() {
			setOne := []string{}
			setTwo := []string{}

			for i := 0; i < 3; i++ {
				setOne = append(setOne, gen.RandomNoun())
				// We wait for 3 millisecond because the seed we use to generate the
				// randomness has a unit of 1 nanosecond plus random test flakiness
				time.Sleep(3)
				setTwo = append(setTwo, gen.RandomNoun())
			}
			Expect(setOne).ToNot(ConsistOf(setTwo))
		})
	})

	Describe("Babble", func() {
		It("generates a random adjective noun pair each time it is called", func() {
			wordPair := gen.Babble()
			Expect(wordPair).To(MatchRegexp("^\\w+-\\w+$"))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher_test

import (
	"crypto/aes"
	"crypto/cipher"
	"testing"
)

func benchmarkAESGCMSign(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		out = aesgcm.Seal(out[:0], nonce[:], nil, buf)
	}
}

func benchmarkAESGCMSeal(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	var ad [13]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])
	}
}

func benchmarkAESGCMOpen(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	var ad [13]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte
	out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err := aesgcm.Open(buf[:0], nonce[:], out, ad[:])
		if err != nil {
			b.Errorf("Open: %v", err)
		}
	}
}

func BenchmarkAESGCMSeal1K(b *testing.B) {
	benchmarkAESGCMSeal(b, make([]byte, 1024))
}

func BenchmarkAESGCMOpen1K(b *testing.B) {
	benchmarkAESGCMOpen(b, make([]byte, 1024))
}

func BenchmarkAESGCMSign8K(b *testing.B) {
	benchmarkAESGCMSign(b, make([]byte, 8*1024))
}

func BenchmarkAESGCMSeal8K(b *testing.B) {
	benchmarkAESGCMSeal(b, make([]byte, 8*1024))
}

func BenchmarkAESGCMOpen8K(b *testing.B) {
	benchmarkAESGCMOpen(b, make([]byte, 8*1024))
}

// If we test exactly 1K blocks, we would generate exact multiples of
// the cipher's block size, and the cipher stream fragments would
// always be wordsize aligned, whereas non-aligned is a more typical
// use-case.
const almost1K = 1024 - 5

func BenchmarkAESCFBEncrypt1K(b *testing.B) {
	buf := make([]byte, almost1K)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	ctr := cipher.NewCFBEncrypter(aes, iv[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ctr.XORKeyStream(buf, buf)
	}
}

func BenchmarkAESCFBDecrypt1K(b *testing.B) {
	buf := make([]byte, almost1K)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	ctr := cipher.NewCFBDecrypter(aes, iv[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ctr.XORKeyStream(buf, buf)
	}
}

func BenchmarkAESOFB1K(b *testing.B) {
	buf := make([]byte, almost1K)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	ctr := cipher.NewOFB(aes, iv[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ctr.XORKeyStream(buf, buf)
	}
}

func BenchmarkAESCTR1K(b *testing.B) {
	buf := make([]byte, almost1K)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	ctr := cipher.NewCTR(aes, iv[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		ctr.XORKeyStream(buf, buf)
	}
}

func BenchmarkAESCBCEncrypt1K(b *testing.B) {
	buf := make([]byte, 1024)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	cbc := cipher.NewCBCEncrypter(aes, iv[:])
	for i := 0; i < b.N; i++ {
		cbc.CryptBlocks(buf, buf)
	}
}

func BenchmarkAESCBCDecrypt1K(b *testing.B) {
	buf := make([]byte, 1024)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	cbc := cipher.NewCBCDecrypter(aes, iv[:])
	for i := 0; i < b.N; i++ {
		cbc.CryptBlocks(buf, buf)
	}
}
<commit_msg>crypto/cipher: 8K benchmarks for AES stream modes<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cipher_test

import (
	"crypto/aes"
	"crypto/cipher"
	"testing"
)

func benchmarkAESGCMSign(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		out = aesgcm.Seal(out[:0], nonce[:], nil, buf)
	}
}

func benchmarkAESGCMSeal(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	var ad [13]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])
	}
}

func benchmarkAESGCMOpen(b *testing.B, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var nonce [12]byte
	var ad [13]byte
	aes, _ := aes.NewCipher(key[:])
	aesgcm, _ := cipher.NewGCM(aes)
	var out []byte
	out = aesgcm.Seal(out[:0], nonce[:], buf, ad[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, err := aesgcm.Open(buf[:0], nonce[:], out, ad[:])
		if err != nil {
			b.Errorf("Open: %v", err)
		}
	}
}

func BenchmarkAESGCMSeal1K(b *testing.B) {
	benchmarkAESGCMSeal(b, make([]byte, 1024))
}

func BenchmarkAESGCMOpen1K(b *testing.B) {
	benchmarkAESGCMOpen(b, make([]byte, 1024))
}

func BenchmarkAESGCMSign8K(b *testing.B) {
	benchmarkAESGCMSign(b, make([]byte, 8*1024))
}

func BenchmarkAESGCMSeal8K(b *testing.B) {
	benchmarkAESGCMSeal(b, make([]byte, 8*1024))
}

func BenchmarkAESGCMOpen8K(b *testing.B) {
	benchmarkAESGCMOpen(b, make([]byte, 8*1024))
}

func benchmarkAESStream(b *testing.B, mode func(cipher.Block, []byte) cipher.Stream, buf []byte) {
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	stream := mode(aes, iv[:])

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		stream.XORKeyStream(buf, buf)
	}
}

// If we test exactly 1K blocks, we would generate exact multiples of
// the cipher's block size, and the cipher stream fragments would
// always be wordsize aligned, whereas non-aligned is a more typical
// use-case.
const almost1K = 1024 - 5
const almost8K = 8*1024 - 5

func BenchmarkAESCFBEncrypt1K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewCFBEncrypter, make([]byte, almost1K))
}

func BenchmarkAESCFBDecrypt1K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewCFBDecrypter, make([]byte, almost1K))
}

func BenchmarkAESCFBDecrypt8K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewCFBDecrypter, make([]byte, almost8K))
}

func BenchmarkAESOFB1K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewOFB, make([]byte, almost1K))
}

func BenchmarkAESCTR1K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewCTR, make([]byte, almost1K))
}

func BenchmarkAESCTR8K(b *testing.B) {
	benchmarkAESStream(b, cipher.NewCTR, make([]byte, almost8K))
}

func BenchmarkAESCBCEncrypt1K(b *testing.B) {
	buf := make([]byte, 1024)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	cbc := cipher.NewCBCEncrypter(aes, iv[:])
	for i := 0; i < b.N; i++ {
		cbc.CryptBlocks(buf, buf)
	}
}

func BenchmarkAESCBCDecrypt1K(b *testing.B) {
	buf := make([]byte, 1024)
	b.SetBytes(int64(len(buf)))

	var key [16]byte
	var iv [16]byte
	aes, _ := aes.NewCipher(key[:])
	cbc := cipher.NewCBCDecrypter(aes, iv[:])
	for i := 0; i < b.N; i++ {
		cbc.CryptBlocks(buf, buf)
	}
}
<|endoftext|>


<commit_before>/*
 * EliasDB
 *
 * Copyright 2016 Matthias Ladkau. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"devt.de/common/fileutil"
	"devt.de/common/httputil"
	"devt.de/eliasdb/api"
	"devt.de/eliasdb/graph/graphstorage"
)

const testdb = "testdb"

const invalidFileName = "**" + string(0x0)

var printLog = []string{}
var errorLog = []string{}

var printLogging = false

func TestMain(m *testing.M) {
	flag.Parse()

	basepath = testdb + "/"

	// Log all print and error messagesí

	print = func(v ...interface{}) {
		if printLogging {
			fmt.Println(v...)
		}
		printLog = append(printLog, fmt.Sprint(v...))
	}
	fatal = func(v ...interface{}) {
		if printLogging {
			fmt.Println(v...)
		}
		errorLog = append(errorLog, fmt.Sprint(v...))
	}

	defer func() {
		fatal = log.Fatal
		basepath = ""
	}()

	if res, _ := fileutil.PathExists(testdb); res {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
	}

	ensurePath(testdb)

	// Run the tests

	res := m.Run()

	if res, _ := fileutil.PathExists(testdb); res {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
	}

	os.Exit(res)
}

func TestMainNormalCase(t *testing.T) {

	// Make sure to reset the DefaultServeMux

	defer func() { http.DefaultServeMux = http.NewServeMux() }()

	// Make sure to remove any files

	defer func() {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
		time.Sleep(time.Duration(100) * time.Millisecond)
		ensurePath(testdb)
	}()

	// Reset logs

	printLog = []string{}
	errorLog = []string{}

	errorChan := make(chan error)

	// Test handle function

	api.HandleFunc = func(pattern string, handler func(http.ResponseWriter, *http.Request)) {
		defer func() {
			if r := recover(); r == nil {
				t.Error("Request with nil request and nil response should cause a panic.")
			}
		}()
		handler(nil, nil)
	}

	// Kick off main function

	go func() {
		main()
		errorChan <- nil
	}()

	// To exit the main function the lock watcher thread
	// has to recognise that the lockfile was modified

	time.Sleep(time.Duration(2) * time.Second)

	// Do a normal shutdown with a log file

	if err := shutdownWithLogFile(); err != nil {
		t.Error(err)
		return
	}

	// Wait for the main function to end

	if err := <-errorChan; err != nil || len(errorLog) != 0 {
		t.Error("Unexpected ending of main thread:", err, errorLog)
		return
	}

	// Check the print log

	if logString := strings.Join(printLog, "\n"); logString != `
EliasDB 0.8
Starting datastore in testdb/db
Creating GraphManager instance
Creating key (key.pem) and certificate (cert.pem) in: ssl
Ensuring web folder: testdb/web
Ensuring web termminal: testdb/web/db/term.html
Starting server on: localhost:9090
Writing fingerprint file: testdb/web/fingerprint.json
Waiting for shutdown
Lockfile was modified
Shutting down
Closing datastore`[1:] {
		t.Error("Unexpected log:", logString)
		return
	}

}

func TestMainErrorCases(t *testing.T) {

	// Make sure to reset the DefaultServeMux

	defer func() { http.DefaultServeMux = http.NewServeMux() }()

	// Make sure to remove any files

	defer func() {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
		time.Sleep(time.Duration(100) * time.Millisecond)
		ensurePath(testdb)
	}()

	// Setup config and logs

	Config = nil
	origConfFile := ConfigFile
	origBasePath := basepath

	basepath = ""
	ConfigFile = invalidFileName

	printLog = []string{}
	errorLog = []string{}

	main()

	// Check that an error happened

	if len(errorLog) != 1 || (errorLog[0] != "stat **"+string(0)+": invalid argument" &&
		errorLog[0] != "Lstat **"+string(0)+": invalid argument") {

		t.Error("Unexpected error:", errorLog)
		return
	}

	// Set back variables

	basepath = origBasePath
	ConfigFile = origConfFile
	printLog = []string{}
	errorLog = []string{}

	data := make(map[string]interface{})
	for k, v := range DefaultConfig {
		data[k] = v
	}

	Config = data

	// Test db access error

	Config[LocationDatastore] = invalidFileName

	main()

	// Check that an error happened

	if len(errorLog) != 2 ||
		!strings.Contains(errorLog[0], "Could not create directory") ||
		!strings.Contains(errorLog[1], "Failed to open graph storage") {
		t.Error("Unexpected error:", errorLog)
		return
	}

	// Set back logs

	printLog = []string{}
	errorLog = []string{}

	// Use memory only storage

	Config[MemoryOnlyStorage] = true

	// Test failed ssl key generation

	Config[HTTPSKey] = invalidFileName

	main()

	// Check that an error happened

	if len(errorLog) != 1 ||
		!strings.Contains(errorLog[0], "Failed to generate ssl key and certificate") {
		t.Error("Unexpected error:", errorLog)
		return
	}

	Config[HTTPSKey] = DefaultConfig[HTTPSKey]

	// Set back logs

	printLog = []string{}
	errorLog = []string{}

	// Special error when closing the store

	graphstorage.MgsRetClose = errors.New("Testerror")

	// Use 9090

	Config[HTTPSPort] = "9090"

	ths := httputil.HTTPServer{}
	go ths.RunHTTPServer(":9090", nil)

	time.Sleep(time.Duration(1) * time.Second)

	main()

	ths.Shutdown()

	time.Sleep(time.Duration(1) * time.Second)

	if ths.Running {
		t.Error("Server should not be running")
		return
	}

	if len(errorLog) != 2 || (errorLog[0] != "listen tcp :9090"+
		": bind: address already in use" && errorLog[0] != "listen tcp :9090"+
		": bind: Only one usage of each socket address (protocol/network address/port) is normally permitted.") ||
		errorLog[1] != "Testerror" {
		t.Error("Unexpected error:", errorLog)
		return
	}

	Config = nil
}

func shutdownWithLogFile() error {
	file, err := os.OpenFile(basepath+config(LockFile), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660)
	defer file.Close()
	if err != nil {
		fmt.Println(errorLog)
		return err
	}

	_, err = file.Write([]byte("a"))
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Adding rev output in test<commit_after>/*
 * EliasDB
 *
 * Copyright 2016 Matthias Ladkau. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"devt.de/common/fileutil"
	"devt.de/common/httputil"
	"devt.de/eliasdb/api"
	"devt.de/eliasdb/graph/graphstorage"
)

const testdb = "testdb"

const invalidFileName = "**" + string(0x0)

var printLog = []string{}
var errorLog = []string{}

var printLogging = false

func TestMain(m *testing.M) {
	flag.Parse()

	basepath = testdb + "/"

	// Log all print and error messagesí

	print = func(v ...interface{}) {
		if printLogging {
			fmt.Println(v...)
		}
		printLog = append(printLog, fmt.Sprint(v...))
	}
	fatal = func(v ...interface{}) {
		if printLogging {
			fmt.Println(v...)
		}
		errorLog = append(errorLog, fmt.Sprint(v...))
	}

	defer func() {
		fatal = log.Fatal
		basepath = ""
	}()

	if res, _ := fileutil.PathExists(testdb); res {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
	}

	ensurePath(testdb)

	// Run the tests

	res := m.Run()

	if res, _ := fileutil.PathExists(testdb); res {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
	}

	os.Exit(res)
}

func TestMainNormalCase(t *testing.T) {

	// Make sure to reset the DefaultServeMux

	defer func() { http.DefaultServeMux = http.NewServeMux() }()

	// Make sure to remove any files

	defer func() {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
		time.Sleep(time.Duration(100) * time.Millisecond)
		ensurePath(testdb)
	}()

	// Reset logs

	printLog = []string{}
	errorLog = []string{}

	errorChan := make(chan error)

	// Test handle function

	api.HandleFunc = func(pattern string, handler func(http.ResponseWriter, *http.Request)) {
		defer func() {
			if r := recover(); r == nil {
				t.Error("Request with nil request and nil response should cause a panic.")
			}
		}()
		handler(nil, nil)
	}

	// Kick off main function

	go func() {
		main()
		errorChan <- nil
	}()

	// To exit the main function the lock watcher thread
	// has to recognise that the lockfile was modified

	time.Sleep(time.Duration(2) * time.Second)

	// Do a normal shutdown with a log file

	if err := shutdownWithLogFile(); err != nil {
		t.Error(err)
		return
	}

	// Wait for the main function to end

	if err := <-errorChan; err != nil || len(errorLog) != 0 {
		t.Error("Unexpected ending of main thread:", err, errorLog)
		return
	}

	// Check the print log

	if logString := strings.Join(printLog, "\n"); logString != `
EliasDB 0.8.0
Starting datastore in testdb/db
Creating GraphManager instance
Creating key (key.pem) and certificate (cert.pem) in: ssl
Ensuring web folder: testdb/web
Ensuring web termminal: testdb/web/db/term.html
Starting server on: localhost:9090
Writing fingerprint file: testdb/web/fingerprint.json
Waiting for shutdown
Lockfile was modified
Shutting down
Closing datastore`[1:] {
		t.Error("Unexpected log:", logString)
		return
	}

}

func TestMainErrorCases(t *testing.T) {

	// Make sure to reset the DefaultServeMux

	defer func() { http.DefaultServeMux = http.NewServeMux() }()

	// Make sure to remove any files

	defer func() {
		if err := os.RemoveAll(testdb); err != nil {
			fmt.Print("Could not remove test directory:", err.Error())
		}
		time.Sleep(time.Duration(100) * time.Millisecond)
		ensurePath(testdb)
	}()

	// Setup config and logs

	Config = nil
	origConfFile := ConfigFile
	origBasePath := basepath

	basepath = ""
	ConfigFile = invalidFileName

	printLog = []string{}
	errorLog = []string{}

	main()

	// Check that an error happened

	if len(errorLog) != 1 || (errorLog[0] != "stat **"+string(0)+": invalid argument" &&
		errorLog[0] != "Lstat **"+string(0)+": invalid argument") {

		t.Error("Unexpected error:", errorLog)
		return
	}

	// Set back variables

	basepath = origBasePath
	ConfigFile = origConfFile
	printLog = []string{}
	errorLog = []string{}

	data := make(map[string]interface{})
	for k, v := range DefaultConfig {
		data[k] = v
	}

	Config = data

	// Test db access error

	Config[LocationDatastore] = invalidFileName

	main()

	// Check that an error happened

	if len(errorLog) != 2 ||
		!strings.Contains(errorLog[0], "Could not create directory") ||
		!strings.Contains(errorLog[1], "Failed to open graph storage") {
		t.Error("Unexpected error:", errorLog)
		return
	}

	// Set back logs

	printLog = []string{}
	errorLog = []string{}

	// Use memory only storage

	Config[MemoryOnlyStorage] = true

	// Test failed ssl key generation

	Config[HTTPSKey] = invalidFileName

	main()

	// Check that an error happened

	if len(errorLog) != 1 ||
		!strings.Contains(errorLog[0], "Failed to generate ssl key and certificate") {
		t.Error("Unexpected error:", errorLog)
		return
	}

	Config[HTTPSKey] = DefaultConfig[HTTPSKey]

	// Set back logs

	printLog = []string{}
	errorLog = []string{}

	// Special error when closing the store

	graphstorage.MgsRetClose = errors.New("Testerror")

	// Use 9090

	Config[HTTPSPort] = "9090"

	ths := httputil.HTTPServer{}
	go ths.RunHTTPServer(":9090", nil)

	time.Sleep(time.Duration(1) * time.Second)

	main()

	ths.Shutdown()

	time.Sleep(time.Duration(1) * time.Second)

	if ths.Running {
		t.Error("Server should not be running")
		return
	}

	if len(errorLog) != 2 || (errorLog[0] != "listen tcp :9090"+
		": bind: address already in use" && errorLog[0] != "listen tcp :9090"+
		": bind: Only one usage of each socket address (protocol/network address/port) is normally permitted.") ||
		errorLog[1] != "Testerror" {
		t.Error("Unexpected error:", errorLog)
		return
	}

	Config = nil
}

func shutdownWithLogFile() error {
	file, err := os.OpenFile(basepath+config(LockFile), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660)
	defer file.Close()
	if err != nil {
		fmt.Println(errorLog)
		return err
	}

	_, err = file.Write([]byte("a"))
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
)

func initTestDb(t *testing.T) *sql.DB {
	user := os.Getenv("DB_USER")
	pass := os.Getenv("DB_PASS")
	db_name := os.Getenv("DB_TEST_DB")
	if len(db_name) == 0 {
		t.Fatal("Must specify a db so we don't accidentally overwrite anything.")
	}
	db, err := sql.Open("postgres", fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, db_name))
	if err != nil {
		t.Fatal("Couldn't connect to DB.", err)
	}
	return db
}

func make_user(id int, username, email string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO auth_user VALUES (
                   $1, $2,
                   'first', 'last',
                   $3, 'pass',
                   TRUE, TRUE, TRUE, now(), now()
                 )`, id, username, email)
	if err != nil {
		t.Fatal("Unable to create user.", err)
	}
}

func make_repo(id int, user, project string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_repo VALUES (
                   $1, $2, $3, NULL, NULL
                 )`, id, user, project)
	if err != nil {
		t.Fatal("Unable to create repo.", err)
	}
}

func make_userprofile(id, uid int, time_interval string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile
                 (id, user_id, max_time_interval_between_emails) VALUES (
                   $1, $2, $3
                 )`, id, uid, time_interval)
	if err != nil {
		t.Fatal("Unable to create userprofile.", err)
	}
}

func make_userprofile_repo(id, userprofile_id, repo_id int, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile_repos
                 (id, userprofile_id, repo_id) VALUES (
                   $1, $2, $3
                 )`, id, userprofile_id, repo_id)
	if err != nil {
		t.Fatal("Unable to create userprofile_repo.", err)
	}
}

func TestGetUser_Exists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	defer db.Exec("DELETE from auth_user")
	make_user(1, "username", "em@a.il", db, t)

	u, err := dbc.GetUser(1)
	if err != nil {
		t.Fatal(err)
	}

	if u.Id != 1 {
		t.Fatal("Incorrect user id.")
		return
	}
	if u.username != "username" {
		t.Fatal("Incorrect username.")
		return
	}
	if u.Email != "em@a.il" {
		t.Fatal("Incorrect email.")
		return
	}
}

func TestGetUser_NoExists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	_, err := dbc.GetUser(1)
	if err == nil {
		t.Fatal("No error when accessing invalid user id.")
		return
	}
}

func TestGetUserRepos_None(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()
	uid := 1

	defer db.Exec("DELETE FROM auth_user;")
	make_user(uid, "username", "em@a.il", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile;")
	make_userprofile(2, uid, "D", db, t) // Daily, should really be enum or something.
	defer db.Exec("DELETE FROM streamer_repo;")
	make_repo(3, "user", "repo", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile_repos;")
	make_userprofile_repo(4, 2, 3, db, t)

	ur, err := dbc.GetUserRepos(uid)
	if err != nil {
		t.Fatal("Can't get user's repos.", err)
		return
	}
	if len(ur) != 1 {
		fmt.Println(ur)
		t.Fatal("Got user's repos, but expected len 1, but got ", len(ur))
		return
	}
}

func testGetUserRepos_One(t *testing.T) {
}

func testGetUserRepos_Many(t *testing.T) {
}

func testGetRepoActivity_OtherRepo(t *testing.T) {

}

func testGetRepoActivity_OtherUser(t *testing.T) {

}

func testGetRepoActivity_Old(t *testing.T) {

}

func testGetRepoActivity_AlreadySent(t *testing.T) {

}

func testGetRepoActivity_NeverSent(t *testing.T) {

}

func testMarkUserRepoSent(t *testing.T) {

}
<commit_msg>add zero userrepo test to the thing with the correct name & add single case.<commit_after>package main

import (
	"database/sql"
	"fmt"
	"os"
	"testing"
)

func initTestDb(t *testing.T) *sql.DB {
	user := os.Getenv("DB_USER")
	pass := os.Getenv("DB_PASS")
	db_name := os.Getenv("DB_TEST_DB")
	if len(db_name) == 0 {
		t.Fatal("Must specify a db so we don't accidentally overwrite anything.")
	}
	db, err := sql.Open("postgres", fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable", user, pass, db_name))
	if err != nil {
		t.Fatal("Couldn't connect to DB.", err)
	}
	return db
}

func make_user(id int, username, email string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO auth_user VALUES (
                   $1, $2,
                   'first', 'last',
                   $3, 'pass',
                   TRUE, TRUE, TRUE, now(), now()
                 )`, id, username, email)
	if err != nil {
		t.Fatal("Unable to create user.", err)
	}
}

func make_repo(id int, user, project string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_repo VALUES (
                   $1, $2, $3, NULL, NULL
                 )`, id, user, project)
	if err != nil {
		t.Fatal("Unable to create repo.", err)
	}
}

func make_userprofile(id, uid int, time_interval string, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile
                 (id, user_id, max_time_interval_between_emails) VALUES (
                   $1, $2, $3
                 )`, id, uid, time_interval)
	if err != nil {
		t.Fatal("Unable to create userprofile.", err)
	}
}

func make_userprofile_repo(id, userprofile_id, repo_id int, db *sql.DB, t *testing.T) {
	_, err := db.Exec(`INSERT INTO streamer_userprofile_repos
                 (id, userprofile_id, repo_id) VALUES (
                   $1, $2, $3
                 )`, id, userprofile_id, repo_id)
	if err != nil {
		t.Fatal("Unable to create userprofile_repo.", err)
	}
}

func TestGetUser_Exists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	defer db.Exec("DELETE from auth_user")
	make_user(1, "username", "em@a.il", db, t)

	u, err := dbc.GetUser(1)
	if err != nil {
		t.Fatal(err)
	}

	if u.Id != 1 {
		t.Fatal("Incorrect user id.")
		return
	}
	if u.username != "username" {
		t.Fatal("Incorrect username.")
		return
	}
	if u.Email != "em@a.il" {
		t.Fatal("Incorrect email.")
		return
	}
}

func TestGetUser_NoExists(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()

	_, err := dbc.GetUser(1)
	if err == nil {
		t.Fatal("No error when accessing invalid user id.")
		return
	}
}

func TestGetUserRepos_None(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()
	uid := 1

	defer db.Exec("DELETE FROM auth_user;")
	make_user(uid, "username", "em@a.il", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile;")
	make_userprofile(2, uid, "D", db, t) // Daily, should really be enum or something.
	defer db.Exec("DELETE FROM streamer_repo;")
	make_repo(3, "user", "repo", db, t)

	ur, err := dbc.GetUserRepos(uid)
	if err != nil {
		t.Fatal("Can't get user's repos.", err)
	}
	if len(ur) != 0 {
		fmt.Println(ur)
		t.Fatal("Got user's repos, but expected len 0, but got ", len(ur))
	}
}

func testGetUserRepos_One(t *testing.T) {
	db := initTestDb(t)
	dbc := DbController{db}
	defer dbc.Close()
	uid := 1

	defer db.Exec("DELETE FROM auth_user;")
	make_user(uid, "username", "em@a.il", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile;")
	make_userprofile(2, uid, "D", db, t) // Daily, should really be enum or something.
	defer db.Exec("DELETE FROM streamer_repo;")
	make_repo(3, "user", "repo", db, t)
	defer db.Exec("DELETE FROM streamer_userprofile_repos;")
	make_userprofile_repo(4, 2, 3, db, t)

	ur, err := dbc.GetUserRepos(uid)
	if err != nil {
		t.Fatal("Can't get user's repos.", err)
		return
	}
	if len(ur) != 1 {
		fmt.Println(ur)
		t.Fatal("Got user's repos, but expected len 1, but got ", len(ur))
		return
	}
}

func testGetUserRepos_Many(t *testing.T) {
}

func testGetRepoActivity_OtherRepo(t *testing.T) {

}

func testGetRepoActivity_OtherUser(t *testing.T) {

}

func testGetRepoActivity_Old(t *testing.T) {

}

func testGetRepoActivity_AlreadySent(t *testing.T) {

}

func testGetRepoActivity_NeverSent(t *testing.T) {

}

func testMarkUserRepoSent(t *testing.T) {

}
<|endoftext|>


<commit_before>package integrationtest

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/jacksontj/dataman/src/query"
	"github.com/jacksontj/dataman/src/router_node"
	"github.com/jacksontj/dataman/src/router_node/metadata"
	"github.com/jacksontj/dataman/src/storage_node"
	"github.com/jacksontj/dataman/src/task_node"
)

type Data map[string]map[string][]map[string]interface{}

type Queries []*query.Query

// For this we assume these are empty and we can do whatever we want to them!
func RunIntegrationTests(t *testing.T, task *tasknode.TaskNode, router *routernode.RouterNode, datasource *storagenode.DatasourceInstance) {

	// Find all tests
	files, err := ioutil.ReadDir("tests")
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if !file.IsDir() {
			continue
		}
		// TODO: subtest stuff
		t.Run(file.Name(), func(t *testing.T) {
			runIntegrationTest(file.Name(), t, task, router, datasource)
		})
	}
}

func runIntegrationTest(testDir string, t *testing.T, task *tasknode.TaskNode, router *routernode.RouterNode, datasource *storagenode.DatasourceInstance) {
	// Load the various files
	schema := make(map[string]*metadata.Database)
	schemaBytes, err := ioutil.ReadFile(path.Join("tests", testDir, "/schema.json"))
	if err != nil {
		t.Fatalf("Unable to read schema for test %s: %v", testDir, err)
	}
	if err := json.Unmarshal(schemaBytes, &schema); err != nil {
		t.Fatalf("Unable to load schema for test %s: %v", testDir, err)
	}

	// Load the schema
	for _, database := range schema {
		if err := task.EnsureExistsDatabase(context.Background(), database); err != nil {
			t.Fatalf("Unable to ensureSchema in test %s for database %s: %v", testDir, database.Name, err)
		}
	}

	// Block the router waiting on an update from the tasknode
	router.FetchMeta()

	// Load data
	data := make(Data)
	dataString, err := ioutil.ReadFile(path.Join("tests", testDir, "/data.json"))
	if err != nil {
		t.Fatalf("Unable to read data for test %s: %v", testDir, err)
	}
	if err := json.Unmarshal([]byte(dataString), &data); err != nil {
		t.Fatalf("Unable to load data for test %s: %v", testDir, err)
	}

	for databaseName, collectionMap := range data {
		for collectionName, recordList := range collectionMap {
			for _, record := range recordList {
				result := router.HandleQuery(context.Background(), &query.Query{
					Type: query.Insert,
					Args: map[string]interface{}{
						"db":         databaseName,
						"collection": collectionName,
						"record":     record,
					},
				})
				if result.ValidationError != nil {
					t.Fatalf("Valdiation error loading data into %s.%s: %v", databaseName, collectionName, result.ValidationError)
				}
				if result.Error != "" {
					t.Fatalf("Error loading data into %s.%s: %v", databaseName, collectionName, result.Error)
				}
			}
		}
	}
	testsDir := path.Join("tests", testDir, "/")

	walkFunc := func(fpath string, info os.FileInfo, err error) error {
		// If its a directory, skip it-- we'll let something else grab it
		if !info.IsDir() {
			return nil
		}

		// if this is a test directory it must have a query.json file
		// Load the query
		q := &query.Query{}
		queryBytes, err := ioutil.ReadFile(path.Join(fpath, "query.json"))
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			t.Fatalf("Unable to read queryBytes for test %s.%s: %v", testDir, info.Name(), err)
		}
		if err := json.Unmarshal([]byte(queryBytes), &q); err != nil {
			t.Fatalf("Unable to load queries for test %s.%s: %v", testDir, info.Name(), err)
		}

		relFilePath, err := filepath.Rel(testsDir, fpath)
		if err != nil {
			t.Fatalf("Error getting relative path? Shouldn't be possible: %v", err)
		}
		t.Run(relFilePath, func(t *testing.T) {

			// Run the query
			result := router.HandleQuery(context.Background(), q)

			// Check result

			// write out results
			resultPath := path.Join(fpath, "result.json")
			resultBytes, _ := json.MarshalIndent(result, "", "  ")
			ioutil.WriteFile(resultPath, resultBytes, 0644)

			// compare against baseline if it exists
			baselinePath := path.Join(fpath, "baseline.json")
			baselineResultBytes, err := ioutil.ReadFile(baselinePath)
			if err != nil {
				t.Skip("No baseline.json found, skipping comparison")
			} else {
				baselineResultBytes = bytes.TrimSpace(baselineResultBytes)
				resultBytes = bytes.TrimSpace(resultBytes)
				if !bytes.Equal(baselineResultBytes, resultBytes) {
					t.Fatalf("Mismatch of results and baseline!")
				}
			}
		})
		return nil
	}

	if err := filepath.Walk(testsDir, walkFunc); err != nil {
		t.Errorf("Error walking: %v", err)
	}

}
<commit_msg>Update tests to cleanup after themselves<commit_after>package integrationtest

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/jacksontj/dataman/src/query"
	"github.com/jacksontj/dataman/src/router_node"
	"github.com/jacksontj/dataman/src/router_node/metadata"
	"github.com/jacksontj/dataman/src/storage_node"
	"github.com/jacksontj/dataman/src/task_node"
)

type Data map[string]map[string][]map[string]interface{}

type Queries []*query.Query

// For this we assume these are empty and we can do whatever we want to them!
func RunIntegrationTests(t *testing.T, task *tasknode.TaskNode, router *routernode.RouterNode, datasource *storagenode.DatasourceInstance) {

	// Find all tests
	files, err := ioutil.ReadDir("tests")
	if err != nil {
		log.Fatal(err)
	}

	for _, file := range files {
		if !file.IsDir() {
			continue
		}
		// TODO: subtest stuff
		t.Run(file.Name(), func(t *testing.T) {
			runIntegrationTest(file.Name(), t, task, router, datasource)
		})
	}
}

func runIntegrationTest(testDir string, t *testing.T, task *tasknode.TaskNode, router *routernode.RouterNode, datasource *storagenode.DatasourceInstance) {
	// Load the various files
	schema := make(map[string]*metadata.Database)
	schemaBytes, err := ioutil.ReadFile(path.Join("tests", testDir, "/schema.json"))
	if err != nil {
		t.Fatalf("Unable to read schema for test %s: %v", testDir, err)
	}
	if err := json.Unmarshal(schemaBytes, &schema); err != nil {
		t.Fatalf("Unable to load schema for test %s: %v", testDir, err)
	}

	// Load the schema
	for _, database := range schema {
		if err := task.EnsureExistsDatabase(context.Background(), database); err != nil {
			t.Fatalf("Unable to ensureSchema in test %s for database %s: %v", testDir, database.Name, err)
		}
	}

	// Block the router waiting on an update from the tasknode
	router.FetchMeta()

	// Load data
	data := make(Data)
	dataString, err := ioutil.ReadFile(path.Join("tests", testDir, "/data.json"))
	if err != nil {
		t.Fatalf("Unable to read data for test %s: %v", testDir, err)
	}
	if err := json.Unmarshal([]byte(dataString), &data); err != nil {
		t.Fatalf("Unable to load data for test %s: %v", testDir, err)
	}

	for databaseName, collectionMap := range data {
		for collectionName, recordList := range collectionMap {
			for _, record := range recordList {
				result := router.HandleQuery(context.Background(), &query.Query{
					Type: query.Insert,
					Args: map[string]interface{}{
						"db":         databaseName,
						"collection": collectionName,
						"record":     record,
					},
				})
				if result.ValidationError != nil {
					t.Fatalf("Valdiation error loading data into %s.%s: %v", databaseName, collectionName, result.ValidationError)
				}
				if result.Error != "" {
					t.Fatalf("Error loading data into %s.%s: %v", databaseName, collectionName, result.Error)
				}
			}
		}
	}
	testsDir := path.Join("tests", testDir, "/")

	walkFunc := func(fpath string, info os.FileInfo, err error) error {
		// If its a directory, skip it-- we'll let something else grab it
		if !info.IsDir() {
			return nil
		}

		// if this is a test directory it must have a query.json file
		// Load the query
		q := &query.Query{}
		queryBytes, err := ioutil.ReadFile(path.Join(fpath, "query.json"))
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			t.Fatalf("Unable to read queryBytes for test %s.%s: %v", testDir, info.Name(), err)
		}
		if err := json.Unmarshal([]byte(queryBytes), &q); err != nil {
			t.Fatalf("Unable to load queries for test %s.%s: %v", testDir, info.Name(), err)
		}

		relFilePath, err := filepath.Rel(testsDir, fpath)
		if err != nil {
			t.Fatalf("Error getting relative path? Shouldn't be possible: %v", err)
		}
		t.Run(relFilePath, func(t *testing.T) {

			// Run the query
			result := router.HandleQuery(context.Background(), q)

			// Check result

			// write out results
			resultPath := path.Join(fpath, "result.json")
			resultBytes, _ := json.MarshalIndent(result, "", "  ")
			ioutil.WriteFile(resultPath, resultBytes, 0644)

			// compare against baseline if it exists
			baselinePath := path.Join(fpath, "baseline.json")
			baselineResultBytes, err := ioutil.ReadFile(baselinePath)
			if err != nil {
				t.Skip("No baseline.json found, skipping comparison")
			} else {
				baselineResultBytes = bytes.TrimSpace(baselineResultBytes)
				resultBytes = bytes.TrimSpace(resultBytes)
				if !bytes.Equal(baselineResultBytes, resultBytes) {
					t.Fatalf("Mismatch of results and baseline!")
				}
			}
		})
		return nil
	}

	if err := filepath.Walk(testsDir, walkFunc); err != nil {
		t.Errorf("Error walking: %v", err)
	}

	// Assuming we finished properly, lets remove the things we added
	for _, database := range schema {
		if err := task.EnsureDoesntExistDatabase(context.Background(), database.Name); err != nil {
			t.Fatalf("Unable to remove in test %s for database %s: %v", testDir, database.Name, err)
		}
	}

	// Block the router waiting on an update from the tasknode
	router.FetchMeta()

}
<|endoftext|>


<commit_before>package cmds

import (
	"fmt"
	"os"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/transaction"
)

// getActiveTransaction will read the active transaction from the config file
// (if it exists) and return it.  If the config file is uninitialized or the
// active transaction is unset, `nil` will be returned.
func getActiveTransaction() (*transaction.Transaction, error) {
	cfg, err := config.Read(false)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading Pachyderm config:")
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return nil, errors.Wrapf(err, "error getting the active context")
	}
	if context.ActiveTransaction == "" {
		return nil, nil
	}
	return &transaction.Transaction{ID: context.ActiveTransaction}, nil
}

func requireActiveTransaction() (*transaction.Transaction, error) {
	txn, err := getActiveTransaction()
	if err != nil {
		return nil, err
	} else if txn == nil {
		return nil, errors.Errorf("no active transaction")
	}
	return txn, nil
}

func setActiveTransaction(txn *transaction.Transaction) error {
	cfg, err := config.Read(false)
	if err != nil {
		return errors.Wrapf(err, "error reading Pachyderm config")
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return errors.Wrapf(err, "error getting the active context:")
	}
	if txn == nil {
		context.ActiveTransaction = ""
	} else {
		context.ActiveTransaction = txn.ID
	}
	if err := cfg.Write(); err != nil {
		return errors.Wrapf(err, "error writing Pachyderm config:")
	}
	return nil
}

// ClearActiveTransaction will remove the active transaction from the pachctl
// config file - used by the 'delete all' command.
func ClearActiveTransaction() error {
	return setActiveTransaction(nil)
}

// WithActiveTransaction is a helper function that will attach the given
// transaction to the given client (resulting in a new client), then run the
// given callback with the new client.  This is for executing RPCs that can be
// run inside a transaction - if this isn't supported, it will have no effect.
func WithActiveTransaction(c *client.APIClient, callback func(*client.APIClient) error) error {
	txn, err := getActiveTransaction()
	if err != nil {
		return err
	}
	if txn != nil {
		c = c.WithTransaction(txn)
	}
	err = callback(c)
	if err == nil && txn != nil {
		fmt.Fprintf(os.Stderr, "Added to transaction: %s\n", txn.ID)
	}
	return err
}
<commit_msg>Update src/server/transaction/cmds/util.go<commit_after>package cmds

import (
	"fmt"
	"os"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/transaction"
)

// getActiveTransaction will read the active transaction from the config file
// (if it exists) and return it.  If the config file is uninitialized or the
// active transaction is unset, `nil` will be returned.
func getActiveTransaction() (*transaction.Transaction, error) {
	cfg, err := config.Read(false)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading Pachyderm config:")
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return nil, errors.Wrapf(err, "error getting the active context")
	}
	if context.ActiveTransaction == "" {
		return nil, nil
	}
	return &transaction.Transaction{ID: context.ActiveTransaction}, nil
}

func requireActiveTransaction() (*transaction.Transaction, error) {
	txn, err := getActiveTransaction()
	if err != nil {
		return nil, err
	} else if txn == nil {
		return nil, errors.Errorf("no active transaction")
	}
	return txn, nil
}

func setActiveTransaction(txn *transaction.Transaction) error {
	cfg, err := config.Read(false)
	if err != nil {
		return errors.Wrapf(err, "error reading Pachyderm config")
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return errors.Wrapf(err, "error getting the active context")
	}
	if txn == nil {
		context.ActiveTransaction = ""
	} else {
		context.ActiveTransaction = txn.ID
	}
	if err := cfg.Write(); err != nil {
		return errors.Wrapf(err, "error writing Pachyderm config:")
	}
	return nil
}

// ClearActiveTransaction will remove the active transaction from the pachctl
// config file - used by the 'delete all' command.
func ClearActiveTransaction() error {
	return setActiveTransaction(nil)
}

// WithActiveTransaction is a helper function that will attach the given
// transaction to the given client (resulting in a new client), then run the
// given callback with the new client.  This is for executing RPCs that can be
// run inside a transaction - if this isn't supported, it will have no effect.
func WithActiveTransaction(c *client.APIClient, callback func(*client.APIClient) error) error {
	txn, err := getActiveTransaction()
	if err != nil {
		return err
	}
	if txn != nil {
		c = c.WithTransaction(txn)
	}
	err = callback(c)
	if err == nil && txn != nil {
		fmt.Fprintf(os.Stderr, "Added to transaction: %s\n", txn.ID)
	}
	return err
}
<|endoftext|>