<commit_before>// Copyright 2018 The Jadep Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fsresolver

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"

	"context"
	"github.com/bazelbuild/tools_jvm_autodeps/bazel"
	"github.com/bazelbuild/tools_jvm_autodeps/jadeplib"
	"github.com/bazelbuild/tools_jvm_autodeps/loadertest"
	"github.com/bazelbuild/tools_jvm_autodeps/pkgloaderfakes"
	"github.com/google/go-cmp/cmp"
)

func TestClassToFiles(t *testing.T) {
	var tests = []struct {
		input jadeplib.ClassName
		want  []string
	}{
		{
			"Foo",
			[]string{"java/Foo.java", "test/Foo.java"},
		},
		{
			"com.google.devtools.build.lib.view.python.PythonProtoAspect",
			[]string{
				"java/com/google/devtools/build/lib/view/python/PythonProtoAspect.java",
				"test/com/google/devtools/build/lib/view/python/PythonProtoAspect.java",
			},
		},
		{
			"com.google.hello",
			[]string{"java/com/google/hello.java", "test/com/google/hello.java"},
		},
		{
			"foo.java",
			[]string{"java/foo/java.java", "test/foo/java.java"},
		},
		{
			"hello.Foo.ok",
			[]string{"java/hello/Foo/ok.java", "test/hello/Foo/ok.java"},
		},
		{
			"com.google.Foo",
			[]string{"java/com/google/Foo.java", "test/com/google/Foo.java"},
		},
	}
	for _, test := range tests {
		got := classToFiles([]string{"java/", "test"}, test.input)
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("classToFile(%s) = %s, want %s", test.input, got, test.want)
		}
	}
}

func TestResolve(t *testing.T) {
	type Attrs = map[string]interface{}

	workDir, err := createWorkspace()
	if err != nil {
		t.Error(err)
	}
	var tests = []struct {
		classnames   []jadeplib.ClassName
		existingPkgs map[string]*bazel.Package
		want         map[jadeplib.ClassName][]*bazel.Rule
	}{
		// Basic test where there is only one rule with the class name in its sources.
		{
			[]jadeplib.ClassName{"x.Foo"},
			map[string]*bazel.Package{
				"java/x": pkgloaderfakes.Pkg([]*bazel.Rule{pkgloaderfakes.JavaLibrary("java/x", "Foo", []string{"Foo.java"}, nil, nil)}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"x.Foo": {pkgloaderfakes.JavaLibrary("java/x", "Foo", []string{"Foo.java"}, nil, nil)},
			},
		},
		// Basic test where there is no rule with the class name in its sources.
		{
			[]jadeplib.ClassName{"m.Foo"},
			map[string]*bazel.Package{
				"java/m": pkgloaderfakes.Pkg([]*bazel.Rule{pkgloaderfakes.JavaLibrary("java/m", "Boo", nil, nil, nil)}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{},
		},
		// Assert that the unresolved classnames slice gets populated when the classname has no rule that can satisfy it.
		{
			[]jadeplib.ClassName{"f.Foo"},
			nil,
			map[jadeplib.ClassName][]*bazel.Rule{},
		},
		// This tests the exports functionality.
		{
			[]jadeplib.ClassName{"y.Foo"},
			map[string]*bazel.Package{
				"java/y": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/y", "Foo", []string{"Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/y", "Bar", nil, nil, []string{"Foo"}),
					pkgloaderfakes.JavaLibrary("java/y", "Bez", nil, nil, []string{"Foo", "Bar"}),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"y.Foo": {
					pkgloaderfakes.JavaLibrary("java/y", "Bar", nil, nil, []string{"Foo"}),
					pkgloaderfakes.JavaLibrary("java/y", "Bez", nil, nil, []string{"Foo", "Bar"}),
					pkgloaderfakes.JavaLibrary("java/y", "Foo", []string{"Foo.java"}, nil, nil),
				},
			},
		},
		// Tests that we only return java_library rules.
		{
			[]jadeplib.ClassName{"a.Foo"},
			map[string]*bazel.Package{
				"java/a": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/a", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil),
					pkgloaderfakes.JavaBinary("java/a", "D", []string{"Foo.java", "Foo_test.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"a.Foo": {pkgloaderfakes.JavaLibrary("java/a", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil)},
			},
		},
		// This tests resolving multiple class names.
		{
			[]jadeplib.ClassName{"b.Foo", "b.Jadep"},
			map[string]*bazel.Package{
				"java/b": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/b", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/b", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"b.Foo":   {pkgloaderfakes.JavaLibrary("java/b", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil)},
				"b.Jadep": {pkgloaderfakes.JavaLibrary("java/b", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
			},
		},
		// This tests multiple class name inputs with multiple build files.
		{
			[]jadeplib.ClassName{"d.Foo", "d.Jadep", "c.Foo", "c.Jadep"},
			map[string]*bazel.Package{
				"java/c": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/c", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/c", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
				"java/d": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/d", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/d", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"c.Foo":   {pkgloaderfakes.JavaLibrary("java/c", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil)},
				"c.Jadep": {pkgloaderfakes.JavaLibrary("java/c", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
				"d.Foo":   {pkgloaderfakes.JavaLibrary("java/d", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil)},
				"d.Jadep": {pkgloaderfakes.JavaLibrary("java/d", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
			},
		},
		// A java_library srcs a filegroup which srcs Foo.java. Assert that we return the java_library as if it srcs-ed Foo.java directly.
		{
			classnames: []jadeplib.ClassName{"x.Foo"},
			existingPkgs: map[string]*bazel.Package{
				"java/x": pkgloaderfakes.Pkg([]*bazel.Rule{
					bazel.NewRule("java_library", "java/x", "Foo", Attrs{"srcs": []string{"Bar"}}),
					bazel.NewRule("filegroup", "java/x", "Bar", Attrs{"srcs": []string{"Foo.java"}}),
				}),
			},
			want: map[jadeplib.ClassName][]*bazel.Rule{
				"x.Foo": {bazel.NewRule("java_library", "java/x", "Foo", Attrs{"srcs": []string{"Bar"}})},
			},
		},
	}
	for i, test := range tests {
		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
			var pkgNames []string
			for p := range test.existingPkgs {
				pkgNames = append(pkgNames, p)
			}
			cleanup, err := createBuildFileDir(t, pkgNames, workDir)
			if err != nil {
				t.Error(err)
			}
			defer cleanup()
			resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: test.existingPkgs})
			actual, err := resolver.Resolve(context.Background(), test.classnames, nil)
			if err != nil {
				t.Errorf("Resolve(%s) failed: %v. On iteration %v.", test.classnames, err, i)
				return
			}
			for _, rules := range actual {
				sort.Slice(rules, func(i, j int) bool { return rules[i].Label() < rules[j].Label() })
			}
			if diff := cmp.Diff(actual, test.want); diff != "" {
				t.Errorf("Resolve(%s) diff: (-got +want)\n%s", test.classnames, diff)
			}
		})
	}
}

// TestResolvePackageNotReturned tests that Resolve() tolerates the Loader not returning a package it requested.
// It creates java/x/BUILD, which makes Resolve request java/x from Loader. But, the Loader returns nothing.
// This simulates loading packages that are so malformed that a PackageLoader will return nothing for them.
func TestResolvePackageNotReturned(t *testing.T) {
	workDir, err := createWorkspace()
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(workDir)
	err = os.MkdirAll(filepath.Join(workDir, filepath.FromSlash("java/x")), 0700)
	if err != nil {
		t.Fatal(err)
	}
	err = ioutil.WriteFile(filepath.Join(workDir, filepath.FromSlash("java/x/BUILD")), []byte("#"), 0500)
	if err != nil {
		t.Fatal(err)
	}

	resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: nil})
	got, err := resolver.Resolve(context.Background(), []jadeplib.ClassName{"x.Foo"}, nil)
	if err != nil {
		t.Fatalf("Resolve returned error %v, want nil", err)
	}
	if len(got) != 0 {
		t.Errorf("Resolve returned %v, want empty map", got)
	}
}

func BenchmarkResolve(b *testing.B) {
	existingPkgs := make(map[string]*bazel.Package)
	for i := 0; i < 100; i++ {
		pkgName := fmt.Sprintf("java/x%d", i)
		var rules []*bazel.Rule
		for j := 0; j < 100; j++ {
			rules = append(rules, pkgloaderfakes.JavaLibrary(pkgName, fmt.Sprintf("Foo%d", j), []string{fmt.Sprintf("Foo%d.java", j)}, nil, nil))
		}
		existingPkgs[pkgName] = pkgloaderfakes.Pkg(rules)
	}

	var classNames []jadeplib.ClassName
	for i := 0; i < 500; i++ {
		classNames = append(classNames, jadeplib.ClassName(fmt.Sprintf("x%d.Foo0", i)))
	}

	workDir, err := createWorkspace()
	if err != nil {
		b.Error(err)
	}
	var pkgNames []string
	for p := range existingPkgs {
		pkgNames = append(pkgNames, p)
	}
	cleanup, err := createBuildFileDir(nil, pkgNames, workDir)
	if err != nil {
		b.Error(err)
	}
	defer cleanup()
	resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: existingPkgs})

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		resolver.Resolve(context.Background(), classNames, nil)
	}
}

func createWorkspace() (string, error) {
	root, err := ioutil.TempDir("", "jadep")
	if err != nil {
		return "", fmt.Errorf("error called ioutil.TempDir: %v", err)
	}
	workDir := filepath.Join(root, "google3")
	if err := os.MkdirAll(filepath.Join(workDir, "tools/build_rules"), 0700); err != nil {
		return "", fmt.Errorf("error called MkdirAll: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(workDir, "tools/build_rules/BUILD"), nil, 0666); err != nil {
		return "", fmt.Errorf("error called WriteFile: %v", err)
	}
	if err := ioutil.WriteFile(filepath.Join(workDir, "tools/build_rules/prelude_-redacted-"), []byte("# must be non-empty"), 0666); err != nil {
		return "", fmt.Errorf("error called WriteFile: %v", err)
	}
	return workDir, nil
}

func createBuildFileDir(t *testing.T, pkgNames []string, workDir string) (func(), error) {
	for _, p := range pkgNames {
		err := os.MkdirAll(filepath.Join(workDir, filepath.FromSlash(p)), 0700)
		if err != nil {
			return nil, err
		}
		err = ioutil.WriteFile(filepath.Join(workDir, filepath.FromSlash(p+"/BUILD")), []byte("#"), 0500)
		if err != nil {
			return nil, err
		}
	}
	return func() {
		for _, p := range pkgNames {
			os.RemoveAll(filepath.Join(workDir, p))
		}
	}, nil
}
<commit_msg>fsresolver_test: we don't need a real-looking Bazel workspace, so don't create one.<commit_after>// Copyright 2018 The Jadep Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fsresolver

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"testing"

	"context"
	"github.com/bazelbuild/tools_jvm_autodeps/bazel"
	"github.com/bazelbuild/tools_jvm_autodeps/jadeplib"
	"github.com/bazelbuild/tools_jvm_autodeps/loadertest"
	"github.com/bazelbuild/tools_jvm_autodeps/pkgloaderfakes"
	"github.com/google/go-cmp/cmp"
)

func TestClassToFiles(t *testing.T) {
	var tests = []struct {
		input jadeplib.ClassName
		want  []string
	}{
		{
			"Foo",
			[]string{"java/Foo.java", "test/Foo.java"},
		},
		{
			"com.google.devtools.build.lib.view.python.PythonProtoAspect",
			[]string{
				"java/com/google/devtools/build/lib/view/python/PythonProtoAspect.java",
				"test/com/google/devtools/build/lib/view/python/PythonProtoAspect.java",
			},
		},
		{
			"com.google.hello",
			[]string{"java/com/google/hello.java", "test/com/google/hello.java"},
		},
		{
			"foo.java",
			[]string{"java/foo/java.java", "test/foo/java.java"},
		},
		{
			"hello.Foo.ok",
			[]string{"java/hello/Foo/ok.java", "test/hello/Foo/ok.java"},
		},
		{
			"com.google.Foo",
			[]string{"java/com/google/Foo.java", "test/com/google/Foo.java"},
		},
	}
	for _, test := range tests {
		got := classToFiles([]string{"java/", "test"}, test.input)
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("classToFile(%s) = %s, want %s", test.input, got, test.want)
		}
	}
}

func TestResolve(t *testing.T) {
	type Attrs = map[string]interface{}

	workDir, err := ioutil.TempDir("", "jadep")
	if err != nil {
		t.Error(err)
	}
	var tests = []struct {
		classnames   []jadeplib.ClassName
		existingPkgs map[string]*bazel.Package
		want         map[jadeplib.ClassName][]*bazel.Rule
	}{
		// Basic test where there is only one rule with the class name in its sources.
		{
			[]jadeplib.ClassName{"x.Foo"},
			map[string]*bazel.Package{
				"java/x": pkgloaderfakes.Pkg([]*bazel.Rule{pkgloaderfakes.JavaLibrary("java/x", "Foo", []string{"Foo.java"}, nil, nil)}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"x.Foo": {pkgloaderfakes.JavaLibrary("java/x", "Foo", []string{"Foo.java"}, nil, nil)},
			},
		},
		// Basic test where there is no rule with the class name in its sources.
		{
			[]jadeplib.ClassName{"m.Foo"},
			map[string]*bazel.Package{
				"java/m": pkgloaderfakes.Pkg([]*bazel.Rule{pkgloaderfakes.JavaLibrary("java/m", "Boo", nil, nil, nil)}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{},
		},
		// Assert that the unresolved classnames slice gets populated when the classname has no rule that can satisfy it.
		{
			[]jadeplib.ClassName{"f.Foo"},
			nil,
			map[jadeplib.ClassName][]*bazel.Rule{},
		},
		// This tests the exports functionality.
		{
			[]jadeplib.ClassName{"y.Foo"},
			map[string]*bazel.Package{
				"java/y": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/y", "Foo", []string{"Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/y", "Bar", nil, nil, []string{"Foo"}),
					pkgloaderfakes.JavaLibrary("java/y", "Bez", nil, nil, []string{"Foo", "Bar"}),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"y.Foo": {
					pkgloaderfakes.JavaLibrary("java/y", "Bar", nil, nil, []string{"Foo"}),
					pkgloaderfakes.JavaLibrary("java/y", "Bez", nil, nil, []string{"Foo", "Bar"}),
					pkgloaderfakes.JavaLibrary("java/y", "Foo", []string{"Foo.java"}, nil, nil),
				},
			},
		},
		// Tests that we only return java_library rules.
		{
			[]jadeplib.ClassName{"a.Foo"},
			map[string]*bazel.Package{
				"java/a": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/a", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil),
					pkgloaderfakes.JavaBinary("java/a", "D", []string{"Foo.java", "Foo_test.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"a.Foo": {pkgloaderfakes.JavaLibrary("java/a", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil)},
			},
		},
		// This tests resolving multiple class names.
		{
			[]jadeplib.ClassName{"b.Foo", "b.Jadep"},
			map[string]*bazel.Package{
				"java/b": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/b", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/b", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"b.Foo":   {pkgloaderfakes.JavaLibrary("java/b", "Foo", []string{"Foo.java", "Bazel.java"}, nil, nil)},
				"b.Jadep": {pkgloaderfakes.JavaLibrary("java/b", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
			},
		},
		// This tests multiple class name inputs with multiple build files.
		{
			[]jadeplib.ClassName{"d.Foo", "d.Jadep", "c.Foo", "c.Jadep"},
			map[string]*bazel.Package{
				"java/c": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/c", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/c", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
				"java/d": pkgloaderfakes.Pkg([]*bazel.Rule{
					pkgloaderfakes.JavaLibrary("java/d", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil),
					pkgloaderfakes.JavaLibrary("java/d", "G", []string{"Hello.java", "Jadep.java"}, nil, nil),
				}),
			},
			map[jadeplib.ClassName][]*bazel.Rule{
				"c.Foo":   {pkgloaderfakes.JavaLibrary("java/c", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil)},
				"c.Jadep": {pkgloaderfakes.JavaLibrary("java/c", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
				"d.Foo":   {pkgloaderfakes.JavaLibrary("java/d", "Foo", []string{"Bazel.java", "Foo.java"}, nil, nil)},
				"d.Jadep": {pkgloaderfakes.JavaLibrary("java/d", "G", []string{"Hello.java", "Jadep.java"}, nil, nil)},
			},
		},
		// A java_library srcs a filegroup which srcs Foo.java. Assert that we return the java_library as if it srcs-ed Foo.java directly.
		{
			classnames: []jadeplib.ClassName{"x.Foo"},
			existingPkgs: map[string]*bazel.Package{
				"java/x": pkgloaderfakes.Pkg([]*bazel.Rule{
					bazel.NewRule("java_library", "java/x", "Foo", Attrs{"srcs": []string{"Bar"}}),
					bazel.NewRule("filegroup", "java/x", "Bar", Attrs{"srcs": []string{"Foo.java"}}),
				}),
			},
			want: map[jadeplib.ClassName][]*bazel.Rule{
				"x.Foo": {bazel.NewRule("java_library", "java/x", "Foo", Attrs{"srcs": []string{"Bar"}})},
			},
		},
	}
	for i, test := range tests {
		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
			var pkgNames []string
			for p := range test.existingPkgs {
				pkgNames = append(pkgNames, p)
			}
			cleanup, err := createBuildFileDir(t, pkgNames, workDir)
			if err != nil {
				t.Error(err)
			}
			defer cleanup()
			resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: test.existingPkgs})
			actual, err := resolver.Resolve(context.Background(), test.classnames, nil)
			if err != nil {
				t.Errorf("Resolve(%s) failed: %v. On iteration %v.", test.classnames, err, i)
				return
			}
			for _, rules := range actual {
				sort.Slice(rules, func(i, j int) bool { return rules[i].Label() < rules[j].Label() })
			}
			if diff := cmp.Diff(actual, test.want); diff != "" {
				t.Errorf("Resolve(%s) diff: (-got +want)\n%s", test.classnames, diff)
			}
		})
	}
}

// TestResolvePackageNotReturned tests that Resolve() tolerates the Loader not returning a package it requested.
// It creates java/x/BUILD, which makes Resolve request java/x from Loader. But, the Loader returns nothing.
// This simulates loading packages that are so malformed that a PackageLoader will return nothing for them.
func TestResolvePackageNotReturned(t *testing.T) {
	workDir, err := ioutil.TempDir("", "jadep")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(workDir)
	err = os.MkdirAll(filepath.Join(workDir, filepath.FromSlash("java/x")), 0700)
	if err != nil {
		t.Fatal(err)
	}
	err = ioutil.WriteFile(filepath.Join(workDir, filepath.FromSlash("java/x/BUILD")), []byte("#"), 0500)
	if err != nil {
		t.Fatal(err)
	}

	resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: nil})
	got, err := resolver.Resolve(context.Background(), []jadeplib.ClassName{"x.Foo"}, nil)
	if err != nil {
		t.Fatalf("Resolve returned error %v, want nil", err)
	}
	if len(got) != 0 {
		t.Errorf("Resolve returned %v, want empty map", got)
	}
}

func BenchmarkResolve(b *testing.B) {
	existingPkgs := make(map[string]*bazel.Package)
	for i := 0; i < 100; i++ {
		pkgName := fmt.Sprintf("java/x%d", i)
		var rules []*bazel.Rule
		for j := 0; j < 100; j++ {
			rules = append(rules, pkgloaderfakes.JavaLibrary(pkgName, fmt.Sprintf("Foo%d", j), []string{fmt.Sprintf("Foo%d.java", j)}, nil, nil))
		}
		existingPkgs[pkgName] = pkgloaderfakes.Pkg(rules)
	}

	var classNames []jadeplib.ClassName
	for i := 0; i < 500; i++ {
		classNames = append(classNames, jadeplib.ClassName(fmt.Sprintf("x%d.Foo0", i)))
	}

	workDir, err := ioutil.TempDir("", "jadep")
	if err != nil {
		b.Error(err)
	}
	var pkgNames []string
	for p := range existingPkgs {
		pkgNames = append(pkgNames, p)
	}
	cleanup, err := createBuildFileDir(nil, pkgNames, workDir)
	if err != nil {
		b.Error(err)
	}
	defer cleanup()
	resolver := NewResolver([]string{"java/", "javatest"}, workDir, &loadertest.StubLoader{Pkgs: existingPkgs})

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		resolver.Resolve(context.Background(), classNames, nil)
	}
}

func createBuildFileDir(t *testing.T, pkgNames []string, workDir string) (func(), error) {
	for _, p := range pkgNames {
		err := os.MkdirAll(filepath.Join(workDir, filepath.FromSlash(p)), 0700)
		if err != nil {
			return nil, err
		}
		err = ioutil.WriteFile(filepath.Join(workDir, filepath.FromSlash(p+"/BUILD")), []byte("#"), 0500)
		if err != nil {
			return nil, err
		}
	}
	return func() {
		for _, p := range pkgNames {
			os.RemoveAll(filepath.Join(workDir, p))
		}
	}, nil
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
)

var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$")

// HTTPCache ...
type HTTPCache interface {
	CacheHandler(w http.ResponseWriter, r *http.Request)
	StatusPageHandler(w http.ResponseWriter, r *http.Request)
}

type httpCache struct {
	cache        *disk.Cache
	accessLogger cache.Logger
	errorLogger  cache.Logger
	validateAC   bool
	mangleACKeys bool
	gitCommit    string
}

type statusPageData struct {
	CurrSize         int64
	UncompressedSize int64
	ReservedSize     int64
	MaxSize          int64
	NumFiles         int
	ServerTime       int64
	GitCommit        string
	NumGoroutines    int
}

// NewHTTPCache returns a new instance of the cache.
// accessLogger will print one line for each HTTP request to stdout.
// errorLogger will print unexpected server errors. Inexistent files and malformed URLs will not
// be reported.
func NewHTTPCache(cache *disk.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool, mangleACKeys bool, commit string) HTTPCache {

	_, _, numItems, _ := cache.Stats()

	errorLogger.Printf("Loaded %d existing disk cache items.", numItems)

	hc := &httpCache{
		cache:        cache,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		validateAC:   validateAC,
		mangleACKeys: mangleACKeys,
	}

	if commit != "{STABLE_GIT_COMMIT}" {
		hc.gitCommit = commit
	}

	return hc
}

// Parse cache artifact information from the request URL
func parseRequestURL(url string, validateAC bool) (kind cache.EntryKind, hash string, instance string, err error) {
	m := blobNameSHA256.FindStringSubmatch(url)
	if m == nil {
		err := fmt.Errorf("resource name must be a SHA256 hash in hex. "+
			"got '%s'", html.EscapeString(url))
		return 0, "", "", err
	}

	instance = strings.TrimSuffix(m[1], "/")

	parts := m[2:]
	if len(parts) != 2 {
		err := fmt.Errorf("the path '%s' is invalid. expected (ac/|cas/)sha256",
			html.EscapeString(url))
		return 0, "", "", err
	}

	// The regex ensures that parts[0] can only be "ac/" or "cas/"
	hash = parts[1]
	if parts[0] == "cas/" {
		return cache.CAS, hash, instance, nil
	}

	if validateAC {
		return cache.AC, hash, instance, nil
	}

	return cache.RAW, hash, instance, nil
}
func (h *httpCache) handleContainsValidAC(w http.ResponseWriter, r *http.Request, hash string) {
	_, data, err := h.cache.GetValidatedActionResult(hash)
	if err != nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if data == nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data)), 10))
	w.WriteHeader(http.StatusOK)
	h.logResponse(http.StatusOK, r)
}

func (h *httpCache) handleGetValidAC(w http.ResponseWriter, r *http.Request, hash string) {
	_, data, err := h.cache.GetValidatedActionResult(hash)
	if err != nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if data == nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if r.Header.Get("Accept") == "application/json" {
		ar := &pb.ActionResult{}
		err = proto.Unmarshal(data, ar)
		if err != nil {
			h.logResponse(http.StatusInternalServerError, r)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		marshaler := jsonpb.Marshaler{}
		err = marshaler.Marshal(w, ar)
		if err != nil {
			h.logResponse(http.StatusInternalServerError, r)
			return
		}

		return
	}

	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data)), 10))
	bytesWritten, err := w.Write(data)

	if err != nil {
		h.logResponse(http.StatusInternalServerError, r)
		return
	}

	if bytesWritten != len(data) {
		h.logResponse(http.StatusInternalServerError, r)
		return
	}
}

// Helper function for logging responses
func (h *httpCache) logResponse(code int, r *http.Request) {
	// Parse the client ip:port
	var clientAddress string
	var err error
	clientAddress, _, err = net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		clientAddress = r.RemoteAddr
	}
	h.accessLogger.Printf("%4s %d %15s %s", r.Method, code, clientAddress, r.URL.Path)
}

func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	kind, hash, instance, err := parseRequestURL(r.URL.Path, h.validateAC)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		h.logResponse(http.StatusBadRequest, r)
		return
	}

	if h.mangleACKeys && kind == cache.AC {
		hash = cache.TransformActionCacheKey(hash, instance, h.accessLogger)
	}

	switch m := r.Method; m {
	case http.MethodGet:

		if h.validateAC && kind == cache.AC {
			h.handleGetValidAC(w, r, hash)
			return
		}

		var rdr io.ReadCloser
		var sizeBytes int64

		zstdCompressed := false
		if kind == cache.CAS && strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") {
			rdr, sizeBytes, err = h.cache.GetZstd(hash, -1, 0)
			zstdCompressed = true
		} else {
			rdr, sizeBytes, err = h.cache.Get(kind, hash, -1, 0)
		}
		if err != nil {
			if e, ok := err.(*cache.Error); ok {
				http.Error(w, e.Error(), e.Code)
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			h.errorLogger.Printf("GET %s: %s", path(kind, hash), err)
			return
		}

		if rdr == nil {
			http.Error(w, "Not found", http.StatusNotFound)
			h.logResponse(http.StatusNotFound, r)
			return
		}
		defer rdr.Close()

		w.Header().Set("Content-Type", "application/octet-stream")
		if zstdCompressed {
			// TODO: calculate Content-Length for compressed blobs too
			// (unless compressing on the fly).
			w.Header().Set("Content-Encoding", "zstd")
		} else {
			w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10))
		}

		io.Copy(w, rdr)

		h.logResponse(http.StatusOK, r)

	case http.MethodPut:
		contentLength := r.ContentLength

		if contentLength == -1 {
			// We need the content-length header to make sure we have enough disk space.
			msg := fmt.Sprintf("PUT without Content-Length (key = %s)", path(kind, hash))
			http.Error(w, msg, http.StatusBadRequest)
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
			return
		}

		if contentLength == 0 && kind == cache.CAS && hash != emptySha256 {
			msg := fmt.Sprintf("Invalid empty blob hash: \"%s\"", hash)
			http.Error(w, msg, http.StatusBadRequest)
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
			return
		}

		rc := r.Body
		if h.validateAC && kind == cache.AC {
			// verify that this is a valid ActionResult

			data, err := ioutil.ReadAll(rc)
			if err != nil {
				msg := "failed to read request body"
				http.Error(w, msg, http.StatusInternalServerError)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			if int64(len(data)) != contentLength {
				msg := fmt.Sprintf("sizes don't match. Expected %d, found %d",
					contentLength, len(data))
				http.Error(w, msg, http.StatusBadRequest)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			// Ensure that the serialized ActionResult has non-zero length.
			data, code, err := addWorkerMetadataHTTP(r.RemoteAddr, r.Header.Get("Content-Type"), data)
			if err != nil {
				http.Error(w, err.Error(), code)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err.Error())
				return
			}
			contentLength = int64(len(data))

			// Note: we do not currently verify that the blobs exist
			// in the CAS.

			rc = ioutil.NopCloser(bytes.NewReader(data))
		}

		err := h.cache.Put(kind, hash, contentLength, rc)
		if err != nil {
			if cerr, ok := err.(*cache.Error); ok {
				http.Error(w, err.Error(), cerr.Code)
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err)
		} else {
			h.logResponse(http.StatusOK, r)
		}

	case http.MethodHead:

		if h.validateAC && kind == cache.AC {
			h.handleContainsValidAC(w, r, hash)
			return
		}

		// Unvalidated path:

		ok, size := h.cache.Contains(kind, hash, -1)
		if !ok {
			http.Error(w, "Not found", http.StatusNotFound)
			h.logResponse(http.StatusNotFound, r)
			return
		}

		w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
		w.WriteHeader(http.StatusOK)
		h.logResponse(http.StatusOK, r)

	default:
		msg := fmt.Sprintf("Method '%s' not supported.", html.EscapeString(m))
		http.Error(w, msg, http.StatusMethodNotAllowed)
		h.logResponse(http.StatusMethodNotAllowed, r)
	}
}

func addWorkerMetadataHTTP(addr string, ct string, orig []byte) (data []byte, code int, err error) {
	ar := &pb.ActionResult{}
	if ct == "application/json" {
		err = jsonpb.Unmarshal(bytes.NewReader(orig), ar)
	} else {
		err = proto.Unmarshal(orig, ar)
	}
	if err != nil {
		return orig, http.StatusBadRequest, err
	}

	if ar.ExecutionMetadata == nil {
		ar.ExecutionMetadata = &pb.ExecutedActionMetadata{}
	} else if ar.ExecutionMetadata.Worker != "" {
		return orig, http.StatusOK, nil
	}

	worker := addr
	if worker == "" {
		worker, _, err = net.SplitHostPort(addr)
		if err != nil || worker == "" {
			worker = "unknown"
		}
	}

	ar.ExecutionMetadata.Worker = worker

	data, err = proto.Marshal(ar)
	if err != nil {
		return orig, http.StatusInternalServerError, err
	}

	return data, http.StatusOK, nil
}

// Produce a debugging page with some stats about the cache.
func (h *httpCache) StatusPageHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	totalSize, reservedSize, numItems, uncompressedSize := h.cache.Stats()

	goroutines := runtime.NumGoroutine()

	w.Header().Set("Content-Type", "application/json")
	enc := json.NewEncoder(w)
	enc.SetIndent("", " ")
	enc.Encode(statusPageData{
		MaxSize:          h.cache.MaxSize(),
		CurrSize:         totalSize,
		UncompressedSize: uncompressedSize,
		ReservedSize:     reservedSize,
		NumFiles:         numItems,
		ServerTime:       time.Now().Unix(),
		GitCommit:        h.gitCommit,
		NumGoroutines:    goroutines,
	})
}

func path(kind cache.EntryKind, hash string) string {
	return fmt.Sprintf("/%s/%s", kind, hash)
}
<commit_msg>[http] accept zstd-encoded PUTs<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	pb "github.com/buchgr/bazel-remote/genproto/build/bazel/remote/execution/v2"
	"github.com/klauspost/compress/zstd"
	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
)

var blobNameSHA256 = regexp.MustCompile("^/?(.*/)?(ac/|cas/)([a-f0-9]{64})$")

var decoder, _ = zstd.NewReader(nil)

// HTTPCache ...
type HTTPCache interface {
	CacheHandler(w http.ResponseWriter, r *http.Request)
	StatusPageHandler(w http.ResponseWriter, r *http.Request)
}

type httpCache struct {
	cache        *disk.Cache
	accessLogger cache.Logger
	errorLogger  cache.Logger
	validateAC   bool
	mangleACKeys bool
	gitCommit    string
}

type statusPageData struct {
	CurrSize         int64
	UncompressedSize int64
	ReservedSize     int64
	MaxSize          int64
	NumFiles         int
	ServerTime       int64
	GitCommit        string
	NumGoroutines    int
}

// NewHTTPCache returns a new instance of the cache.
// accessLogger will print one line for each HTTP request to stdout.
// errorLogger will print unexpected server errors. Inexistent files and malformed URLs will not
// be reported.
func NewHTTPCache(cache *disk.Cache, accessLogger cache.Logger, errorLogger cache.Logger, validateAC bool, mangleACKeys bool, commit string) HTTPCache {

	_, _, numItems, _ := cache.Stats()

	errorLogger.Printf("Loaded %d existing disk cache items.", numItems)

	hc := &httpCache{
		cache:        cache,
		accessLogger: accessLogger,
		errorLogger:  errorLogger,
		validateAC:   validateAC,
		mangleACKeys: mangleACKeys,
	}

	if commit != "{STABLE_GIT_COMMIT}" {
		hc.gitCommit = commit
	}

	return hc
}

// Parse cache artifact information from the request URL
func parseRequestURL(url string, validateAC bool) (kind cache.EntryKind, hash string, instance string, err error) {
	m := blobNameSHA256.FindStringSubmatch(url)
	if m == nil {
		err := fmt.Errorf("resource name must be a SHA256 hash in hex. "+
			"got '%s'", html.EscapeString(url))
		return 0, "", "", err
	}

	instance = strings.TrimSuffix(m[1], "/")

	parts := m[2:]
	if len(parts) != 2 {
		err := fmt.Errorf("the path '%s' is invalid. expected (ac/|cas/)sha256",
			html.EscapeString(url))
		return 0, "", "", err
	}

	// The regex ensures that parts[0] can only be "ac/" or "cas/"
	hash = parts[1]
	if parts[0] == "cas/" {
		return cache.CAS, hash, instance, nil
	}

	if validateAC {
		return cache.AC, hash, instance, nil
	}

	return cache.RAW, hash, instance, nil
}
func (h *httpCache) handleContainsValidAC(w http.ResponseWriter, r *http.Request, hash string) {
	_, data, err := h.cache.GetValidatedActionResult(hash)
	if err != nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if data == nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data)), 10))
	w.WriteHeader(http.StatusOK)
	h.logResponse(http.StatusOK, r)
}

func (h *httpCache) handleGetValidAC(w http.ResponseWriter, r *http.Request, hash string) {
	_, data, err := h.cache.GetValidatedActionResult(hash)
	if err != nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if data == nil {
		http.Error(w, "Not found", http.StatusNotFound)
		h.logResponse(http.StatusNotFound, r)
		return
	}

	if r.Header.Get("Accept") == "application/json" {
		ar := &pb.ActionResult{}
		err = proto.Unmarshal(data, ar)
		if err != nil {
			h.logResponse(http.StatusInternalServerError, r)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		marshaler := jsonpb.Marshaler{}
		err = marshaler.Marshal(w, ar)
		if err != nil {
			h.logResponse(http.StatusInternalServerError, r)
			return
		}

		return
	}

	w.Header().Set("Content-Type", "application/octet-stream")
	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(data)), 10))
	bytesWritten, err := w.Write(data)

	if err != nil {
		h.logResponse(http.StatusInternalServerError, r)
		return
	}

	if bytesWritten != len(data) {
		h.logResponse(http.StatusInternalServerError, r)
		return
	}
}

// Helper function for logging responses
func (h *httpCache) logResponse(code int, r *http.Request) {
	// Parse the client ip:port
	var clientAddress string
	var err error
	clientAddress, _, err = net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		clientAddress = r.RemoteAddr
	}
	h.accessLogger.Printf("%4s %d %15s %s", r.Method, code, clientAddress, r.URL.Path)
}

func (h *httpCache) CacheHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	kind, hash, instance, err := parseRequestURL(r.URL.Path, h.validateAC)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		h.logResponse(http.StatusBadRequest, r)
		return
	}

	if h.mangleACKeys && kind == cache.AC {
		hash = cache.TransformActionCacheKey(hash, instance, h.accessLogger)
	}

	switch m := r.Method; m {
	case http.MethodGet:

		if h.validateAC && kind == cache.AC {
			h.handleGetValidAC(w, r, hash)
			return
		}

		var rdr io.ReadCloser
		var sizeBytes int64

		zstdCompressed := false
		if kind == cache.CAS && strings.Contains(r.Header.Get("Accept-Encoding"), "zstd") {
			rdr, sizeBytes, err = h.cache.GetZstd(hash, -1, 0)
			zstdCompressed = true
		} else {
			rdr, sizeBytes, err = h.cache.Get(kind, hash, -1, 0)
		}
		if err != nil {
			if e, ok := err.(*cache.Error); ok {
				http.Error(w, e.Error(), e.Code)
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			h.errorLogger.Printf("GET %s: %s", path(kind, hash), err)
			return
		}

		if rdr == nil {
			http.Error(w, "Not found", http.StatusNotFound)
			h.logResponse(http.StatusNotFound, r)
			return
		}
		defer rdr.Close()

		w.Header().Set("Content-Type", "application/octet-stream")
		if zstdCompressed {
			// TODO: calculate Content-Length for compressed blobs too
			// (unless compressing on the fly).
			w.Header().Set("Content-Encoding", "zstd")
		} else {
			w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10))
		}

		io.Copy(w, rdr)

		h.logResponse(http.StatusOK, r)

	case http.MethodPut:
		contentLength := r.ContentLength

		// If custom header X-Digest-SizeBytes is set, use that for the
		// size of the blob instead of Content-Length (which only works
		// for uncompressed PUTs).
		sb := r.Header.Get("X-Digest-SizeBytes")
		if sb != "" {
			cl, err := strconv.Atoi(sb)
			if err != nil {
				msg := fmt.Sprintf("PUT with unparseable X-Digest-SizeBytes header: %v", sb)
				http.Error(w, msg, http.StatusBadRequest)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			contentLength = int64(cl)
		}

		if contentLength == -1 {
			// We need the content-length header to make sure we have enough disk space.
			msg := fmt.Sprintf("PUT without Content-Length (key = %s)", path(kind, hash))
			http.Error(w, msg, http.StatusBadRequest)
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
			return
		}

		if contentLength == 0 && kind == cache.CAS && hash != emptySha256 {
			msg := fmt.Sprintf("Invalid empty blob hash: \"%s\"", hash)
			http.Error(w, msg, http.StatusBadRequest)
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
			return
		}

		zstdCompressed := false

		// Content-Encoding must be one of "identity", "zstd" or not present.
		ce := r.Header.Get("Content-Encoding")
		if ce == "zstd" {
			zstdCompressed = true
		} else if ce != "" && ce != "identity" {
			msg := fmt.Sprintf("Unsupported content-encoding: %q", ce)
			http.Error(w, msg, http.StatusBadRequest)
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
			return
		}

		var rdr io.Reader = r.Body
		if h.validateAC && kind == cache.AC {
			// verify that this is a valid ActionResult

			data, err := ioutil.ReadAll(rdr)
			if err != nil {
				msg := "failed to read request body"
				http.Error(w, msg, http.StatusInternalServerError)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			if zstdCompressed {
				uncompressed, err := decoder.DecodeAll(data, nil)
				if err != nil {
					msg := fmt.Sprintf("failed to uncompress zstd-encoded request body: %v", err)
					http.Error(w, msg, http.StatusBadRequest)
					h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
					return
				}

				data = uncompressed
				zstdCompressed = false
			}

			if int64(len(data)) != contentLength {
				msg := fmt.Sprintf("sizes don't match. Expected %d, found %d",
					contentLength, len(data))
				http.Error(w, msg, http.StatusBadRequest)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			// Ensure that the serialized ActionResult has non-zero length.
			data, code, err := addWorkerMetadataHTTP(r.RemoteAddr, r.Header.Get("Content-Type"), data)
			if err != nil {
				http.Error(w, err.Error(), code)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err.Error())
				return
			}
			contentLength = int64(len(data))

			// Note: we do not currently verify that the blobs exist
			// in the CAS.

			rdr = bytes.NewReader(data)
		}

		if zstdCompressed {
			z, err := zstd.NewReader(rdr) // TODO: use a pool.
			if err != nil {
				if z != nil {
					defer z.Close()
				}
				msg := fmt.Sprintf("Failed to create zstd reader: %v", err)
				http.Error(w, msg, http.StatusInternalServerError)
				h.errorLogger.Printf("PUT %s: %s", path(kind, hash), msg)
				return
			}

			rc := z.IOReadCloser()
			defer rc.Close()
			rdr = rc
		}

		err := h.cache.Put(kind, hash, contentLength, rdr)
		if err != nil {
			if cerr, ok := err.(*cache.Error); ok {
				http.Error(w, err.Error(), cerr.Code)
			} else {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			}
			h.errorLogger.Printf("PUT %s: %s", path(kind, hash), err)
		} else {
			h.logResponse(http.StatusOK, r)
		}

	case http.MethodHead:

		if h.validateAC && kind == cache.AC {
			h.handleContainsValidAC(w, r, hash)
			return
		}

		// Unvalidated path:

		ok, size := h.cache.Contains(kind, hash, -1)
		if !ok {
			http.Error(w, "Not found", http.StatusNotFound)
			h.logResponse(http.StatusNotFound, r)
			return
		}

		w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
		w.WriteHeader(http.StatusOK)
		h.logResponse(http.StatusOK, r)

	default:
		msg := fmt.Sprintf("Method '%s' not supported.", html.EscapeString(m))
		http.Error(w, msg, http.StatusMethodNotAllowed)
		h.logResponse(http.StatusMethodNotAllowed, r)
	}
}

func addWorkerMetadataHTTP(addr string, ct string, orig []byte) (data []byte, code int, err error) {
	ar := &pb.ActionResult{}
	if ct == "application/json" {
		err = jsonpb.Unmarshal(bytes.NewReader(orig), ar)
	} else {
		err = proto.Unmarshal(orig, ar)
	}
	if err != nil {
		return orig, http.StatusBadRequest, err
	}

	if ar.ExecutionMetadata == nil {
		ar.ExecutionMetadata = &pb.ExecutedActionMetadata{}
	} else if ar.ExecutionMetadata.Worker != "" {
		return orig, http.StatusOK, nil
	}

	worker := addr
	if worker == "" {
		worker, _, err = net.SplitHostPort(addr)
		if err != nil || worker == "" {
			worker = "unknown"
		}
	}

	ar.ExecutionMetadata.Worker = worker

	data, err = proto.Marshal(ar)
	if err != nil {
		return orig, http.StatusInternalServerError, err
	}

	return data, http.StatusOK, nil
}

// Produce a debugging page with some stats about the cache.
func (h *httpCache) StatusPageHandler(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()

	totalSize, reservedSize, numItems, uncompressedSize := h.cache.Stats()

	goroutines := runtime.NumGoroutine()

	w.Header().Set("Content-Type", "application/json")
	enc := json.NewEncoder(w)
	enc.SetIndent("", " ")
	enc.Encode(statusPageData{
		MaxSize:          h.cache.MaxSize(),
		CurrSize:         totalSize,
		UncompressedSize: uncompressedSize,
		ReservedSize:     reservedSize,
		NumFiles:         numItems,
		ServerTime:       time.Now().Unix(),
		GitCommit:        h.gitCommit,
		NumGoroutines:    goroutines,
	})
}

func path(kind cache.EntryKind, hash string) string {
	return fmt.Sprintf("/%s/%s", kind, hash)
}
<|endoftext|>


<commit_before>package api

import (
    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
    "time"
    "net/url"
    "strconv"
)
func testHandlerFn(vars map[string]string, form url.Values, db *mgo.Session) *apiResponse{
  res := map[string]interface{}{
    "error": false,
    "text": "this is a test",
  }
  return newOKRes(res)
}

func examHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{

  c := session.DB("ebisc").C("exam")
  m := make(bson.M)
  if dateStr := vars["date"]; len(dateStr) > 0 {
    if date, err := time.Parse(time.RFC3339Nano, dateStr); err != nil {
      return newBadRequestRes("Date not valid RFC3339Nano")
    } else {
      if err := c.Find(bson.M{"date": date}).One(&m); err != nil {
        if (err == mgo.ErrNotFound) {
          return newNotFoundRes()
        }
        panic(newApiError(err, "Database find error"))
      }
    }
  } else {
    if err := c.Find(nil).Sort("-date").One(&m); err != nil {
      panic(newApiError(err, "Database find error"))
    }
  }

  if qSlice, ok := m["questions"].([]interface{}); ok {
    for _, q := range qSlice {
      if bsonQ, ok := q.(bson.M); !ok {
        panic(newApiError("type conversion of Question to a map", "Unexpected json structure"))
      } else {
        expandQuestion(bsonQ, session)
      }
    }
  }

  delete(m, "_id")
  m["error"] = false

  return newOKRes(m)
}

func expandQuestion(q bson.M, session *mgo.Session) {
  c := session.DB("ebisc").C("question")
  if err := c.Find(bson.M{"module": q["module"]}).Select(bson.M{"title": 1, "description": 1, "_id": 0}).One(&q); err != nil {
    if (err == mgo.ErrNotFound) {
      return // Not an error
    }
    panic(newApiError(err, "Database find error"))
  }
}

func examListHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("exam")
  var query bson.M = nil
  if dateStr := form.Get("date"); len(dateStr) > 0 {
    if date, err := time.Parse(time.RFC3339Nano, dateStr); err != nil {
      return newBadRequestRes("Date not valid RFC3339Nano")
    } else {
      query = bson.M{"date": bson.M{"$lt": date}};
    }
  }
  var items []interface{}
  if err := c.Find(query).Sort("-date").Limit(10).Select(bson.M{"date": 1, "_id": 0}).All(&items); err != nil {
    panic(newApiError(err, "Database find error"))
  }
  m := bson.M{"items": items, "error": false}
  return newOKRes(m)
}

func failHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("question_fail")
  queryParams := bson.M{};

  if date, err := time.Parse(time.RFC3339Nano, vars["date"]); err != nil {
    return newBadRequestRes("Date not valid RFC3339Nano")
  } else {
    queryParams["date"] = date
  }
  if str := form.Get("cell_line"); len(str) > 0 {
    queryParams["cellLine"] = str
  }
  if str := form.Get("batch"); len(str) > 0 {
    queryParams["batch"] = str
  }
  if str := form.Get("module"); len(str) > 0 {
    queryParams["module"] = str
  }

  query := c.Find(queryParams);
  m := bson.M{}

  if n, err := query.Count(); err != nil {
    panic(newApiError(err, "Database error"))
  } else {
    m["total"] = n
  }

  query = query.Sort("_id").Limit(100).Select(bson.M{"_id": 0})

  if skipStr := form.Get("offset"); len(skipStr) > 0 {
    if skip, err := strconv.Atoi(skipStr); err != nil{
      return newBadRequestRes("skip not a valid integer")
    } else {
      query = query.Skip(skip)
      m["pageOffset"] = skip
    }
  } else {
    m["pageOffset"] = 0
  }

  if limitStr := form.Get("limit"); len(limitStr) > 0 {
    if limit, err := strconv.Atoi(limitStr); err != nil{
      return newBadRequestRes("limit not a valid integer")
    } else {
      m["pageLimit"] = limit
      query = query.Limit(limit)
    }
  } else {
    m["pageLimit"] = 100
    query = query.Limit(100)
  }

  var items []interface{}
  if err := query.All(&items); err != nil {
    panic(newApiError(err, "Database find error"))
  }
  m["items"] = items
  m["error"] = false
  return newOKRes(m)
}

func lineFailHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("question_fail")
  m := bson.M{}

  var o1 bson.M
  if date, err := time.Parse(time.RFC3339Nano, vars["date"]); err != nil {
    return newBadRequestRes("Date not valid RFC3339Nano")
  } else {
    o1 = bson.M{"$match": bson.M{"date": date}}
  }

  o2 := bson.M{
    "$group": bson.M{"_id": "$cellLine",  "modules": bson.M{"$addToSet": "$module"}},
  }
  o3 := bson.M{
    "$project": bson.M{"cellLine": "$_id", "modules": 1, "count": bson.M{"$size": "$modules"}},
  }
  o4 := bson.M{
    "$sort": bson.D{{"count", 1}, {"cellLine", 1}},
  }

  var o5 bson.M
  if skipStr := form.Get("offset"); len(skipStr) > 0 {
    if skip, err := strconv.Atoi(skipStr); err != nil{
      return newBadRequestRes("skip not a valid integer")
    } else {
      m["pageOffset"] = skip
      o5 = bson.M{"$skip": skip}
    }
  } else {
    m["pageOffset"] = 0
    o5 = bson.M{"$skip": 0}
  }

  var o6 bson.M
  if limitStr := form.Get("limit"); len(limitStr) > 0 {
    if limit, err := strconv.Atoi(limitStr); err != nil{
      return newBadRequestRes("limit not a valid integer")
    } else {
      m["pageLimit"] = limit
      o6 = bson.M{"$limit": limit}
    }
  } else {
    m["pageLimit"] = 100
    o6 = bson.M{"$limit": 100}
  }

  o7 := bson.M{
    "$project": bson.M{"cellLine": 1, "modules": 1, "_id": 0},
  }

  operations := []bson.M{o1,o2,o3,o4,o5,o6,o7}
  pipe := c.Pipe(operations)

  var items []interface{}
  if err := pipe.All(&items); err != nil {
    panic(newApiError(err, "Database find error"))
  }
  m["items"] = items
  m["error"] = false
  return newOKRes(m)
}

<commit_msg>line_fails endpoint<commit_after>package api

import (
    "gopkg.in/mgo.v2"
    "gopkg.in/mgo.v2/bson"
    "time"
    "net/url"
    "strconv"
)
func testHandlerFn(vars map[string]string, form url.Values, db *mgo.Session) *apiResponse{
  res := map[string]interface{}{
    "error": false,
    "text": "this is a test",
  }
  return newOKRes(res)
}

func examHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{

  c := session.DB("ebisc").C("exam")
  m := make(bson.M)
  if dateStr := vars["date"]; len(dateStr) > 0 {
    if date, err := time.Parse(time.RFC3339Nano, dateStr); err != nil {
      return newBadRequestRes("Date not valid RFC3339Nano")
    } else {
      if err := c.Find(bson.M{"date": date}).One(&m); err != nil {
        if (err == mgo.ErrNotFound) {
          return newNotFoundRes()
        }
        panic(newApiError(err, "Database find error"))
      }
    }
  } else {
    if err := c.Find(nil).Sort("-date").One(&m); err != nil {
      panic(newApiError(err, "Database find error"))
    }
  }

  if qSlice, ok := m["questions"].([]interface{}); ok {
    for _, q := range qSlice {
      if bsonQ, ok := q.(bson.M); !ok {
        panic(newApiError("type conversion of Question to a map", "Unexpected json structure"))
      } else {
        expandQuestion(bsonQ, session)
      }
    }
  }

  delete(m, "_id")
  m["error"] = false

  return newOKRes(m)
}

func expandQuestion(q bson.M, session *mgo.Session) {
  c := session.DB("ebisc").C("question")
  if err := c.Find(bson.M{"module": q["module"]}).Select(bson.M{"title": 1, "description": 1, "_id": 0}).One(&q); err != nil {
    if (err == mgo.ErrNotFound) {
      return // Not an error
    }
    panic(newApiError(err, "Database find error"))
  }
}

func examListHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("exam")
  var query bson.M = nil
  if dateStr := form.Get("date"); len(dateStr) > 0 {
    if date, err := time.Parse(time.RFC3339Nano, dateStr); err != nil {
      return newBadRequestRes("Date not valid RFC3339Nano")
    } else {
      query = bson.M{"date": bson.M{"$lt": date}};
    }
  }
  var items []interface{}
  if err := c.Find(query).Sort("-date").Limit(10).Select(bson.M{"date": 1, "_id": 0}).All(&items); err != nil {
    panic(newApiError(err, "Database find error"))
  }
  m := bson.M{"items": items, "error": false}
  return newOKRes(m)
}

func failHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("question_fail")
  queryParams := bson.M{};

  if date, err := time.Parse(time.RFC3339Nano, vars["date"]); err != nil {
    return newBadRequestRes("Date not valid RFC3339Nano")
  } else {
    queryParams["date"] = date
  }
  if str := form.Get("cell_line"); len(str) > 0 {
    queryParams["cellLine"] = str
  }
  if str := form.Get("batch"); len(str) > 0 {
    queryParams["batch"] = str
  }
  if str := form.Get("module"); len(str) > 0 {
    queryParams["module"] = str
  }

  query := c.Find(queryParams);
  m := bson.M{}

  if n, err := query.Count(); err != nil {
    panic(newApiError(err, "Database error"))
  } else {
    m["total"] = n
  }

  query = query.Sort("_id").Limit(100).Select(bson.M{"_id": 0})

  if skipStr := form.Get("offset"); len(skipStr) > 0 {
    if skip, err := strconv.Atoi(skipStr); err != nil{
      return newBadRequestRes("skip not a valid integer")
    } else {
      query = query.Skip(skip)
      m["pageOffset"] = skip
    }
  } else {
    m["pageOffset"] = 0
  }

  if limitStr := form.Get("limit"); len(limitStr) > 0 {
    if limit, err := strconv.Atoi(limitStr); err != nil{
      return newBadRequestRes("limit not a valid integer")
    } else {
      m["pageLimit"] = limit
      query = query.Limit(limit)
    }
  } else {
    m["pageLimit"] = 100
    query = query.Limit(100)
  }

  var items []interface{}
  if err := query.All(&items); err != nil {
    panic(newApiError(err, "Database find error"))
  }
  m["items"] = items
  m["error"] = false
  return newOKRes(m)
}

func lineFailHandlerFn(vars map[string]string, form url.Values, session *mgo.Session) *apiResponse{
  c := session.DB("ebisc").C("question_fail")
  m := bson.M{}
  var items []interface{}

  queryParams := bson.M{}
  if date, err := time.Parse(time.RFC3339Nano, vars["date"]); err != nil {
    return newBadRequestRes("Date not valid RFC3339Nano")
  } else {
    queryParams["date"] = date
  }
  if moduleStr := form.Get("module"); len(moduleStr) > 0 {
    queryParams["module"] = moduleStr
  }

  skip := 0
  if skipStr := form.Get("offset"); len(skipStr) > 0 {
    var err error
    if skip, err = strconv.Atoi(skipStr); err != nil{
      return newBadRequestRes("skip not a valid integer")
    }
  }
  limit := 100
  if limitStr := form.Get("limit"); len(limitStr) > 0 {
    var err error
    if limit, err = strconv.Atoi(limitStr); err != nil{
      return newBadRequestRes("limit not a valid integer")
    }
  }

  o1 := bson.M{"$match": queryParams}

  oDistinct2 := bson.M{"$group": bson.M{"_id": "$cellLine"}}
  oDistinct3 := bson.M{"$group": bson.M{"_id": nil, "count": bson.M{"$sum": 1}}}
  operationsDistinct := []bson.M{o1,oDistinct2, oDistinct3}
  pipeDistinct := c.Pipe(operationsDistinct)
  distinct := bson.M{}
  if err := pipeDistinct.One(&distinct); err != nil {
    if (err == mgo.ErrNotFound) {
      m["total"] = 0
    } else {
      panic(newApiError(err, "Database find error"))
    }
  } else {
    m["total"] = distinct["count"]

    o2 := bson.M{
      "$group": bson.M{"_id": "$cellLine",  "modules": bson.M{"$addToSet": "$module"}},
    }
    o3 := bson.M{
      "$project": bson.M{"cellLine": "$_id", "modules": 1, "count": bson.M{"$size": "$modules"}},
    }
    o4 := bson.M{
      "$sort": bson.D{{"count", 1}, {"cellLine", 1}},
    }
    o5 := bson.M{"$skip": skip}
    o6 := bson.M{"$limit": limit}

    o7 := bson.M{
      "$project": bson.M{"cellLine": 1, "modules": 1, "_id": 0},
    }

    operations := []bson.M{o1,o2,o3,o4,o5,o6,o7}
    pipe := c.Pipe(operations)

    if err := pipe.All(&items); err != nil {
      panic(newApiError(err, "Database find error"))
    }
  }

  m["items"] = items
  m["pageLimit"] = limit
  m["pageOffset"] = skip
  m["error"] = false
  return newOKRes(m)
}

<|endoftext|>


<commit_before>package models

import (
	"fmt"
	"math/rand"
	"sort"

	"golang.org/x/tools/container/intsets"

	"github.com/mrap/stringutil"
	"github.com/mrap/wordpatterns"
)

const (
	MinCombosCount = 0
	MinMaxComboLen = 2
	MinMinComboLen = 2
)

// TODO: Might provide other languages/wordlists in the future.
//       We would need to store lookups/wordmaps in a db
var (
	_sharedLookup = make(wordpatterns.Wordmap)
)

func init() {
	StoreWords("wordlists/google-10000-english.txt")
}

type Combo struct {
	S     string
	Count int
}

func GenerateCombos(chars string, count, minLen, maxLen int) []string {
	// Validate and normalize params
	if len(chars) == 0 {
		return []string{}
	}

	if count < MinCombosCount {
		count = MinCombosCount
	}

	if minLen < MinMinComboLen {
		minLen = MinMinComboLen
	}

	if maxLen < MinMaxComboLen {
		maxLen = MinMaxComboLen
	}

	if maxLen < minLen {
		maxLen = minLen
	}

	combos := SortedCombos(_sharedLookup, chars, minLen, maxLen)
	return generateComboList(combos, count)
}

// StoreWords stores words from a word list file into
func StoreWords(filename string) {
	wm := wordpatterns.CreateWordmap(filename)
	_sharedLookup = CreateCharWordLookup(wm)
}

// CreateCharWordLookup creates a lookup structure that makes it efficient
// to query for sub-words that only contain specific characters
func CreateCharWordLookup(wm wordpatterns.Wordmap) wordpatterns.Wordmap {
	var (
		lookup = make(wordpatterns.Wordmap)
		sorted string
	)

	for subword, _ := range wm {
		sorted = stringutil.SortString(subword)
		lookup[sorted] = append(lookup[sorted], subword)
	}
	return lookup
}

func CombosWithChars(lookup wordpatterns.Wordmap, chars string, minLen int, maxLen int) []Combo {
	if maxLen < 0 {
		maxLen = intsets.MaxInt
	}
	if minLen < 0 {
		minLen = 0
	}

	var combos []Combo
	for _, sub := range sortedSubstrs(chars, minLen) {
		if (len(sub) >= minLen) && (len(sub) <= maxLen) {
			combos = append(combos, Combo{sub, len(lookup[sub])})
		}
	}
	return combos
}

func sortedSubstrs(chars string, minLen int) []string {
	sortedChars := stringutil.SortString(chars)
	return stringutil.Substrs(sortedChars, minLen)
}

// SortedCombos returns combos ranked by most occuring in the wordmap to least.
// i.e. a combo is more occurant if it is a substring of more words in the wordmap.
func SortedCombos(lookup wordpatterns.Wordmap, chars string, minLen, maxLen int) []Combo {
	combos := CombosWithChars(lookup, chars, minLen, maxLen)
	sort.Sort(ByCount(combos))
	return combos
}

// generateComboList returns a slice of combos that are distributed based on rank
// More occurring combos will appear more frequently
func generateComboList(combos []Combo, count int) []string {
	matchCount := sumCount(combos)
	list := make([]string, count)
	for i := 0; i < count; i++ {
		list[i] = probableCombo(combos, matchCount).S
	}
	return list
}

func sumCount(combos []Combo) int {
	total := 0
	for _, c := range combos {
		total += c.Count
	}
	return total
}

// probableCombo returns probable combo based on occurances.
// i.e. the higher the combo count, the more likely to chose it.
func probableCombo(sortedCombos []Combo, totalMatches int) Combo {
	i := 0
	for rem := rand.Intn(totalMatches) + 1; rem > 0; i++ {
		rem -= sortedCombos[i].Count
	}
	return sortedCombos[i]
}

type ByCount []Combo

func (a ByCount) Len() int      { return len(a) }
func (a ByCount) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByCount) Less(i, j int) bool {
	return a[i].Count < a[j].Count
}

func printComboListCounts(combos []string) {
	counts := make(map[string]int)
	for _, c := range combos {
		counts[c]++
	}
	for key, count := range counts {
		fmt.Printf("%s -> %d\n", key, count)
	}
	fmt.Println()
}
<commit_msg>Revert "Simplifies combo ranking"<commit_after>package models

import (
	"fmt"
	"math/rand"
	"sort"

	"github.com/mrap/stringutil"
	wordpatterns "github.com/mrap/wordpatterns"
)

const (
	MinCombosCount = 0
	MinMaxComboLen = 2
	MinMinComboLen = 2
)

// TODO: Might provide other languages/wordlists in the future.
//       We would need to store lookups/wordmaps in a db
var (
	_sharedLookup  = make(wordpatterns.Wordmap)
	_sharedWordmap = make(wordpatterns.Wordmap)
)

func init() {
	StoreWords("wordlists/google-10000-english.txt")
}

func GenerateCombos(chars string, count, minLen, maxLen int) []string {
	// Validate and normalize params
	if len(chars) == 0 {
		return []string{}
	}

	if count < MinCombosCount {
		count = MinCombosCount
	}

	if minLen < MinMinComboLen {
		minLen = MinMinComboLen
	}

	if maxLen < MinMaxComboLen {
		maxLen = MinMaxComboLen
	}

	if maxLen < minLen {
		maxLen = minLen
	}

	combos := RankedCombos(_sharedLookup, chars, minLen, maxLen)
	return generateComboList(_sharedWordmap, combos, count)
}

// StoreWords stores words from a word list file into
func StoreWords(filename string) {
	wm := wordpatterns.CreateWordmap(filename)
	_sharedWordmap = wm
	_sharedLookup = CreateCharWordLookup(_sharedWordmap)
}

// CreateCharWordLookup creates a lookup structure that makes it efficient
// to query for sub-words that only contain specific characters
func CreateCharWordLookup(wm wordpatterns.Wordmap) wordpatterns.Wordmap {
	var (
		lookup = make(wordpatterns.Wordmap)
		sorted string
	)

	for subword, _ := range wm {
		sorted = stringutil.SortString(subword)
		lookup[sorted] = append(lookup[sorted], subword)
	}
	return lookup
}

func CombosWithChars(lookup wordpatterns.Wordmap, chars string, minLen int, maxLen int) []string {
	if maxLen < 0 {
		maxLen = 0
	}
	if minLen < 0 {
		minLen = 0
	}

	var combos []string
	sortedChars := stringutil.SortString(chars)

	for _, subsorted := range stringutil.Substrs(sortedChars, 2) {
		if (minLen != 0 && len(subsorted) < minLen) || (maxLen != 0 && len(subsorted) > maxLen) {
			continue
		} else {
			combos = append(combos, lookup[subsorted]...)
		}
	}
	return combos
}

func RankedCombos(lookup wordpatterns.Wordmap, chars string, minLen, maxLen int) []string {
	combos := CombosWithChars(lookup, chars, minLen, maxLen)
	sort.Sort(ByWordCount(combos))
	return combos
}

// generateComboList returns a slice of combos that are distributed based on rank
// More occurring combos will appear more frequently
func generateComboList(wm wordpatterns.Wordmap, combos []string, count int) []string {
	total := 0
	for _, c := range combos {
		total += len(wm[c])
	}

	list := make([]string, count)
	var r, cur int
	for i := 0; i < count; i++ {
		cur = 0
		r = rand.Intn(total) + 1
		for _, c := range combos {
			cur += len(wm[c])
			if r <= cur {
				list[i] = c
				break
			}
		}
	}
	return list
}

type ByWordCount []string

func (a ByWordCount) Len() int      { return len(a) }
func (a ByWordCount) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByWordCount) Less(i, j int) bool {
	return _sharedWordmap.Compare(a[i], a[j]) >= 0
}

func printComboListCounts(combos []string) {
	counts := make(map[string]int)
	for _, c := range combos {
		counts[c]++
	}
	for key, count := range counts {
		fmt.Printf("%s -> %d\n", key, count)
	}
	fmt.Println()
}
<|endoftext|>


<commit_before>package tests

var TestFuncs = `// package testfunc contains various helpers to be used in tests. Included
// from: https://github.com/benbjohnson/testing
package tests

import (
    "fmt"
    "path/filepath"
    "reflect"
    "runtime"
    "testing"
)

// Assert fails the test if the condition is false.
func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
    if !condition {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
        tb.FailNow()
    }
}

// Ok fails the test if an err is not nil.
func Ok(tb testing.TB, err error) {
    if err != nil {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
        tb.FailNow()
    }
}

// Equals fails the test if exp is not equal to act.
func Equals(tb testing.TB, exp, act interface{}) {
    if !reflect.DeepEqual(exp, act) {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
        tb.FailNow()
    }
}`

var MainTestsTemplate = `
{{$ModuleName := ToLower .Title}}

package {{$ModuleName}}tests

import (
    "testing"
    "net/http"

    "github.com/youtube/vitess/go/rpcplus"
    "github.com/youtube/vitess/go/rpcplus/jsonrpc"
    "github.com/youtube/vitess/go/rpcwrap"
    "golang.org/x/net/context"
)

func createClient(tb testing.TB) *rpcplus.Client {
    client, err := rpcwrap.DialHTTP(
        "tcp",                  // network
        "localhost:3000",       // address
        "json",                 // codec name
        jsonrpc.NewClientCodec, // codec factory
        time.Second*10,         // timeout
        nil,                    // TLS config
    )
    tests.Assert(tb, err != nil, "Err while creating the client")
    return client
}

{{range $defKey, $def := .Definitions}}

func with{{$defKey}}Client(tb testing.TB, f func(*{{$ModuleName}}client.{{$defKey}})) {
    client := createClient()
    defer client.Close()

    f({{$ModuleName}}client.New{{$defKey}}(client))
}

{{end}}
`

var TestsTemplate = `
{{$Name := ToUpperFirst .Name}}
{{$ModuleName := ToLower .ModuleName}}

package {{ToLower .ModuleName}}tests

import (
    "testing"

    "golang.org/x/net/context"
)

func Test{{$Name}}One(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.One(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err != nil, "Err should be nil while testing {{$Name}}.One")
    })
}

func Test{{$Name}}Create(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Create(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err != nil, "Err should be nil while testing {{$Name}}.Create")
    })
}

func Test{{$Name}}Update(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Update(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err != nil, "Err should be nil while testing {{$Name}}.Update")
    })
}

func Test{{$Name}}Delete(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Delete(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err != nil, "Err should be nil while testing {{$Name}}.Delete")
    })
}

func Test{{$Name}}Some(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Some(context.Background(), request.Options{}, make(*[]*models.{{$Name}}{},0))
        tests.Assert(t, err != nil, "Err should be nil while testing {{$Name}}.Some")
    })
}`
<commit_msg>tests: fix assertions<commit_after>package tests

var TestFuncs = `// package testfunc contains various helpers to be used in tests. Included
// from: https://github.com/benbjohnson/testing
package tests

import (
    "fmt"
    "path/filepath"
    "reflect"
    "runtime"
    "testing"
)

// Assert fails the test if the condition is false.
func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) {
    if !condition {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...)
        tb.FailNow()
    }
}

// Ok fails the test if an err is not nil.
func Ok(tb testing.TB, err error) {
    if err != nil {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error())
        tb.FailNow()
    }
}

// Equals fails the test if exp is not equal to act.
func Equals(tb testing.TB, exp, act interface{}) {
    if !reflect.DeepEqual(exp, act) {
        _, file, line, _ := runtime.Caller(1)
        fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
        tb.FailNow()
    }
}`

var MainTestsTemplate = `
{{$ModuleName := ToLower .Title}}

package {{$ModuleName}}tests

import (
    "testing"
    "net/http"

    "github.com/youtube/vitess/go/rpcplus"
    "github.com/youtube/vitess/go/rpcplus/jsonrpc"
    "github.com/youtube/vitess/go/rpcwrap"
    "golang.org/x/net/context"
)

func createClient(tb testing.TB) *rpcplus.Client {
    client, err := rpcwrap.DialHTTP(
        "tcp",                  // network
        "localhost:3000",       // address
        "json",                 // codec name
        jsonrpc.NewClientCodec, // codec factory
        time.Second*10,         // timeout
        nil,                    // TLS config
    )
    tests.Assert(tb, err == nil, "Err while creating the client")
    return client
}

{{range $defKey, $def := .Definitions}}

func with{{$defKey}}Client(tb testing.TB, f func(*{{$ModuleName}}client.{{$defKey}})) {
    client := createClient(tb)
    defer client.Close()

    f({{$ModuleName}}client.New{{$defKey}}(client))
}

{{end}}
`

var TestsTemplate = `
{{$Name := ToUpperFirst .Name}}
{{$ModuleName := ToLower .ModuleName}}

package {{ToLower .ModuleName}}tests

import (
    "testing"

    "golang.org/x/net/context"
)

func Test{{$Name}}One(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.One(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err == nil, "Err should be nil while testing {{$Name}}.One")
    })
}

func Test{{$Name}}Create(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Create(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err == nil, "Err should be nil while testing {{$Name}}.Create")
    })
}

func Test{{$Name}}Update(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Update(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err == nil, "Err should be nil while testing {{$Name}}.Update")
    })
}

func Test{{$Name}}Delete(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        err := c.Delete(context.Background(), models.New{{$Name}}(), models.New{{$Name}}())
        tests.Assert(t, err == nil, "Err should be nil while testing {{$Name}}.Delete")
    })
}

func Test{{$Name}}Some(t *testing.T) {
    with{{$Name}}Client(t, func(c *{{$ModuleName}}client.{{$Name}}){
        res := make([]*models.{{$Name}}, 0)
        err := c.Some(context.Background(), &request.Options{}, &res)
        tests.Assert(t, err == nil, "Err should be nil while testing {{$Name}}.Some")
    })
}`
<|endoftext|>


<commit_before>package xog

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/andreluzz/cas-xog/constant"
	"github.com/andreluzz/cas-xog/migration"
	"github.com/andreluzz/cas-xog/model"
	"github.com/andreluzz/cas-xog/transform"
	"github.com/andreluzz/cas-xog/util"
	"github.com/andreluzz/cas-xog/validate"
	"github.com/beevik/etree"
	"io/ioutil"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

var driverXOG *model.Driver

//LoadDriver load an specific driver defined by a path
func LoadDriver(path string) (int, error) {
	driverXOG = &model.Driver{}
	driverXOG.Clear()
	xmlFile, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, errors.New("Error loading driver file - " + err.Error())
	}

	driverXOGTypePattern := model.DriverTypesPattern{}
	xml.Unmarshal(xmlFile, &driverXOGTypePattern)

	v, err := strconv.ParseFloat(driverXOGTypePattern.Version, 64)
	if err != nil || v < constant.Version {
		return 0, fmt.Errorf("invalid driver(%s) version, expected version %.1f or greater", path, constant.Version)
	}

	if len(driverXOGTypePattern.Files) > 0 {
		return 0, fmt.Errorf("invalid driver(%s) tag <file> is no longer supported", path)
	}

	types := reflect.ValueOf(&driverXOGTypePattern).Elem()
	typeOfT := types.Type()
	for i := 0; i < types.NumField(); i++ {
		t := types.Field(i)
		if t.Kind() == reflect.Slice {
			for _, f := range t.Interface().([]model.DriverFile) {
				f.Type = typeOfT.Field(i).Name
				f.ExecutionOrder = -1
				driverXOG.Files = append(driverXOG.Files, f)
			}
		}
	}

	doc := etree.NewDocument()
	doc.ReadFromBytes(xmlFile)

	for i, e := range doc.FindElements("//xogdriver/*") {
		tag := e.Tag
		path := e.SelectAttrValue("path", constant.Undefined)
		code := e.SelectAttrValue("code", constant.Undefined)
		for y, f := range driverXOG.Files {
			if f.ExecutionOrder == -1 && (strings.ToLower(f.GetXMLType()) == strings.ToLower(tag)) && (f.Path == path) && (f.Code == code) {
				driverXOG.Files[y].ExecutionOrder = i
				break
			}
		}
	}

	sort.Sort(model.ByExecutionOrder(driverXOG.Files))

	driverXOG.Version = driverXOGTypePattern.Version
	driverXOG.FilePath = path

	return len(driverXOG.Files), nil
}

//GetLoadedDriver returns the pointer to the loaded driver
func GetLoadedDriver() *model.Driver {
	return driverXOG
}

//ValidateLoadedDriver verify if the driver was loaded successfully
func ValidateLoadedDriver() bool {
	if driverXOG == nil {
		return false
	}
	return len(driverXOG.Files) > 0
}

//GetDriversList returns a list of available drivers in the defined folder
func GetDriversList(folder string) ([]model.Driver, error) {
	driversFileList, err := ioutil.ReadDir(folder)

	if err != nil || len(driversFileList) == 0 {
		return nil, errors.New("driver folder not found or empty")
	}

	var driversList []model.Driver
	for _, f := range driversFileList {
		driver := new(model.Driver)
		driver.Info = f
		driver.FilePath = folder + f.Name()
		driversList = append(driversList, *driver)
	}

	return driversList, nil
}

//ProcessDriverFile execute a xog files and return the output
func ProcessDriverFile(file *model.DriverFile, action, sourceFolder, outputFolder string, environments *model.Environments, soapFunc util.Soap) model.Output {
	output := model.Output{Code: constant.OutputSuccess, Debug: constant.Undefined}

	if action == constant.Migrate && file.Type != constant.TypeMigration {
		output.Code = constant.OutputWarning
		output.Debug = "Use action 'r' to this type(" + file.Type + ") of file"
		return output
	} else if action == constant.Read && file.Type == constant.TypeMigration {
		output.Code = constant.OutputWarning
		output.Debug = "Use action 'm' to this type(" + file.Type + ") of file"
		return output
	}

	if action == constant.Migrate {
		resp, err := migration.ReadDataFromExcel(file)
		if err != nil {
			output.Code = constant.OutputError
			output.Debug = err.Error()
			return output
		}
		file.SetXML(resp)
		file.Write(outputFolder)
		return output
	}

	err := file.InitXML(action, sourceFolder)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	if action == constant.Write {
		iniTagRegexpStr, endTagRegexpStr := file.TagCDATA()
		if iniTagRegexpStr != constant.Undefined && endTagRegexpStr != constant.Undefined {
			transformedString := transform.IncludeCDATA(file.GetXML(), iniTagRegexpStr, endTagRegexpStr)
			file.SetXML(transformedString)
		}
	}
	err = file.RunXML(action, sourceFolder, environments, soapFunc)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	xogResponse := etree.NewDocument()
	xogResponse.ReadFromString(file.GetXML())
	output, err = validate.Check(xogResponse)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	if action == constant.Read {
		output := processDriverFileRead(file, xogResponse, outputFolder)
		if output.Code != constant.OutputSuccess {
			return output
		}
	}

	file.Write(outputFolder)
	return output
}

func processDriverFileRead(file *model.DriverFile, xogResponse *etree.Document, outputFolder string) model.Output {
	output := model.Output{Code: constant.OutputSuccess, Debug: constant.Undefined}

	var auxResponse *etree.Document
	if file.NeedAuxXML() {
		auxResponse = etree.NewDocument()
		auxResponse.ReadFromString(file.GetAuxXML())
		output, err := validate.Check(auxResponse)
		if err != nil {
			output.Code = constant.OutputError
			output.Debug = "aux validation - " + err.Error()
			return output
		}
	}
	err := transform.Execute(xogResponse, auxResponse, file)

	if file.GetInstanceTag() != constant.Undefined && file.InstancesPerFile > 0 {
		instanceTagPath := "//" + file.GetInstanceTag()

		parentTagPath := "//" + xogResponse.FindElement(instanceTagPath).Parent().Tag

		splitXogResponse := xogResponse.Copy()
		for _, e := range splitXogResponse.FindElements(instanceTagPath) {
			e.Parent().RemoveChild(e)
		}
		instances := xogResponse.FindElements(instanceTagPath)

		totalFiles := math.Ceil(float64(len(instances)) / float64(file.InstancesPerFile))
		path := util.GetPathWithoutExtension(file.Path)
		for i := 0; i < int(totalFiles); i++ {
			file.Path = fmt.Sprintf("%s_%03d.xml", path, i+1)
			xog := splitXogResponse.Copy()
			elementParent := xog.FindElement(parentTagPath)
			for z := file.InstancesPerFile * i; z < file.InstancesPerFile*(i+1); z++ {
				if z < len(instances) {
					elementParent.AddChild(instances[z].Copy())
				}
			}
			xog.IndentTabs()
			s, _ := xog.WriteToString()
			file.SetXML(s)
			file.Write(outputFolder)
		}
		file.Path = "complete_" + path + ".xml"
	}

	str, _ := xogResponse.WriteToString()
	file.SetXML(str)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	iniTagRegexpStr, endTagRegexpStr := file.TagCDATA()
	if iniTagRegexpStr != constant.Undefined && endTagRegexpStr != constant.Undefined {
		transformedString := transform.IncludeCDATA(file.GetXML(), iniTagRegexpStr, endTagRegexpStr)
		file.SetXML(transformedString)
	}

	if file.ExportToExcel {
		migration.ExportInstancesToExcel(xogResponse, file, constant.FolderMigration)
	}

	return output
}

//CreateFileFolder creates the folders according to the action (read, write or migrate)
func CreateFileFolder(action, fileType, path string) (string, string) {
	sourceFolder := constant.Undefined
	outputFolder := constant.Undefined

	switch action {
	case constant.Read:
		sourceFolder = constant.FolderRead
		outputFolder = constant.FolderWrite
		util.ValidateFolder(constant.FolderRead + fileType + util.GetPathFolder(path))
	case constant.Write:
		sourceFolder = constant.FolderWrite
		outputFolder = constant.FolderDebug
	case constant.Migrate:
		sourceFolder = constant.FolderWrite
		outputFolder = constant.FolderMigration
	}

	util.ValidateFolder(outputFolder + fileType + util.GetPathFolder(path))

	return sourceFolder, outputFolder
}
<commit_msg>Validating driver xml root tag <xogdriver><commit_after>package xog

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/andreluzz/cas-xog/constant"
	"github.com/andreluzz/cas-xog/migration"
	"github.com/andreluzz/cas-xog/model"
	"github.com/andreluzz/cas-xog/transform"
	"github.com/andreluzz/cas-xog/util"
	"github.com/andreluzz/cas-xog/validate"
	"github.com/beevik/etree"
	"io/ioutil"
	"math"
	"reflect"
	"sort"
	"strconv"
	"strings"
)

var driverXOG *model.Driver

//LoadDriver load an specific driver defined by a path
func LoadDriver(path string) (int, error) {
	driverXOG = &model.Driver{}
	driverXOG.Clear()
	xmlFile, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, errors.New("Error loading driver file - " + err.Error())
	}

	driverXOGTypePattern := model.DriverTypesPattern{}
	xml.Unmarshal(xmlFile, &driverXOGTypePattern)

	v, err := strconv.ParseFloat(driverXOGTypePattern.Version, 64)
	if err != nil || v < constant.Version {
		return 0, fmt.Errorf("invalid driver(%s) version, expected version %.1f or greater", path, constant.Version)
	}

	if len(driverXOGTypePattern.Files) > 0 {
		return 0, fmt.Errorf("invalid driver(%s) tag <file> is no longer supported", path)
	}

	types := reflect.ValueOf(&driverXOGTypePattern).Elem()
	typeOfT := types.Type()
	for i := 0; i < types.NumField(); i++ {
		t := types.Field(i)
		if t.Kind() == reflect.Slice {
			for _, f := range t.Interface().([]model.DriverFile) {
				f.Type = typeOfT.Field(i).Name
				f.ExecutionOrder = -1
				driverXOG.Files = append(driverXOG.Files, f)
			}
		}
	}

	doc := etree.NewDocument()
	doc.ReadFromBytes(xmlFile)

	if doc.Root().Tag != "xogdriver" {
		return 0, fmt.Errorf("invalid driver(%s) tag <%s> is incorrect", path, doc.Root().Tag)
	}

	for i, e := range doc.FindElements("//xogdriver/*") {
		tag := e.Tag
		path := e.SelectAttrValue("path", constant.Undefined)
		code := e.SelectAttrValue("code", constant.Undefined)
		for y, f := range driverXOG.Files {
			if f.ExecutionOrder == -1 && (strings.ToLower(f.GetXMLType()) == strings.ToLower(tag)) && (f.Path == path) && (f.Code == code) {
				driverXOG.Files[y].ExecutionOrder = i
				break
			}
		}
	}

	sort.Sort(model.ByExecutionOrder(driverXOG.Files))

	driverXOG.Version = driverXOGTypePattern.Version
	driverXOG.FilePath = path

	return len(driverXOG.Files), nil
}

//GetLoadedDriver returns the pointer to the loaded driver
func GetLoadedDriver() *model.Driver {
	return driverXOG
}

//ValidateLoadedDriver verify if the driver was loaded successfully
func ValidateLoadedDriver() bool {
	if driverXOG == nil {
		return false
	}
	return len(driverXOG.Files) > 0
}

//GetDriversList returns a list of available drivers in the defined folder
func GetDriversList(folder string) ([]model.Driver, error) {
	driversFileList, err := ioutil.ReadDir(folder)

	if err != nil || len(driversFileList) == 0 {
		return nil, errors.New("driver folder not found or empty")
	}

	var driversList []model.Driver
	for _, f := range driversFileList {
		driver := new(model.Driver)
		driver.Info = f
		driver.FilePath = folder + f.Name()
		driversList = append(driversList, *driver)
	}

	return driversList, nil
}

//ProcessDriverFile execute a xog files and return the output
func ProcessDriverFile(file *model.DriverFile, action, sourceFolder, outputFolder string, environments *model.Environments, soapFunc util.Soap) model.Output {
	output := model.Output{Code: constant.OutputSuccess, Debug: constant.Undefined}

	if action == constant.Migrate && file.Type != constant.TypeMigration {
		output.Code = constant.OutputWarning
		output.Debug = "Use action 'r' to this type(" + file.Type + ") of file"
		return output
	} else if action == constant.Read && file.Type == constant.TypeMigration {
		output.Code = constant.OutputWarning
		output.Debug = "Use action 'm' to this type(" + file.Type + ") of file"
		return output
	}

	if action == constant.Migrate {
		resp, err := migration.ReadDataFromExcel(file)
		if err != nil {
			output.Code = constant.OutputError
			output.Debug = err.Error()
			return output
		}
		file.SetXML(resp)
		file.Write(outputFolder)
		return output
	}

	err := file.InitXML(action, sourceFolder)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	if action == constant.Write {
		iniTagRegexpStr, endTagRegexpStr := file.TagCDATA()
		if iniTagRegexpStr != constant.Undefined && endTagRegexpStr != constant.Undefined {
			transformedString := transform.IncludeCDATA(file.GetXML(), iniTagRegexpStr, endTagRegexpStr)
			file.SetXML(transformedString)
		}
	}
	err = file.RunXML(action, sourceFolder, environments, soapFunc)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	xogResponse := etree.NewDocument()
	xogResponse.ReadFromString(file.GetXML())
	output, err = validate.Check(xogResponse)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	if action == constant.Read {
		output := processDriverFileRead(file, xogResponse, outputFolder)
		if output.Code != constant.OutputSuccess {
			return output
		}
	}

	file.Write(outputFolder)
	return output
}

func processDriverFileRead(file *model.DriverFile, xogResponse *etree.Document, outputFolder string) model.Output {
	output := model.Output{Code: constant.OutputSuccess, Debug: constant.Undefined}

	var auxResponse *etree.Document
	if file.NeedAuxXML() {
		auxResponse = etree.NewDocument()
		auxResponse.ReadFromString(file.GetAuxXML())
		output, err := validate.Check(auxResponse)
		if err != nil {
			output.Code = constant.OutputError
			output.Debug = "aux validation - " + err.Error()
			return output
		}
	}
	err := transform.Execute(xogResponse, auxResponse, file)

	if file.GetInstanceTag() != constant.Undefined && file.InstancesPerFile > 0 {
		instanceTagPath := "//" + file.GetInstanceTag()

		parentTagPath := "//" + xogResponse.FindElement(instanceTagPath).Parent().Tag

		splitXogResponse := xogResponse.Copy()
		for _, e := range splitXogResponse.FindElements(instanceTagPath) {
			e.Parent().RemoveChild(e)
		}
		instances := xogResponse.FindElements(instanceTagPath)

		totalFiles := math.Ceil(float64(len(instances)) / float64(file.InstancesPerFile))
		path := util.GetPathWithoutExtension(file.Path)
		for i := 0; i < int(totalFiles); i++ {
			file.Path = fmt.Sprintf("%s_%03d.xml", path, i+1)
			xog := splitXogResponse.Copy()
			elementParent := xog.FindElement(parentTagPath)
			for z := file.InstancesPerFile * i; z < file.InstancesPerFile*(i+1); z++ {
				if z < len(instances) {
					elementParent.AddChild(instances[z].Copy())
				}
			}
			xog.IndentTabs()
			s, _ := xog.WriteToString()
			file.SetXML(s)
			file.Write(outputFolder)
		}
		file.Path = "complete_" + path + ".xml"
	}

	str, _ := xogResponse.WriteToString()
	file.SetXML(str)
	if err != nil {
		output.Code = constant.OutputError
		output.Debug = err.Error()
		return output
	}
	iniTagRegexpStr, endTagRegexpStr := file.TagCDATA()
	if iniTagRegexpStr != constant.Undefined && endTagRegexpStr != constant.Undefined {
		transformedString := transform.IncludeCDATA(file.GetXML(), iniTagRegexpStr, endTagRegexpStr)
		file.SetXML(transformedString)
	}

	if file.ExportToExcel {
		migration.ExportInstancesToExcel(xogResponse, file, constant.FolderMigration)
	}

	return output
}

//CreateFileFolder creates the folders according to the action (read, write or migrate)
func CreateFileFolder(action, fileType, path string) (string, string) {
	sourceFolder := constant.Undefined
	outputFolder := constant.Undefined

	switch action {
	case constant.Read:
		sourceFolder = constant.FolderRead
		outputFolder = constant.FolderWrite
		util.ValidateFolder(constant.FolderRead + fileType + util.GetPathFolder(path))
	case constant.Write:
		sourceFolder = constant.FolderWrite
		outputFolder = constant.FolderDebug
	case constant.Migrate:
		sourceFolder = constant.FolderWrite
		outputFolder = constant.FolderMigration
	}

	util.ValidateFolder(outputFolder + fileType + util.GetPathFolder(path))

	return sourceFolder, outputFolder
}
<|endoftext|>