<commit_before>package main

import (
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/urfave/cli"
	"golang.org/x/xerrors"
)

func TestCommandList(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestCommandListUnique(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		flagSet.Parse([]string{"--unique"})
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestCommandListUnknown(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		flagSet.Parse([]string{"--unknown-flag"})
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func sortLines(s string) string {
	ss := strings.Split(s, "\n")
	sort.Strings(ss)
	return strings.Join(ss, "\n")
}

func TestDoList_query(t *testing.T) {
	gitRepos := []string{
		"github.com/motemen/ghq",
		"github.com/motemen/gobump",
		"github.com/motemen/gore",
		"github.com/Songmu/gobump",
		"golang.org/x/crypt",
		"golang.org/x/image",
	}
	svnRepos := []string{
		"github.com/msh5/svntest",
	}
	testCases := []struct {
		name   string
		args   []string
		expect string
	}{{
		name:   "repo match",
		args:   []string{"ghq"},
		expect: "github.com/motemen/ghq\n",
	}, {
		name:   "unique",
		args:   []string{"--unique", "ghq"},
		expect: "ghq\n",
	}, {
		name:   "host only doesn't match",
		args:   []string{"github.com"},
		expect: "",
	}, {
		name:   "host and slash match",
		args:   []string{"golang.org/"},
		expect: "golang.org/x/crypt\ngolang.org/x/image\n",
	}, {
		name:   "host and user",
		args:   []string{"github.com/Songmu"},
		expect: "github.com/Songmu/gobump\n",
	}, {
		name:   "with scheme",
		args:   []string{"https://github.com/motemen/ghq"},
		expect: "github.com/motemen/ghq\n",
	}, {
		name:   "exact",
		args:   []string{"-e", "gobump"},
		expect: "github.com/Songmu/gobump\ngithub.com/motemen/gobump\n",
	}, {
		name:   "query",
		args:   []string{"men/go"},
		expect: "github.com/motemen/gobump\ngithub.com/motemen/gore\n",
	}, {
		name:   "exact query",
		args:   []string{"-e", "men/go"},
		expect: "",
	}, {
		name:   "vcs",
		args:   []string{"--vcs", "svn"},
		expect: "github.com/msh5/svntest\n",
	}}

	withFakeGitBackend(t, func(t *testing.T, tmproot string, _ *_cloneArgs, _ *_updateArgs) {
		for _, r := range gitRepos {
			os.MkdirAll(filepath.Join(tmproot, r, ".git"), 0755)
		}
		for _, r := range svnRepos {
			os.MkdirAll(filepath.Join(tmproot, r, ".svn"), 0755)
		}
		for _, tc := range testCases {
			t.Run(tc.name, func(t *testing.T) {
				args := append([]string{"ghq", "list"}, tc.args...)
				out, _, _ := capture(func() {
					newApp().Run(args)
				})
				if sortLines(out) != sortLines(tc.expect) {
					t.Errorf("got:\n%s\nexpect:\n%s", out, tc.expect)
				}
				if strings.Contains(tc.name, "unique") {
					return
				}
				argsFull := append([]string{"ghq", "list", "--full-path"}, tc.args...)
				fullExpect := tc.expect
				if fullExpect != "" {
					fullExpect = tmproot + string(filepath.Separator) + strings.TrimSpace(fullExpect)
					fullExpect = strings.ReplaceAll(fullExpect, "\n", "\n"+tmproot+string(filepath.Separator))
					fullExpect += "\n"
				}
				out, _, _ = capture(func() {
					newApp().Run(argsFull)
				})
				if sortLines(out) != sortLines(fullExpect) {
					t.Errorf("got:\n%s\nexpect:\n%s", out, fullExpect)
				}
			})
		}
	})
}

func TestDoList_unique(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	defer func(orig string) { os.Setenv("GHQ_ROOT", orig) }(os.Getenv("GHQ_ROOT"))

	tmp1 := newTempDir(t)
	defer os.RemoveAll(tmp1)
	tmp2 := newTempDir(t)
	defer os.RemoveAll(tmp2)

	_localRepositoryRoots = nil
	rootPaths := []string{tmp1, tmp2}
	os.Setenv("GHQ_ROOT", strings.Join(rootPaths, string(os.PathListSeparator)))
	for _, rootPath := range rootPaths {
		os.MkdirAll(filepath.Join(rootPath, "github.com/motemen/ghq/.git"), 0755)
	}
	out, _, _ := capture(func() {
		newApp().Run([]string{"ghq", "list", "--unique"})
	})
	if out != "ghq\n" {
		t.Errorf("got: %s, expect: ghq\n", out)
	}
}

func TestDoList_unknownRoot(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	defer tmpEnv("GHQ_ROOT", "/path/to/unknown-ghq")()
	_localRepositoryRoots = nil

	err := newApp().Run([]string{"ghq", "list"})
	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestDoList_notPermittedRoot(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	tmpdir := newTempDir(t)
	defer func(dir string) {
		os.Chmod(dir, 0755)
		os.RemoveAll(dir)
	}(tmpdir)
	defer tmpEnv("GHQ_ROOT", tmpdir)()

	_localRepositoryRoots = nil
	os.Chmod(tmpdir, 0000)

	err := newApp().Run([]string{"ghq", "list"})
	if !os.IsPermission(xerrors.Unwrap(err)) {
		t.Errorf("error should be ErrPermission, but: %#+v", err)
	}
}

func TestDoList_withSystemHiddenDir(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	tmpdir := newTempDir(t)
	systemHidden := filepath.Join(tmpdir, ".system")
	os.MkdirAll(systemHidden, 0000)
	defer func(dir string) {
		os.Chmod(systemHidden, 0755)
		os.RemoveAll(dir)
	}(tmpdir)
	defer tmpEnv("GHQ_ROOT", tmpdir)()

	_localRepositoryRoots = nil

	err := newApp().Run([]string{"ghq", "list"})
	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}
<commit_msg>Fix test<commit_after>package main

import (
	"os"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/urfave/cli"
	"golang.org/x/xerrors"
)

func TestCommandList(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestCommandListUnique(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		flagSet.Parse([]string{"--unique"})
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestCommandListUnknown(t *testing.T) {
	_, _, err := capture(func() {
		app := cli.NewApp()
		flagSet := flagSet("list", commandList.Flags)
		flagSet.Parse([]string{"--unknown-flag"})
		c := cli.NewContext(app, flagSet, nil)

		doList(c)
	})

	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func sortLines(s string) string {
	ss := strings.Split(s, "\n")
	sort.Strings(ss)
	return strings.Join(ss, "\n")
}

func TestDoList_query(t *testing.T) {
	gitRepos := []string{
		"github.com/motemen/ghq",
		"github.com/motemen/gobump",
		"github.com/motemen/gore",
		"github.com/Songmu/gobump",
		"golang.org/x/crypt",
		"golang.org/x/image",
	}
	svnRepos := []string{
		"github.com/msh5/svntest",
	}
	testCases := []struct {
		name   string
		args   []string
		expect string
	}{{
		name:   "repo match",
		args:   []string{"ghq"},
		expect: "github.com/motemen/ghq\n",
	}, {
		name:   "unique",
		args:   []string{"--unique", "ghq"},
		expect: "ghq\n",
	}, {
		name:   "host only doesn't match",
		args:   []string{"github.com"},
		expect: "",
	}, {
		name:   "host and slash match",
		args:   []string{"golang.org/"},
		expect: "golang.org/x/crypt\ngolang.org/x/image\n",
	}, {
		name:   "host and user",
		args:   []string{"github.com/Songmu"},
		expect: "github.com/Songmu/gobump\n",
	}, {
		name:   "with scheme",
		args:   []string{"https://github.com/motemen/ghq"},
		expect: "github.com/motemen/ghq\n",
	}, {
		name:   "exact",
		args:   []string{"-e", "gobump"},
		expect: "github.com/Songmu/gobump\ngithub.com/motemen/gobump\n",
	}, {
		name:   "query",
		args:   []string{"men/go"},
		expect: "github.com/motemen/gobump\ngithub.com/motemen/gore\n",
	}, {
		name:   "exact query",
		args:   []string{"-e", "men/go"},
		expect: "",
	}, {
		name:   "vcs",
		args:   []string{"--vcs", "svn"},
		expect: "github.com/msh5/svntest\n",
	}}

	withFakeGitBackend(t, func(t *testing.T, tmproot string, _ *_cloneArgs, _ *_updateArgs) {
		for _, r := range gitRepos {
			os.MkdirAll(filepath.Join(tmproot, r, ".git"), 0755)
		}
		for _, r := range svnRepos {
			os.MkdirAll(filepath.Join(tmproot, r, ".svn"), 0755)
		}
		for _, tc := range testCases {
			t.Run(tc.name, func(t *testing.T) {
				args := append([]string{"ghq", "list"}, tc.args...)
				out, _, _ := capture(func() {
					newApp().Run(args)
				})
				if sortLines(out) != sortLines(tc.expect) {
					t.Errorf("got:\n%s\nexpect:\n%s", out, tc.expect)
				}
				if strings.Contains(tc.name, "unique") {
					return
				}
				argsFull := append([]string{"ghq", "list", "--full-path"}, tc.args...)
				fullExpect := tc.expect
				if fullExpect != "" {
					fullExpect = tmproot + string(filepath.Separator) + strings.TrimSpace(fullExpect)
					fullExpect = strings.ReplaceAll(fullExpect, "\n", "\n"+tmproot+string(filepath.Separator))
					fullExpect += "\n"
				}
				out, _, _ = capture(func() {
					newApp().Run(argsFull)
				})
				if sortLines(out) != sortLines(fullExpect) {
					t.Errorf("got:\n%s\nexpect:\n%s", out, fullExpect)
				}
			})
		}
	})
}

func TestDoList_unique(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	defer func(orig string) { os.Setenv("GHQ_ROOT", orig) }(os.Getenv("GHQ_ROOT"))

	tmp1 := newTempDir(t)
	defer os.RemoveAll(tmp1)
	tmp2 := newTempDir(t)
	defer os.RemoveAll(tmp2)

	_localRepositoryRoots = nil
	rootPaths := []string{tmp1, tmp2}
	os.Setenv("GHQ_ROOT", strings.Join(rootPaths, string(os.PathListSeparator)))
	for _, rootPath := range rootPaths {
		os.MkdirAll(filepath.Join(rootPath, "github.com/motemen/ghq/.git"), 0755)
	}
	out, _, _ := capture(func() {
		newApp().Run([]string{"ghq", "list", "--unique"})
	})
	if out != "ghq\n" {
		t.Errorf("got: %s, expect: ghq\n", out)
	}
}

func TestDoList_unknownRoot(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	defer tmpEnv("GHQ_ROOT", "/path/to/unknown-ghq")()
	_localRepositoryRoots = nil

	err := newApp().Run([]string{"ghq", "list"})
	if err != nil {
		t.Errorf("error should be nil, but: %v", err)
	}
}

func TestDoList_notPermittedRoot(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	tmpdir := newTempDir(t)
	defer func(dir string) {
		os.Chmod(dir, 0755)
		os.RemoveAll(dir)
	}(tmpdir)
	defer tmpEnv("GHQ_ROOT", tmpdir)()

	_localRepositoryRoots = nil
	os.Chmod(tmpdir, 0000)

	err := newApp().Run([]string{"ghq", "list"})
	if !os.IsPermission(xerrors.Unwrap(err)) {
		t.Errorf("error should be ErrPermission, but: %#+v", err)
	}
}

func TestDoList_withSystemHiddenDir(t *testing.T) {
	defer func(orig []string) { _localRepositoryRoots = orig }(_localRepositoryRoots)
	tmpdir := newTempDir(t)
	systemHidden := filepath.Join(tmpdir, ".system")
	os.MkdirAll(systemHidden, 0000)
	defer func(dir string) {
		os.Chmod(systemHidden, 0755)
		os.RemoveAll(dir)
	}(tmpdir)
	defer tmpEnv("GHQ_ROOT", tmpdir)()

	_localRepositoryRoots = nil

	err := newApp().Run([]string{"ghq", "list"})
	if err == nil {
		t.Errorf("should not be error")
	}
}
<|endoftext|>


<commit_before>package blocktree

import (
	"errors"
	"fmt"
	"github.com/balzaczyy/golucene/core/codec"
	. "github.com/balzaczyy/golucene/core/codec/spi"
	. "github.com/balzaczyy/golucene/core/index/model"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"log"
)

// BlockTreeTermsReader.java

const (
	BTT_OUTPUT_FLAGS_NUM_BITS = 2
	BTT_OUTPUT_FLAG_IS_FLOOR  = 1
	BTT_OUTPUT_FLAG_HAS_TERMS = 2

	// BTT_INDEX_EXTENSION           = "tip"
	// BTT_INDEX_CODEC_NAME          = "BLOCK_TREE_TERMS_INDEX"
	// BTT_INDEX_VERSION_START       = 0
	// BTT_INDEX_VERSION_APPEND_ONLY = 1
	// BTT_INDEX_VERSION_CURRENT     = BTT_INDEX_VERSION_APPEND_ONLY
)

/* A block-based terms index and dictionary that assigns
terms to variable length blocks according to how they
share prefixes. The terms index is a prefix trie
whose leaves are term blocks. The advantage of this
approach is that seekExact is often able to
determine a term cannot exist without doing any IO, and
intersection with Automata is very fast. NOte that this
terms dictionary has its own fixed terms index (ie, it
does not support a pluggable terms index
implementation).

NOTE: this terms dictionary does not support
index divisor when opening an IndexReader. Instead, you
can change the min/maxItemsPerBlock during indexing.

The data strucure used by this implementation is very
similar to a [burst trie]
(http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.18.3499),
but with added logic to break up too-large blocks of all
terms sharing a given prefix into smaller ones.

Use CheckIndex with the -verbose
option to see summary statistics on the blocks in the
dictionary. */
type BlockTreeTermsReader struct {
	// Open input to the main terms dict file (_X.tib)
	in store.IndexInput
	// Reads the terms dict entries, to gather state to
	// produce DocsEnum on demand
	postingsReader PostingsReaderBase
	fields         map[string]FieldReader
	// File offset where the directory starts in the terms file.
	dirOffset int64
	// File offset where the directory starts in the index file.
	indexDirOffset int64
	segment        string
	version        int
}

func NewBlockTreeTermsReader(dir store.Directory,
	fieldInfos FieldInfos, info *SegmentInfo,
	postingsReader PostingsReaderBase, ctx store.IOContext,
	segmentSuffix string, indexDivisor int) (p FieldsProducer, err error) {

	log.Print("Initializing BlockTreeTermsReader...")
	fp := &BlockTreeTermsReader{
		postingsReader: postingsReader,
		fields:         make(map[string]FieldReader),
		segment:        info.Name,
	}
	fp.in, err = dir.OpenInput(util.SegmentFileName(info.Name, segmentSuffix, TERMS_EXTENSION), ctx)
	if err != nil {
		return nil, err
	}

	success := false
	var indexIn store.IndexInput
	defer func() {
		if !success {
			fmt.Println("Failed to initialize BlockTreeTermsReader.")
			if err != nil {
				fmt.Println("DEBUG ", err)
			}
			// this.close() will close in:
			util.CloseWhileSuppressingError(indexIn, fp)
		}
	}()

	fp.version, err = fp.readHeader(fp.in)
	if err != nil {
		return nil, err
	}
	log.Printf("Version: %v", fp.version)

	if indexDivisor != -1 {
		indexIn, err = dir.OpenInput(util.SegmentFileName(info.Name, segmentSuffix, TERMS_EXTENSION), ctx)
		if err != nil {
			return nil, err
		}

		indexVersion, err := fp.readIndexHeader(indexIn)
		if err != nil {
			return nil, err
		}
		log.Printf("Index version: %v", indexVersion)
		if int(indexVersion) != fp.version {
			return nil, errors.New(fmt.Sprintf("mixmatched version files: %v=%v,%v=%v", fp.in, fp.version, indexIn, indexVersion))
		}
	}

	// verify
	if indexIn != nil && fp.version >= TERMS_VERSION_CURRENT {
		if _, err = store.ChecksumEntireFile(indexIn); err != nil {
			return nil, err
		}
	}

	// Have PostingsReader init itself
	postingsReader.Init(fp.in)

	// Read per-field details
	fp.seekDir(fp.in, fp.dirOffset)
	if indexDivisor != -1 {
		fp.seekDir(indexIn, fp.indexDirOffset)
	}

	numFields, err := fp.in.ReadVInt()
	if err != nil {
		return nil, err
	}
	log.Printf("Fields number: %v", numFields)
	if numFields < 0 {
		return nil, errors.New(fmt.Sprintf("invalid numFields: %v (resource=%v)", numFields, fp.in))
	}

	for i := int32(0); i < numFields; i++ {
		log.Printf("Next field...")
		field, err := fp.in.ReadVInt()
		if err != nil {
			return nil, err
		}
		log.Printf("Field: %v", field)

		numTerms, err := fp.in.ReadVLong()
		if err != nil {
			return nil, err
		}
		// assert numTerms >= 0
		log.Printf("Terms number: %v", numTerms)

		numBytes, err := fp.in.ReadVInt()
		if err != nil {
			return nil, err
		}
		log.Printf("Bytes number: %v", numBytes)

		rootCode := make([]byte, numBytes)
		err = fp.in.ReadBytes(rootCode)
		if err != nil {
			return nil, err
		}
		fieldInfo := fieldInfos.FieldInfoByNumber(int(field))
		// assert fieldInfo != nil
		var sumTotalTermFreq int64
		if fieldInfo.IndexOptions() == INDEX_OPT_DOCS_ONLY {
			sumTotalTermFreq = -1
		} else {
			sumTotalTermFreq, err = fp.in.ReadVLong()
			if err != nil {
				return nil, err
			}
		}
		sumDocFreq, err := fp.in.ReadVLong()
		if err != nil {
			return nil, err
		}
		var docCount int
		if docCount, err = asInt(fp.in.ReadVInt()); err != nil {
			return nil, err
		}
		fmt.Printf("DocCount: %v\n", docCount)
		var longsSize int
		if fp.version >= TERMS_VERSION_META_ARRAY {
			if longsSize, err = asInt(fp.in.ReadVInt()); err != nil {
				return nil, err
			}
		}
		var minTerm, maxTerm []byte
		if fp.version >= TERMS_VERSION_MIN_MAX_TERMS {
			if minTerm, err = readBytesRef(fp.in); err != nil {
				return nil, err
			}
			if maxTerm, err = readBytesRef(fp.in); err != nil {
				return nil, err
			}
		}
		if docCount < 0 || int(docCount) > info.DocCount() { // #docs with field must be <= #docs
			return nil, errors.New(fmt.Sprintf(
				"invalid docCount: %v maxDoc: %v (resource=%v)",
				docCount, info.DocCount(), fp.in))
		}
		if sumDocFreq < int64(docCount) { // #postings must be >= #docs with field
			return nil, errors.New(fmt.Sprintf(
				"invalid sumDocFreq: %v docCount: %v (resource=%v)",
				sumDocFreq, docCount, fp.in))
		}
		if sumTotalTermFreq != -1 && sumTotalTermFreq < sumDocFreq { // #positions must be >= #postings
			return nil, errors.New(fmt.Sprintf(
				"invalid sumTotalTermFreq: %v sumDocFreq: %v (resource=%v)",
				sumTotalTermFreq, sumDocFreq, fp.in))
		}

		var indexStartFP int64
		if indexDivisor != -1 {
			if indexStartFP, err = indexIn.ReadVLong(); err != nil {
				return nil, err
			}
		}
		log.Printf("indexStartFP: %v", indexStartFP)
		if _, ok := fp.fields[fieldInfo.Name]; ok {
			return nil, errors.New(fmt.Sprintf(
				"duplicate field: %v (resource=%v)", fieldInfo.Name, fp.in))
		}
		if fp.fields[fieldInfo.Name], err = newFieldReader(fp,
			fieldInfo, numTerms, rootCode, sumTotalTermFreq,
			sumDocFreq, docCount, indexStartFP, longsSize,
			indexIn, minTerm, maxTerm); err != nil {
			return nil, err
		}
	}

	if indexDivisor != -1 {
		if err = indexIn.Close(); err != nil {
			return nil, err
		}
	}

	success = true

	return fp, nil
}

func asInt(n int32, err error) (n2 int, err2 error) {
	return int(n), err
}

func readBytesRef(in store.IndexInput) ([]byte, error) {
	panic("not implemented yet")
}

func (r *BlockTreeTermsReader) readHeader(input store.IndexInput) (version int, err error) {
	fmt.Println("DEBUG1")
	version, err = asInt(codec.CheckHeader(input, TERMS_CODEC_NAME, TERMS_VERSION_START, TERMS_VERSION_CURRENT))
	if err != nil {
		return int(version), err
	}
	fmt.Println("DEBUG2")
	if version < TERMS_VERSION_APPEND_ONLY {
		r.dirOffset, err = input.ReadLong()
		if err != nil {
			return int(version), err
		}
	}
	return int(version), nil
}

func (r *BlockTreeTermsReader) readIndexHeader(input store.IndexInput) (version int, err error) {
	version, err = asInt(codec.CheckHeader(input, TERMS_CODEC_NAME, TERMS_VERSION_START, TERMS_VERSION_CURRENT))
	if err != nil {
		return version, err
	}
	if version < TERMS_VERSION_APPEND_ONLY {
		r.indexDirOffset, err = input.ReadLong()
		if err != nil {
			return version, err
		}
	}
	return version, nil
}

func (r *BlockTreeTermsReader) seekDir(input store.IndexInput, dirOffset int64) (err error) {
	log.Printf("Seeking to: %v", dirOffset)
	if r.version >= TERMS_VERSION_CHECKSUM {
		if err = input.Seek(input.Length() - codec.FOOTER_LENGTH - 8); err != nil {
			return
		}
		if dirOffset, err = input.ReadLong(); err != nil {
			return
		}
	} else if r.version >= TERMS_VERSION_APPEND_ONLY {
		input.Seek(input.Length() - 8)
		if dirOffset, err = input.ReadLong(); err != nil {
			return
		}
	}
	return input.Seek(dirOffset)
}

func (r *BlockTreeTermsReader) Terms(field string) Terms {
	ans := r.fields[field]
	return &ans
}

func (r *BlockTreeTermsReader) Close() error {
	defer func() {
		// Clear so refs to terms index is GCable even if
		// app hangs onto us:
		r.fields = make(map[string]FieldReader)
	}()
	return util.Close(r.in, r.postingsReader)
}
<commit_msg>implement BTT/readBytesRef()<commit_after>package blocktree

import (
	"errors"
	"fmt"
	"github.com/balzaczyy/golucene/core/codec"
	. "github.com/balzaczyy/golucene/core/codec/spi"
	. "github.com/balzaczyy/golucene/core/index/model"
	"github.com/balzaczyy/golucene/core/store"
	"github.com/balzaczyy/golucene/core/util"
	"log"
)

// BlockTreeTermsReader.java

const (
	BTT_OUTPUT_FLAGS_NUM_BITS = 2
	BTT_OUTPUT_FLAG_IS_FLOOR  = 1
	BTT_OUTPUT_FLAG_HAS_TERMS = 2

	// BTT_INDEX_EXTENSION           = "tip"
	// BTT_INDEX_CODEC_NAME          = "BLOCK_TREE_TERMS_INDEX"
	// BTT_INDEX_VERSION_START       = 0
	// BTT_INDEX_VERSION_APPEND_ONLY = 1
	// BTT_INDEX_VERSION_CURRENT     = BTT_INDEX_VERSION_APPEND_ONLY
)

/* A block-based terms index and dictionary that assigns
terms to variable length blocks according to how they
share prefixes. The terms index is a prefix trie
whose leaves are term blocks. The advantage of this
approach is that seekExact is often able to
determine a term cannot exist without doing any IO, and
intersection with Automata is very fast. NOte that this
terms dictionary has its own fixed terms index (ie, it
does not support a pluggable terms index
implementation).

NOTE: this terms dictionary does not support
index divisor when opening an IndexReader. Instead, you
can change the min/maxItemsPerBlock during indexing.

The data strucure used by this implementation is very
similar to a [burst trie]
(http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.18.3499),
but with added logic to break up too-large blocks of all
terms sharing a given prefix into smaller ones.

Use CheckIndex with the -verbose
option to see summary statistics on the blocks in the
dictionary. */
type BlockTreeTermsReader struct {
	// Open input to the main terms dict file (_X.tib)
	in store.IndexInput
	// Reads the terms dict entries, to gather state to
	// produce DocsEnum on demand
	postingsReader PostingsReaderBase
	fields         map[string]FieldReader
	// File offset where the directory starts in the terms file.
	dirOffset int64
	// File offset where the directory starts in the index file.
	indexDirOffset int64
	segment        string
	version        int
}

func NewBlockTreeTermsReader(dir store.Directory,
	fieldInfos FieldInfos, info *SegmentInfo,
	postingsReader PostingsReaderBase, ctx store.IOContext,
	segmentSuffix string, indexDivisor int) (p FieldsProducer, err error) {

	log.Print("Initializing BlockTreeTermsReader...")
	fp := &BlockTreeTermsReader{
		postingsReader: postingsReader,
		fields:         make(map[string]FieldReader),
		segment:        info.Name,
	}
	fp.in, err = dir.OpenInput(util.SegmentFileName(info.Name, segmentSuffix, TERMS_EXTENSION), ctx)
	if err != nil {
		return nil, err
	}

	success := false
	var indexIn store.IndexInput
	defer func() {
		if !success {
			fmt.Println("Failed to initialize BlockTreeTermsReader.")
			if err != nil {
				fmt.Println("DEBUG ", err)
			}
			// this.close() will close in:
			util.CloseWhileSuppressingError(indexIn, fp)
		}
	}()

	fp.version, err = fp.readHeader(fp.in)
	if err != nil {
		return nil, err
	}
	log.Printf("Version: %v", fp.version)

	if indexDivisor != -1 {
		indexIn, err = dir.OpenInput(util.SegmentFileName(info.Name, segmentSuffix, TERMS_EXTENSION), ctx)
		if err != nil {
			return nil, err
		}

		indexVersion, err := fp.readIndexHeader(indexIn)
		if err != nil {
			return nil, err
		}
		log.Printf("Index version: %v", indexVersion)
		if int(indexVersion) != fp.version {
			return nil, errors.New(fmt.Sprintf("mixmatched version files: %v=%v,%v=%v", fp.in, fp.version, indexIn, indexVersion))
		}
	}

	// verify
	if indexIn != nil && fp.version >= TERMS_VERSION_CURRENT {
		if _, err = store.ChecksumEntireFile(indexIn); err != nil {
			return nil, err
		}
	}

	// Have PostingsReader init itself
	postingsReader.Init(fp.in)

	// Read per-field details
	fp.seekDir(fp.in, fp.dirOffset)
	if indexDivisor != -1 {
		fp.seekDir(indexIn, fp.indexDirOffset)
	}

	numFields, err := fp.in.ReadVInt()
	if err != nil {
		return nil, err
	}
	log.Printf("Fields number: %v", numFields)
	if numFields < 0 {
		return nil, errors.New(fmt.Sprintf("invalid numFields: %v (resource=%v)", numFields, fp.in))
	}

	for i := int32(0); i < numFields; i++ {
		log.Printf("Next field...")
		field, err := fp.in.ReadVInt()
		if err != nil {
			return nil, err
		}
		log.Printf("Field: %v", field)

		numTerms, err := fp.in.ReadVLong()
		if err != nil {
			return nil, err
		}
		// assert numTerms >= 0
		log.Printf("Terms number: %v", numTerms)

		numBytes, err := fp.in.ReadVInt()
		if err != nil {
			return nil, err
		}
		log.Printf("Bytes number: %v", numBytes)

		rootCode := make([]byte, numBytes)
		err = fp.in.ReadBytes(rootCode)
		if err != nil {
			return nil, err
		}
		fieldInfo := fieldInfos.FieldInfoByNumber(int(field))
		// assert fieldInfo != nil
		var sumTotalTermFreq int64
		if fieldInfo.IndexOptions() == INDEX_OPT_DOCS_ONLY {
			sumTotalTermFreq = -1
		} else {
			sumTotalTermFreq, err = fp.in.ReadVLong()
			if err != nil {
				return nil, err
			}
		}
		sumDocFreq, err := fp.in.ReadVLong()
		if err != nil {
			return nil, err
		}
		var docCount int
		if docCount, err = asInt(fp.in.ReadVInt()); err != nil {
			return nil, err
		}
		fmt.Printf("DocCount: %v\n", docCount)
		var longsSize int
		if fp.version >= TERMS_VERSION_META_ARRAY {
			if longsSize, err = asInt(fp.in.ReadVInt()); err != nil {
				return nil, err
			}
		}
		var minTerm, maxTerm []byte
		if fp.version >= TERMS_VERSION_MIN_MAX_TERMS {
			if minTerm, err = readBytesRef(fp.in); err != nil {
				return nil, err
			}
			if maxTerm, err = readBytesRef(fp.in); err != nil {
				return nil, err
			}
		}
		if docCount < 0 || int(docCount) > info.DocCount() { // #docs with field must be <= #docs
			return nil, errors.New(fmt.Sprintf(
				"invalid docCount: %v maxDoc: %v (resource=%v)",
				docCount, info.DocCount(), fp.in))
		}
		if sumDocFreq < int64(docCount) { // #postings must be >= #docs with field
			return nil, errors.New(fmt.Sprintf(
				"invalid sumDocFreq: %v docCount: %v (resource=%v)",
				sumDocFreq, docCount, fp.in))
		}
		if sumTotalTermFreq != -1 && sumTotalTermFreq < sumDocFreq { // #positions must be >= #postings
			return nil, errors.New(fmt.Sprintf(
				"invalid sumTotalTermFreq: %v sumDocFreq: %v (resource=%v)",
				sumTotalTermFreq, sumDocFreq, fp.in))
		}

		var indexStartFP int64
		if indexDivisor != -1 {
			if indexStartFP, err = indexIn.ReadVLong(); err != nil {
				return nil, err
			}
		}
		log.Printf("indexStartFP: %v", indexStartFP)
		if _, ok := fp.fields[fieldInfo.Name]; ok {
			return nil, errors.New(fmt.Sprintf(
				"duplicate field: %v (resource=%v)", fieldInfo.Name, fp.in))
		}
		if fp.fields[fieldInfo.Name], err = newFieldReader(fp,
			fieldInfo, numTerms, rootCode, sumTotalTermFreq,
			sumDocFreq, docCount, indexStartFP, longsSize,
			indexIn, minTerm, maxTerm); err != nil {
			return nil, err
		}
	}

	if indexDivisor != -1 {
		if err = indexIn.Close(); err != nil {
			return nil, err
		}
	}

	success = true

	return fp, nil
}

func asInt(n int32, err error) (n2 int, err2 error) {
	return int(n), err
}

func readBytesRef(in store.IndexInput) ([]byte, error) {
	length, err := asInt(in.ReadVInt())
	if err != nil {
		return nil, err
	}
	bytes := make([]byte, length)
	if err = in.ReadBytes(bytes); err != nil {
		return nil, err
	}
	return bytes, nil
}

func (r *BlockTreeTermsReader) readHeader(input store.IndexInput) (version int, err error) {
	version, err = asInt(codec.CheckHeader(input, TERMS_CODEC_NAME, TERMS_VERSION_START, TERMS_VERSION_CURRENT))
	if err != nil {
		return int(version), err
	}
	if version < TERMS_VERSION_APPEND_ONLY {
		r.dirOffset, err = input.ReadLong()
		if err != nil {
			return int(version), err
		}
	}
	return int(version), nil
}

func (r *BlockTreeTermsReader) readIndexHeader(input store.IndexInput) (version int, err error) {
	version, err = asInt(codec.CheckHeader(input, TERMS_CODEC_NAME, TERMS_VERSION_START, TERMS_VERSION_CURRENT))
	if err != nil {
		return version, err
	}
	if version < TERMS_VERSION_APPEND_ONLY {
		r.indexDirOffset, err = input.ReadLong()
		if err != nil {
			return version, err
		}
	}
	return version, nil
}

func (r *BlockTreeTermsReader) seekDir(input store.IndexInput, dirOffset int64) (err error) {
	log.Printf("Seeking to: %v", dirOffset)
	if r.version >= TERMS_VERSION_CHECKSUM {
		if err = input.Seek(input.Length() - codec.FOOTER_LENGTH - 8); err != nil {
			return
		}
		if dirOffset, err = input.ReadLong(); err != nil {
			return
		}
	} else if r.version >= TERMS_VERSION_APPEND_ONLY {
		input.Seek(input.Length() - 8)
		if dirOffset, err = input.ReadLong(); err != nil {
			return
		}
	}
	return input.Seek(dirOffset)
}

func (r *BlockTreeTermsReader) Terms(field string) Terms {
	ans := r.fields[field]
	return &ans
}

func (r *BlockTreeTermsReader) Close() error {
	defer func() {
		// Clear so refs to terms index is GCable even if
		// app hangs onto us:
		r.fields = make(map[string]FieldReader)
	}()
	return util.Close(r.in, r.postingsReader)
}
<|endoftext|>


<commit_before>// Package iface defines IPFS Core API which is a set of interfaces used to
// interact with IPFS nodes.
package iface

import (
	"context"
	"errors"
	"io"
	"time"

	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"

	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)

// Path is a generic wrapper for paths used in the API. A path can be resolved
// to a CID using one of Resolve functions in the API.
type Path interface {
	// String returns the path as a string.
	String() string
	// Cid returns cid referred to by path
	Cid() *cid.Cid
	// Root returns cid of root path
	Root() *cid.Cid
	// Resolved returns whether path has been fully resolved
	Resolved() bool
}

// TODO: should we really copy these?
//       if we didn't, godoc would generate nice links straight to go-ipld-format
type Node ipld.Node
type Link ipld.Link

type Reader interface {
	io.ReadSeeker
	io.Closer
}

// IpnsEntry specifies the interface to IpnsEntries
type IpnsEntry interface {
	// Name returns IpnsEntry name
	Name() string
	// Value returns IpnsEntry value
	Value() Path
}

// Key specifies the interface to Keys in KeyAPI Keystore
type Key interface {
	// Key returns key name
	Name() string
	// Path returns key path
	Path() Path
}

type BlockStat interface {
	Size() int
	Path() Path
}

// CoreAPI defines an unified interface to IPFS for Go programs.
type CoreAPI interface {
	// Unixfs returns an implementation of Unixfs API.
	Unixfs() UnixfsAPI
	// Block returns an implementation of Block API.
	Block() BlockAPI
	// Dag returns an implementation of Dag API.
	Dag() DagAPI
	// Name returns an implementation of Name API.
	Name() NameAPI
	// Key returns an implementation of Key API.
	Key() KeyAPI

	// ObjectAPI returns an implementation of Object API
	Object() ObjectAPI

	// ResolvePath resolves the path using Unixfs resolver
	ResolvePath(context.Context, Path) (Path, error)

	// ResolveNode resolves the path (if not resolved already) using Unixfs
	// resolver, gets and returns the resolved Node
	ResolveNode(context.Context, Path) (Node, error)
}

// UnixfsAPI is the basic interface to immutable files in IPFS
type UnixfsAPI interface {
	// Add imports the data from the reader into merkledag file
	Add(context.Context, io.Reader) (Path, error)

	// Cat returns a reader for the file
	Cat(context.Context, Path) (Reader, error)

	// Ls returns the list of links in a directory
	Ls(context.Context, Path) ([]*Link, error)
}

type BlockAPI interface {
	Put(context.Context, io.Reader, ...options.BlockPutOption) (Path, error)
	WithFormat(codec string) options.BlockPutOption
	WithHash(mhType uint64, mhLen int) options.BlockPutOption

	Get(context.Context, Path) (io.Reader, error)

	Rm(context.Context, Path, ...options.BlockRmOption) error
	WithForce(force bool) options.BlockRmOption

	Stat(context.Context, Path) (BlockStat, error)
}

// DagAPI specifies the interface to IPLD
type DagAPI interface {
	// Put inserts data using specified format and input encoding.
	// Unless used with WithCodec or WithHash, the defaults "dag-cbor" and
	// "sha256" are used.
	Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (Path, error)

	// WithInputEnc is an option for Put which specifies the input encoding of the
	// data. Default is "json", most formats/codecs support "raw"
	WithInputEnc(enc string) options.DagPutOption

	// WithCodec is an option for Put which specifies the multicodec to use to
	// serialize the object. Default is cid.DagCBOR (0x71)
	WithCodec(codec uint64) options.DagPutOption

	// WithHash is an option for Put which specifies the multihash settings to use
	// when hashing the object. Default is based on the codec used
	// (mh.SHA2_256 (0x12) for DagCBOR). If mhLen is set to -1, default length for
	// the hash will be used
	WithHash(mhType uint64, mhLen int) options.DagPutOption

	// Get attempts to resolve and get the node specified by the path
	Get(ctx context.Context, path Path) (Node, error)

	// Tree returns list of paths within a node specified by the path.
	Tree(ctx context.Context, path Path, opts ...options.DagTreeOption) ([]Path, error)

	// WithDepth is an option for Tree which specifies maximum depth of the
	// returned tree. Default is -1 (no depth limit)
	WithDepth(depth int) options.DagTreeOption
}

// NameAPI specifies the interface to IPNS.
//
// IPNS is a PKI namespace, where names are the hashes of public keys, and the
// private key enables publishing new (signed) values. In both publish and
// resolve, the default name used is the node's own PeerID, which is the hash of
// its public key.
//
// You can use .Key API to list and generate more names and their respective keys.
type NameAPI interface {
	// Publish announces new IPNS name
	Publish(ctx context.Context, path Path, opts ...options.NamePublishOption) (IpnsEntry, error)

	// WithValidTime is an option for Publish which specifies for how long the
	// entry will remain valid. Default value is 24h
	WithValidTime(validTime time.Duration) options.NamePublishOption

	// WithKey is an option for Publish which specifies the key to use for
	// publishing. Default value is "self" which is the node's own PeerID.
	// The key parameter must be either PeerID or keystore key alias.
	//
	// You can use KeyAPI to list and generate more names and their respective keys.
	WithKey(key string) options.NamePublishOption

	// Resolve attempts to resolve the newest version of the specified name
	Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (Path, error)

	// WithRecursive is an option for Resolve which specifies whether to perform a
	// recursive lookup. Default value is false
	WithRecursive(recursive bool) options.NameResolveOption

	// WithLocal is an option for Resolve which specifies if the lookup should be
	// offline. Default value is false
	WithLocal(local bool) options.NameResolveOption

	// WithCache is an option for Resolve which specifies if cache should be used.
	// Default value is true
	WithCache(cache bool) options.NameResolveOption
}

// KeyAPI specifies the interface to Keystore
type KeyAPI interface {
	// Generate generates new key, stores it in the keystore under the specified
	// name and returns a base58 encoded multihash of it's public key
	Generate(ctx context.Context, name string, opts ...options.KeyGenerateOption) (Key, error)

	// WithType is an option for Generate which specifies which algorithm
	// should be used for the key. Default is options.RSAKey
	//
	// Supported key types:
	// * options.RSAKey
	// * options.Ed25519Key
	WithType(algorithm string) options.KeyGenerateOption

	// WithSize is an option for Generate which specifies the size of the key to
	// generated. Default is -1
	//
	// value of -1 means 'use default size for key type':
	//  * 2048 for RSA
	WithSize(size int) options.KeyGenerateOption

	// Rename renames oldName key to newName. Returns the key and whether another
	// key was overwritten, or an error
	Rename(ctx context.Context, oldName string, newName string, opts ...options.KeyRenameOption) (Key, bool, error)

	// WithForce is an option for Rename which specifies whether to allow to
	// replace existing keys.
	WithForce(force bool) options.KeyRenameOption

	// List lists keys stored in keystore
	List(ctx context.Context) ([]Key, error)

	// Remove removes keys from keystore. Returns ipns path of the removed key
	Remove(ctx context.Context, name string) (Path, error)
}

// ObjectAPI specifies the interface to MerkleDAG and contains useful utilities
// for manipulating MerkleDAG data structures.
type ObjectAPI interface {
	// New creates new, empty (by default) dag-node.
	New(context.Context, ...options.ObjectNewOption) (Node, error)

	// WithType is an option for New which allows to change the type of created
	// dag node.
	//
	// Supported types:
	// * 'empty' - Empty node
	// * 'unixfs-dir' - Empty UnixFS directory
	WithType(string) options.ObjectNewOption

	// Put imports the data into merkledag
	Put(context.Context, io.Reader, ...options.ObjectPutOption) (Path, error)

	// WithInputEnc is an option for Put which specifies the input encoding of the
	// data. Default is "json".
	//
	// Supported encodings:
	// * "protobuf"
	// * "json"
	WithInputEnc(e string) options.ObjectPutOption

	// WithDataType specifies the encoding of data field when using Josn or XML
	// input encoding.
	//
	// Supported types:
	// * "text" (default)
	// * "base64"
	WithDataType(t string) options.ObjectPutOption

	// Get returns the node for the path
	Get(context.Context, Path) (Node, error)

	// Data returns reader for data of the node
	Data(context.Context, Path) (io.Reader, error)

	// Links returns lint or links the node contains
	Links(context.Context, Path) ([]*Link, error)

	// Stat returns information about the node
	Stat(context.Context, Path) (*ObjectStat, error)

	// AddLink adds a link under the specified path. child path can point to a
	// subdirectory within the patent which must be present (can be overridden
	// with WithCreate option).
	AddLink(ctx context.Context, base Path, name string, child Path, opts ...options.ObjectAddLinkOption) (Path, error)

	// WithCreate is an option for AddLink which specifies whether create required
	// directories for the child
	WithCreate(create bool) options.ObjectAddLinkOption

	// RmLink removes a link from the node
	RmLink(ctx context.Context, base Path, link string) (Path, error)

	// AppendData appends data to the node
	AppendData(context.Context, Path, io.Reader) (Path, error)

	// SetData sets the data contained in the node
	SetData(context.Context, Path, io.Reader) (Path, error)
}

// ObjectStat provides information about dag nodes
type ObjectStat struct {
	// Cid is the CID of the node
	Cid *cid.Cid

	// NumLinks is number of links the node contains
	NumLinks int

	// BlockSize is size of the raw serialized node
	BlockSize int

	// LinksSize is size of the links block section
	LinksSize int

	// DataSize is the size of data block section
	DataSize int

	// CumulativeSize is size of the tree (BlockSize + link sizes)
	CumulativeSize int
}

var ErrIsDir = errors.New("object is a directory")
var ErrOffline = errors.New("can't resolve, ipfs node is offline")
<commit_msg>corapi: block docs<commit_after>// Package iface defines IPFS Core API which is a set of interfaces used to
// interact with IPFS nodes.
package iface

import (
	"context"
	"errors"
	"io"
	"time"

	options "github.com/ipfs/go-ipfs/core/coreapi/interface/options"

	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)

// Path is a generic wrapper for paths used in the API. A path can be resolved
// to a CID using one of Resolve functions in the API.
type Path interface {
	// String returns the path as a string.
	String() string
	// Cid returns cid referred to by path
	Cid() *cid.Cid
	// Root returns cid of root path
	Root() *cid.Cid
	// Resolved returns whether path has been fully resolved
	Resolved() bool
}

// TODO: should we really copy these?
//       if we didn't, godoc would generate nice links straight to go-ipld-format
type Node ipld.Node
type Link ipld.Link

type Reader interface {
	io.ReadSeeker
	io.Closer
}

// IpnsEntry specifies the interface to IpnsEntries
type IpnsEntry interface {
	// Name returns IpnsEntry name
	Name() string
	// Value returns IpnsEntry value
	Value() Path
}

// Key specifies the interface to Keys in KeyAPI Keystore
type Key interface {
	// Key returns key name
	Name() string
	// Path returns key path
	Path() Path
}

type BlockStat interface {
	Size() int
	Path() Path
}

// CoreAPI defines an unified interface to IPFS for Go programs.
type CoreAPI interface {
	// Unixfs returns an implementation of Unixfs API.
	Unixfs() UnixfsAPI
	// Block returns an implementation of Block API.
	Block() BlockAPI
	// Dag returns an implementation of Dag API.
	Dag() DagAPI
	// Name returns an implementation of Name API.
	Name() NameAPI
	// Key returns an implementation of Key API.
	Key() KeyAPI

	// ObjectAPI returns an implementation of Object API
	Object() ObjectAPI

	// ResolvePath resolves the path using Unixfs resolver
	ResolvePath(context.Context, Path) (Path, error)

	// ResolveNode resolves the path (if not resolved already) using Unixfs
	// resolver, gets and returns the resolved Node
	ResolveNode(context.Context, Path) (Node, error)
}

// UnixfsAPI is the basic interface to immutable files in IPFS
type UnixfsAPI interface {
	// Add imports the data from the reader into merkledag file
	Add(context.Context, io.Reader) (Path, error)

	// Cat returns a reader for the file
	Cat(context.Context, Path) (Reader, error)

	// Ls returns the list of links in a directory
	Ls(context.Context, Path) ([]*Link, error)
}

// BlockAPI specifies the interface to the block layer
type BlockAPI interface {
	// Put imports raw block data, hashing it using specified settings.
	Put(context.Context, io.Reader, ...options.BlockPutOption) (Path, error)

	// WithFormat is an option for Put which specifies the multicodec to use to
	// serialize the object. Default is "v0"
	WithFormat(codec string) options.BlockPutOption

	// WithHash is an option for Put which specifies the multihash settings to use
	// when hashing the object. Default is mh.SHA2_256 (0x12).
	// If mhLen is set to -1, default length for the hash will be used
	WithHash(mhType uint64, mhLen int) options.BlockPutOption

	// Get attempts to resolve the path and return a reader for data in the block
	Get(context.Context, Path) (io.Reader, error)

	// Rm removes the block specified by the path from local blockstore.
	// By default an error will be returned if the block can't be found locally.
	//
	// NOTE: If the specified block is pinned it won't be removed and no error
	// will be returned
	Rm(context.Context, Path, ...options.BlockRmOption) error

	// WithForce is an option for Rm which, when set to true, will ignore
	// non-existing blocks
	WithForce(force bool) options.BlockRmOption

	// Stat returns information on
	Stat(context.Context, Path) (BlockStat, error)
}

// DagAPI specifies the interface to IPLD
type DagAPI interface {
	// Put inserts data using specified format and input encoding.
	// Unless used with WithCodec or WithHash, the defaults "dag-cbor" and
	// "sha256" are used.
	Put(ctx context.Context, src io.Reader, opts ...options.DagPutOption) (Path, error)

	// WithInputEnc is an option for Put which specifies the input encoding of the
	// data. Default is "json", most formats/codecs support "raw"
	WithInputEnc(enc string) options.DagPutOption

	// WithCodec is an option for Put which specifies the multicodec to use to
	// serialize the object. Default is cid.DagCBOR (0x71)
	WithCodec(codec uint64) options.DagPutOption

	// WithHash is an option for Put which specifies the multihash settings to use
	// when hashing the object. Default is based on the codec used
	// (mh.SHA2_256 (0x12) for DagCBOR). If mhLen is set to -1, default length for
	// the hash will be used
	WithHash(mhType uint64, mhLen int) options.DagPutOption

	// Get attempts to resolve and get the node specified by the path
	Get(ctx context.Context, path Path) (Node, error)

	// Tree returns list of paths within a node specified by the path.
	Tree(ctx context.Context, path Path, opts ...options.DagTreeOption) ([]Path, error)

	// WithDepth is an option for Tree which specifies maximum depth of the
	// returned tree. Default is -1 (no depth limit)
	WithDepth(depth int) options.DagTreeOption
}

// NameAPI specifies the interface to IPNS.
//
// IPNS is a PKI namespace, where names are the hashes of public keys, and the
// private key enables publishing new (signed) values. In both publish and
// resolve, the default name used is the node's own PeerID, which is the hash of
// its public key.
//
// You can use .Key API to list and generate more names and their respective keys.
type NameAPI interface {
	// Publish announces new IPNS name
	Publish(ctx context.Context, path Path, opts ...options.NamePublishOption) (IpnsEntry, error)

	// WithValidTime is an option for Publish which specifies for how long the
	// entry will remain valid. Default value is 24h
	WithValidTime(validTime time.Duration) options.NamePublishOption

	// WithKey is an option for Publish which specifies the key to use for
	// publishing. Default value is "self" which is the node's own PeerID.
	// The key parameter must be either PeerID or keystore key alias.
	//
	// You can use KeyAPI to list and generate more names and their respective keys.
	WithKey(key string) options.NamePublishOption

	// Resolve attempts to resolve the newest version of the specified name
	Resolve(ctx context.Context, name string, opts ...options.NameResolveOption) (Path, error)

	// WithRecursive is an option for Resolve which specifies whether to perform a
	// recursive lookup. Default value is false
	WithRecursive(recursive bool) options.NameResolveOption

	// WithLocal is an option for Resolve which specifies if the lookup should be
	// offline. Default value is false
	WithLocal(local bool) options.NameResolveOption

	// WithCache is an option for Resolve which specifies if cache should be used.
	// Default value is true
	WithCache(cache bool) options.NameResolveOption
}

// KeyAPI specifies the interface to Keystore
type KeyAPI interface {
	// Generate generates new key, stores it in the keystore under the specified
	// name and returns a base58 encoded multihash of it's public key
	Generate(ctx context.Context, name string, opts ...options.KeyGenerateOption) (Key, error)

	// WithType is an option for Generate which specifies which algorithm
	// should be used for the key. Default is options.RSAKey
	//
	// Supported key types:
	// * options.RSAKey
	// * options.Ed25519Key
	WithType(algorithm string) options.KeyGenerateOption

	// WithSize is an option for Generate which specifies the size of the key to
	// generated. Default is -1
	//
	// value of -1 means 'use default size for key type':
	//  * 2048 for RSA
	WithSize(size int) options.KeyGenerateOption

	// Rename renames oldName key to newName. Returns the key and whether another
	// key was overwritten, or an error
	Rename(ctx context.Context, oldName string, newName string, opts ...options.KeyRenameOption) (Key, bool, error)

	// WithForce is an option for Rename which specifies whether to allow to
	// replace existing keys.
	WithForce(force bool) options.KeyRenameOption

	// List lists keys stored in keystore
	List(ctx context.Context) ([]Key, error)

	// Remove removes keys from keystore. Returns ipns path of the removed key
	Remove(ctx context.Context, name string) (Path, error)
}

// ObjectAPI specifies the interface to MerkleDAG and contains useful utilities
// for manipulating MerkleDAG data structures.
type ObjectAPI interface {
	// New creates new, empty (by default) dag-node.
	New(context.Context, ...options.ObjectNewOption) (Node, error)

	// WithType is an option for New which allows to change the type of created
	// dag node.
	//
	// Supported types:
	// * 'empty' - Empty node
	// * 'unixfs-dir' - Empty UnixFS directory
	WithType(string) options.ObjectNewOption

	// Put imports the data into merkledag
	Put(context.Context, io.Reader, ...options.ObjectPutOption) (Path, error)

	// WithInputEnc is an option for Put which specifies the input encoding of the
	// data. Default is "json".
	//
	// Supported encodings:
	// * "protobuf"
	// * "json"
	WithInputEnc(e string) options.ObjectPutOption

	// WithDataType specifies the encoding of data field when using Josn or XML
	// input encoding.
	//
	// Supported types:
	// * "text" (default)
	// * "base64"
	WithDataType(t string) options.ObjectPutOption

	// Get returns the node for the path
	Get(context.Context, Path) (Node, error)

	// Data returns reader for data of the node
	Data(context.Context, Path) (io.Reader, error)

	// Links returns lint or links the node contains
	Links(context.Context, Path) ([]*Link, error)

	// Stat returns information about the node
	Stat(context.Context, Path) (*ObjectStat, error)

	// AddLink adds a link under the specified path. child path can point to a
	// subdirectory within the patent which must be present (can be overridden
	// with WithCreate option).
	AddLink(ctx context.Context, base Path, name string, child Path, opts ...options.ObjectAddLinkOption) (Path, error)

	// WithCreate is an option for AddLink which specifies whether create required
	// directories for the child
	WithCreate(create bool) options.ObjectAddLinkOption

	// RmLink removes a link from the node
	RmLink(ctx context.Context, base Path, link string) (Path, error)

	// AppendData appends data to the node
	AppendData(context.Context, Path, io.Reader) (Path, error)

	// SetData sets the data contained in the node
	SetData(context.Context, Path, io.Reader) (Path, error)
}

// ObjectStat provides information about dag nodes
type ObjectStat struct {
	// Cid is the CID of the node
	Cid *cid.Cid

	// NumLinks is number of links the node contains
	NumLinks int

	// BlockSize is size of the raw serialized node
	BlockSize int

	// LinksSize is size of the links block section
	LinksSize int

	// DataSize is the size of data block section
	DataSize int

	// CumulativeSize is size of the tree (BlockSize + link sizes)
	CumulativeSize int
}

var ErrIsDir = errors.New("object is a directory")
var ErrOffline = errors.New("can't resolve, ipfs node is offline")
<|endoftext|>


<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package disk provides a storage.Storage that stores data on local disk.
package disk // import "upspin.io/cloud/storage/disk"

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"upspin.io/cloud/storage"
	"upspin.io/cloud/storage/disk/internal/local"
	"upspin.io/errors"
	"upspin.io/upspin"
)

// New initializes and returns a disk-backed storage.Storage with the given
// options. The single, required option is "basePath" that must be an absolute
// path under which all objects should be stored.
func New(opts *storage.Opts) (storage.Storage, error) {
	const op errors.Op = "cloud/storage/disk.New"

	base, ok := opts.Opts["basePath"]
	if !ok {
		return nil, errors.E(op, "the basePath option must be specified")
	}
	if err := os.MkdirAll(base, 0700); err != nil {
		return nil, errors.E(op, errors.IO, err)
	}

	if err := guaranteeNewEncoding(base); err != nil {
		return nil, errors.E(op, errors.IO, err)
	}

	return &storageImpl{base: base}, nil
}

// guaranteeNewEncoding makes sure we are using the new, safe path encoding.
// If we're not, it prints a recipe to update it and errors out.
func guaranteeNewEncoding(base string) error {
	// Make sure the disk tree is or will be using the new path encoding.
	// Three cases:
	// 1) Directory is empty. Use new encoding, and add "++" directory.
	// 2) Directory contains subdirectory "++". Use new encoding.
	// 3) Directory is non-empty and does not contain "++". Give error.

	// The "++" directory is used as an indicator that we are using the new
	// encoding. This might hold storage one day but will never exist if
	// using the old one, so it serves as a good marker.
	plusDir := filepath.Join(base, "++")
	empty, err := isEmpty(base)
	if err != nil {
		return err
	}
	if empty {
		// New directory tree. Create the "++" directory as a marker.
		return os.MkdirAll(plusDir, 0700)
	}
	// Directory is not empty. It must contain "++".
	if _, err := os.Stat(plusDir); err != nil {
		// Return a very long error explaining what to do.
		format := "Base directory %[1]q uses a deprecated path encoding.\n" +
			"It must be updated before serving again.\n" +
			"To update, move the tree aside to a backup location, and run:\n" +
			"\tgo run upspin.io/cloud/storage/disk/convert.go -old=<backup-location> -new=%[1]q\n" +
			"Then restart the server.\n"
		return errors.Errorf(format, base)
	}
	return nil
}

// isEmpty reports whether the directory is empty.
// The directory must exist; we have already created it if we needed to.
func isEmpty(dir string) (bool, error) {
	fd, err := os.Open(dir)
	if err != nil {
		return true, err
	}
	defer fd.Close()
	names, err := fd.Readdirnames(0)
	if err != nil {
		return true, err
	}
	return len(names) == 0, nil
}

func init() {
	storage.Register("Disk", New)
}

type storageImpl struct {
	base string
}

var (
	_ storage.Storage = (*storageImpl)(nil)
	_ storage.Lister  = (*storageImpl)(nil)
)

// LinkBase implements storage.Storage.
func (s *storageImpl) LinkBase() (base string, err error) {
	return "", upspin.ErrNotSupported
}

// Download implements storage.Storage.
func (s *storageImpl) Download(ref string) ([]byte, error) {
	const op errors.Op = "cloud/storage/disk.Download"
	b, err := ioutil.ReadFile(s.path(ref))
	if os.IsNotExist(err) {
		return nil, errors.E(op, errors.NotExist, ref)
	} else if err != nil {
		return nil, errors.E(op, errors.IO, ref)
	}
	return b, nil
}

// Put implements storage.Storage.
func (s *storageImpl) Put(ref string, contents []byte) error {
	const op errors.Op = "cloud/storage/disk.Put"
	p := s.path(ref)
	if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
		return errors.E(op, errors.IO, err)
	}
	if err := ioutil.WriteFile(p, contents, 0600); err != nil {
		return errors.E(op, errors.IO, err)
	}
	return nil
}

// Delete implements storage.Storage.
func (s *storageImpl) Delete(ref string) error {
	const op errors.Op = "cloud/storage/disk.Delete"
	if err := os.Remove(s.path(ref)); os.IsNotExist(err) {
		return errors.E(op, errors.NotExist, ref)
	} else if err != nil {
		return errors.E(op, errors.IO, ref)
	}
	return nil
}

var maxRefsPerCall = 1000 // A variable so that it may be overridden by tests.

// List implements storage.Lister.
func (s *storageImpl) List(token string) (refs []upspin.ListRefsItem, next string, err error) {
	const op errors.Op = "cloud/storage/disk.List"
	err = filepath.Walk(s.base, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Convert path into its base path.
		path = strings.TrimPrefix(strings.TrimPrefix(path, s.base), string(filepath.Separator))

		// Ignore the root.
		if path == "" {
			return nil
		}

		// Stop walking when we've gathered enough refs.
		if len(refs) >= maxRefsPerCall {
			if next == "" {
				next = path
			}
			return filepath.SkipDir
		}

		// Don't process paths that come before our pagination token.
		if path < token {
			if fi.IsDir() && !strings.HasPrefix(token, path) {
				// Don't descend into irrelevant directories.
				return filepath.SkipDir
			}
			return nil
		}

		if fi.IsDir() {
			// Nothing more to do for directories.
			return nil
		}

		// Convert the file path into its reference name
		// and append it to refs.
		ref, err := local.Ref(path)
		if err != nil {
			return err
		}
		refs = append(refs, upspin.ListRefsItem{
			Ref:  upspin.Reference(ref),
			Size: fi.Size(),
		})
		return nil
	})
	if err != nil {
		return nil, "", errors.E(op, err)
	}
	return refs, next, nil
}

// path returns the absolute path that should contain ref.
func (s *storageImpl) path(ref string) string {
	return local.Path(s.base, ref)
}
<commit_msg>cloud/storage/disk: wrap references in errors.Str when constructing errors<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package disk provides a storage.Storage that stores data on local disk.
package disk // import "upspin.io/cloud/storage/disk"

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"upspin.io/cloud/storage"
	"upspin.io/cloud/storage/disk/internal/local"
	"upspin.io/errors"
	"upspin.io/upspin"
)

// New initializes and returns a disk-backed storage.Storage with the given
// options. The single, required option is "basePath" that must be an absolute
// path under which all objects should be stored.
func New(opts *storage.Opts) (storage.Storage, error) {
	const op errors.Op = "cloud/storage/disk.New"

	base, ok := opts.Opts["basePath"]
	if !ok {
		return nil, errors.E(op, "the basePath option must be specified")
	}
	if err := os.MkdirAll(base, 0700); err != nil {
		return nil, errors.E(op, errors.IO, err)
	}

	if err := guaranteeNewEncoding(base); err != nil {
		return nil, errors.E(op, errors.IO, err)
	}

	return &storageImpl{base: base}, nil
}

// guaranteeNewEncoding makes sure we are using the new, safe path encoding.
// If we're not, it prints a recipe to update it and errors out.
func guaranteeNewEncoding(base string) error {
	// Make sure the disk tree is or will be using the new path encoding.
	// Three cases:
	// 1) Directory is empty. Use new encoding, and add "++" directory.
	// 2) Directory contains subdirectory "++". Use new encoding.
	// 3) Directory is non-empty and does not contain "++". Give error.

	// The "++" directory is used as an indicator that we are using the new
	// encoding. This might hold storage one day but will never exist if
	// using the old one, so it serves as a good marker.
	plusDir := filepath.Join(base, "++")
	empty, err := isEmpty(base)
	if err != nil {
		return err
	}
	if empty {
		// New directory tree. Create the "++" directory as a marker.
		return os.MkdirAll(plusDir, 0700)
	}
	// Directory is not empty. It must contain "++".
	if _, err := os.Stat(plusDir); err != nil {
		// Return a very long error explaining what to do.
		format := "Base directory %[1]q uses a deprecated path encoding.\n" +
			"It must be updated before serving again.\n" +
			"To update, move the tree aside to a backup location, and run:\n" +
			"\tgo run upspin.io/cloud/storage/disk/convert.go -old=<backup-location> -new=%[1]q\n" +
			"Then restart the server.\n"
		return errors.Errorf(format, base)
	}
	return nil
}

// isEmpty reports whether the directory is empty.
// The directory must exist; we have already created it if we needed to.
func isEmpty(dir string) (bool, error) {
	fd, err := os.Open(dir)
	if err != nil {
		return true, err
	}
	defer fd.Close()
	names, err := fd.Readdirnames(0)
	if err != nil {
		return true, err
	}
	return len(names) == 0, nil
}

func init() {
	storage.Register("Disk", New)
}

type storageImpl struct {
	base string
}

var (
	_ storage.Storage = (*storageImpl)(nil)
	_ storage.Lister  = (*storageImpl)(nil)
)

// LinkBase implements storage.Storage.
func (s *storageImpl) LinkBase() (base string, err error) {
	return "", upspin.ErrNotSupported
}

// Download implements storage.Storage.
func (s *storageImpl) Download(ref string) ([]byte, error) {
	const op errors.Op = "cloud/storage/disk.Download"
	b, err := ioutil.ReadFile(s.path(ref))
	if os.IsNotExist(err) {
		return nil, errors.E(op, errors.NotExist, errors.Str(ref))
	} else if err != nil {
		return nil, errors.E(op, errors.IO, err)
	}
	return b, nil
}

// Put implements storage.Storage.
func (s *storageImpl) Put(ref string, contents []byte) error {
	const op errors.Op = "cloud/storage/disk.Put"
	p := s.path(ref)
	if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
		return errors.E(op, errors.IO, err)
	}
	if err := ioutil.WriteFile(p, contents, 0600); err != nil {
		return errors.E(op, errors.IO, err)
	}
	return nil
}

// Delete implements storage.Storage.
func (s *storageImpl) Delete(ref string) error {
	const op errors.Op = "cloud/storage/disk.Delete"
	if err := os.Remove(s.path(ref)); os.IsNotExist(err) {
		return errors.E(op, errors.NotExist, errors.Str(ref))
	} else if err != nil {
		return errors.E(op, errors.IO, err)
	}
	return nil
}

var maxRefsPerCall = 1000 // A variable so that it may be overridden by tests.

// List implements storage.Lister.
func (s *storageImpl) List(token string) (refs []upspin.ListRefsItem, next string, err error) {
	const op errors.Op = "cloud/storage/disk.List"
	err = filepath.Walk(s.base, func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Convert path into its base path.
		path = strings.TrimPrefix(strings.TrimPrefix(path, s.base), string(filepath.Separator))

		// Ignore the root.
		if path == "" {
			return nil
		}

		// Stop walking when we've gathered enough refs.
		if len(refs) >= maxRefsPerCall {
			if next == "" {
				next = path
			}
			return filepath.SkipDir
		}

		// Don't process paths that come before our pagination token.
		if path < token {
			if fi.IsDir() && !strings.HasPrefix(token, path) {
				// Don't descend into irrelevant directories.
				return filepath.SkipDir
			}
			return nil
		}

		if fi.IsDir() {
			// Nothing more to do for directories.
			return nil
		}

		// Convert the file path into its reference name
		// and append it to refs.
		ref, err := local.Ref(path)
		if err != nil {
			return err
		}
		refs = append(refs, upspin.ListRefsItem{
			Ref:  upspin.Reference(ref),
			Size: fi.Size(),
		})
		return nil
	})
	if err != nil {
		return nil, "", errors.E(op, err)
	}
	return refs, next, nil
}

// path returns the absolute path that should contain ref.
func (s *storageImpl) path(ref string) string {
	return local.Path(s.base, ref)
}
<|endoftext|>


<commit_before>package recommended

import (
	"net/http"
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

const (
	maxRecommendations = 10
)

// Anime shows a list of recommended anime.
func Anime(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	nick := ctx.Get("nick")
	viewUser, err := arn.GetUserByNick(nick)

	if err != nil {
		return ctx.Error(http.StatusUnauthorized, "Not logged in", err)
	}

	animeList := viewUser.AnimeList()

	// Get all anime
	var tv []*arn.Anime
	var movies []*arn.Anime
	allAnime := arn.AllAnime()

	// Affinity maps an anime ID to a number that indicates how likely a user is going to enjoy that anime.
	affinity := map[string]float64{}

	// Calculate affinity for each anime
	for _, anime := range allAnime {
		// Skip anime that are upcoming or tba
		if anime.Status == "upcoming" || anime.Status == "tba" {
			continue
		}

		if anime.Type == "tv" {
			tv = append(tv, anime)
		} else if anime.Type == "movie" {
			movies = append(movies, anime)
		}

		// Skip anime from my list (except planned anime)
		existing := animeList.Find(anime.ID)

		if existing != nil && existing.Status != arn.AnimeListStatusPlanned {
			continue
		}

		animeAffinity := anime.Score()

		// Planned anime go higher
		if existing != nil && existing.Status == arn.AnimeListStatusPlanned {
			animeAffinity += 15.0
		}

		affinity[anime.ID] = animeAffinity
	}

	// Sort
	sort.Slice(tv, func(i, j int) bool {
		affinityA := affinity[tv[i].ID]
		affinityB := affinity[tv[j].ID]

		if affinityA == affinityB {
			return tv[i].Title.Canonical < tv[j].Title.Canonical
		}

		return affinityA > affinityB
	})

	sort.Slice(movies, func(i, j int) bool {
		affinityA := affinity[movies[i].ID]
		affinityB := affinity[movies[j].ID]

		if affinityA == affinityB {
			return movies[i].Title.Canonical < movies[j].Title.Canonical
		}

		return affinityA > affinityB
	})

	// Take the top 10
	if len(tv) > maxRecommendations {
		tv = tv[:maxRecommendations]
	}

	if len(movies) > maxRecommendations {
		movies = movies[:maxRecommendations]
	}

	return ctx.HTML(components.RecommendedAnime(tv, movies, viewUser, user))
}
<commit_msg>Recommendations don't show sequels (unless prequel completed)<commit_after>package recommended

import (
	"net/http"
	"sort"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

const (
	maxRecommendations = 10
)

// Anime shows a list of recommended anime.
func Anime(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	nick := ctx.Get("nick")
	viewUser, err := arn.GetUserByNick(nick)

	if err != nil {
		return ctx.Error(http.StatusUnauthorized, "Not logged in", err)
	}

	animeList := viewUser.AnimeList()

	// Get all anime
	var tv []*arn.Anime
	var movies []*arn.Anime
	allAnime := arn.AllAnime()

	// Affinity maps an anime ID to a number that indicates how likely a user is going to enjoy that anime.
	affinity := map[string]float64{}

	// Calculate affinity for each anime
	for _, anime := range allAnime {
		// Skip anime that are upcoming or tba
		if anime.Status == "upcoming" || anime.Status == "tba" {
			continue
		}

		if anime.Type == "tv" {
			tv = append(tv, anime)
		} else if anime.Type == "movie" {
			movies = append(movies, anime)
		}

		// Skip anime from my list (except planned anime)
		existing := animeList.Find(anime.ID)

		if existing != nil && existing.Status != arn.AnimeListStatusPlanned {
			continue
		}

		animeAffinity := anime.Score()

		// Planned anime go higher
		if existing != nil && existing.Status == arn.AnimeListStatusPlanned {
			animeAffinity += 10.0
		}

		// Anime whose high-ranked prequel you did not see are lower ranked
		prequels := anime.Prequels()

		for _, prequel := range prequels {
			item := animeList.Find(prequel.ID)

			// Filter out unimportant prequels
			if prequel.Score() < anime.Score()/2 {
				continue
			}

			if item == nil || item.Status != arn.AnimeListStatusCompleted {
				animeAffinity -= 20.0
			}
		}

		affinity[anime.ID] = animeAffinity
	}

	// Sort
	sort.Slice(tv, func(i, j int) bool {
		affinityA := affinity[tv[i].ID]
		affinityB := affinity[tv[j].ID]

		if affinityA == affinityB {
			return tv[i].Title.Canonical < tv[j].Title.Canonical
		}

		return affinityA > affinityB
	})

	sort.Slice(movies, func(i, j int) bool {
		affinityA := affinity[movies[i].ID]
		affinityB := affinity[movies[j].ID]

		if affinityA == affinityB {
			return movies[i].Title.Canonical < movies[j].Title.Canonical
		}

		return affinityA > affinityB
	})

	// Take the top 10
	if len(tv) > maxRecommendations {
		tv = tv[:maxRecommendations]
	}

	if len(movies) > maxRecommendations {
		movies = movies[:maxRecommendations]
	}

	return ctx.HTML(components.RecommendedAnime(tv, movies, viewUser, user))
}
<|endoftext|>


<commit_before>package restic_test

import (
	"os"
	"testing"
	"time"

	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/repository"
	. "github.com/restic/restic/test"
)

func TestLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock.Unlock())
}

func TestDoubleUnlock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock.Unlock())

	err = lock.Unlock()
	Assert(t, err != nil,
		"double unlock didn't return an error, got %v", err)
}

func TestMultipleLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock1, err := restic.NewLock(repo)
	OK(t, err)

	lock2, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock1.Unlock())
	OK(t, lock2.Unlock())
}

func TestLockExclusive(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewExclusiveLock(repo)
	OK(t, err)
	OK(t, elock.Unlock())
}

func TestLockOnExclusiveLockedRepo(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewExclusiveLock(repo)
	OK(t, err)

	lock, err := restic.NewLock(repo)
	Assert(t, err != nil,
		"create normal lock with exclusively locked repo didn't return an error")
	Assert(t, restic.IsAlreadyLocked(err),
		"create normal lock with exclusively locked repo didn't return the correct error")

	OK(t, lock.Unlock())
	OK(t, elock.Unlock())
}

func TestExclusiveLockOnLockedRepo(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewLock(repo)
	OK(t, err)

	lock, err := restic.NewExclusiveLock(repo)
	Assert(t, err != nil,
		"create normal lock with exclusively locked repo didn't return an error")
	Assert(t, restic.IsAlreadyLocked(err),
		"create normal lock with exclusively locked repo didn't return the correct error")

	OK(t, lock.Unlock())
	OK(t, elock.Unlock())
}

func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) {
	newLock := &restic.Lock{Time: t, PID: pid}
	return repo.SaveJSONUnpacked(backend.Lock, &newLock)
}

func removeLock(repo *repository.Repository, id backend.ID) error {
	return repo.Backend().Remove(backend.Lock, id.String())
}

var staleLockTests = []struct {
	timestamp time.Time
	stale     bool
	pid       int
}{
	{
		timestamp: time.Now(),
		stale:     false,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now().Add(-time.Hour),
		stale:     true,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now().Add(3 * time.Minute),
		stale:     false,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now(),
		stale:     true,
		pid:       os.Getpid() + 500,
	},
}

func TestLockStale(t *testing.T) {
	for i, test := range staleLockTests {
		lock := restic.Lock{
			Time: test.timestamp,
			PID:  test.pid,
		}

		Assert(t, lock.Stale() == test.stale,
			"TestStaleLock: test %d failed: expected stale: %v, got %v",
			i, test.stale, !test.stale)
	}
}

func lockExists(repo *repository.Repository, t testing.TB, id backend.ID) bool {
	exists, err := repo.Backend().Test(backend.Lock, id.String())
	OK(t, err)

	return exists
}

func TestLockWithStaleLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
	OK(t, err)

	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
	OK(t, err)

	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500)
	OK(t, err)

	OK(t, restic.RemoveStaleLocks(repo))

	Assert(t, lockExists(repo, t, id1) == false,
		"stale lock still exists after RemoveStaleLocks was called")
	Assert(t, lockExists(repo, t, id2) == true,
		"non-stale lock was removed by RemoveStaleLocks")
	Assert(t, lockExists(repo, t, id3) == false,
		"stale lock still exists after RemoveStaleLocks was called")

	OK(t, removeLock(repo, id2))
}

func TestRemoveAllLocks(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
	OK(t, err)

	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
	OK(t, err)

	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500)
	OK(t, err)

	OK(t, restic.RemoveAllLocks(repo))

	Assert(t, lockExists(repo, t, id1) == false,
		"lock still exists after RemoveAllLocks was called")
	Assert(t, lockExists(repo, t, id2) == false,
		"lock still exists after RemoveAllLocks was called")
	Assert(t, lockExists(repo, t, id3) == false,
		"lock still exists after RemoveAllLocks was called")
}

func TestLockRefresh(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	var lockID *backend.ID
	for id := range repo.List(backend.Lock, nil) {
		if lockID != nil {
			t.Error("more than one lock found")
		}
		lockID = &id
	}

	OK(t, lock.Refresh())

	var lockID2 *backend.ID
	for id := range repo.List(backend.Lock, nil) {
		if lockID2 != nil {
			t.Error("more than one lock found")
		}
		lockID2 = &id
	}

	Assert(t, !lockID.Equal(*lockID2),
		"expected a new ID after lock refresh, got the same")
	OK(t, lock.Unlock())
}
<commit_msg>Use bigger offset in test to avoid random test failures.<commit_after>package restic_test

import (
	"os"
	"testing"
	"time"

	"github.com/restic/restic"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/repository"
	. "github.com/restic/restic/test"
)

func TestLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock.Unlock())
}

func TestDoubleUnlock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock.Unlock())

	err = lock.Unlock()
	Assert(t, err != nil,
		"double unlock didn't return an error, got %v", err)
}

func TestMultipleLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock1, err := restic.NewLock(repo)
	OK(t, err)

	lock2, err := restic.NewLock(repo)
	OK(t, err)

	OK(t, lock1.Unlock())
	OK(t, lock2.Unlock())
}

func TestLockExclusive(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewExclusiveLock(repo)
	OK(t, err)
	OK(t, elock.Unlock())
}

func TestLockOnExclusiveLockedRepo(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewExclusiveLock(repo)
	OK(t, err)

	lock, err := restic.NewLock(repo)
	Assert(t, err != nil,
		"create normal lock with exclusively locked repo didn't return an error")
	Assert(t, restic.IsAlreadyLocked(err),
		"create normal lock with exclusively locked repo didn't return the correct error")

	OK(t, lock.Unlock())
	OK(t, elock.Unlock())
}

func TestExclusiveLockOnLockedRepo(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	elock, err := restic.NewLock(repo)
	OK(t, err)

	lock, err := restic.NewExclusiveLock(repo)
	Assert(t, err != nil,
		"create normal lock with exclusively locked repo didn't return an error")
	Assert(t, restic.IsAlreadyLocked(err),
		"create normal lock with exclusively locked repo didn't return the correct error")

	OK(t, lock.Unlock())
	OK(t, elock.Unlock())
}

func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) {
	newLock := &restic.Lock{Time: t, PID: pid}
	return repo.SaveJSONUnpacked(backend.Lock, &newLock)
}

func removeLock(repo *repository.Repository, id backend.ID) error {
	return repo.Backend().Remove(backend.Lock, id.String())
}

var staleLockTests = []struct {
	timestamp time.Time
	stale     bool
	pid       int
}{
	{
		timestamp: time.Now(),
		stale:     false,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now().Add(-time.Hour),
		stale:     true,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now().Add(3 * time.Minute),
		stale:     false,
		pid:       os.Getpid(),
	},
	{
		timestamp: time.Now(),
		stale:     true,
		pid:       os.Getpid() + 500000,
	},
}

func TestLockStale(t *testing.T) {
	for i, test := range staleLockTests {
		lock := restic.Lock{
			Time: test.timestamp,
			PID:  test.pid,
		}

		Assert(t, lock.Stale() == test.stale,
			"TestStaleLock: test %d failed: expected stale: %v, got %v",
			i, test.stale, !test.stale)
	}
}

func lockExists(repo *repository.Repository, t testing.TB, id backend.ID) bool {
	exists, err := repo.Backend().Test(backend.Lock, id.String())
	OK(t, err)

	return exists
}

func TestLockWithStaleLock(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
	OK(t, err)

	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
	OK(t, err)

	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
	OK(t, err)

	OK(t, restic.RemoveStaleLocks(repo))

	Assert(t, lockExists(repo, t, id1) == false,
		"stale lock still exists after RemoveStaleLocks was called")
	Assert(t, lockExists(repo, t, id2) == true,
		"non-stale lock was removed by RemoveStaleLocks")
	Assert(t, lockExists(repo, t, id3) == false,
		"stale lock still exists after RemoveStaleLocks was called")

	OK(t, removeLock(repo, id2))
}

func TestRemoveAllLocks(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
	OK(t, err)

	id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid())
	OK(t, err)

	id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500)
	OK(t, err)

	OK(t, restic.RemoveAllLocks(repo))

	Assert(t, lockExists(repo, t, id1) == false,
		"lock still exists after RemoveAllLocks was called")
	Assert(t, lockExists(repo, t, id2) == false,
		"lock still exists after RemoveAllLocks was called")
	Assert(t, lockExists(repo, t, id3) == false,
		"lock still exists after RemoveAllLocks was called")
}

func TestLockRefresh(t *testing.T) {
	repo := SetupRepo()
	defer TeardownRepo(repo)

	lock, err := restic.NewLock(repo)
	OK(t, err)

	var lockID *backend.ID
	for id := range repo.List(backend.Lock, nil) {
		if lockID != nil {
			t.Error("more than one lock found")
		}
		lockID = &id
	}

	OK(t, lock.Refresh())

	var lockID2 *backend.ID
	for id := range repo.List(backend.Lock, nil) {
		if lockID2 != nil {
			t.Error("more than one lock found")
		}
		lockID2 = &id
	}

	Assert(t, !lockID.Equal(*lockID2),
		"expected a new ID after lock refresh, got the same")
	OK(t, lock.Unlock())
}
<|endoftext|>


<commit_before>package log2

import (
	"testing"
)

func TestSimple(t *testing.T) {
	l := New()
	l.Warn("Hello world")
	l.Warnf("Hello %s", "srid")
	l.Print("xyz")
	Fatal("oops")
}
<commit_msg>make drain errors/retry appear in cloud events.<commit_after>package log2

import (
	"testing"
)

func TestSimple(t *testing.T) {
	l := New()
	l.Warn("Hello world")
	l.Warnf("Hello %s", "srid")
	l.Print("xyz")
	// Fatal("oops")
}
<|endoftext|>


<commit_before>package raft

import (
	"sync"
)

// LogCache wraps a logstore with a ring buffer providing fast access to the
// last n raft log entries.
type LogCache struct {
	store      LogStore
	cache      []*Log
	current    int
	lastlogidx uint64
	l          sync.RWMutex
}

func NewLogCache(capacity int, logstore LogStore) *LogCache {
	return &LogCache{
		cache: make([]*Log, capacity),
		store: logstore,
	}
}

func (c *LogCache) getLogFromCache(logidx uint64) (*Log, bool) {
	c.l.RLock()
	defer c.l.RUnlock()

	// 'last' is the index of the element we cached last,
	// its raft log index is 'lastlogidx'
	last := (c.current - 1)
	m := last - int(c.lastlogidx-logidx)

	// See https://golang.org/issue/448 for why (m % n) is not enough.
	n := len(c.cache)
	log := c.cache[((m%n)+n)%n]
	if log == nil {
		return nil, false
	}
	// If the index does not match, cacheLog’s expected access pattern was
	// violated and we need to fall back to reading from the LogStore.
	return log, log.Index == logidx
}

// cacheLogs should be called with strictly monotonically increasing logidx
// values, otherwise the cache will not be effective.
func (c *LogCache) cacheLogs(logs []*Log) {
	c.l.Lock()
	defer c.l.Unlock()

	for _, l := range logs {
		c.cache[c.current] = l
		c.lastlogidx = l.Index
		c.current = (c.current + 1) % len(c.cache)
	}
}

func (c *LogCache) GetLog(logidx uint64, log *Log) error {
	if cached, ok := c.getLogFromCache(logidx); ok {
		*log = *cached
		return nil
	}
	return c.store.GetLog(logidx, log)
}

func (c *LogCache) StoreLog(log *Log) error {
	return c.StoreLogs([]*Log{log})
}

func (c *LogCache) StoreLogs(logs []*Log) error {
	c.cacheLogs(logs)
	return c.store.StoreLogs(logs)
}

func (c *LogCache) FirstIndex() (uint64, error) {
	return c.store.FirstIndex()
}

func (c *LogCache) LastIndex() (uint64, error) {
	return c.store.LastIndex()
}

func (c *LogCache) DeleteRange(min, max uint64) error {
	c.l.Lock()
	defer c.l.Unlock()

	c.lastlogidx = 0
	c.current = 0
	c.cache = make([]*Log, len(c.cache))

	return c.store.DeleteRange(min, max)
}
<commit_msg>Minor cleanups<commit_after>package raft

import (
	"sync"
)

// LogCache wraps any LogStore implementation to provide an
// in-memory ring buffer. This is used to cache access to
// the recently written entries. For implementations that do not
// cache themselves, this can provide a substantial boost by
// avoiding disk I/O on recent entries.
type LogCache struct {
	store LogStore

	cache      []*Log
	current    int
	lastLogIdx uint64
	l          sync.RWMutex
}

// NewLogCache is used to create a new LogCache with the
// given capacity and backend store.
func NewLogCache(capacity int, store LogStore) *LogCache {
	return &LogCache{
		store: store,
		cache: make([]*Log, capacity),
	}
}

func (c *LogCache) getLogFromCache(logidx uint64) (*Log, bool) {
	c.l.RLock()
	defer c.l.RUnlock()

	// 'last' is the index of the element we cached last,
	// its raft log index is 'lastLogIdx'
	last := (c.current - 1)
	m := last - int(c.lastLogIdx-logidx)

	// See https://golang.org/issue/448 for why (m % n) is not enough.
	n := len(c.cache)
	log := c.cache[((m%n)+n)%n]
	if log == nil {
		return nil, false
	}
	// If the index does not match, cacheLog’s expected access pattern was
	// violated and we need to fall back to reading from the LogStore.
	return log, log.Index == logidx
}

// cacheLogs should be called with strictly monotonically increasing logidx
// values, otherwise the cache will not be effective.
func (c *LogCache) cacheLogs(logs []*Log) {
	c.l.Lock()
	defer c.l.Unlock()

	for _, l := range logs {
		c.cache[c.current] = l
		c.lastLogIdx = l.Index
		c.current = (c.current + 1) % len(c.cache)
	}
}

func (c *LogCache) GetLog(idx uint64, log *Log) error {
	if cached, ok := c.getLogFromCache(idx); ok {
		*log = *cached
		return nil
	}
	return c.store.GetLog(idx, log)
}

func (c *LogCache) StoreLog(log *Log) error {
	return c.StoreLogs([]*Log{log})
}

func (c *LogCache) StoreLogs(logs []*Log) error {
	c.cacheLogs(logs)
	return c.store.StoreLogs(logs)
}

func (c *LogCache) FirstIndex() (uint64, error) {
	return c.store.FirstIndex()
}

func (c *LogCache) LastIndex() (uint64, error) {
	return c.store.LastIndex()
}

func (c *LogCache) DeleteRange(min, max uint64) error {
	c.l.Lock()
	defer c.l.Unlock()

	// Invalidate the cache on deletes
	c.lastLogIdx = 0
	c.current = 0
	c.cache = make([]*Log, len(c.cache))

	return c.store.DeleteRange(min, max)
}
<|endoftext|>


<commit_before>package dialekt

// Evaluate an expression against a set of tags.
type Evaluator struct {
	// True if tag matching should be case-sensitive; otherwise, false.
	caseSensitive bool

	// True if an empty expression matches all tags; or false to match none.
	emptyIsWildcard bool

	// The set of tags to evaluate against.
	tags []string

	// The expression results.
	expressionResults []ExpressionResult
}

// Case sensitive is true if tag matching should be case-sensitive; otherwise, false.
// Empty is wildcard is true if an empty expression matches all tags; or false to match none.
func NewEvaluator(caseSensitive, emptyIsWildcard bool) *Evaluator {
	return &Evaluator{caseSensitive, emptyIsWildcard, nil, nil}
}

// Evaluate an expression against a set of tags.
// Return the result of the evaluation.
func (ev *Evaluator) Evaluate(expression ExpressionInterface, tags []string) (result *EvaluationResult) {
	ev.tags = tags
	ev.expressionResults = make([]ExpressionResult)

	result = NewEvaluationResult(expression.Accept(ev).IsMatch(), ev.expressionResults)

	ev.tags = nil
	ev.expressionResults = nil

	return result
}

// Visit a LogicalAnd node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalAnd(node *LogicalAnd) (interface{}) {
	matchedTags := make(map[string]string)
	unmatchedTags := make(map[string]string)
	isMatch := false

	for n := range node.Children() {
		result := n.Accept(ev)

		if (!result.IsMatch()) {
			isMatch = false;
		}

		for tag := range result.MatchedTags() {
			matchedTags[tag] = true
		}

		// TODO: how to port these?

        // $matchedTags = array_keys($matchedTags);

		// XXXXX = array_values(
		// 	array_diff($this->tags, $matchedTags)
		// )

		append(ev.expressionResults, NewExpressionResult(node, isMatch, matchedTags, unmatchedTags);

		return ev.expressionResults
	}
}

// Visit a LogicalOr node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalOr(node LogicalOr) (result interface{}) {
	// TODO
}

// Visit a LogicalNot node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalNot(node LogicalNot) (result interface{}) {
	// TODO
}

// Visit a Tag node.
// The result will be returned.
func (ev *Evaluator) VisitTag(node Tag) (result interface{}) {
	// TODO
}

// Visit a pattern node.
// The result will be returned.
func (ev *Evaluator) VisitPattern(node Pattern) (result interface{}) {
	// TODO
}

// Visit a PatternLiteral node.
// The result will be returned.
func (ev *Evaluator) VisitPatternLiteral(node PatternLiteral) (result interface{}) {
	// TODO
}

// Visit a PatternWildcard node.
// The result will be returned.
func (ev *Evaluator) VisitPatternWildcard(node PatternWildcard) (result interface{}) {
	// TODO
}

// Visit a EmptyExpression node.
// The result will be returned.
func (ev *Evaluator) VisitEmptyExpression(node EmptyExpression) (result interface{}) {
	// TODO
}

func (ev *Evaluator) matchTags(expression ExpressionInterface, predicate func(string) bool) []ExpressionResult {
	// TODO
}
<commit_msg>Working on Evaluator.<commit_after>package dialekt

// import "github.com/deckarep/golang-set"

// Evaluate an expression against a set of tags.
type Evaluator struct {
	// True if tag matching should be case-sensitive; otherwise, false.
	caseSensitive bool

	// True if an empty expression matches all tags; or false to match none.
	emptyIsWildcard bool

	// The set of tags to evaluate against.
	tags []string

	// The expression results.
	expressionResults []ExpressionResult
}

// Case sensitive is true if tag matching should be case-sensitive; otherwise, false.
// Empty is wildcard is true if an empty expression matches all tags; or false to match none.
func NewEvaluator(caseSensitive, emptyIsWildcard bool) *Evaluator {
	return &Evaluator{caseSensitive, emptyIsWildcard, nil, nil}
}

// Evaluate an expression against a set of tags.
// Return the result of the evaluation.
func (ev *Evaluator) Evaluate(expression ExpressionInterface, tags []string) (result *EvaluationResult) {
	ev.tags = tags
	ev.expressionResults = make([]ExpressionResult)

	result = NewEvaluationResult(expression.Accept(ev).IsMatch(), ev.expressionResults)

	ev.tags = nil
	ev.expressionResults = nil

	return result
}

// Visit a LogicalAnd node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalAnd(node *LogicalAnd) interface{} {
	matchedTags := make([]string)
	isMatch := false

	for n := range node.Children() {
		result := n.Accept(ev)

		if !result.IsMatch() {
			isMatch = false
		}

		matchedTags = append(matchedTags, result.MatchedTags()...)
	}

	unmatchedTags := stringsDiff(ev.tags, matchedTags)

	ev.expressionResults = append(ev.expressionResults, NewExpressionResult(node, isMatch, matchedTags, unmatchedTags))

	return ev.expressionResults
}

// Visit a LogicalOr node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalOr(node LogicalOr) (result interface{}) {
	matchedTags := make([]string)
	isMatch := false

	for n := range node.Children() {
		result := n.Accept(ev)

		if !result.IsMatch() {
			isMatch = false
		}

		matchedTags = append(matchedTags, result.MatchedTags()...)
	}

	unmatchedTags := stringsDiff(ev.tags, matchedTags)

	newExpResult := NewExpressionResult(node, isMatch, matchedTags, unmatchedTags)

	ev.expressionResults = append(ev.expressionResults, newExpResult)

	return ev.expressionResults
}

// Visit a LogicalNot node.
// The result will be returned.
func (ev *Evaluator) VisitLogicalNot(node LogicalNot) (result interface{}) {
	childResult := node.Child().Accept(ev)

	newExpResult := NewExpressionResult(node, !childResult.IsMatch(), childResult.UnmatchedTags(), childResult.MatchedTags())

	ev.expressionResults = append(ev.expressionResults, newExpResult)

	return ev.expressionResults
}

// Visit a Tag node.
// The result will be returned.
func (ev *Evaluator) VisitTag(node Tag) (result interface{}) {
	if ev.caseSensitive {
		predicate := func(tag string) bool {
			return node.name() == tag
		}

		return ev.matchTags(node, predicate)
	} else {
		// TODO: compare them case insensitive
		predicate := func(tag string) bool {
			return node.name() == tag
		}

		return ev.matchTags(node, predicate)
	}
}

// Visit a pattern node.
// The result will be returned.
func (ev *Evaluator) VisitPattern(node Pattern) (result interface{}) {
	pattern := "/^"

	for n := range node.Children() {
		pattern += n.Accept(ev)
	}

	pattern += "$/"

	if !ev.caseSensitive {
		pattern += "i"
	}

	predicate := func(tag string) bool {
		// TODO: regex
		return false
	}

	return ev.matchTags(node, predicate)
}

// Visit a PatternLiteral node.
// The result will be returned.
func (ev *Evaluator) VisitPatternLiteral(node PatternLiteral) (result interface{}) {
	// TODO: regex quote
	return node.String()
}

// Visit a PatternWildcard node.
// The result will be returned.
func (ev *Evaluator) VisitPatternWildcard(node PatternWildcard) (result interface{}) {
	// TODO: regex wildcard
	return ".*"
}

// Visit a EmptyExpression node.
// The result will be returned.
func (ev *Evaluator) VisitEmptyExpression(node EmptyExpression) (result interface{}) {
	otherTags := make([]string)

	if ev.emptyIsWildcard {
		newExpResult := NewExpressionResult(node, ev.emptyIsWildcard, ev.tags, otherTags)
	} else {
		newExpResult := NewExpressionResult(node, ev.emptyIsWildcard, otherTags, ev.tags)
	}

	ev.expressionResults = append(ev.expressionResults, newExpResult)

	return ev.expressionResults
}

func (ev *Evaluator) matchTags(expression ExpressionInterface, predicate func(string) bool) []ExpressionResult {
	matchedTags := make([]string)
	unmatchedTags := make([]string)

	for tag := range ev.tags {
		if predicate(tag) {
			matchedTags = append(matchedTags, tag)
		} else {
			unmatchedTags = append(unmatchedTags, tag)
		}
	}

	newExpResult := NewExpressionResult(expression, len(matchedTags) > 0, matchedTags, unmatchedTags)

	ev.expressionResults = append(ev.expressionResults, newExpResult)

	return ev.expressionResults
}

// Helper function that works like PHP array_diff()
// TODO: remove this and use the imported golang-set third part lib?
func stringsDiff(left, right []string) []string {
	result := make([]string)

	for l := range left {
		addToResult := true

		for r := range right {
			if l == r {
				addToResult = false
				break
			}
		}

		if addToResult {
			result = append(result, l)
		}
	}

	return result
}
<|endoftext|>


<commit_before>package transform_data

import (
	"bytes"
	"encoding/json"
	"text/template"

	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/logger"
)

var (
	TemplateFile string
	TemplateStr  string
)

func ConvertJsonToMap(data string) interface{} {
	// if the data is an array of json objects then we need to store that in an array of maps and not just a map
	if data[0] == '[' {
		var res []map[string]interface{}
		err := json.Unmarshal([]byte(data), &res)
		if err != nil {
			logger.Fatalf("Error unmarshalling %v\n", err)
		}
		return res
	}
	var res map[string]interface{}
	err := json.Unmarshal([]byte(data), &res)
	if err != nil {
		logger.Fatalf("Error unmarshalling %v\n", err)
	}
	return res
}

func ProcessTemplateFile(mp interface{}) string {
	path := []string{TemplateFile}
	tmpl := myMust(template.ParseFiles(path...))
	return process(tmpl, mp)
}

func ProcessTemplateStr(mp interface{}) string {
	tmpl := myMust(template.New("tmpl").Parse(TemplateStr))
	return process(tmpl, mp)
}

func process(t *template.Template, vars interface{}) string {
	var tmplBytes bytes.Buffer
	err := t.Execute(&tmplBytes, vars)
	if err != nil {
		logger.Fatalf("Error applying parsed template to data object: %v\n", err)
	}
	return tmplBytes.String()
}

// myMust() provides the same functionality as https://cs.opensource.google/go/go/+/refs/tags/go1.17:src/text/template/helper.go;l=23;drc=4f1b0a44cb46f3df28f5ef82e5769ebeac1bc493
// but does not "panic" and print the stack trace, instead it logs and error message and exits the application if the template syntax is not valid
func myMust(t *template.Template, err error) *template.Template {
	if err != nil {
		logger.Fatalf("Error invalid syntax in %v\n", err)
	}
	return t
}
<commit_msg>renaming a helper function in transform_data.go<commit_after>package transform_data

import (
	"bytes"
	"encoding/json"
	"text/template"

	"github.com/mypurecloud/platform-client-sdk-cli/build/gc/logger"
)

var (
	TemplateFile string
	TemplateStr  string
)

func ConvertJsonToMap(data string) interface{} {
	// if the data is an array of json objects then we need to store that in an array of maps and not just a map
	if data[0] == '[' {
		var res []map[string]interface{}
		err := json.Unmarshal([]byte(data), &res)
		if err != nil {
			logger.Fatalf("Error unmarshalling %v\n", err)
		}
		return res
	}
	var res map[string]interface{}
	err := json.Unmarshal([]byte(data), &res)
	if err != nil {
		logger.Fatalf("Error unmarshalling %v\n", err)
	}
	return res
}

func ProcessTemplateFile(mp interface{}) string {
	path := []string{TemplateFile}
	tmpl := handleParse(template.ParseFiles(path...))
	return process(tmpl, mp)
}

func ProcessTemplateStr(mp interface{}) string {
	tmpl := handleParse(template.New("tmpl").Parse(TemplateStr))
	return process(tmpl, mp)
}

func process(t *template.Template, vars interface{}) string {
	var tmplBytes bytes.Buffer
	err := t.Execute(&tmplBytes, vars)
	if err != nil {
		logger.Fatalf("Error applying parsed template to data object: %v\n", err)
	}
	return tmplBytes.String()
}

// handleParse() provides similar functionality to Must() from the text/template package
// https://cs.opensource.google/go/go/+/refs/tags/go1.17:src/text/template/helper.go;l=23;drc=4f1b0a44cb46f3df28f5ef82e5769ebeac1bc493
// but does not "panic" and print the stack trace if there is an error, instead it prints an error message and exits the application if the template syntax is not valid
// otherwise, it returns the template
func handleParse(t *template.Template, err error) *template.Template {
	if err != nil {
		logger.Fatalf("Error invalid syntax in %v\n", err)
	}
	return t
}
<|endoftext|>


<commit_before>package cmap

import (
	"unsafe"
)

type ListNode struct {
	unsafe.Pointer
	value Any
}<commit_msg>ConcurrentList update....<commit_after>package cmap

import (
	"unsafe"
	"sync/atomic"
)

var deleteMark = "delete"
var head = "head"

type ListNode struct {
	unsafe.Pointer
	value Any
}

type ConcurrentList struct {
	*ListNode
	size int64
}

func NewConcurrentList() *ConcurrentList {
	return &ConcurrentList{&ListNode{nil, &head}, 0}
}

func (list *ConcurrentList) Insert(value Any) bool {

}

func (list *ConcurrentList) Size() int {
	return int(atomic.LoadInt64(&list.size))
}

func (node *ListNode) next() *ListNode {
	next := atomic.LoadPointer(&node.Pointer)

	for next != nil {
		nextNode := (*ListNode)(next)

		if sp, ok := nextNode.value.(*string); ok && sp == &deleteMark {
			return nextNode.next()
		}

		if nextNode.isDeleted() {
			atomic.CompareAndSwapPointer(&node.Pointer, next, unsafe.Pointer(nextNode.next()))
			next = atomic.LoadPointer(&node.Pointer)
		} else {
			return nextNode
		}
	}

	return nil
}

func (node *ListNode) isDeleted() bool {
	next := atomic.LoadPointer(&node.Pointer)
	if next == nil {
		return false
	}

	if sp, ok := (*ListNode)(next).value.(*string); ok && sp == &deleteMark {
		return true
	}

	return false
}<|endoftext|>