<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"context"
	"fmt"
	"log"
	"sort"
	"strings"

	"golang.org/x/net/trace"

	"github.com/google/zoekt/query"
)

func (p *contentProvider) evalContentMatches(s *substrMatchTree) {
	if !s.coversContent {
		pruned := s.current[:0]
		for _, m := range s.current {
			if p.matchContent(m) {
				pruned = append(pruned, m)
			}
		}
		s.current = pruned
	} else {
		// TODO - this side effect is kind of hidden and surprising.
		for _, cm := range s.current {
			cm.byteOffset = p.findOffset(cm.fileName, cm.runeOffset)
		}
	}
	s.contEvaluated = true
}

func (p *contentProvider) evalRegexpMatches(s *regexpMatchTree) {
	idxs := s.regexp.FindAllIndex(p.data(s.fileName), -1)
	s.found = make([]*candidateMatch, 0, len(idxs))
	for _, idx := range idxs {
		s.found = append(s.found, &candidateMatch{
			byteOffset:  uint32(idx[0]),
			byteMatchSz: uint32(idx[1] - idx[0]),
			fileName:    s.fileName,
		})
	}
	s.reEvaluated = true
}

func (d *indexData) simplify(in query.Q) query.Q {
	eval := query.Map(in, func(q query.Q) query.Q {
		if r, ok := q.(*query.Repo); ok {
			return &query.Const{Value: strings.Contains(d.repoMetaData.Name, r.Pattern)}
		}
		if l, ok := q.(*query.Language); ok {
			_, has := d.metaData.LanguageMap[l.Language]
			if !has {
				return &query.Const{Value: false}
			}
		}
		return q
	})
	return query.Simplify(eval)
}

func (o *SearchOptions) SetDefaults() {
	if o.ShardMaxMatchCount == 0 {
		// We cap the total number of matches, so overly broad
		// searches don't crash the machine.
		o.ShardMaxMatchCount = 100000
	}
	if o.TotalMaxMatchCount == 0 {
		o.TotalMaxMatchCount = 10 * o.ShardMaxMatchCount
	}
	if o.ShardMaxImportantMatch == 0 {
		o.ShardMaxImportantMatch = 10
	}
	if o.TotalMaxImportantMatch == 0 {
		o.TotalMaxImportantMatch = 10 * o.ShardMaxImportantMatch
	}
}

func (d *indexData) Search(ctx context.Context, q query.Q, opts *SearchOptions) (sr *SearchResult, err error) {
	copyOpts := *opts
	opts = &copyOpts
	opts.SetDefaults()
	importantMatchCount := 0

	var res SearchResult
	if len(d.fileNameIndex) == 0 {
		return &res, nil
	}

	tr := trace.New("indexData.Search", d.file.Name())
	tr.LazyPrintf("opts: %+v", opts)
	defer func() {
		if sr != nil {
			tr.LazyPrintf("num files: %d", len(sr.Files))
			tr.LazyPrintf("stats: %+v", sr.Stats)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	q = d.simplify(q)
	tr.LazyLog(q, true)
	if c, ok := q.(*query.Const); ok && !c.Value {
		return &res, nil
	}

	if opts.EstimateDocCount {
		res.Stats.ShardFilesConsidered = len(d.fileBranchMasks)
		return &res, nil
	}

	select {
	case <-ctx.Done():
		res.Stats.ShardsSkipped++
		return &res, nil
	default:
	}

	q = query.Map(q, query.ExpandFileContent)

	mt, err := d.newMatchTree(q, &res.Stats)
	if err != nil {
		return nil, err
	}

	totalAtomCount := 0
	var substrAtoms, fileAtoms []*substrMatchTree
	var regexpAtoms []*regexpMatchTree

	collectAtoms(mt, func(t matchTree) {
		totalAtomCount++
		if st, ok := t.(*substrMatchTree); ok {
			if st.fileName {
				fileAtoms = append(fileAtoms, st)
			} else {
				substrAtoms = append(substrAtoms, st)
			}

		}
		if re, ok := t.(*regexpMatchTree); ok {
			regexpAtoms = append(regexpAtoms, re)
		}
	})

	cp := contentProvider{
		id:    d,
		stats: &res.Stats,
	}

	docCount := uint32(len(d.fileBranchMasks))
	lastDoc := int(-1)

nextFileMatch:
	for {
		canceled := false
		select {
		case <-ctx.Done():
			canceled = true
		default:
		}

		nextDoc := mt.nextDoc()
		if int(nextDoc) <= lastDoc {
			nextDoc = uint32(lastDoc + 1)
		}
		if nextDoc >= docCount {
			break
		}
		lastDoc = int(nextDoc)

		if canceled || res.Stats.MatchCount >= opts.ShardMaxMatchCount ||
			importantMatchCount >= opts.ShardMaxImportantMatch {
			res.Stats.FilesSkipped += d.repoListEntry.Stats.Documents - lastDoc
			break
		}

		res.Stats.FilesConsidered++
		mt.prepare(nextDoc)

		cp.setDocument(nextDoc)

		known := make(map[matchTree]bool)
		if v, ok := evalMatchTree(known, mt); ok && !v {
			continue nextFileMatch
		}

		// Files are cheap to match. Do them first.
		if len(fileAtoms) > 0 {
			for _, st := range fileAtoms {
				cp.evalContentMatches(st)
			}
			if v, ok := evalMatchTree(known, mt); ok && !v {
				continue nextFileMatch
			}
		}

		for _, st := range substrAtoms {
			// TODO - this may evaluate too much.
			cp.evalContentMatches(st)
		}
		if len(regexpAtoms) > 0 {
			if v, ok := evalMatchTree(known, mt); ok && !v {
				continue nextFileMatch
			}

			for _, re := range regexpAtoms {
				cp.evalRegexpMatches(re)
			}
		}

		if v, ok := evalMatchTree(known, mt); !ok {
			log.Panicf("did not decide. Repo %s, doc %d, known %v",
				d.repoMetaData.Name, nextDoc, known)
		} else if !v {
			continue nextFileMatch
		}

		fileMatch := FileMatch{
			Repository: d.repoMetaData.Name,
			FileName:   string(d.fileName(nextDoc)),
			// Maintain ordering of input files. This
			// strictly dominates the in-file ordering of
			// the matches.
			Score:    10 * float64(nextDoc) / float64(len(d.boundaries)),
			Checksum: d.getChecksum(nextDoc),
			Language: d.languageMap[d.languages[nextDoc]],
		}

		if s := d.subRepos[nextDoc]; s > 0 {
			if s >= uint32(len(d.subRepoPaths)) {
				log.Panicf("corrupt index: subrepo %d beyond %v", s, d.subRepoPaths)
			}
			path := d.subRepoPaths[s]
			fileMatch.SubRepositoryPath = path
			sr := d.repoMetaData.SubRepoMap[path]
			fileMatch.SubRepositoryName = sr.Name
			if idx := d.branchIndex(nextDoc); idx >= 0 {
				fileMatch.Version = sr.Branches[idx].Version
			}
		} else {
			idx := d.branchIndex(nextDoc)
			if idx >= 0 {
				fileMatch.Version = d.repoMetaData.Branches[idx].Version
			}
		}

		atomMatchCount := 0
		visitMatches(mt, known, func(mt matchTree) {
			atomMatchCount++
		})
		fileMatch.Score += float64(atomMatchCount) / float64(totalAtomCount) * scoreFactorAtomMatch
		finalCands := gatherMatches(mt, known)

		if len(finalCands) == 0 {
			nm := d.fileName(nextDoc)
			finalCands = append(finalCands,
				&candidateMatch{
					caseSensitive: false,
					fileName:      true,
					substrBytes:   nm,
					substrLowered: nm,
					file:          nextDoc,
					runeOffset:    0,
					byteOffset:    0,
					byteMatchSz:   uint32(len(nm)),
				})
		}
		fileMatch.LineMatches = cp.fillMatches(finalCands)

		maxFileScore := 0.0
		for i := range fileMatch.LineMatches {
			if maxFileScore < fileMatch.LineMatches[i].Score {
				maxFileScore = fileMatch.LineMatches[i].Score

			}

			// Order by ordering in file.
			fileMatch.LineMatches[i].Score += 1.0 - (float64(i) / float64(len(fileMatch.LineMatches)))
		}
		fileMatch.Score += maxFileScore

		if fileMatch.Score > scoreImportantThreshold {
			importantMatchCount++
		}
		fileMatch.Branches = d.gatherBranches(nextDoc, mt, known)

		sortMatchesByScore(fileMatch.LineMatches)
		if opts.Whole {
			fileMatch.Content = cp.data(false)
		}

		res.Files = append(res.Files, fileMatch)
		res.Stats.MatchCount += len(fileMatch.LineMatches)
		res.Stats.FileCount++
	}
	SortFilesByScore(res.Files)

	addRepo(&res, &d.repoMetaData)
	for _, v := range d.repoMetaData.SubRepoMap {
		addRepo(&res, v)
	}

	mt.updateStats(&res.Stats)
	return &res, nil
}

func addRepo(res *SearchResult, repo *Repository) {
	if res.RepoURLs == nil {
		res.RepoURLs = map[string]string{}
	}
	res.RepoURLs[repo.Name] = repo.FileURLTemplate

	if res.LineFragments == nil {
		res.LineFragments = map[string]string{}
	}
	res.LineFragments[repo.Name] = repo.LineFragmentTemplate
}

func extractSubstringQueries(q query.Q) []*query.Substring {
	var r []*query.Substring
	switch s := q.(type) {
	case *query.And:
		for _, ch := range s.Children {
			r = append(r, extractSubstringQueries(ch)...)
		}
	case *query.Or:
		for _, ch := range s.Children {
			r = append(r, extractSubstringQueries(ch)...)
		}
	case *query.Not:
		r = append(r, extractSubstringQueries(s.Child)...)
	case *query.Substring:
		r = append(r, s)
	}
	return r
}

type sortByOffsetSlice []*candidateMatch

func (m sortByOffsetSlice) Len() int      { return len(m) }
func (m sortByOffsetSlice) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m sortByOffsetSlice) Less(i, j int) bool {
	return m[i].byteOffset < m[j].byteOffset
}

// Gather matches from this document. This never returns a mixture of
// filename/content matches: if there are content matches, all
// filename matches are trimmed from the result. The matches are
// returned in document order and are non-overlapping.
func gatherMatches(mt matchTree, known map[matchTree]bool) []*candidateMatch {
	var cands []*candidateMatch
	visitMatches(mt, known, func(mt matchTree) {
		if smt, ok := mt.(*substrMatchTree); ok {
			cands = append(cands, smt.current...)
		}
		if rmt, ok := mt.(*regexpMatchTree); ok {
			cands = append(cands, rmt.found...)
		}
	})

	foundContentMatch := false
	for _, c := range cands {
		if !c.fileName {
			foundContentMatch = true
			break
		}
	}

	res := cands[:0]
	for _, c := range cands {
		if !foundContentMatch || !c.fileName {
			res = append(res, c)
		}
	}
	cands = res

	// Merge adjacent candidates. This guarantees that the matches
	// are non-overlapping.
	sort.Sort((sortByOffsetSlice)(cands))
	res = cands[:0]
	for i, c := range cands {
		if i == 0 {
			res = append(res, c)
			continue
		}
		last := res[len(res)-1]
		lastEnd := last.byteOffset + last.byteMatchSz
		end := c.byteOffset + c.byteMatchSz
		if lastEnd >= c.byteOffset {
			if end > lastEnd {
				last.byteMatchSz = end - last.byteOffset
			}
			continue
		}

		res = append(res, c)
	}

	return res
}

func (d *indexData) branchIndex(docID uint32) int {
	mask := d.fileBranchMasks[docID]
	idx := 0
	for mask != 0 {
		if mask&0x1 != 0 {
			return idx
		}
		idx++
		mask >>= 1
	}
	return -1
}

// gatherBranches returns a list of branch names.
func (d *indexData) gatherBranches(docID uint32, mt matchTree, known map[matchTree]bool) []string {
	foundBranchQuery := false
	var branches []string

	visitMatches(mt, known, func(mt matchTree) {
		bq, ok := mt.(*branchQueryMatchTree)
		if ok {
			foundBranchQuery = true
			branches = append(branches,
				d.branchNames[uint(bq.mask)])
		}
	})

	if !foundBranchQuery {
		mask := d.fileBranchMasks[docID]
		id := uint32(1)
		for mask != 0 {
			if mask&0x1 != 0 {
				branches = append(branches, d.branchNames[uint(id)])
			}
			id <<= 1
			mask >>= 1
		}
	}
	return branches
}

func (d *indexData) List(ctx context.Context, q query.Q) (rl *RepoList, err error) {
	tr := trace.New("indexData.List", d.file.Name())
	defer func() {
		if rl != nil {
			tr.LazyPrintf("repos size: %d", len(rl.Repos))
			tr.LazyPrintf("crashes: %d", rl.Crashes)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	q = d.simplify(q)
	tr.LazyLog(q, true)
	c, ok := q.(*query.Const)

	if !ok {
		return nil, fmt.Errorf("List should receive Repo-only query.")
	}

	l := &RepoList{}
	if c.Value {
		l.Repos = append(l.Repos, &d.repoListEntry)
	}
	return l, nil
}
<commit_msg>Delete dead function extractSubstringQueries<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zoekt

import (
	"context"
	"fmt"
	"log"
	"sort"
	"strings"

	"golang.org/x/net/trace"

	"github.com/google/zoekt/query"
)

func (p *contentProvider) evalContentMatches(s *substrMatchTree) {
	if !s.coversContent {
		pruned := s.current[:0]
		for _, m := range s.current {
			if p.matchContent(m) {
				pruned = append(pruned, m)
			}
		}
		s.current = pruned
	} else {
		// TODO - this side effect is kind of hidden and surprising.
		for _, cm := range s.current {
			cm.byteOffset = p.findOffset(cm.fileName, cm.runeOffset)
		}
	}
	s.contEvaluated = true
}

func (p *contentProvider) evalRegexpMatches(s *regexpMatchTree) {
	idxs := s.regexp.FindAllIndex(p.data(s.fileName), -1)
	s.found = make([]*candidateMatch, 0, len(idxs))
	for _, idx := range idxs {
		s.found = append(s.found, &candidateMatch{
			byteOffset:  uint32(idx[0]),
			byteMatchSz: uint32(idx[1] - idx[0]),
			fileName:    s.fileName,
		})
	}
	s.reEvaluated = true
}

func (d *indexData) simplify(in query.Q) query.Q {
	eval := query.Map(in, func(q query.Q) query.Q {
		if r, ok := q.(*query.Repo); ok {
			return &query.Const{Value: strings.Contains(d.repoMetaData.Name, r.Pattern)}
		}
		if l, ok := q.(*query.Language); ok {
			_, has := d.metaData.LanguageMap[l.Language]
			if !has {
				return &query.Const{Value: false}
			}
		}
		return q
	})
	return query.Simplify(eval)
}

func (o *SearchOptions) SetDefaults() {
	if o.ShardMaxMatchCount == 0 {
		// We cap the total number of matches, so overly broad
		// searches don't crash the machine.
		o.ShardMaxMatchCount = 100000
	}
	if o.TotalMaxMatchCount == 0 {
		o.TotalMaxMatchCount = 10 * o.ShardMaxMatchCount
	}
	if o.ShardMaxImportantMatch == 0 {
		o.ShardMaxImportantMatch = 10
	}
	if o.TotalMaxImportantMatch == 0 {
		o.TotalMaxImportantMatch = 10 * o.ShardMaxImportantMatch
	}
}

func (d *indexData) Search(ctx context.Context, q query.Q, opts *SearchOptions) (sr *SearchResult, err error) {
	copyOpts := *opts
	opts = &copyOpts
	opts.SetDefaults()
	importantMatchCount := 0

	var res SearchResult
	if len(d.fileNameIndex) == 0 {
		return &res, nil
	}

	tr := trace.New("indexData.Search", d.file.Name())
	tr.LazyPrintf("opts: %+v", opts)
	defer func() {
		if sr != nil {
			tr.LazyPrintf("num files: %d", len(sr.Files))
			tr.LazyPrintf("stats: %+v", sr.Stats)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	q = d.simplify(q)
	tr.LazyLog(q, true)
	if c, ok := q.(*query.Const); ok && !c.Value {
		return &res, nil
	}

	if opts.EstimateDocCount {
		res.Stats.ShardFilesConsidered = len(d.fileBranchMasks)
		return &res, nil
	}

	select {
	case <-ctx.Done():
		res.Stats.ShardsSkipped++
		return &res, nil
	default:
	}

	q = query.Map(q, query.ExpandFileContent)

	mt, err := d.newMatchTree(q, &res.Stats)
	if err != nil {
		return nil, err
	}

	totalAtomCount := 0
	var substrAtoms, fileAtoms []*substrMatchTree
	var regexpAtoms []*regexpMatchTree

	collectAtoms(mt, func(t matchTree) {
		totalAtomCount++
		if st, ok := t.(*substrMatchTree); ok {
			if st.fileName {
				fileAtoms = append(fileAtoms, st)
			} else {
				substrAtoms = append(substrAtoms, st)
			}

		}
		if re, ok := t.(*regexpMatchTree); ok {
			regexpAtoms = append(regexpAtoms, re)
		}
	})

	cp := contentProvider{
		id:    d,
		stats: &res.Stats,
	}

	docCount := uint32(len(d.fileBranchMasks))
	lastDoc := int(-1)

nextFileMatch:
	for {
		canceled := false
		select {
		case <-ctx.Done():
			canceled = true
		default:
		}

		nextDoc := mt.nextDoc()
		if int(nextDoc) <= lastDoc {
			nextDoc = uint32(lastDoc + 1)
		}
		if nextDoc >= docCount {
			break
		}
		lastDoc = int(nextDoc)

		if canceled || res.Stats.MatchCount >= opts.ShardMaxMatchCount ||
			importantMatchCount >= opts.ShardMaxImportantMatch {
			res.Stats.FilesSkipped += d.repoListEntry.Stats.Documents - lastDoc
			break
		}

		res.Stats.FilesConsidered++
		mt.prepare(nextDoc)

		cp.setDocument(nextDoc)

		known := make(map[matchTree]bool)
		if v, ok := evalMatchTree(known, mt); ok && !v {
			continue nextFileMatch
		}

		// Files are cheap to match. Do them first.
		if len(fileAtoms) > 0 {
			for _, st := range fileAtoms {
				cp.evalContentMatches(st)
			}
			if v, ok := evalMatchTree(known, mt); ok && !v {
				continue nextFileMatch
			}
		}

		for _, st := range substrAtoms {
			// TODO - this may evaluate too much.
			cp.evalContentMatches(st)
		}
		if len(regexpAtoms) > 0 {
			if v, ok := evalMatchTree(known, mt); ok && !v {
				continue nextFileMatch
			}

			for _, re := range regexpAtoms {
				cp.evalRegexpMatches(re)
			}
		}

		if v, ok := evalMatchTree(known, mt); !ok {
			log.Panicf("did not decide. Repo %s, doc %d, known %v",
				d.repoMetaData.Name, nextDoc, known)
		} else if !v {
			continue nextFileMatch
		}

		fileMatch := FileMatch{
			Repository: d.repoMetaData.Name,
			FileName:   string(d.fileName(nextDoc)),
			// Maintain ordering of input files. This
			// strictly dominates the in-file ordering of
			// the matches.
			Score:    10 * float64(nextDoc) / float64(len(d.boundaries)),
			Checksum: d.getChecksum(nextDoc),
			Language: d.languageMap[d.languages[nextDoc]],
		}

		if s := d.subRepos[nextDoc]; s > 0 {
			if s >= uint32(len(d.subRepoPaths)) {
				log.Panicf("corrupt index: subrepo %d beyond %v", s, d.subRepoPaths)
			}
			path := d.subRepoPaths[s]
			fileMatch.SubRepositoryPath = path
			sr := d.repoMetaData.SubRepoMap[path]
			fileMatch.SubRepositoryName = sr.Name
			if idx := d.branchIndex(nextDoc); idx >= 0 {
				fileMatch.Version = sr.Branches[idx].Version
			}
		} else {
			idx := d.branchIndex(nextDoc)
			if idx >= 0 {
				fileMatch.Version = d.repoMetaData.Branches[idx].Version
			}
		}

		atomMatchCount := 0
		visitMatches(mt, known, func(mt matchTree) {
			atomMatchCount++
		})
		fileMatch.Score += float64(atomMatchCount) / float64(totalAtomCount) * scoreFactorAtomMatch
		finalCands := gatherMatches(mt, known)

		if len(finalCands) == 0 {
			nm := d.fileName(nextDoc)
			finalCands = append(finalCands,
				&candidateMatch{
					caseSensitive: false,
					fileName:      true,
					substrBytes:   nm,
					substrLowered: nm,
					file:          nextDoc,
					runeOffset:    0,
					byteOffset:    0,
					byteMatchSz:   uint32(len(nm)),
				})
		}
		fileMatch.LineMatches = cp.fillMatches(finalCands)

		maxFileScore := 0.0
		for i := range fileMatch.LineMatches {
			if maxFileScore < fileMatch.LineMatches[i].Score {
				maxFileScore = fileMatch.LineMatches[i].Score

			}

			// Order by ordering in file.
			fileMatch.LineMatches[i].Score += 1.0 - (float64(i) / float64(len(fileMatch.LineMatches)))
		}
		fileMatch.Score += maxFileScore

		if fileMatch.Score > scoreImportantThreshold {
			importantMatchCount++
		}
		fileMatch.Branches = d.gatherBranches(nextDoc, mt, known)

		sortMatchesByScore(fileMatch.LineMatches)
		if opts.Whole {
			fileMatch.Content = cp.data(false)
		}

		res.Files = append(res.Files, fileMatch)
		res.Stats.MatchCount += len(fileMatch.LineMatches)
		res.Stats.FileCount++
	}
	SortFilesByScore(res.Files)

	addRepo(&res, &d.repoMetaData)
	for _, v := range d.repoMetaData.SubRepoMap {
		addRepo(&res, v)
	}

	mt.updateStats(&res.Stats)
	return &res, nil
}

func addRepo(res *SearchResult, repo *Repository) {
	if res.RepoURLs == nil {
		res.RepoURLs = map[string]string{}
	}
	res.RepoURLs[repo.Name] = repo.FileURLTemplate

	if res.LineFragments == nil {
		res.LineFragments = map[string]string{}
	}
	res.LineFragments[repo.Name] = repo.LineFragmentTemplate
}

type sortByOffsetSlice []*candidateMatch

func (m sortByOffsetSlice) Len() int      { return len(m) }
func (m sortByOffsetSlice) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m sortByOffsetSlice) Less(i, j int) bool {
	return m[i].byteOffset < m[j].byteOffset
}

// Gather matches from this document. This never returns a mixture of
// filename/content matches: if there are content matches, all
// filename matches are trimmed from the result. The matches are
// returned in document order and are non-overlapping.
func gatherMatches(mt matchTree, known map[matchTree]bool) []*candidateMatch {
	var cands []*candidateMatch
	visitMatches(mt, known, func(mt matchTree) {
		if smt, ok := mt.(*substrMatchTree); ok {
			cands = append(cands, smt.current...)
		}
		if rmt, ok := mt.(*regexpMatchTree); ok {
			cands = append(cands, rmt.found...)
		}
	})

	foundContentMatch := false
	for _, c := range cands {
		if !c.fileName {
			foundContentMatch = true
			break
		}
	}

	res := cands[:0]
	for _, c := range cands {
		if !foundContentMatch || !c.fileName {
			res = append(res, c)
		}
	}
	cands = res

	// Merge adjacent candidates. This guarantees that the matches
	// are non-overlapping.
	sort.Sort((sortByOffsetSlice)(cands))
	res = cands[:0]
	for i, c := range cands {
		if i == 0 {
			res = append(res, c)
			continue
		}
		last := res[len(res)-1]
		lastEnd := last.byteOffset + last.byteMatchSz
		end := c.byteOffset + c.byteMatchSz
		if lastEnd >= c.byteOffset {
			if end > lastEnd {
				last.byteMatchSz = end - last.byteOffset
			}
			continue
		}

		res = append(res, c)
	}

	return res
}

func (d *indexData) branchIndex(docID uint32) int {
	mask := d.fileBranchMasks[docID]
	idx := 0
	for mask != 0 {
		if mask&0x1 != 0 {
			return idx
		}
		idx++
		mask >>= 1
	}
	return -1
}

// gatherBranches returns a list of branch names.
func (d *indexData) gatherBranches(docID uint32, mt matchTree, known map[matchTree]bool) []string {
	foundBranchQuery := false
	var branches []string

	visitMatches(mt, known, func(mt matchTree) {
		bq, ok := mt.(*branchQueryMatchTree)
		if ok {
			foundBranchQuery = true
			branches = append(branches,
				d.branchNames[uint(bq.mask)])
		}
	})

	if !foundBranchQuery {
		mask := d.fileBranchMasks[docID]
		id := uint32(1)
		for mask != 0 {
			if mask&0x1 != 0 {
				branches = append(branches, d.branchNames[uint(id)])
			}
			id <<= 1
			mask >>= 1
		}
	}
	return branches
}

func (d *indexData) List(ctx context.Context, q query.Q) (rl *RepoList, err error) {
	tr := trace.New("indexData.List", d.file.Name())
	defer func() {
		if rl != nil {
			tr.LazyPrintf("repos size: %d", len(rl.Repos))
			tr.LazyPrintf("crashes: %d", rl.Crashes)
		}
		if err != nil {
			tr.LazyPrintf("error: %v", err)
			tr.SetError()
		}
		tr.Finish()
	}()

	q = d.simplify(q)
	tr.LazyLog(q, true)
	c, ok := q.(*query.Const)

	if !ok {
		return nil, fmt.Errorf("List should receive Repo-only query.")
	}

	l := &RepoList{}
	if c.Value {
		l.Repos = append(l.Repos, &d.repoListEntry)
	}
	return l, nil
}
<|endoftext|>


<commit_before>package q

import (
	"fmt"
	"reflect"

	"github.com/oov/q/qutil"
)

// Expression represents expressions.
type Expression interface {
	// C creates Column from Expression.
	C(aliasName ...string) Column
	// for internal use.
	WriteExpression(ctx *qutil.Context, buf []byte) []byte
}

func expressionToString(e Expression) string {
	buf, ctx := qutil.NewContext(e, 32, 1, nil)
	buf = e.WriteExpression(ctx, buf)
	return toString(buf, ctx.Args)
}

func interfaceToExpression(x interface{}) Expression {
	if x == nil {
		return nullExpr{}
	}
	if v, ok := x.(Expression); ok {
		return v
	}
	return V(x)
}

func columnExpr(e Expression, aliasName ...string) Column {
	r := &exprAsColumn{e}
	if len(aliasName) == 0 {
		return r
	}
	return &columnAlias{Column: r, Alias: aliasName[0]}
}

// Expressions represents combination of an expression.
// It can use as Expression because Expressions implements Expression interface.
type Expressions interface {
	Expression
	// Add adds conds and returns myself.
	Add(conds ...Expression) Expressions
	// Len returns number of expressions.
	Len() int
}

type nullExpr struct{}

func (e nullExpr) C(aliasName ...string) Column { panic("not implemeneted") }
func (e nullExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	return append(buf, "NULL"...)
}

type simpleExpr struct {
	Op    string
	Left  Expression
	Right Expression
}

func (e *simpleExpr) String() string               { return expressionToString(e) }
func (e *simpleExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *simpleExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	buf = e.Left.WriteExpression(ctx, buf)
	buf = append(buf, e.Op...)
	buf = e.Right.WriteExpression(ctx, buf)
	return buf
}

type eqExpr struct {
	Eq    bool
	Left  Expression
	Right Expression
}

func (e *eqExpr) String() string               { return expressionToString(e) }
func (e *eqExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *eqExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	lv, rv := e.Left, e.Right
	if _, ok := lv.(nullExpr); ok {
		lv, rv = rv, lv
	}
	if _, ok := rv.(nullExpr); ok {
		buf = lv.WriteExpression(ctx, buf)
		if e.Eq {
			return append(buf, " IS NULL"...)
		}
		return append(buf, " IS NOT NULL"...)
	}

	buf = lv.WriteExpression(ctx, buf)
	if e.Eq {
		buf = append(buf, " = "...)
	} else {
		buf = append(buf, " != "...)
	}
	buf = rv.WriteExpression(ctx, buf)
	return buf
}

type inExpr struct {
	Eq    bool
	Left  Expression
	Right inVariable
}

func (e *inExpr) String() string               { return expressionToString(e) }
func (e *inExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *inExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	if len(e.Right) == 0 {
		// x IN () is invaild syntax.
		// But at the same time, a result is a obvious expression.
		// So replace the alternative valid expression which is the same result.
		if e.Eq {
			return append(buf, "'IN' = '()'"...)
		}
		return append(buf, "'IN' != '()'"...)
	}

	buf = e.Left.WriteExpression(ctx, buf)
	if e.Eq {
		buf = append(buf, " IN "...)
	} else {
		buf = append(buf, " NOT IN "...)
	}
	buf = e.Right.WriteExpression(ctx, buf)
	return buf
}

type logicalExpr struct {
	Exprs []Expression
	Op    string
}

func (e *logicalExpr) String() string               { return expressionToString(e) }
func (e *logicalExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *logicalExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	switch len(e.Exprs) {
	case 0:
		buf = append(buf, "('empty' = '"...)
		buf = append(buf, e.Op...)
		buf = append(buf, "')"...)
		return buf
	case 1:
		return e.Exprs[0].WriteExpression(ctx, buf)
	}
	buf = append(buf, '(')
	buf = e.Exprs[0].WriteExpression(ctx, buf)
	buf = append(buf, ')')
	for _, cd := range e.Exprs[1:] {
		buf = append(buf, e.Op...)
		buf = append(buf, '(')
		buf = cd.WriteExpression(ctx, buf)
		buf = append(buf, ')')
	}
	return buf
}

func (e *logicalExpr) Add(exprs ...Expression) Expressions {
	e.Exprs = append(e.Exprs, exprs...)
	return e
}

func (e *logicalExpr) Len() int {
	return len(e.Exprs)
}

func newIn(l interface{}, v reflect.Value, eq bool) Expression {
	var r inVariable
	for i, l := 0, v.Len(); i < l; i++ {
		r = append(r, v.Index(i).Interface())
	}
	return &inExpr{Eq: eq, Left: interfaceToExpression(l), Right: r}
}

// Eq creates Expression such as "l = r".
// But when you pass nil to one of a pair, Eq creates "x IS NULL" instead.
// In the same way, when you pass slice of any type to r, Eq creates "x IN (?)".
func Eq(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, true)
		}
	}
	return &eqExpr{
		Eq:    true,
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Neq creates Expression such as "l != r".
// But when you pass nil to one of a pair, Neq creates "x IS NOT NULL" instead.
// In the same way, when you pass slice of any type to r, Neq creates "x NOT IN (?)".
func Neq(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, false)
		}
	}
	return &eqExpr{
		Eq:    false,
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// In creates Expression such as "l IN r".
func In(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, true)
		}
	}
	return &simpleExpr{
		Op:    " IN ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// NotIn creates Expression such as "l NOT IN r".
func NotIn(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, false)
		}
	}
	return &simpleExpr{
		Op:    " NOT IN ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Gt creates Expression such as "l > r".
func Gt(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " > ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Gte creates Expression such as "l >= r".
func Gte(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " >= ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Lt creates Expression such as "l < r".
func Lt(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " < ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Lte creates Expression such as "l <= r".
func Lte(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " <= ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// And creates Expression such as "(exprs[0])AND(exprs[1])AND(exprs[2])".
//
// If you output expression which isn't adding Expression at all,
// it generates "('empty' = 'AND')".
func And(exprs ...Expression) Expressions {
	return &logicalExpr{Op: "AND", Exprs: exprs}
}

// Or creates Expression such as "(exprs[0])OR(exprs[1])OR(exprs[2])".
//
// If you output expression which isn't adding Expression at all,
// it generates "('empty' = 'OR')".
func Or(exprs ...Expression) Expressions {
	return &logicalExpr{Op: "OR", Exprs: exprs}
}

// Unsafe creates any custom expressions.
//
// But IT DOES NOT ESCAPE so if want to use input from outside, should wrap by V or InV.
//
// The basic how to use is similar to fmt.Print.
// Please refer to the example for more details.
func Unsafe(v ...interface{}) UnsafeExpression {
	return unsafeExpr(v)
}

// UnsafeExpression represents unsafe expression.
type UnsafeExpression Expression

type unsafeExpr []interface{}

func (e unsafeExpr) String() string               { return expressionToString(e) }
func (e unsafeExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e unsafeExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	for _, i := range e {
		switch v := i.(type) {
		case Expression:
			buf = v.WriteExpression(ctx, buf)
		case string:
			buf = append(buf, v...)
		case nil:
			buf = append(buf, "NULL"...)
		default:
			buf = append(buf, fmt.Sprint(i)...)
		}
	}
	return buf
}

// Variable represents the argument to which is given from outside.
type Variable Expression

// V creates Variable from a single input.
func V(v interface{}) Variable {
	return &variable{v}
}

type variable struct {
	V interface{}
}

func (v *variable) String() string               { return expressionToString(v) }
func (v *variable) C(aliasName ...string) Column { return columnExpr(v, aliasName...) }
func (v *variable) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	ctx.Args = append(ctx.Args, v.V)
	return ctx.Placeholder.Next(buf)
}

// InV creates Variable from slice.
// It can be used with IN operator.
func InV(slice interface{}) Variable {
	var v inVariable
	s := reflect.ValueOf(slice)
	if s.Kind() != reflect.Slice {
		v = append(v, slice)
		return v
	}
	if s.Len() == 0 {
		panic("q: need at least one value to create Variable.")
	}
	for i, l := 0, s.Len(); i < l; i++ {
		v = append(v, s.Index(i).Interface())
	}
	return v
}

type inVariable []interface{}

func (v inVariable) String() string               { return expressionToString(v) }
func (v inVariable) C(aliasName ...string) Column { return columnExpr(v, aliasName...) }
func (v inVariable) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	buf = append(buf, '(')
	buf = ctx.Placeholder.Next(buf)
	for i, l := 1, len(v); i < l; i++ {
		buf = append(buf, ',')
		buf = ctx.Placeholder.Next(buf)
	}
	buf = append(buf, ')')
	ctx.Args = append(ctx.Args, v...)
	return buf
}
<commit_msg>optimize newIn<commit_after>package q

import (
	"fmt"
	"reflect"

	"github.com/oov/q/qutil"
)

// Expression represents expressions.
type Expression interface {
	// C creates Column from Expression.
	C(aliasName ...string) Column
	// for internal use.
	WriteExpression(ctx *qutil.Context, buf []byte) []byte
}

func expressionToString(e Expression) string {
	buf, ctx := qutil.NewContext(e, 32, 1, nil)
	buf = e.WriteExpression(ctx, buf)
	return toString(buf, ctx.Args)
}

func interfaceToExpression(x interface{}) Expression {
	if x == nil {
		return nullExpr{}
	}
	if v, ok := x.(Expression); ok {
		return v
	}
	return V(x)
}

func columnExpr(e Expression, aliasName ...string) Column {
	r := &exprAsColumn{e}
	if len(aliasName) == 0 {
		return r
	}
	return &columnAlias{Column: r, Alias: aliasName[0]}
}

// Expressions represents combination of an expression.
// It can use as Expression because Expressions implements Expression interface.
type Expressions interface {
	Expression
	// Add adds conds and returns myself.
	Add(conds ...Expression) Expressions
	// Len returns number of expressions.
	Len() int
}

type nullExpr struct{}

func (e nullExpr) C(aliasName ...string) Column { panic("not implemeneted") }
func (e nullExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	return append(buf, "NULL"...)
}

type simpleExpr struct {
	Op    string
	Left  Expression
	Right Expression
}

func (e *simpleExpr) String() string               { return expressionToString(e) }
func (e *simpleExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *simpleExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	buf = e.Left.WriteExpression(ctx, buf)
	buf = append(buf, e.Op...)
	buf = e.Right.WriteExpression(ctx, buf)
	return buf
}

type eqExpr struct {
	Eq    bool
	Left  Expression
	Right Expression
}

func (e *eqExpr) String() string               { return expressionToString(e) }
func (e *eqExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *eqExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	lv, rv := e.Left, e.Right
	if _, ok := lv.(nullExpr); ok {
		lv, rv = rv, lv
	}
	if _, ok := rv.(nullExpr); ok {
		buf = lv.WriteExpression(ctx, buf)
		if e.Eq {
			return append(buf, " IS NULL"...)
		}
		return append(buf, " IS NOT NULL"...)
	}

	buf = lv.WriteExpression(ctx, buf)
	if e.Eq {
		buf = append(buf, " = "...)
	} else {
		buf = append(buf, " != "...)
	}
	buf = rv.WriteExpression(ctx, buf)
	return buf
}

type inExpr struct {
	Eq    bool
	Left  Expression
	Right inVariable
}

func (e *inExpr) String() string               { return expressionToString(e) }
func (e *inExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *inExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	if len(e.Right) == 0 {
		// x IN () is invaild syntax.
		// But at the same time, a result is a obvious expression.
		// So replace the alternative valid expression which is the same result.
		if e.Eq {
			return append(buf, "'IN' = '()'"...)
		}
		return append(buf, "'IN' != '()'"...)
	}

	buf = e.Left.WriteExpression(ctx, buf)
	if e.Eq {
		buf = append(buf, " IN "...)
	} else {
		buf = append(buf, " NOT IN "...)
	}
	buf = e.Right.WriteExpression(ctx, buf)
	return buf
}

type logicalExpr struct {
	Exprs []Expression
	Op    string
}

func (e *logicalExpr) String() string               { return expressionToString(e) }
func (e *logicalExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e *logicalExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	switch len(e.Exprs) {
	case 0:
		buf = append(buf, "('empty' = '"...)
		buf = append(buf, e.Op...)
		buf = append(buf, "')"...)
		return buf
	case 1:
		return e.Exprs[0].WriteExpression(ctx, buf)
	}
	buf = append(buf, '(')
	buf = e.Exprs[0].WriteExpression(ctx, buf)
	buf = append(buf, ')')
	for _, cd := range e.Exprs[1:] {
		buf = append(buf, e.Op...)
		buf = append(buf, '(')
		buf = cd.WriteExpression(ctx, buf)
		buf = append(buf, ')')
	}
	return buf
}

func (e *logicalExpr) Add(exprs ...Expression) Expressions {
	e.Exprs = append(e.Exprs, exprs...)
	return e
}

func (e *logicalExpr) Len() int {
	return len(e.Exprs)
}

func newIn(l interface{}, v reflect.Value, eq bool) Expression {
	ln := v.Len()
	r := make(inVariable, ln)
	for i := 0; i < ln; i++ {
		r[i] = v.Index(i).Interface()
	}
	return &inExpr{Eq: eq, Left: interfaceToExpression(l), Right: r}
}

// Eq creates Expression such as "l = r".
// But when you pass nil to one of a pair, Eq creates "x IS NULL" instead.
// In the same way, when you pass slice of any type to r, Eq creates "x IN (?)".
func Eq(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, true)
		}
	}
	return &eqExpr{
		Eq:    true,
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Neq creates Expression such as "l != r".
// But when you pass nil to one of a pair, Neq creates "x IS NOT NULL" instead.
// In the same way, when you pass slice of any type to r, Neq creates "x NOT IN (?)".
func Neq(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, false)
		}
	}
	return &eqExpr{
		Eq:    false,
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// In creates Expression such as "l IN r".
func In(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, true)
		}
	}
	return &simpleExpr{
		Op:    " IN ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// NotIn creates Expression such as "l NOT IN r".
func NotIn(l, r interface{}) Expression {
	if rv := reflect.ValueOf(r); rv.Kind() == reflect.Slice {
		if _, ok := r.(Expression); !ok {
			return newIn(l, rv, false)
		}
	}
	return &simpleExpr{
		Op:    " NOT IN ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Gt creates Expression such as "l > r".
func Gt(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " > ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Gte creates Expression such as "l >= r".
func Gte(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " >= ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Lt creates Expression such as "l < r".
func Lt(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " < ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// Lte creates Expression such as "l <= r".
func Lte(l, r interface{}) Expression {
	return &simpleExpr{
		Op:    " <= ",
		Left:  interfaceToExpression(l),
		Right: interfaceToExpression(r),
	}
}

// And creates Expression such as "(exprs[0])AND(exprs[1])AND(exprs[2])".
//
// If you output expression which isn't adding Expression at all,
// it generates "('empty' = 'AND')".
func And(exprs ...Expression) Expressions {
	return &logicalExpr{Op: "AND", Exprs: exprs}
}

// Or creates Expression such as "(exprs[0])OR(exprs[1])OR(exprs[2])".
//
// If you output expression which isn't adding Expression at all,
// it generates "('empty' = 'OR')".
func Or(exprs ...Expression) Expressions {
	return &logicalExpr{Op: "OR", Exprs: exprs}
}

// Unsafe creates any custom expressions.
//
// But IT DOES NOT ESCAPE so if want to use input from outside, should wrap by V or InV.
//
// The basic how to use is similar to fmt.Print.
// Please refer to the example for more details.
func Unsafe(v ...interface{}) UnsafeExpression {
	return unsafeExpr(v)
}

// UnsafeExpression represents unsafe expression.
type UnsafeExpression Expression

type unsafeExpr []interface{}

func (e unsafeExpr) String() string               { return expressionToString(e) }
func (e unsafeExpr) C(aliasName ...string) Column { return columnExpr(e, aliasName...) }
func (e unsafeExpr) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	for _, i := range e {
		switch v := i.(type) {
		case Expression:
			buf = v.WriteExpression(ctx, buf)
		case string:
			buf = append(buf, v...)
		case nil:
			buf = append(buf, "NULL"...)
		default:
			buf = append(buf, fmt.Sprint(i)...)
		}
	}
	return buf
}

// Variable represents the argument to which is given from outside.
type Variable Expression

// V creates Variable from a single input.
func V(v interface{}) Variable {
	return &variable{v}
}

type variable struct {
	V interface{}
}

func (v *variable) String() string               { return expressionToString(v) }
func (v *variable) C(aliasName ...string) Column { return columnExpr(v, aliasName...) }
func (v *variable) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	ctx.Args = append(ctx.Args, v.V)
	return ctx.Placeholder.Next(buf)
}

// InV creates Variable from slice.
// It can be used with IN operator.
func InV(slice interface{}) Variable {
	var v inVariable
	s := reflect.ValueOf(slice)
	if s.Kind() != reflect.Slice {
		v = append(v, slice)
		return v
	}
	if s.Len() == 0 {
		panic("q: need at least one value to create Variable.")
	}
	for i, l := 0, s.Len(); i < l; i++ {
		v = append(v, s.Index(i).Interface())
	}
	return v
}

type inVariable []interface{}

func (v inVariable) String() string               { return expressionToString(v) }
func (v inVariable) C(aliasName ...string) Column { return columnExpr(v, aliasName...) }
func (v inVariable) WriteExpression(ctx *qutil.Context, buf []byte) []byte {
	buf = append(buf, '(')
	buf = ctx.Placeholder.Next(buf)
	for i, l := 1, len(v); i < l; i++ {
		buf = append(buf, ',')
		buf = ctx.Placeholder.Next(buf)
	}
	buf = append(buf, ')')
	ctx.Args = append(ctx.Args, v...)
	return buf
}
<|endoftext|>


<commit_before>package fifo

import (
	"context"
	"io"
	"os"
	"runtime"
	"sync"
	"syscall"

	"github.com/pkg/errors"
)

type fifo struct {
	flag        int
	opened      chan struct{}
	closed      chan struct{}
	closing     chan struct{}
	err         error
	file        *os.File
	closingOnce sync.Once // close has been called
	closedOnce  sync.Once // fifo is closed
	handle      *handle
}

var leakCheckWg *sync.WaitGroup

// OpenFifo opens a fifo. Returns io.ReadWriteCloser.
// Context can be used to cancel this function until open(2) has not returned.
// Accepted flags:
// - syscall.O_CREAT - create new fifo if one doesn't exist
// - syscall.O_RDONLY - open fifo only from reader side
// - syscall.O_WRONLY - open fifo only from writer side
// - syscall.O_RDWR - open fifo from both sides, never block on syscall level
// - syscall.O_NONBLOCK - return io.ReadWriteCloser even if other side of the
//     fifo isn't open. read/write will be connected after the actual fifo is
//     open or after fifo is closed.
func OpenFifo(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
	if _, err := os.Stat(fn); err != nil {
		if os.IsNotExist(err) && flag&syscall.O_CREAT != 0 {
			if err := syscall.Mkfifo(fn, uint32(perm&os.ModePerm)); err != nil && !os.IsExist(err) {
				return nil, errors.Wrapf(err, "error creating fifo %v", fn)
			}
		} else {
			return nil, err
		}
	}

	block := flag&syscall.O_NONBLOCK == 0 || flag&syscall.O_RDWR != 0

	flag &= ^syscall.O_CREAT
	flag &= ^syscall.O_NONBLOCK

	h, err := getHandle(fn)
	if err != nil {
		return nil, err
	}

	f := &fifo{
		handle:  h,
		flag:    flag,
		opened:  make(chan struct{}),
		closed:  make(chan struct{}),
		closing: make(chan struct{}),
	}

	wg := leakCheckWg
	if wg != nil {
		wg.Add(2)
	}

	go func() {
		if wg != nil {
			defer wg.Done()
		}
		select {
		case <-ctx.Done():
			f.Close()
		case <-f.opened:
		case <-f.closed:
		}
	}()
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		var file *os.File
		fn, err := h.Path()
		if err == nil {
			file, err = os.OpenFile(fn, flag, 0)
		}
		select {
		case <-f.closing:
			if err == nil {
				select {
				case <-ctx.Done():
					err = ctx.Err()
				default:
					err = errors.Errorf("fifo %v was closed before opening", fn)
				}
				if file != nil {
					file.Close()
				}
			}
		default:
		}
		if err != nil {
			f.closedOnce.Do(func() {
				f.err = err
				close(f.closed)
			})
			return
		}
		f.file = file
		close(f.opened)
	}()
	if block {
		select {
		case <-f.opened:
		case <-f.closed:
			return nil, f.err
		}
	}
	return f, nil
}

// Read from a fifo to a byte array.
func (f *fifo) Read(b []byte) (int, error) {
	if f.flag&syscall.O_WRONLY > 0 {
		return 0, errors.New("reading from write-only fifo")
	}
	select {
	case <-f.opened:
		return f.file.Read(b)
	default:
	}
	select {
	case <-f.opened:
		return f.file.Read(b)
	case <-f.closed:
		return 0, errors.New("reading from a closed fifo")
	}
}

// Write from byte array to a fifo.
func (f *fifo) Write(b []byte) (int, error) {
	if f.flag&(syscall.O_WRONLY|syscall.O_RDWR) == 0 {
		return 0, errors.New("writing to read-only fifo")
	}
	select {
	case <-f.opened:
		return f.file.Write(b)
	default:
	}
	select {
	case <-f.opened:
		return f.file.Write(b)
	case <-f.closed:
		return 0, errors.New("writing to a closed fifo")
	}
}

// Close the fifo. Next reads/writes will error. This method can also be used
// before open(2) has returned and fifo was never opened.
func (f *fifo) Close() error {
	for {
		select {
		case <-f.closed:
			f.handle.Close()
			return f.err
		default:
			select {
			case <-f.opened:
				f.closedOnce.Do(func() {
					f.err = f.file.Close()
					close(f.closed)
				})
			default:
				if f.flag&syscall.O_RDWR != 0 {
					runtime.Gosched()
					break
				}
				f.closingOnce.Do(func() {
					close(f.closing)
				})
				reverseMode := syscall.O_WRONLY
				if f.flag&syscall.O_WRONLY > 0 {
					reverseMode = syscall.O_RDONLY
				}
				fn, err := f.handle.Path()
				// if Close() is called concurrently(shouldn't) it may cause error
				// because handle is closed
				select {
				case <-f.closed:
				default:
					if err != nil {
						// Path has become invalid. We will leak a goroutine.
						// This case should not happen in linux.
						f.closedOnce.Do(func() {
							f.err = err
							close(f.closed)
						})
						<-f.closed
						break
					}
					f, err := os.OpenFile(fn, reverseMode|syscall.O_NONBLOCK, 0)
					if err == nil {
						f.Close()
					}
					runtime.Gosched()
				}
			}
		}
	}
}
<commit_msg>Replace usage of syscall.Mkfifo with unix.Mkfifo. This is a requirement for the Solaris platform.<commit_after>package fifo

import (
	"context"
	"io"
	"os"
	"runtime"
	"sync"
	"syscall"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
)

type fifo struct {
	flag        int
	opened      chan struct{}
	closed      chan struct{}
	closing     chan struct{}
	err         error
	file        *os.File
	closingOnce sync.Once // close has been called
	closedOnce  sync.Once // fifo is closed
	handle      *handle
}

var leakCheckWg *sync.WaitGroup

// OpenFifo opens a fifo. Returns io.ReadWriteCloser.
// Context can be used to cancel this function until open(2) has not returned.
// Accepted flags:
// - syscall.O_CREAT - create new fifo if one doesn't exist
// - syscall.O_RDONLY - open fifo only from reader side
// - syscall.O_WRONLY - open fifo only from writer side
// - syscall.O_RDWR - open fifo from both sides, never block on syscall level
// - syscall.O_NONBLOCK - return io.ReadWriteCloser even if other side of the
//     fifo isn't open. read/write will be connected after the actual fifo is
//     open or after fifo is closed.
func OpenFifo(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
	if _, err := os.Stat(fn); err != nil {
		if os.IsNotExist(err) && flag&syscall.O_CREAT != 0 {
			if err := unix.Mkfifo(fn, uint32(perm&os.ModePerm)); err != nil && !os.IsExist(err) {
				return nil, errors.Wrapf(err, "error creating fifo %v", fn)
			}
		} else {
			return nil, err
		}
	}

	block := flag&syscall.O_NONBLOCK == 0 || flag&syscall.O_RDWR != 0

	flag &= ^syscall.O_CREAT
	flag &= ^syscall.O_NONBLOCK

	h, err := getHandle(fn)
	if err != nil {
		return nil, err
	}

	f := &fifo{
		handle:  h,
		flag:    flag,
		opened:  make(chan struct{}),
		closed:  make(chan struct{}),
		closing: make(chan struct{}),
	}

	wg := leakCheckWg
	if wg != nil {
		wg.Add(2)
	}

	go func() {
		if wg != nil {
			defer wg.Done()
		}
		select {
		case <-ctx.Done():
			f.Close()
		case <-f.opened:
		case <-f.closed:
		}
	}()
	go func() {
		if wg != nil {
			defer wg.Done()
		}
		var file *os.File
		fn, err := h.Path()
		if err == nil {
			file, err = os.OpenFile(fn, flag, 0)
		}
		select {
		case <-f.closing:
			if err == nil {
				select {
				case <-ctx.Done():
					err = ctx.Err()
				default:
					err = errors.Errorf("fifo %v was closed before opening", fn)
				}
				if file != nil {
					file.Close()
				}
			}
		default:
		}
		if err != nil {
			f.closedOnce.Do(func() {
				f.err = err
				close(f.closed)
			})
			return
		}
		f.file = file
		close(f.opened)
	}()
	if block {
		select {
		case <-f.opened:
		case <-f.closed:
			return nil, f.err
		}
	}
	return f, nil
}

// Read from a fifo to a byte array.
func (f *fifo) Read(b []byte) (int, error) {
	if f.flag&syscall.O_WRONLY > 0 {
		return 0, errors.New("reading from write-only fifo")
	}
	select {
	case <-f.opened:
		return f.file.Read(b)
	default:
	}
	select {
	case <-f.opened:
		return f.file.Read(b)
	case <-f.closed:
		return 0, errors.New("reading from a closed fifo")
	}
}

// Write from byte array to a fifo.
func (f *fifo) Write(b []byte) (int, error) {
	if f.flag&(syscall.O_WRONLY|syscall.O_RDWR) == 0 {
		return 0, errors.New("writing to read-only fifo")
	}
	select {
	case <-f.opened:
		return f.file.Write(b)
	default:
	}
	select {
	case <-f.opened:
		return f.file.Write(b)
	case <-f.closed:
		return 0, errors.New("writing to a closed fifo")
	}
}

// Close the fifo. Next reads/writes will error. This method can also be used
// before open(2) has returned and fifo was never opened.
func (f *fifo) Close() error {
	for {
		select {
		case <-f.closed:
			f.handle.Close()
			return f.err
		default:
			select {
			case <-f.opened:
				f.closedOnce.Do(func() {
					f.err = f.file.Close()
					close(f.closed)
				})
			default:
				if f.flag&syscall.O_RDWR != 0 {
					runtime.Gosched()
					break
				}
				f.closingOnce.Do(func() {
					close(f.closing)
				})
				reverseMode := syscall.O_WRONLY
				if f.flag&syscall.O_WRONLY > 0 {
					reverseMode = syscall.O_RDONLY
				}
				fn, err := f.handle.Path()
				// if Close() is called concurrently(shouldn't) it may cause error
				// because handle is closed
				select {
				case <-f.closed:
				default:
					if err != nil {
						// Path has become invalid. We will leak a goroutine.
						// This case should not happen in linux.
						f.closedOnce.Do(func() {
							f.err = err
							close(f.closed)
						})
						<-f.closed
						break
					}
					f, err := os.OpenFile(fn, reverseMode|syscall.O_NONBLOCK, 0)
					if err == nil {
						f.Close()
					}
					runtime.Gosched()
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package shade

import (
	"encoding/json"
	"fmt"
	"time"
)

// File represents the metadata of a file stored in Shade.  It is stored and
// retrieved by the drive.Client API, and boiled down
type File struct {
	// Filename is a fully qualified path, with no leading slash.
	Filename string
	Filesize int64 // Bytes

	// ModifiedTime represents the "commit" time of this File object.  A given
	// Filename is represented by the valid File with the latest ModifiedTime.
	ModifiedTime time.Time

	// Chunks represets an ordered list of the bytes in the file.
	Chunks []Chunk

	// Chunksize is the size of each plaintext Chunk, in bytes.
	Chunksize int

	// AesKey is a 256 bit key used to encrypt the Chunks with AES-GCM.  If no
	// key is provided, the blocks are not encrypted.  The GCM nonce is stored at
	// the front of the encrypted Chunk using gcm.Seal(); use gcm.Open() to
	// recover the Nonce when decrypting.  Nb: This increases the encrypted
	// Chunk's size by gcm.NonceSize(), currently 12 bytes.
	AesKey []byte `json:",omitempty"`
}

// Chunk represents the a portion of content of the File being stored.
type Chunk struct {
	Index  int
	Sha256 []byte
}

func (f *File) String() string {
	out := fmt.Sprintf("{Filename: %q, Filesize: %d, Chunksize: %d, AesKey: %q, Chunks:", f.Filename, f.Filesize, f.Chunksize, f.AesKey)
	sep := ", "
	if len(f.Chunks) < 2 {
		out += " "
	} else {
		out += "\n"
		sep = ",\n"
	}
	for i, c := range f.Chunks {
		if i == len(f.Chunks) {
			out += c.String() + sep
		} else {
			out += c.String()
		}
	}
	return out
}

// ToJSON returns a JSON representation of the File struct.
func (f *File) ToJSON() ([]byte, error) {
	fj, err := json.Marshal(f)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal file %x: %s", f.Filename, err)
	}
	return fj, nil
}

// FromJSON populates the fields of this File struct from a JSON representation.
// It primarily provides a convenient error message if this fails.
func (f *File) FromJSON(fj []byte) error {
	if err := json.Unmarshal(fj, f); err != nil {
		return fmt.Errorf("failed to unmarshal sha256sum %x: %s", SumString(fj), err)
	}
	return nil
}

func (c *Chunk) String() string {
	return fmt.Sprintf("{Index: %d, Sha256: %x}", c.Index, c.Sha256)
}
<commit_msg>Fix comment grammar<commit_after>package shade

import (
	"encoding/json"
	"fmt"
	"time"
)

// File represents the metadata of a file stored in Shade.  It is stored and
// retrieved by the drive.Client API, and boiled down
type File struct {
	// Filename is a fully qualified path, with no leading slash.
	Filename string
	Filesize int64 // Bytes

	// ModifiedTime represents the "commit" time of this File object.  A given
	// Filename is represented by the valid File with the latest ModifiedTime.
	ModifiedTime time.Time

	// Chunks represets an ordered list of the bytes in the file.
	Chunks []Chunk

	// Chunksize is the size of each plaintext Chunk, in bytes.
	Chunksize int

	// AesKey is a 256 bit key used to encrypt the Chunks with AES-GCM.  If no
	// key is provided, the blocks are not encrypted.  The GCM nonce is stored at
	// the front of the encrypted Chunk using gcm.Seal(); use gcm.Open() to
	// recover the Nonce when decrypting.  Nb: This increases the encrypted
	// Chunk's size by gcm.NonceSize(), currently 12 bytes.
	AesKey []byte `json:",omitempty"`
}

// Chunk represents a portion of the content of the File being stored.
type Chunk struct {
	Index  int
	Sha256 []byte
}

func (f *File) String() string {
	out := fmt.Sprintf("{Filename: %q, Filesize: %d, Chunksize: %d, AesKey: %q, Chunks:", f.Filename, f.Filesize, f.Chunksize, f.AesKey)
	sep := ", "
	if len(f.Chunks) < 2 {
		out += " "
	} else {
		out += "\n"
		sep = ",\n"
	}
	for i, c := range f.Chunks {
		if i == len(f.Chunks) {
			out += c.String() + sep
		} else {
			out += c.String()
		}
	}
	return out
}

// ToJSON returns a JSON representation of the File struct.
func (f *File) ToJSON() ([]byte, error) {
	fj, err := json.Marshal(f)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal file %x: %s", f.Filename, err)
	}
	return fj, nil
}

// FromJSON populates the fields of this File struct from a JSON representation.
// It primarily provides a convenient error message if this fails.
func (f *File) FromJSON(fj []byte) error {
	if err := json.Unmarshal(fj, f); err != nil {
		return fmt.Errorf("failed to unmarshal sha256sum %x: %s", SumString(fj), err)
	}
	return nil
}

func (c *Chunk) String() string {
	return fmt.Sprintf("{Index: %d, Sha256: %x}", c.Index, c.Sha256)
}
<|endoftext|>


<commit_before>package xlsx

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// File is a high level structure providing a slice of Sheet structs
// to the user.
type File struct {
	worksheets     map[string]*zip.File
	referenceTable *RefTable
	Date1904       bool
	styles         *xlsxStyleSheet
	Sheets         []*Sheet
	Sheet          map[string]*Sheet
	theme          *theme
	DefinedNames   []*xlsxDefinedName
}

// Create a new File
func NewFile() *File {
	return &File{
		Sheet:        make(map[string]*Sheet),
		Sheets:       make([]*Sheet, 0),
		DefinedNames: make([]*xlsxDefinedName, 0),
	}
}

// OpenFile() take the name of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenFile(filename string) (file *File, err error) {
	var f *zip.ReadCloser
	f, err = zip.OpenReader(filename)
	if err != nil {
		return nil, err
	}
	file, err = ReadZip(f)
	return
}

// OpenBinary() take bytes of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenBinary(bs []byte) (*File, error) {
	r := bytes.NewReader(bs)
	return OpenReaderAt(r, int64(r.Len()))
}

// OpenReaderAt() take io.ReaderAt of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenReaderAt(r io.ReaderAt, size int64) (*File, error) {
	file, err := zip.NewReader(r, size)
	if err != nil {
		return nil, err
	}
	return ReadZipReader(file)
}

// A convenient wrapper around File.ToSlice, FileToSlice will
// return the raw data contained in an Excel XLSX file as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func FileToSlice(path string) ([][][]string, error) {
	f, err := OpenFile(path)
	if err != nil {
		return nil, err
	}
	return f.ToSlice()
}

// Save the File to an xlsx file at the provided path.
func (f *File) Save(path string) (err error) {
	target, err := os.Create(path)
	if err != nil {
		return err
	}
	err = f.Write(target)
	if err != nil {
		return err
	}
	return target.Close()
}

// Write the File to io.Writer as xlsx
func (f *File) Write(writer io.Writer) (err error) {
	parts, err := f.MarshallParts()
	if err != nil {
		return
	}
	zipWriter := zip.NewWriter(writer)
	for partName, part := range parts {
		w, err := zipWriter.Create(partName)
		if err != nil {
			return err
		}
		_, err = w.Write([]byte(part))
		if err != nil {
			return err
		}
	}
	return zipWriter.Close()
}

// Add a new Sheet, with the provided name, to a File
func (f *File) AddSheet(sheetName string) (*Sheet, error) {
	if _, exists := f.Sheet[sheetName]; exists {
		return nil, fmt.Errorf("duplicate sheet name '%s'.", sheetName)
	}
	sheet := &Sheet{
		Name:     sheetName,
		File:     f,
		Selected: len(f.Sheets) == 0,
	}
	f.Sheet[sheetName] = sheet
	f.Sheets = append(f.Sheets, sheet)
	return sheet, nil
}

func (f *File) makeWorkbook() xlsxWorkbook {
	return xlsxWorkbook{
		FileVersion: xlsxFileVersion{AppName: "Go XLSX"},
		WorkbookPr:  xlsxWorkbookPr{ShowObjects: "all"},
		BookViews: xlsxBookViews{
			WorkBookView: []xlsxWorkBookView{
				{
					ShowHorizontalScroll: true,
					ShowSheetTabs:        true,
					ShowVerticalScroll:   true,
					TabRatio:             204,
					WindowHeight:         8192,
					WindowWidth:          16384,
					XWindow:              "0",
					YWindow:              "0",
				},
			},
		},
		Sheets: xlsxSheets{Sheet: make([]xlsxSheet, len(f.Sheets))},
		CalcPr: xlsxCalcPr{
			IterateCount: 100,
			RefMode:      "A1",
			Iterate:      false,
			IterateDelta: 0.001,
		},
	}
}

// Some tools that read XLSX files have very strict requirements about
// the structure of the input XML.  In particular both Numbers on the Mac
// and SAS dislike inline XML namespace declarations, or namespace
// prefixes that don't match the ones that Excel itself uses.  This is a
// problem because the Go XML library doesn't multiple namespace
// declarations in a single element of a document.  This function is a
// horrible hack to fix that after the XML marshalling is completed.
func replaceRelationshipsNameSpace(workbookMarshal string) string {
	newWorkbook := strings.Replace(workbookMarshal, `xmlns:relationships="http://schemas.openxmlformats.org/officeDocument/2006/relationships" relationships:id`, `r:id`, -1)
	// Dirty hack to fix issues #63 and #91; encoding/xml currently
	// "doesn't allow for additional namespaces to be defined in the
	// root element of the document," as described by @tealeg in the
	// comments for #63.
	oldXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">`
	newXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">`
	return strings.Replace(newWorkbook, oldXmlns, newXmlns, 1)
}

// Construct a map of file name to XML content representing the file
// in terms of the structure of an XLSX file.
func (f *File) MarshallParts() (map[string]string, error) {
	var parts map[string]string
	var refTable *RefTable = NewSharedStringRefTable()
	refTable.isWrite = true
	var workbookRels WorkBookRels = make(WorkBookRels)
	var err error
	var workbook xlsxWorkbook
	var types xlsxTypes = MakeDefaultContentTypes()

	marshal := func(thing interface{}) (string, error) {
		body, err := xml.Marshal(thing)
		if err != nil {
			return "", err
		}
		return xml.Header + string(body), nil
	}

	parts = make(map[string]string)
	workbook = f.makeWorkbook()
	sheetIndex := 1

	if f.styles == nil {
		f.styles = newXlsxStyleSheet(f.theme)
	}
	f.styles.reset()
	for _, sheet := range f.Sheets {
		xSheet := sheet.makeXLSXSheet(refTable, f.styles)
		rId := fmt.Sprintf("rId%d", sheetIndex)
		sheetId := strconv.Itoa(sheetIndex)
		sheetPath := fmt.Sprintf("worksheets/sheet%d.xml", sheetIndex)
		partName := "xl/" + sheetPath
		types.Overrides = append(
			types.Overrides,
			xlsxOverride{
				PartName:    "/" + partName,
				ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"})
		workbookRels[rId] = sheetPath
		workbook.Sheets.Sheet[sheetIndex-1] = xlsxSheet{
			Name:    sheet.Name,
			SheetId: sheetId,
			Id:      rId,
			State:   "visible"}
		parts[partName], err = marshal(xSheet)
		if err != nil {
			return parts, err
		}
		sheetIndex++
	}

	workbookMarshal, err := marshal(workbook)
	if err != nil {
		return parts, err
	}
	workbookMarshal = replaceRelationshipsNameSpace(workbookMarshal)
	parts["xl/workbook.xml"] = workbookMarshal
	if err != nil {
		return parts, err
	}

	parts["_rels/.rels"] = TEMPLATE__RELS_DOT_RELS
	parts["docProps/app.xml"] = TEMPLATE_DOCPROPS_APP
	// TODO - do this properly, modification and revision information
	parts["docProps/core.xml"] = TEMPLATE_DOCPROPS_CORE
	parts["xl/theme/theme1.xml"] = TEMPLATE_XL_THEME_THEME

	xSST := refTable.makeXLSXSST()
	parts["xl/sharedStrings.xml"], err = marshal(xSST)
	if err != nil {
		return parts, err
	}

	xWRel := workbookRels.MakeXLSXWorkbookRels()

	parts["xl/_rels/workbook.xml.rels"], err = marshal(xWRel)
	if err != nil {
		return parts, err
	}

	parts["[Content_Types].xml"], err = marshal(types)
	if err != nil {
		return parts, err
	}

	parts["xl/styles.xml"], err = f.styles.Marshal()
	if err != nil {
		return parts, err
	}

	return parts, nil
}

// Return the raw data contained in the File as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func (file *File) ToSlice() (output [][][]string, err error) {
	output = [][][]string{}
	for _, sheet := range file.Sheets {
		s := [][]string{}
		for _, row := range sheet.Rows {
			if row == nil {
				continue
			}
			r := []string{}
			for _, cell := range row.Cells {
				str, err := cell.String()
				if err != nil {
					return output, err
				}
				r = append(r, str)
			}
			s = append(s, r)
		}
		output = append(output, s)
	}
	return output, nil
}
<commit_msg>issue-259-XLSX file corrupts when saving the file without adding Worksheet<commit_after>package xlsx

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"errors"
)

// File is a high level structure providing a slice of Sheet structs
// to the user.
type File struct {
	worksheets     map[string]*zip.File
	referenceTable *RefTable
	Date1904       bool
	styles         *xlsxStyleSheet
	Sheets         []*Sheet
	Sheet          map[string]*Sheet
	theme          *theme
	DefinedNames   []*xlsxDefinedName
}

// Create a new File
func NewFile() *File {
	return &File{
		Sheet:        make(map[string]*Sheet),
		Sheets:       make([]*Sheet, 0),
		DefinedNames: make([]*xlsxDefinedName, 0),
	}
}

// OpenFile() take the name of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenFile(filename string) (file *File, err error) {
	var f *zip.ReadCloser
	f, err = zip.OpenReader(filename)
	if err != nil {
		return nil, err
	}
	file, err = ReadZip(f)
	return
}

// OpenBinary() take bytes of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenBinary(bs []byte) (*File, error) {
	r := bytes.NewReader(bs)
	return OpenReaderAt(r, int64(r.Len()))
}

// OpenReaderAt() take io.ReaderAt of an XLSX file and returns a populated
// xlsx.File struct for it.
func OpenReaderAt(r io.ReaderAt, size int64) (*File, error) {
	file, err := zip.NewReader(r, size)
	if err != nil {
		return nil, err
	}
	return ReadZipReader(file)
}

// A convenient wrapper around File.ToSlice, FileToSlice will
// return the raw data contained in an Excel XLSX file as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func FileToSlice(path string) ([][][]string, error) {
	f, err := OpenFile(path)
	if err != nil {
		return nil, err
	}
	return f.ToSlice()
}

// Save the File to an xlsx file at the provided path.
func (f *File) Save(path string) (err error) {
	target, err := os.Create(path)
	if err != nil {
		return err
	}
	err = f.Write(target)
	if err != nil {
		return err
	}
	return target.Close()
}

// Write the File to io.Writer as xlsx
func (f *File) Write(writer io.Writer) (err error) {
	parts, err := f.MarshallParts()
	if err != nil {
		return
	}
	zipWriter := zip.NewWriter(writer)
	for partName, part := range parts {
		w, err := zipWriter.Create(partName)
		if err != nil {
			return err
		}
		_, err = w.Write([]byte(part))
		if err != nil {
			return err
		}
	}
	return zipWriter.Close()
}

// Add a new Sheet, with the provided name, to a File
func (f *File) AddSheet(sheetName string) (*Sheet, error) {
	if _, exists := f.Sheet[sheetName]; exists {
		return nil, fmt.Errorf("duplicate sheet name '%s'.", sheetName)
	}
	sheet := &Sheet{
		Name:     sheetName,
		File:     f,
		Selected: len(f.Sheets) == 0,
	}
	f.Sheet[sheetName] = sheet
	f.Sheets = append(f.Sheets, sheet)
	return sheet, nil
}

func (f *File) makeWorkbook() xlsxWorkbook {
	return xlsxWorkbook{
		FileVersion: xlsxFileVersion{AppName: "Go XLSX"},
		WorkbookPr:  xlsxWorkbookPr{ShowObjects: "all"},
		BookViews: xlsxBookViews{
			WorkBookView: []xlsxWorkBookView{
				{
					ShowHorizontalScroll: true,
					ShowSheetTabs:        true,
					ShowVerticalScroll:   true,
					TabRatio:             204,
					WindowHeight:         8192,
					WindowWidth:          16384,
					XWindow:              "0",
					YWindow:              "0",
				},
			},
		},
		Sheets: xlsxSheets{Sheet: make([]xlsxSheet, len(f.Sheets))},
		CalcPr: xlsxCalcPr{
			IterateCount: 100,
			RefMode:      "A1",
			Iterate:      false,
			IterateDelta: 0.001,
		},
	}
}

// Some tools that read XLSX files have very strict requirements about
// the structure of the input XML.  In particular both Numbers on the Mac
// and SAS dislike inline XML namespace declarations, or namespace
// prefixes that don't match the ones that Excel itself uses.  This is a
// problem because the Go XML library doesn't multiple namespace
// declarations in a single element of a document.  This function is a
// horrible hack to fix that after the XML marshalling is completed.
func replaceRelationshipsNameSpace(workbookMarshal string) string {
	newWorkbook := strings.Replace(workbookMarshal, `xmlns:relationships="http://schemas.openxmlformats.org/officeDocument/2006/relationships" relationships:id`, `r:id`, -1)
	// Dirty hack to fix issues #63 and #91; encoding/xml currently
	// "doesn't allow for additional namespaces to be defined in the
	// root element of the document," as described by @tealeg in the
	// comments for #63.
	oldXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">`
	newXmlns := `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">`
	return strings.Replace(newWorkbook, oldXmlns, newXmlns, 1)
}

// Construct a map of file name to XML content representing the file
// in terms of the structure of an XLSX file.
func (f *File) MarshallParts() (map[string]string, error) {
	var parts map[string]string
	var refTable *RefTable = NewSharedStringRefTable()
	refTable.isWrite = true
	var workbookRels WorkBookRels = make(WorkBookRels)
	var err error
	var workbook xlsxWorkbook
	var types xlsxTypes = MakeDefaultContentTypes()

	marshal := func(thing interface{}) (string, error) {
		body, err := xml.Marshal(thing)
		if err != nil {
			return "", err
		}
		return xml.Header + string(body), nil
	}

	parts = make(map[string]string)
	workbook = f.makeWorkbook()
	sheetIndex := 1

	if f.styles == nil {
		f.styles = newXlsxStyleSheet(f.theme)
	}
	f.styles.reset()
	if len(f.Sheets)==0 {
		err:= errors.New("Workbook must contains atleast one worksheet")
		return nil, err
	}
	for _, sheet := range f.Sheets {
		xSheet := sheet.makeXLSXSheet(refTable, f.styles)
		rId := fmt.Sprintf("rId%d", sheetIndex)
		sheetId := strconv.Itoa(sheetIndex)
		sheetPath := fmt.Sprintf("worksheets/sheet%d.xml", sheetIndex)
		partName := "xl/" + sheetPath
		types.Overrides = append(
			types.Overrides,
			xlsxOverride{
				PartName:    "/" + partName,
				ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"})
		workbookRels[rId] = sheetPath
		workbook.Sheets.Sheet[sheetIndex-1] = xlsxSheet{
			Name:    sheet.Name,
			SheetId: sheetId,
			Id:      rId,
			State:   "visible"}
		parts[partName], err = marshal(xSheet)
		if err != nil {
			return parts, err
		}
		sheetIndex++
	}

	workbookMarshal, err := marshal(workbook)
	if err != nil {
		return parts, err
	}
	workbookMarshal = replaceRelationshipsNameSpace(workbookMarshal)
	parts["xl/workbook.xml"] = workbookMarshal
	if err != nil {
		return parts, err
	}

	parts["_rels/.rels"] = TEMPLATE__RELS_DOT_RELS
	parts["docProps/app.xml"] = TEMPLATE_DOCPROPS_APP
	// TODO - do this properly, modification and revision information
	parts["docProps/core.xml"] = TEMPLATE_DOCPROPS_CORE
	parts["xl/theme/theme1.xml"] = TEMPLATE_XL_THEME_THEME

	xSST := refTable.makeXLSXSST()
	parts["xl/sharedStrings.xml"], err = marshal(xSST)
	if err != nil {
		return parts, err
	}

	xWRel := workbookRels.MakeXLSXWorkbookRels()

	parts["xl/_rels/workbook.xml.rels"], err = marshal(xWRel)
	if err != nil {
		return parts, err
	}

	parts["[Content_Types].xml"], err = marshal(types)
	if err != nil {
		return parts, err
	}

	parts["xl/styles.xml"], err = f.styles.Marshal()
	if err != nil {
		return parts, err
	}

	return parts, nil
}

// Return the raw data contained in the File as three
// dimensional slice.  The first index represents the sheet number,
// the second the row number, and the third the cell number.
//
// For example:
//
//    var mySlice [][][]string
//    var value string
//    mySlice = xlsx.FileToSlice("myXLSX.xlsx")
//    value = mySlice[0][0][0]
//
// Here, value would be set to the raw value of the cell A1 in the
// first sheet in the XLSX file.
func (file *File) ToSlice() (output [][][]string, err error) {
	output = [][][]string{}
	for _, sheet := range file.Sheets {
		s := [][]string{}
		for _, row := range sheet.Rows {
			if row == nil {
				continue
			}
			r := []string{}
			for _, cell := range row.Cells {
				str, err := cell.String()
				if err != nil {
					return output, err
				}
				r = append(r, str)
			}
			s = append(s, r)
		}
		output = append(output, s)
	}
	return output, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path"
)

// get filepath base name
func Basename(file string) string {
	return path.Base(file)
}

// get filepath dir name
func Dirname(file string) string {
	return path.Dir(file)
}

// get absolute filepath, based on built excutable file
func RealPath(file string) (string, error) {
	if path.IsAbs(file) {
		return file, nil
	}
	wd, err := os.Getwd()
	return path.Join(wd, file), err
}

// get file modified time
func FileMTime(file string) (int64, error) {
	f, e := os.Stat(file)
	if e != nil {
		return 0, e
	}
	return f.ModTime().Unix(), nil
}

// get file size as how many bytes
func FileSize(file string) (int64, error) {
	f, e := os.Stat(file)
	if e != nil {
		return 0, e
	}
	return f.Size(), nil
}

// copy file to new path
func Copy(file string, to string) (bool, error) {
	sf, e := os.Open(file)
	if e != nil {
		return false, e
	}
	defer sf.Close()
	df, e2 := os.Create(to)
	if e2 != nil {
		return false, e2
	}
	defer df.Close()
	// buffer reader, do chunk transfer
	buf := make([]byte, 1024)
	for {
		// read a chunk
		n, err := sf.Read(buf)
		if err != nil && err != io.EOF {
			return false, err
		}
		if n == 0 {
			break
		}
		// write a chunk
		if _, err := df.Write(buf[:n]); err != nil {
			return false, err
		}
	}
	return true, nil
}

// move file to new path
func Move(file string, to string) (bool, error) {
	// copy
	r, e := Copy(file, to)
	if e != nil {
		return r, e
	}
	// then remove
	e = os.Remove(file)
	if e != nil {
		r = false
	}
	return r, e
}

// delete file
func Unlink(file string) error {
	return os.Remove(file)
}

// rename file name
func Rename(file string, to string) error {
	return os.Rename(file, to)
}

// IsFile checks whether the path is a file,
// it returns false when it's a directory or does not exist.
func IsFile(filePath string) bool {
	f, e := os.Stat(filePath)
	if e != nil {
		return false
	}
	return !f.IsDir()
}

// IsExist returns whether a file or directory exists.
func IsExist(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsExist(err)
}

// SaveFile saves content type '[]byte' to file by given path.
// It returns error when fail to finish operation.
func SaveFile(filePath string, b []byte) (int, error) {
	os.MkdirAll(path.Dir(filePath), os.ModePerm)
	fw, err := os.Create(filePath)
	if err != nil {
		return 0, err
	}
	defer fw.Close()
	return fw.Write(b)
}

// SaveFileS saves content type 'string' to file by given path.
// It returns error when fail to finish operation.
func SaveFileS(filePath string, s string) (int, error) {
	return SaveFile(filePath, []byte(s))
}

// ReadFile reads data type '[]byte' from file by given path.
// It returns error when fail to finish operation.
func ReadFile(filePath string) ([]byte, error) {
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return []byte(""), err
	}
	return b, nil
}

// ReadFileS reads data type 'string' from file by given path.
// It returns error when fail to finish operation.
func ReadFileS(filePath string) (string, error) {
	b, err := ReadFile(filePath)
	return string(b), err
}

// Unzip unzips .zip file to 'destPath' and returns sub-directories.
// It returns error when fail to finish operation.
func Unzip(srcPath, destPath string) ([]string, error) {
	// Open a zip archive for reading
	r, err := zip.OpenReader(srcPath)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	dirs := make([]string, 0, 5)
	// Iterate through the files in the archive
	for _, f := range r.File {
		// Get files from archive
		rc, err := f.Open()
		if err != nil {
			return nil, err
		}

		dir := path.Dir(f.FileInfo().Name())
		// Create diretory before create file
		os.MkdirAll(destPath+"/"+dir, os.ModePerm)
		dirs = AppendStr(dirs, dir)

		if f.FileInfo().IsDir() {
			continue
		}

		// Write data to file
		fw, _ := os.Create(destPath + "/" + f.FileInfo().Name())
		if err != nil {
			return nil, err
		}
		_, err = io.Copy(fw, rc)
		fw.Close()
		if err != nil {
			return nil, err
		}
	}
	return dirs, nil
}

// UnTarGz ungzips and untars .tar.gz file to 'destPath' and returns sub-directories.
// It returns error when fail to finish operation.
func UnTarGz(srcFilePath string, destDirPath string) ([]string, error) {
	// Create destination directory
	os.Mkdir(destDirPath, os.ModePerm)

	fr, err := os.Open(srcFilePath)
	if err != nil {
		return nil, err
	}
	defer fr.Close()

	// Gzip reader
	gr, err := gzip.NewReader(fr)
	if err != nil {
		return nil, err
	}
	defer gr.Close()

	// Tar reader
	tr := tar.NewReader(gr)

	dirs := make([]string, 0, 5)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// End of tar archive
			break
		}

		// Check if it is diretory or file
		if hdr.Typeflag != tar.TypeDir {
			// Get files from archive
			// Create diretory before create file
			dir := path.Dir(hdr.Name)
			os.MkdirAll(destDirPath+"/"+dir, os.ModePerm)
			dirs = AppendStr(dirs, dir)

			// Write data to file
			fw, _ := os.Create(destDirPath + "/" + hdr.Name)
			if err != nil {
				return nil, err
			}
			_, err = io.Copy(fw, tr)
			if err != nil {
				return nil, err
			}
		}
	}
	return dirs, nil
}
<commit_msg>Updated func: Unzip<commit_after>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"io"
	"io/ioutil"
	"os"
	"path"
)

// get filepath base name
func Basename(file string) string {
	return path.Base(file)
}

// get filepath dir name
func Dirname(file string) string {
	return path.Dir(file)
}

// get absolute filepath, based on built excutable file
func RealPath(file string) (string, error) {
	if path.IsAbs(file) {
		return file, nil
	}
	wd, err := os.Getwd()
	return path.Join(wd, file), err
}

// get file modified time
func FileMTime(file string) (int64, error) {
	f, e := os.Stat(file)
	if e != nil {
		return 0, e
	}
	return f.ModTime().Unix(), nil
}

// get file size as how many bytes
func FileSize(file string) (int64, error) {
	f, e := os.Stat(file)
	if e != nil {
		return 0, e
	}
	return f.Size(), nil
}

// copy file to new path
func Copy(file string, to string) (bool, error) {
	sf, e := os.Open(file)
	if e != nil {
		return false, e
	}
	defer sf.Close()
	df, e2 := os.Create(to)
	if e2 != nil {
		return false, e2
	}
	defer df.Close()
	// buffer reader, do chunk transfer
	buf := make([]byte, 1024)
	for {
		// read a chunk
		n, err := sf.Read(buf)
		if err != nil && err != io.EOF {
			return false, err
		}
		if n == 0 {
			break
		}
		// write a chunk
		if _, err := df.Write(buf[:n]); err != nil {
			return false, err
		}
	}
	return true, nil
}

// move file to new path
func Move(file string, to string) (bool, error) {
	// copy
	r, e := Copy(file, to)
	if e != nil {
		return r, e
	}
	// then remove
	e = os.Remove(file)
	if e != nil {
		r = false
	}
	return r, e
}

// delete file
func Unlink(file string) error {
	return os.Remove(file)
}

// rename file name
func Rename(file string, to string) error {
	return os.Rename(file, to)
}

// IsFile checks whether the path is a file,
// it returns false when it's a directory or does not exist.
func IsFile(filePath string) bool {
	f, e := os.Stat(filePath)
	if e != nil {
		return false
	}
	return !f.IsDir()
}

// IsExist returns whether a file or directory exists.
func IsExist(path string) bool {
	_, err := os.Stat(path)
	return err == nil || os.IsExist(err)
}

// SaveFile saves content type '[]byte' to file by given path.
// It returns error when fail to finish operation.
func SaveFile(filePath string, b []byte) (int, error) {
	os.MkdirAll(path.Dir(filePath), os.ModePerm)
	fw, err := os.Create(filePath)
	if err != nil {
		return 0, err
	}
	defer fw.Close()
	return fw.Write(b)
}

// SaveFileS saves content type 'string' to file by given path.
// It returns error when fail to finish operation.
func SaveFileS(filePath string, s string) (int, error) {
	return SaveFile(filePath, []byte(s))
}

// ReadFile reads data type '[]byte' from file by given path.
// It returns error when fail to finish operation.
func ReadFile(filePath string) ([]byte, error) {
	b, err := ioutil.ReadFile(filePath)
	if err != nil {
		return []byte(""), err
	}
	return b, nil
}

// ReadFileS reads data type 'string' from file by given path.
// It returns error when fail to finish operation.
func ReadFileS(filePath string) (string, error) {
	b, err := ReadFile(filePath)
	return string(b), err
}

// Unzip unzips .zip file to 'destPath' and returns sub-directories.
// It returns error when fail to finish operation.
func Unzip(srcPath, destPath string) ([]string, error) {
	// Open a zip archive for reading
	r, err := zip.OpenReader(srcPath)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	dirs := make([]string, 0, 5)
	// Iterate through the files in the archive
	for _, f := range r.File {
		// Get files from archive
		rc, err := f.Open()
		if err != nil {
			return nil, err
		}

		dir := path.Dir(f.FileInfo().Name())
		// Create diretory before create file
		os.MkdirAll(destPath+"/"+dir, os.ModePerm)
		dirs = AppendStr(dirs, dir)

		if f.FileInfo().IsDir() {
			continue
		}

		// Write data to file
		fw, _ := os.Create(destPath + "/" + f.FileInfo().Name())
		if err != nil {
			return nil, err
		}
		_, err = io.Copy(fw, rc)

		if fw != nil {
			fw.Close()
		}
		if err != nil {
			return nil, err
		}
	}
	return dirs, nil
}

// UnTarGz ungzips and untars .tar.gz file to 'destPath' and returns sub-directories.
// It returns error when fail to finish operation.
func UnTarGz(srcFilePath string, destDirPath string) ([]string, error) {
	// Create destination directory
	os.Mkdir(destDirPath, os.ModePerm)

	fr, err := os.Open(srcFilePath)
	if err != nil {
		return nil, err
	}
	defer fr.Close()

	// Gzip reader
	gr, err := gzip.NewReader(fr)
	if err != nil {
		return nil, err
	}
	defer gr.Close()

	// Tar reader
	tr := tar.NewReader(gr)

	dirs := make([]string, 0, 5)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// End of tar archive
			break
		}

		// Check if it is diretory or file
		if hdr.Typeflag != tar.TypeDir {
			// Get files from archive
			// Create diretory before create file
			dir := path.Dir(hdr.Name)
			os.MkdirAll(destDirPath+"/"+dir, os.ModePerm)
			dirs = AppendStr(dirs, dir)

			// Write data to file
			fw, _ := os.Create(destDirPath + "/" + hdr.Name)
			if err != nil {
				return nil, err
			}
			_, err = io.Copy(fw, tr)
			if err != nil {
				return nil, err
			}
		}
	}
	return dirs, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

var flags = struct {
	Package string
	Var     string
	Abs     bool
}{}

var stdout = log.New(os.Stdout, "", 0)
var stderr = log.New(os.Stderr, "", 0)

const helpString = `fmap generates a go source file containing a map[string][]byte
for the specified directory trees. The keys are the paths
of files and the values are the contents of the file at that path.

The generated go file is printed to stdout. Empty directories are
ignored, and symlinks are not followed.

usage:
  fmap [flags] path/to/dir path/to/dir2 ...

flags:
  -package  package name to use in generated file (default: "main")
  -var      variable name of the map (default: "files")
  -abs      use absolute paths for map keys

example:
  fmap static/css static/js | gofmt > static_files.go`

const fileTmpl = `package << .Package >>

var << .Var >> = map[string][]byte{
	<<- range $k, $v := .M >>
	"<< $k >>": << $v | printf "%#v" | trimPrefix >>,
	<<- end >>
}`

func main() {
	flag.StringVar(&flags.Package, "package", "main", "package name")
	flag.StringVar(&flags.Var, "var", "files", "variable name of map")
	flag.BoolVar(&flags.Abs, "abs", false, "use absolute path for keys")
	flag.Usage = func() {
		stderr.Println(helpString)
		os.Exit(2)
	}
	flag.Parse()

	roots := flag.Args()
	if len(roots) == 0 {
		stderr.Println(errors.New(`fmap: error: require path argument`))
		stderr.Println(helpString)
		os.Exit(2)
	}
	for _, dirRoot := range roots {
		info, err := os.Stat(dirRoot)
		if err != nil {
			stderr.Println(err)
			os.Exit(1)
		}
		if !info.IsDir() {
			stderr.Println(errors.New(`fmap: error: path argument must be a directory`))
			os.Exit(1)
		}
	}

	tmpl, err := template.New("file").Funcs(template.FuncMap{
		"trimPrefix": func(s string) string { return strings.TrimPrefix(s, "[]byte") },
	}).Delims("<<", ">>").Parse(fileTmpl)
	if err != nil {
		stderr.Println(err)
		os.Exit(1)
	}

	wg := sync.WaitGroup{}
	m := make(map[string][]byte)
	merged := make(chan File)

	for _, dirRoot := range roots {
		dirRoot := dirRoot
		wg.Add(1)
		go func() {
			defer wg.Done()
			out := fileContents(dirRoot)
			for f := range out {
				merged <- f
			}
		}()
	}

	go func() {
		for f := range merged {
			if f.Err != nil {
				stderr.Println(f.Err)
				os.Exit(1)
			}

			k := f.Path
			if flags.Abs {
				absP, err := filepath.Abs(f.Path)
				if err != nil {
					stderr.Println(err)
					os.Exit(1)
				}
				k = absP
			}
			m[filepath.ToSlash(k)] = f.Content
		}
	}()

	wg.Wait()
	close(merged)

	if err := tmpl.Execute(os.Stdout, struct {
		Package string
		Var     string
		M       map[string][]byte
	}{
		flags.Package,
		flags.Var,
		m,
	}); err != nil {
		stderr.Println(err)
		os.Exit(1)
	}
}

// File represents the path and contents of a file.
// Err has no real relation, but makes it convenient to
// pass any errors encountered in fileContents.
type File struct {
	Path    string
	Content []byte
	Err     error
}

func fileContents(root string) <-chan File {
	out := make(chan File)

	go func() {
		wg := sync.WaitGroup{}

		// Ignore error since all errors are sent on the channel anyway.
		_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				out <- File{Err: err}
				return err
			}
			if info.IsDir() {
				return nil
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				b, err := ioutil.ReadFile(p)
				out <- File{p, b, err}
			}()
			return nil
		})

		go func() {
			wg.Wait()
			close(out)
		}()
	}()

	return out
}
<commit_msg>Fix sublte race<commit_after>package main

import (
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

var flags = struct {
	Package string
	Var     string
	Abs     bool
}{}

var stdout = log.New(os.Stdout, "", 0)
var stderr = log.New(os.Stderr, "", 0)

const helpString = `fmap generates a go source file containing a map[string][]byte
for the specified directory trees. The keys are the paths
of files and the values are the contents of the file at that path.

The generated go file is printed to stdout. Empty directories are
ignored, and symlinks are not followed.

usage:
  fmap [flags] path/to/dir path/to/dir2 ...

flags:
  -package  package name to use in generated file (default: "main")
  -var      variable name of the map (default: "files")
  -abs      use absolute paths for map keys

example:
  fmap static/css static/js | gofmt > static_files.go`

const fileTmpl = `package << .Package >>

var << .Var >> = map[string][]byte{
	<<- range $k, $v := .M >>
	"<< $k >>": << $v | printf "%#v" | trimPrefix >>,
	<<- end >>
}`

func main() {
	flag.StringVar(&flags.Package, "package", "main", "package name")
	flag.StringVar(&flags.Var, "var", "files", "variable name of map")
	flag.BoolVar(&flags.Abs, "abs", false, "use absolute path for keys")
	flag.Usage = func() {
		stderr.Println(helpString)
		os.Exit(2)
	}
	flag.Parse()

	roots := flag.Args()
	if len(roots) == 0 {
		stderr.Println(errors.New(`fmap: error: require path argument`))
		stderr.Println(helpString)
		os.Exit(2)
	}
	for _, dirRoot := range roots {
		info, err := os.Stat(dirRoot)
		if err != nil {
			stderr.Println(err)
			os.Exit(1)
		}
		if !info.IsDir() {
			stderr.Println(errors.New(`fmap: error: path argument must be a directory`))
			os.Exit(1)
		}
	}

	tmpl, err := template.New("file").Funcs(template.FuncMap{
		"trimPrefix": func(s string) string { return strings.TrimPrefix(s, "[]byte") },
	}).Delims("<<", ">>").Parse(fileTmpl)
	if err != nil {
		stderr.Println(err)
		os.Exit(1)
	}

	wg := sync.WaitGroup{}
	m := make(map[string][]byte)
	merged := make(chan File)

	for _, dirRoot := range roots {
		dirRoot := dirRoot
		wg.Add(1)
		go func() {
			defer wg.Done()
			out := fileContents(dirRoot)
			for f := range out {
				merged <- f
			}
		}()
	}

	go func() {
		wg.Wait()
		close(merged)
	}()

	for f := range merged {
		if f.Err != nil {
			stderr.Println(f.Err)
			os.Exit(1)
		}

		k := f.Path
		if flags.Abs {
			absP, err := filepath.Abs(f.Path)
			if err != nil {
				stderr.Println(err)
				os.Exit(1)
			}
			k = absP
		}
		m[filepath.ToSlash(k)] = f.Content
	}

	if err := tmpl.Execute(os.Stdout, struct {
		Package string
		Var     string
		M       map[string][]byte
	}{
		flags.Package,
		flags.Var,
		m,
	}); err != nil {
		stderr.Println(err)
		os.Exit(1)
	}
}

// File represents the path and contents of a file.
// Err has no real relation, but makes it convenient to
// pass any errors encountered in fileContents.
type File struct {
	Path    string
	Content []byte
	Err     error
}

func fileContents(root string) <-chan File {
	out := make(chan File)

	go func() {
		wg := sync.WaitGroup{}

		// Ignore error since all errors are sent on the channel anyway.
		_ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				out <- File{Err: err}
				return err
			}
			if info.IsDir() {
				return nil
			}
			wg.Add(1)
			go func() {
				defer wg.Done()
				b, err := ioutil.ReadFile(p)
				out <- File{p, b, err}
			}()
			return nil
		})

		go func() {
			wg.Wait()
			close(out)
		}()
	}()

	return out
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
)

type Game struct {
	clock    InstanceClock
	handlers HandlerContainer
	units    UnitContainer

	w InstanceOutputWriter
}

// NewGame returns a Game
func NewGame(clock InstanceClock, w InstanceOutputWriter) *Game {
	return &Game{
		clock:    clock,
		handlers: MakeHandlerSet(),
		units:    NewUnitMap(),

		w: w,
	}
}

// Join creates a Unit and adds it to the game
func (g *Game) Join(group UnitGroup, name UnitName, class *Class) (id UnitID, err error) {
	u, err := g.units.Join(group, name, class)
	if err != nil {
		return
	}
	id = u.ID()
	g.w.Write(OutputUnitJoin{
		UnitID:    u.ID(),
		UnitGroup: u.Group(),
		UnitName:  u.Name(),
		ClassName: u.ClassName(),
		Health:    u.Health(),
		HealthMax: u.HealthMax(),
		Mana:      u.Mana(),
		ManaMax:   u.ManaMax(),
	})
	return
}

// Leave removes the Unit
func (g *Game) Leave(id UnitID) (err error) {
	if err = g.units.Leave(id); err != nil {
		return
	}
	g.w.Write(OutputUnitLeave{
		UnitID: id,
	})
	return
}

// Ability activates the ability
func (g *Game) Ability(sid UnitID, oid *UnitID, abilityName string) error {
	s := g.units.Find(sid)
	if s == nil {
		return errors.New("Unknown subject UnitID")
	}
	var o *Unit
	if oid != nil {
		o = g.units.Find(*oid)
		if o == nil {
			return errors.New("Unknown object UnitID")
		}
	}
	a := s.Ability(abilityName)
	if a == nil {
		return errors.New("Unknown ability name")
	}
	g.Activating(s, o, a)
	return nil
}

// PerformGameTick performs the game tick routine
func (g *Game) PerformGameTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.TriggerEvent(EventGameTick)
	})
}

// PerformPeriodicalTick performs the periodical rick routine
func (g *Game) PerformPeriodicalTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.TriggerEvent(EventPeriodicalTick)
	})
}

// PerformRegenerationTick performs the regeneration tick routine
func (g *Game) PerformRegenerationTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.ModifyHealth(g.w, u.HealthRegeneration())
		u.ModifyMana(g.w, u.ManaRegeneration())
	})
}
<commit_msg>Add Game.Sync<commit_after>package main

import (
	"errors"
)

type Game struct {
	clock    InstanceClock
	handlers HandlerContainer
	units    UnitContainer

	w InstanceOutputWriter
}

// NewGame returns a Game
func NewGame(clock InstanceClock, w InstanceOutputWriter) *Game {
	return &Game{
		clock:    clock,
		handlers: MakeHandlerSet(),
		units:    NewUnitMap(),

		w: w,
	}
}

// Sync sends the game state
func (g *Game) Sync(w InstanceOutputWriter) {
	g.units.Each(func(u *Unit) {
		w.Write(OutputUnitJoin{
			UnitID:    u.ID(),
			UnitGroup: u.Group(),
			UnitName:  u.Name(),
			ClassName: u.ClassName(),
			Health:    u.Health(),
			HealthMax: u.HealthMax(),
			Mana:      u.Mana(),
			ManaMax:   u.ManaMax(),
		})
	})
}

// Join creates a Unit and adds it to the game
func (g *Game) Join(group UnitGroup, name UnitName, class *Class) (id UnitID, err error) {
	u, err := g.units.Join(group, name, class)
	if err != nil {
		return
	}
	id = u.ID()
	g.w.Write(OutputUnitJoin{
		UnitID:    u.ID(),
		UnitGroup: u.Group(),
		UnitName:  u.Name(),
		ClassName: u.ClassName(),
		Health:    u.Health(),
		HealthMax: u.HealthMax(),
		Mana:      u.Mana(),
		ManaMax:   u.ManaMax(),
	})
	return
}

// Leave removes the Unit
func (g *Game) Leave(id UnitID) (err error) {
	if err = g.units.Leave(id); err != nil {
		return
	}
	g.w.Write(OutputUnitLeave{
		UnitID: id,
	})
	return
}

// Ability activates the ability
func (g *Game) Ability(sid UnitID, oid *UnitID, abilityName string) error {
	s := g.units.Find(sid)
	if s == nil {
		return errors.New("Unknown subject UnitID")
	}
	var o *Unit
	if oid != nil {
		o = g.units.Find(*oid)
		if o == nil {
			return errors.New("Unknown object UnitID")
		}
	}
	a := s.Ability(abilityName)
	if a == nil {
		return errors.New("Unknown ability name")
	}
	g.Activating(s, o, a)
	return nil
}

// PerformGameTick performs the game tick routine
func (g *Game) PerformGameTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.TriggerEvent(EventGameTick)
	})
}

// PerformPeriodicalTick performs the periodical rick routine
func (g *Game) PerformPeriodicalTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.TriggerEvent(EventPeriodicalTick)
	})
}

// PerformRegenerationTick performs the regeneration tick routine
func (g *Game) PerformRegenerationTick() {
	g.units.Each(func(u *Unit) {
		if u.IsDead() {
			return
		}
		u.ModifyHealth(g.w, u.HealthRegeneration())
		u.ModifyMana(g.w, u.ManaRegeneration())
	})
}
<|endoftext|>