<commit_before>package main

// TODO refactor tests to use testify/assert library like `bitrot_test.go` and
// testify/suite to extract common before/after hooks
import (
	"encoding/json"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"
)

var helloWorldString = "hello! world\n"
var helloWorldChecksum = "87b3fe7479c73ae4246dbe8081550f52e2cf9e59"

func writeTestFile(dir, name, content string) string {
	testFile := filepath.Join(dir, name)
	err := ioutil.WriteFile(testFile, []byte(content), 0644)
	check(err)
	return testFile
}

func populateTestDirectory(tempDir string) (map[string]string, time.Time) {
	writeTestFile(tempDir, "foo", helloWorldString)
	subdir := filepath.Join(tempDir, "bar", "baz", "stuff")
	check(os.MkdirAll(subdir, 0755))
	writeTestFile(subdir, "foo", helloWorldString)

	expectedChecksums := map[string]string{
		"bar/baz/stuff/foo": helloWorldChecksum,
		"foo":               helloWorldChecksum,
	}

	expectedCreationTime := time.Now()

	return expectedChecksums, expectedCreationTime
}

func TestDirectoryManifest(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	expectedChecksums, expectedCreationTime := populateTestDirectory(tempDir)

	config := Config{}
	manifest, err := NewManifest(tempDir, &config)
	check(err)

	if manifest.Path != tempDir {
		t.Fatalf("expected manifest path %s, got %s", tempDir, manifest.Path)
	}

	if math.Abs(float64(manifest.CreatedAt.Unix()-expectedCreationTime.Unix())) > 0 {
		t.Fatalf("expected manifest createdAt within 5s of %v, got %v", expectedCreationTime, manifest.CreatedAt)
	}

	if len(manifest.Entries) != len(expectedChecksums) {
		t.Fatalf(
			"unexpected number of checksums! expected %d, got %d (%v)",
			len(expectedChecksums),
			len(manifest.Entries),
			manifest.Entries,
		)
	}

	for path, fileChecksum := range manifest.Entries {
		if fileChecksum.Checksum != expectedChecksums[path] {
			t.Fatalf("checksum mismatch; expected %s, got %s", expectedChecksums[path], fileChecksum.Checksum)
		}
	}
}

func TestManifestExclusionOnFile(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	populateTestDirectory(tempDir)

	config := Config{
		ExcludedFiles: []string{"foo"},
	}

	manifest, err := NewManifest(tempDir, &config)
	check(err)

	if !reflect.DeepEqual(manifest.Entries, map[string]ChecksumRecord{}) {
		t.Fatalf("Entries mismatch; expected %v, got %v", map[string]ChecksumRecord{}, manifest.Entries)
	}
}

func TestManifestExclusionOnFolder(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	populateTestDirectory(tempDir)

	config := Config{
		ExcludedFiles: []string{"baz"},
	}

	manifest, err := NewManifest(tempDir, &config)
	check(err)

	entryPaths := []string{}
	for path := range manifest.Entries {
		entryPaths = append(entryPaths, path)
	}
	expectedEntryPaths := []string{"foo"}

	if !reflect.DeepEqual(entryPaths, expectedEntryPaths) {
		t.Fatalf("Entries mismatch; expected %v, got %v", expectedEntryPaths, entryPaths)
	}
}

func TestManifestJSON(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	expectedChecksums, expectedCreationTime := populateTestDirectory(tempDir)

	config := Config{}
	manifest, err := NewManifest(tempDir, &config)
	check(err)

	jsonBytes, err := json.Marshal(manifest)

	var recreatedManifest Manifest
	err = json.Unmarshal(jsonBytes, &recreatedManifest)
	check(err)

	if recreatedManifest.Path != tempDir {
		t.Fatalf("expected JSON path %s, got %s", tempDir, recreatedManifest.Path)
	}

	if math.Abs(float64(recreatedManifest.CreatedAt.Unix()-expectedCreationTime.Unix())) > 0 {
		t.Fatalf("expected manifest created_at within 0s of %v, got %v", expectedCreationTime, recreatedManifest.CreatedAt)
	}

	entries := recreatedManifest.Entries
	if len(entries) != len(expectedChecksums) {
		t.Fatalf(
			"unexpected number of checksums! expected %d, got %d (%v)",
			len(expectedChecksums),
			len(entries),
			entries,
		)
	}

	for path, fileChecksum := range entries {
		if fileChecksum.Checksum != expectedChecksums[path] {
			t.Fatalf("checksum mismatch; expected %s, got %s", expectedChecksums[path], fileChecksum.Checksum)
		}
	}
}

func TestManifestComparison(t *testing.T) {
	newCreatedAt := time.Now()
	newModTime := time.Now()
	oldCreatedAt := newCreatedAt.Add(-24 * time.Hour)
	oldModTime := newModTime.Add(-24 * time.Hour)
	oldManifest := Manifest{
		Path:      "/old/stuff",
		CreatedAt: oldCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "asdf", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "qwer", ModTime: oldModTime},
			"touched":            {Checksum: "olkm", ModTime: oldModTime},
			"deleted":            {Checksum: "jklh", ModTime: oldModTime},
			"renamedOld":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	newManifest := Manifest{
		Path:      "/new/thing",
		CreatedAt: newCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "zzzz", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "tyui", ModTime: newModTime},
			"touched":            {Checksum: "olkm", ModTime: newModTime},
			"added":              {Checksum: "bnmv", ModTime: newModTime},
			"renamedNew":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	comparison := CompareManifests(&oldManifest, &newManifest)

	expectedDeletedPaths := []string{"deleted"}
	if !reflect.DeepEqual(comparison.DeletedPaths, expectedDeletedPaths) {
		t.Fatalf("expected DeletedPaths %v; got %v", expectedDeletedPaths, comparison.DeletedPaths)
	}

	expectedModifiedPaths := []string{"modified"}
	if !reflect.DeepEqual(comparison.ModifiedPaths, expectedModifiedPaths) {
		t.Fatalf("expected ModifiedPaths %v; got %v", expectedModifiedPaths, comparison.ModifiedPaths)
	}

	expectedFlaggedPaths := []string{"silently_corrupted"}
	if !reflect.DeepEqual(comparison.FlaggedPaths, expectedFlaggedPaths) {
		t.Fatalf("expected FlaggedPaths %v; got %v", expectedFlaggedPaths, comparison.FlaggedPaths)
	}

	expectedAddedPaths := []string{"added"}
	if !reflect.DeepEqual(comparison.AddedPaths, expectedAddedPaths) {
		t.Fatalf("expected AddedPaths %v; got %v", expectedAddedPaths, comparison.AddedPaths)
	}

	expectedRenamedPaths := []RenamedPath{{OldPath: "renamedOld", NewPath: "renamedNew"}}
	if !reflect.DeepEqual(comparison.RenamedPaths, expectedRenamedPaths) {
		t.Fatalf("expected RenamedPaths %v; got %v", expectedRenamedPaths, comparison.RenamedPaths)
	}
}
<commit_msg>Fix flaky test with less strict time comparison<commit_after>package main

// TODO refactor tests to use testify/assert library like `bitrot_test.go` and
// testify/suite to extract common before/after hooks
import (
	"encoding/json"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"reflect"
	"testing"
	"time"
)

var helloWorldString = "hello! world\n"
var helloWorldChecksum = "87b3fe7479c73ae4246dbe8081550f52e2cf9e59"

func writeTestFile(dir, name, content string) string {
	testFile := filepath.Join(dir, name)
	err := ioutil.WriteFile(testFile, []byte(content), 0644)
	check(err)
	return testFile
}

func populateTestDirectory(tempDir string) (map[string]string, time.Time) {
	writeTestFile(tempDir, "foo", helloWorldString)
	subdir := filepath.Join(tempDir, "bar", "baz", "stuff")
	check(os.MkdirAll(subdir, 0755))
	writeTestFile(subdir, "foo", helloWorldString)

	expectedChecksums := map[string]string{
		"bar/baz/stuff/foo": helloWorldChecksum,
		"foo":               helloWorldChecksum,
	}

	expectedCreationTime := time.Now()

	return expectedChecksums, expectedCreationTime
}

func TestDirectoryManifest(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	expectedChecksums, expectedCreationTime := populateTestDirectory(tempDir)

	config := Config{}
	manifest, err := NewManifest(tempDir, &config)
	check(err)

	if manifest.Path != tempDir {
		t.Fatalf("expected manifest path %s, got %s", tempDir, manifest.Path)
	}

	if math.Abs(float64(manifest.CreatedAt.Unix()-expectedCreationTime.Unix())) > 5 {
		t.Fatalf("expected manifest createdAt within 5s of %v, got %v", expectedCreationTime, manifest.CreatedAt)
	}

	if len(manifest.Entries) != len(expectedChecksums) {
		t.Fatalf(
			"unexpected number of checksums! expected %d, got %d (%v)",
			len(expectedChecksums),
			len(manifest.Entries),
			manifest.Entries,
		)
	}

	for path, fileChecksum := range manifest.Entries {
		if fileChecksum.Checksum != expectedChecksums[path] {
			t.Fatalf("checksum mismatch; expected %s, got %s", expectedChecksums[path], fileChecksum.Checksum)
		}
	}
}

func TestManifestExclusionOnFile(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	populateTestDirectory(tempDir)

	config := Config{
		ExcludedFiles: []string{"foo"},
	}

	manifest, err := NewManifest(tempDir, &config)
	check(err)

	if !reflect.DeepEqual(manifest.Entries, map[string]ChecksumRecord{}) {
		t.Fatalf("Entries mismatch; expected %v, got %v", map[string]ChecksumRecord{}, manifest.Entries)
	}
}

func TestManifestExclusionOnFolder(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	populateTestDirectory(tempDir)

	config := Config{
		ExcludedFiles: []string{"baz"},
	}

	manifest, err := NewManifest(tempDir, &config)
	check(err)

	entryPaths := []string{}
	for path := range manifest.Entries {
		entryPaths = append(entryPaths, path)
	}
	expectedEntryPaths := []string{"foo"}

	if !reflect.DeepEqual(entryPaths, expectedEntryPaths) {
		t.Fatalf("Entries mismatch; expected %v, got %v", expectedEntryPaths, entryPaths)
	}
}

func TestManifestJSON(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "checksum")
	check(err)

	defer os.RemoveAll(tempDir)

	expectedChecksums, expectedCreationTime := populateTestDirectory(tempDir)

	config := Config{}
	manifest, err := NewManifest(tempDir, &config)
	check(err)

	jsonBytes, err := json.Marshal(manifest)

	var recreatedManifest Manifest
	err = json.Unmarshal(jsonBytes, &recreatedManifest)
	check(err)

	if recreatedManifest.Path != tempDir {
		t.Fatalf("expected JSON path %s, got %s", tempDir, recreatedManifest.Path)
	}

	if math.Abs(float64(recreatedManifest.CreatedAt.Unix()-expectedCreationTime.Unix())) > 5 {
		t.Fatalf("expected manifest created_at within 5s of %v, got %v", expectedCreationTime, recreatedManifest.CreatedAt)
	}

	entries := recreatedManifest.Entries
	if len(entries) != len(expectedChecksums) {
		t.Fatalf(
			"unexpected number of checksums! expected %d, got %d (%v)",
			len(expectedChecksums),
			len(entries),
			entries,
		)
	}

	for path, fileChecksum := range entries {
		if fileChecksum.Checksum != expectedChecksums[path] {
			t.Fatalf("checksum mismatch; expected %s, got %s", expectedChecksums[path], fileChecksum.Checksum)
		}
	}
}

func TestManifestComparison(t *testing.T) {
	newCreatedAt := time.Now()
	newModTime := time.Now()
	oldCreatedAt := newCreatedAt.Add(-24 * time.Hour)
	oldModTime := newModTime.Add(-24 * time.Hour)
	oldManifest := Manifest{
		Path:      "/old/stuff",
		CreatedAt: oldCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "asdf", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "qwer", ModTime: oldModTime},
			"touched":            {Checksum: "olkm", ModTime: oldModTime},
			"deleted":            {Checksum: "jklh", ModTime: oldModTime},
			"renamedOld":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	newManifest := Manifest{
		Path:      "/new/thing",
		CreatedAt: newCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "zzzz", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "tyui", ModTime: newModTime},
			"touched":            {Checksum: "olkm", ModTime: newModTime},
			"added":              {Checksum: "bnmv", ModTime: newModTime},
			"renamedNew":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	comparison := CompareManifests(&oldManifest, &newManifest)

	expectedDeletedPaths := []string{"deleted"}
	if !reflect.DeepEqual(comparison.DeletedPaths, expectedDeletedPaths) {
		t.Fatalf("expected DeletedPaths %v; got %v", expectedDeletedPaths, comparison.DeletedPaths)
	}

	expectedModifiedPaths := []string{"modified"}
	if !reflect.DeepEqual(comparison.ModifiedPaths, expectedModifiedPaths) {
		t.Fatalf("expected ModifiedPaths %v; got %v", expectedModifiedPaths, comparison.ModifiedPaths)
	}

	expectedFlaggedPaths := []string{"silently_corrupted"}
	if !reflect.DeepEqual(comparison.FlaggedPaths, expectedFlaggedPaths) {
		t.Fatalf("expected FlaggedPaths %v; got %v", expectedFlaggedPaths, comparison.FlaggedPaths)
	}

	expectedAddedPaths := []string{"added"}
	if !reflect.DeepEqual(comparison.AddedPaths, expectedAddedPaths) {
		t.Fatalf("expected AddedPaths %v; got %v", expectedAddedPaths, comparison.AddedPaths)
	}

	expectedRenamedPaths := []RenamedPath{{OldPath: "renamedOld", NewPath: "renamedNew"}}
	if !reflect.DeepEqual(comparison.RenamedPaths, expectedRenamedPaths) {
		t.Fatalf("expected RenamedPaths %v; got %v", expectedRenamedPaths, comparison.RenamedPaths)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package index

import (
	"bytes"
	"encoding/json"
	"fmt"

	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index/store"
)

var ErrorUnknownStorageType = fmt.Errorf("unknown storage type")

type Index interface {
	Open() error
	Close() error

	Update(doc *document.Document) error
	Delete(id string) error
	Batch(batch *Batch) error

	SetInternal(key, val []byte) error
	DeleteInternal(key []byte) error

	// Reader returns a low-level accessor on the index data. Close it to
	// release associated resources.
	Reader() (IndexReader, error)

	Stats() json.Marshaler
	StatsMap() map[string]interface{}

	Analyze(d *document.Document) *AnalysisResult

	Advanced() (store.KVStore, error)
}

type IndexReader interface {
	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)

	// DocIDReader returns an iterator over all doc ids
	// The caller must close returned instance to release associated resources.
	DocIDReaderAll() (DocIDReader, error)

	DocIDReaderOnly(ids []string) (DocIDReader, error)

	FieldDict(field string) (FieldDict, error)

	// FieldDictRange is currently defined to include the start and end terms
	FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)

	Document(id string) (*document.Document, error)
	DocumentFieldTerms(id IndexInternalID, fields []string) (FieldTerms, error)

	Fields() ([]string, error)

	GetInternal(key []byte) ([]byte, error)

	DocCount() (uint64, error)

	ExternalID(id IndexInternalID) (string, error)
	InternalID(id string) (IndexInternalID, error)

	DumpAll() chan interface{}
	DumpDoc(id string) chan interface{}
	DumpFields() chan interface{}

	Close() error
}

// FieldTerms contains the terms used by a document, keyed by field
type FieldTerms map[string][]string

// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields
func (f FieldTerms) FieldsNotYetCached(fields []string) []string {
	rv := make([]string, 0, len(fields))
	for _, field := range fields {
		if _, ok := f[field]; !ok {
			rv = append(rv, field)
		}
	}
	return rv
}

// Merge will combine two FieldTerms
// it assumes that the terms lists are complete (thus do not need to be merged)
// field terms from the other list always replace the ones in the receiver
func (f FieldTerms) Merge(other FieldTerms) {
	for field, terms := range other {
		f[field] = terms
	}
}

type TermFieldVector struct {
	Field          string
	ArrayPositions []uint64
	Pos            uint64
	Start          uint64
	End            uint64
}

// IndexInternalID is an opaque document identifier interal to the index impl
type IndexInternalID []byte

func (id IndexInternalID) Equals(other IndexInternalID) bool {
	return id.Compare(other) == 0
}

func (id IndexInternalID) Compare(other IndexInternalID) int {
	return bytes.Compare(id, other)
}

type TermFieldDoc struct {
	Term    string
	ID      IndexInternalID
	Freq    uint64
	Norm    float64
	Vectors []*TermFieldVector
}

// Reset allows an already allocated TermFieldDoc to be reused
func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
	// remember the []byte used for the ID
	id := tfd.ID
	// idiom to copy over from empty TermFieldDoc (0 allocations)
	*tfd = TermFieldDoc{}
	// reuse the []byte already allocated (and reset len to 0)
	tfd.ID = id[:0]
	return tfd
}

// TermFieldReader is the interface exposing the enumeration of documents
// containing a given term in a given field. Documents are returned in byte
// lexicographic order over their identifiers.
type TermFieldReader interface {
	// Next returns the next document containing the term in this field, or nil
	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
	// is optional, and when non-nil, will be used instead of allocating memory.
	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Advance resets the enumeration at specified document or its immediate
	// follower.
	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Count returns the number of documents contains the term in this field.
	Count() uint64
	Close() error
}

type DictEntry struct {
	Term  string
	Count uint64
}

type FieldDict interface {
	Next() (*DictEntry, error)
	Close() error
}

// DocIDReader is the interface exposing enumeration of documents identifiers.
// Close the reader to release associated resources.
type DocIDReader interface {
	// Next returns the next document internal identifier in the natural
	// index order, or io.EOF when the end of the sequence is reached.
	Next() (IndexInternalID, error)

	// Advance resets the iteration to the first internal identifier greater than
	// or equal to ID. If ID is smaller than the start of the range, the iteration
	// will start there instead. If ID is greater than or equal to the end of
	// the range, Next() call will return io.EOF.
	Advance(ID IndexInternalID) (IndexInternalID, error)
	Close() error
}

type Batch struct {
	IndexOps    map[string]*document.Document
	InternalOps map[string][]byte
}

func NewBatch() *Batch {
	return &Batch{
		IndexOps:    make(map[string]*document.Document),
		InternalOps: make(map[string][]byte),
	}
}

func (b *Batch) Update(doc *document.Document) {
	b.IndexOps[doc.ID] = doc
}

func (b *Batch) Delete(id string) {
	b.IndexOps[id] = nil
}

func (b *Batch) SetInternal(key, val []byte) {
	b.InternalOps[string(key)] = val
}

func (b *Batch) DeleteInternal(key []byte) {
	b.InternalOps[string(key)] = nil
}

func (b *Batch) String() string {
	rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps))
	for k, v := range b.IndexOps {
		if v != nil {
			rv += fmt.Sprintf("\tINDEX - '%s'\n", k)
		} else {
			rv += fmt.Sprintf("\tDELETE - '%s'\n", k)
		}
	}
	for k, v := range b.InternalOps {
		if v != nil {
			rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k)
		} else {
			rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k)
		}
	}
	return rv
}

func (b *Batch) Reset() {
	b.IndexOps = make(map[string]*document.Document)
	b.InternalOps = make(map[string][]byte)
}
<commit_msg>index: DocIDReader.Next() returns nil when done not io.EOF<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package index

import (
	"bytes"
	"encoding/json"
	"fmt"

	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index/store"
)

var ErrorUnknownStorageType = fmt.Errorf("unknown storage type")

type Index interface {
	Open() error
	Close() error

	Update(doc *document.Document) error
	Delete(id string) error
	Batch(batch *Batch) error

	SetInternal(key, val []byte) error
	DeleteInternal(key []byte) error

	// Reader returns a low-level accessor on the index data. Close it to
	// release associated resources.
	Reader() (IndexReader, error)

	Stats() json.Marshaler
	StatsMap() map[string]interface{}

	Analyze(d *document.Document) *AnalysisResult

	Advanced() (store.KVStore, error)
}

type IndexReader interface {
	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)

	// DocIDReader returns an iterator over all doc ids
	// The caller must close returned instance to release associated resources.
	DocIDReaderAll() (DocIDReader, error)

	DocIDReaderOnly(ids []string) (DocIDReader, error)

	FieldDict(field string) (FieldDict, error)

	// FieldDictRange is currently defined to include the start and end terms
	FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)

	Document(id string) (*document.Document, error)
	DocumentFieldTerms(id IndexInternalID, fields []string) (FieldTerms, error)

	Fields() ([]string, error)

	GetInternal(key []byte) ([]byte, error)

	DocCount() (uint64, error)

	ExternalID(id IndexInternalID) (string, error)
	InternalID(id string) (IndexInternalID, error)

	DumpAll() chan interface{}
	DumpDoc(id string) chan interface{}
	DumpFields() chan interface{}

	Close() error
}

// FieldTerms contains the terms used by a document, keyed by field
type FieldTerms map[string][]string

// FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields
func (f FieldTerms) FieldsNotYetCached(fields []string) []string {
	rv := make([]string, 0, len(fields))
	for _, field := range fields {
		if _, ok := f[field]; !ok {
			rv = append(rv, field)
		}
	}
	return rv
}

// Merge will combine two FieldTerms
// it assumes that the terms lists are complete (thus do not need to be merged)
// field terms from the other list always replace the ones in the receiver
func (f FieldTerms) Merge(other FieldTerms) {
	for field, terms := range other {
		f[field] = terms
	}
}

type TermFieldVector struct {
	Field          string
	ArrayPositions []uint64
	Pos            uint64
	Start          uint64
	End            uint64
}

// IndexInternalID is an opaque document identifier interal to the index impl
type IndexInternalID []byte

func (id IndexInternalID) Equals(other IndexInternalID) bool {
	return id.Compare(other) == 0
}

func (id IndexInternalID) Compare(other IndexInternalID) int {
	return bytes.Compare(id, other)
}

type TermFieldDoc struct {
	Term    string
	ID      IndexInternalID
	Freq    uint64
	Norm    float64
	Vectors []*TermFieldVector
}

// Reset allows an already allocated TermFieldDoc to be reused
func (tfd *TermFieldDoc) Reset() *TermFieldDoc {
	// remember the []byte used for the ID
	id := tfd.ID
	// idiom to copy over from empty TermFieldDoc (0 allocations)
	*tfd = TermFieldDoc{}
	// reuse the []byte already allocated (and reset len to 0)
	tfd.ID = id[:0]
	return tfd
}

// TermFieldReader is the interface exposing the enumeration of documents
// containing a given term in a given field. Documents are returned in byte
// lexicographic order over their identifiers.
type TermFieldReader interface {
	// Next returns the next document containing the term in this field, or nil
	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
	// is optional, and when non-nil, will be used instead of allocating memory.
	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Advance resets the enumeration at specified document or its immediate
	// follower.
	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Count returns the number of documents contains the term in this field.
	Count() uint64
	Close() error
}

type DictEntry struct {
	Term  string
	Count uint64
}

type FieldDict interface {
	Next() (*DictEntry, error)
	Close() error
}

// DocIDReader is the interface exposing enumeration of documents identifiers.
// Close the reader to release associated resources.
type DocIDReader interface {
	// Next returns the next document internal identifier in the natural
	// index order, nil when the end of the sequence is reached.
	Next() (IndexInternalID, error)

	// Advance resets the iteration to the first internal identifier greater than
	// or equal to ID. If ID is smaller than the start of the range, the iteration
	// will start there instead. If ID is greater than or equal to the end of
	// the range, Next() call will return io.EOF.
	Advance(ID IndexInternalID) (IndexInternalID, error)
	Close() error
}

type Batch struct {
	IndexOps    map[string]*document.Document
	InternalOps map[string][]byte
}

func NewBatch() *Batch {
	return &Batch{
		IndexOps:    make(map[string]*document.Document),
		InternalOps: make(map[string][]byte),
	}
}

func (b *Batch) Update(doc *document.Document) {
	b.IndexOps[doc.ID] = doc
}

func (b *Batch) Delete(id string) {
	b.IndexOps[id] = nil
}

func (b *Batch) SetInternal(key, val []byte) {
	b.InternalOps[string(key)] = val
}

func (b *Batch) DeleteInternal(key []byte) {
	b.InternalOps[string(key)] = nil
}

func (b *Batch) String() string {
	rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps))
	for k, v := range b.IndexOps {
		if v != nil {
			rv += fmt.Sprintf("\tINDEX - '%s'\n", k)
		} else {
			rv += fmt.Sprintf("\tDELETE - '%s'\n", k)
		}
	}
	for k, v := range b.InternalOps {
		if v != nil {
			rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k)
		} else {
			rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k)
		}
	}
	return rv
}

func (b *Batch) Reset() {
	b.IndexOps = make(map[string]*document.Document)
	b.InternalOps = make(map[string][]byte)
}
<|endoftext|>


<commit_before>package mcstore

import (
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// ProjectService is a service for manipulating projects.
type ProjectService interface {
	createProject(projectName, owner string, mustNotExist bool) (*schema.Project, bool, error)
	getProject(projectName, owner string, create bool) (*schema.Project, bool, error)
}

// projectService implements the ProjectService interface
type projectService struct {
	projects dai.Projects
	dirs     dai.Dirs
}

// newProjectService creates a new instance of the ProjectService. NewProjectService will panic
// if it cannot attach to the database.
func newProjectService() *projectService {
	session := db.RSessionMust()
	return &projectService{
		projects: dai.NewRProjects(session),
		dirs:     dai.NewRDirs(session),
	}
}

// createProject will create a project. If mustNotExist is false, CreateProject will return the
// the project if it already exists. If mustNotExist is true and a matching project is found then
// CreateProject will return ErrExists, and the project will be nil. If CreateProject returns an
// existing project it will return true, otherwise it will return false.
func (s *projectService) createProject(projectName, owner string, mustNotExist bool) (*schema.Project, bool, error) {
	project, err := s.projects.ByName(projectName, owner)
	switch {
	case err != nil:
		return nil, false, err

	case project == nil:
		proj, err := s.createNewProject(projectName, owner)
		return proj, false, err

	default:
		if mustNotExist {
			return nil, false, app.ErrExists
		}

		return project, true, nil
	}
}

// getProject attempts to get a project by project name for the given user. If the create flag is
// set then it will create the project if it doesn't exist. If GetProject creates a project it
// will return true, otherwise it will return false.
func (s *projectService) getProject(projectName, owner string, create bool) (*schema.Project, bool, error) {
	project, err := s.projects.ByName(projectName, owner)

	switch {
	case err != nil:
		return nil, false, err

	case project == nil:
		if create {
			proj, err := s.createNewProject(projectName, owner)
			return proj, true, err
		}
		return nil, false, app.ErrNotFound

	default:
		return project, false, nil
	}
}

// createNewProject will create a new project.
func (s *projectService) createNewProject(projectName, owner string) (*schema.Project, error) {
	project := schema.NewProject(projectName, "", owner)
	return s.projects.Insert(&project)
}
<commit_msg>Fix getProject to handle case where project doesn't exist and we want to create it.<commit_after>package mcstore

import (
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// ProjectService is a service for manipulating projects.
type ProjectService interface {
	createProject(projectName, owner string, mustNotExist bool) (*schema.Project, bool, error)
	getProject(projectName, owner string, create bool) (*schema.Project, bool, error)
}

// projectService implements the ProjectService interface
type projectService struct {
	projects dai.Projects
	dirs     dai.Dirs
}

// newProjectService creates a new instance of the ProjectService. NewProjectService will panic
// if it cannot attach to the database.
func newProjectService() *projectService {
	session := db.RSessionMust()
	return &projectService{
		projects: dai.NewRProjects(session),
		dirs:     dai.NewRDirs(session),
	}
}

// createProject will create a project. If mustNotExist is false, CreateProject will return the
// the project if it already exists. If mustNotExist is true and a matching project is found then
// CreateProject will return ErrExists, and the project will be nil. If CreateProject returns an
// existing project it will return true, otherwise it will return false.
func (s *projectService) createProject(projectName, owner string, mustNotExist bool) (*schema.Project, bool, error) {
	project, err := s.projects.ByName(projectName, owner)
	switch {
	case err != nil:
		return nil, false, err

	case project == nil:
		proj, err := s.createNewProject(projectName, owner)
		return proj, false, err

	default:
		if mustNotExist {
			return nil, true, app.ErrExists
		}

		return project, true, nil
	}
}

// getProject attempts to get a project by project name for the given user. If the create flag is
// set then it will create the project if it doesn't exist. If GetProject creates a project it
// will return true, otherwise it will return false.
func (s *projectService) getProject(projectName, owner string, create bool) (*schema.Project, bool, error) {
	project, err := s.projects.ByName(projectName, owner)

	switch {
	case err == app.ErrNotFound:
		if create {
			proj, err := s.createNewProject(projectName, owner)
			return proj, true, err
		}
		return nil, false, app.ErrNotFound

	case err != nil:
		return nil, false, err

	default:
		return project, false, nil
	}
}

// createNewProject will create a new project.
func (s *projectService) createNewProject(projectName, owner string) (*schema.Project, error) {
	project := schema.NewProject(projectName, "", owner)
	return s.projects.Insert(&project)
}
<|endoftext|>


<commit_before>package kocha

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/naoina/kocha/util"
)

// Render returns result of template.
//
// The data variadic argument must be without specified or only one.
// A data to used will be determined the according to the following rules.
//
// 1. If data of the Data type is given, it will be merged with Context.Data and it will be used.
//
// 2. If data of an other type is given and Context.Data hasn't been set, it will be used as it is.
//    Or it panics if Context.Data has been set.
//
// 3. If data isn't given, Context.Data will be used.
//
// Render retrieve a template file from controller name and c.Response.ContentType.
// e.g. If controller name is "root" and ContentType is "application/xml", Render will
// try to retrieve the template file "root.xml".
// Also ContentType set to "text/html" if not specified.
func Render(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("text/html")
	if err := c.setFormatFromContentTypeIfNotExists(); err != nil {
		panic(err)
	}
	t := c.App.Template.Get(c.App.Config.AppName, c.Layout, c.Name, c.Format)
	if t == nil {
		panic(errors.New("kocha: no such template: " + c.App.Template.Ident(c.App.Config.AppName, c.Layout, c.Name, c.Format)))
	}
	var buf bytes.Buffer
	if err := t.Execute(&buf, c); err != nil {
		panic(err)
	}
	return &resultContent{
		Body: &buf,
	}
}

// RenderJSON returns result of JSON.
//
// RenderJSON is similar to Render but data will be encoded to JSON.
// ContentType set to "application/json" if not specified.
func RenderJSON(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("application/json")
	buf, err := json.Marshal(c.Data)
	if err != nil {
		panic(err)
	}
	return &resultContent{
		Body: bytes.NewReader(buf),
	}
}

// RenderXML returns result of XML.
//
// RenderXML is similar to Render but data will be encoded to XML.
// ContentType set to "application/xml" if not specified.
func RenderXML(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("application/xml")
	buf, err := xml.Marshal(c.Data)
	if err != nil {
		panic(err)
	}
	return &resultContent{
		Body: bytes.NewReader(buf),
	}
}

// RenderText returns result of text.
//
// ContentType set to "text/plain" if not specified.
func RenderText(c *Context, content string) Result {
	c.setContentTypeIfNotExists("text/plain")
	return &resultContent{
		Body: strings.NewReader(content),
	}
}

// RenderError returns result of error.
//
// RenderError is similar to Render, but there is a point where some different.
// Render retrieve a template file from statusCode and c.Response.ContentType.
// e.g. If statusCode is 500 and ContentType is "application/xml", Render will
// try to retrieve the template file "errors/500.xml".
// If failed to retrieve the template file, it returns result of text with statusCode.
// Also ContentType set to "text/html" if not specified.
func RenderError(c *Context, statusCode int, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("text/html")
	if err := c.setFormatFromContentTypeIfNotExists(); err != nil {
		panic(err)
	}
	c.Response.StatusCode = statusCode
	c.Name = errorTemplateName(statusCode)
	t := c.App.Template.Get(c.App.Config.AppName, c.Layout, c.Name, c.Format)
	if t == nil {
		c.Response.ContentType = "text/plain"
		return &resultContent{
			Body: bytes.NewReader([]byte(http.StatusText(statusCode))),
		}
	}
	var buf bytes.Buffer
	if err := t.Execute(&buf, c); err != nil {
		panic(err)
	}
	return &resultContent{
		Body: &buf,
	}
}

// SendFile returns result of any content.
//
// The path argument specifies an absolute or relative path.
// If absolute path, read the content from the path as it is.
// If relative path, First, Try to get the content from included resources and
// returns it if successful. Otherwise, Add AppPath and StaticDir to the prefix
// of the path and then will read the content from the path that.
// Also, set ContentType detect from content if c.Response.ContentType is empty.
func SendFile(c *Context, path string) Result {
	var file io.ReadSeeker
	path = filepath.FromSlash(path)
	if rc := c.App.ResourceSet.Get(path); rc != nil {
		switch b := rc.(type) {
		case string:
			file = strings.NewReader(b)
		case []byte:
			file = bytes.NewReader(b)
		}
	}
	if file == nil {
		if !filepath.IsAbs(path) {
			path = filepath.Join(c.App.Config.AppPath, StaticDir, path)
		}
		if _, err := os.Stat(path); err != nil {
			return RenderError(c, http.StatusNotFound)
		}
		var err error
		if file, err = os.Open(path); err != nil {
			panic(err)
		}
	}
	c.Response.ContentType = util.DetectContentTypeByExt(path)
	if c.Response.ContentType == "" {
		c.Response.ContentType = util.DetectContentTypeByBody(file)
	}
	return &resultContent{
		Body: file,
	}
}

// Redirect returns result of redirect.
//
// If permanently is true, redirect to url with 301. (http.StatusMovedPermanently)
// Otherwise redirect to url with 302. (http.StatusFound)
func Redirect(c *Context, url string, permanently bool) Result {
	return &resultRedirect{
		Request:     c.Request,
		URL:         url,
		Permanently: permanently,
	}
}
<commit_msg>renderer: Fix godoc<commit_after>package kocha

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"errors"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/naoina/kocha/util"
)

// Render returns result of template.
//
// The data variadic argument must be without specified or only one.
// A data to used will be determined the according to the following rules.
//
// 1. If data of the Data type is given, it will be merged with Context.Data and it will be used.
//
// 2. If data of an other type is given, it will be used as it is.
//
// 3. If data isn't given, Context.Data will be used.
//
// Render retrieve a template file from controller name and c.Response.ContentType.
// e.g. If controller name is "root" and ContentType is "application/xml", Render will
// try to retrieve the template file "root.xml".
// Also ContentType set to "text/html" if not specified.
func Render(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("text/html")
	if err := c.setFormatFromContentTypeIfNotExists(); err != nil {
		panic(err)
	}
	t := c.App.Template.Get(c.App.Config.AppName, c.Layout, c.Name, c.Format)
	if t == nil {
		panic(errors.New("kocha: no such template: " + c.App.Template.Ident(c.App.Config.AppName, c.Layout, c.Name, c.Format)))
	}
	var buf bytes.Buffer
	if err := t.Execute(&buf, c); err != nil {
		panic(err)
	}
	return &resultContent{
		Body: &buf,
	}
}

// RenderJSON returns result of JSON.
//
// RenderJSON is similar to Render but data will be encoded to JSON.
// ContentType set to "application/json" if not specified.
func RenderJSON(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("application/json")
	buf, err := json.Marshal(c.Data)
	if err != nil {
		panic(err)
	}
	return &resultContent{
		Body: bytes.NewReader(buf),
	}
}

// RenderXML returns result of XML.
//
// RenderXML is similar to Render but data will be encoded to XML.
// ContentType set to "application/xml" if not specified.
func RenderXML(c *Context, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("application/xml")
	buf, err := xml.Marshal(c.Data)
	if err != nil {
		panic(err)
	}
	return &resultContent{
		Body: bytes.NewReader(buf),
	}
}

// RenderText returns result of text.
//
// ContentType set to "text/plain" if not specified.
func RenderText(c *Context, content string) Result {
	c.setContentTypeIfNotExists("text/plain")
	return &resultContent{
		Body: strings.NewReader(content),
	}
}

// RenderError returns result of error.
//
// RenderError is similar to Render, but there is a point where some different.
// Render retrieve a template file from statusCode and c.Response.ContentType.
// e.g. If statusCode is 500 and ContentType is "application/xml", Render will
// try to retrieve the template file "errors/500.xml".
// If failed to retrieve the template file, it returns result of text with statusCode.
// Also ContentType set to "text/html" if not specified.
func RenderError(c *Context, statusCode int, data ...interface{}) Result {
	if err := c.setData(data); err != nil {
		panic(err)
	}
	c.setContentTypeIfNotExists("text/html")
	if err := c.setFormatFromContentTypeIfNotExists(); err != nil {
		panic(err)
	}
	c.Response.StatusCode = statusCode
	c.Name = errorTemplateName(statusCode)
	t := c.App.Template.Get(c.App.Config.AppName, c.Layout, c.Name, c.Format)
	if t == nil {
		c.Response.ContentType = "text/plain"
		return &resultContent{
			Body: bytes.NewReader([]byte(http.StatusText(statusCode))),
		}
	}
	var buf bytes.Buffer
	if err := t.Execute(&buf, c); err != nil {
		panic(err)
	}
	return &resultContent{
		Body: &buf,
	}
}

// SendFile returns result of any content.
//
// The path argument specifies an absolute or relative path.
// If absolute path, read the content from the path as it is.
// If relative path, First, Try to get the content from included resources and
// returns it if successful. Otherwise, Add AppPath and StaticDir to the prefix
// of the path and then will read the content from the path that.
// Also, set ContentType detect from content if c.Response.ContentType is empty.
func SendFile(c *Context, path string) Result {
	var file io.ReadSeeker
	path = filepath.FromSlash(path)
	if rc := c.App.ResourceSet.Get(path); rc != nil {
		switch b := rc.(type) {
		case string:
			file = strings.NewReader(b)
		case []byte:
			file = bytes.NewReader(b)
		}
	}
	if file == nil {
		if !filepath.IsAbs(path) {
			path = filepath.Join(c.App.Config.AppPath, StaticDir, path)
		}
		if _, err := os.Stat(path); err != nil {
			return RenderError(c, http.StatusNotFound)
		}
		var err error
		if file, err = os.Open(path); err != nil {
			panic(err)
		}
	}
	c.Response.ContentType = util.DetectContentTypeByExt(path)
	if c.Response.ContentType == "" {
		c.Response.ContentType = util.DetectContentTypeByBody(file)
	}
	return &resultContent{
		Body: file,
	}
}

// Redirect returns result of redirect.
//
// If permanently is true, redirect to url with 301. (http.StatusMovedPermanently)
// Otherwise redirect to url with 302. (http.StatusFound)
func Redirect(c *Context, url string, permanently bool) Result {
	return &resultRedirect{
		Request:     c.Request,
		URL:         url,
		Permanently: permanently,
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package piazza

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

//----------------------------------------------------------

const (
	// ContentTypeJSON is the http content-type for JSON.
	ContentTypeJSON = "application/json"

	// ContentTypeText is the http content-type for plain text.
	ContentTypeText = "text/plain"
)

//----------------------------------------------------------

// Put, because there is no http.Put.
func HTTPPut(url string, contentType string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest("PUT", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", contentType)
	client := &http.Client{}
	return client.Do(req)
}

// Delete, because there is no http.Delete.
func HTTPDelete(url string) (*http.Response, error) {
	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return nil, err
	}

	client := &http.Client{}
	return client.Do(req)
}

//----------------------------------------------------------
/*
// expects endpoint to return JSON
func HttpJsonGetObject(url string, output interface{}) (int, error) {
	h := &Http{}
	return h.Get(url, output)
}

// expects endpoint to take in and return JSON
func HttpJsonPostObject(url string, input interface{}, output interface{}) (int, error) {
	h := &Http{}
	return h.Post(url, input, output)
}

// expects endpoint to take in and return JSON
func HttpJsonPutObject(url string, input interface{}, output interface{}) (int, error) {
	h := &Http{}
	return h.Put(url, input, output)
}

// expects endpoint to return JSON (or empty)
func HttpJsonDeleteObject(url string) (int, error) {
	h := &Http{}
	return h.Delete(url)
}
*/
//----------------------------------------------------------

// We don't want to pass http.Request objects into the Services classes
// (and certainly not a gin.Context!), and we need this kind of information
// a lot, so we'll keep a special data structure which actually understands
// the semantics as well as the syntax.

type HttpQueryParams struct {
	raw map[string]string
}

func NewQueryParams(request *http.Request) *HttpQueryParams {
	params := HttpQueryParams{raw: make(map[string]string)}

	for k, v := range request.URL.Query() {
		params.raw[k] = v[0]
	}
	return &params
}

func (params *HttpQueryParams) AsInt(key string, defalt *int) (*int, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	i, err := strconv.Atoi(value)
	if err != nil {
		return nil, err
	}

	return &i, nil
}

func (params *HttpQueryParams) AsString(key string, defalt *string) (*string, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	s := value
	return &s, nil
}

func (params *HttpQueryParams) AsOrder(key string, defalt *PaginationOrder) (*PaginationOrder, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	var order PaginationOrder
	switch strings.ToLower(value) {
	case "desc":
		order = PaginationOrderDescending
	case "asc":
		order = PaginationOrderDescending
	default:
		s := fmt.Sprintf("query argument for '?%s' must be \"asc\" or \"desc\"", value)
		err := errors.New(s)
		return nil, err
	}

	return &order, nil
}

func (params *HttpQueryParams) AsTime(key string, defalt *time.Time) (*time.Time, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	t, err := time.Parse(time.RFC3339, value)
	if err != nil {
		return nil, err
	}

	return &t, nil
}

func (params *HttpQueryParams) GetAfter(defalt *time.Time) (*time.Time, error) {
	return params.AsTime("after", defalt)
}

func (params *HttpQueryParams) GetBefore(defalt *time.Time) (*time.Time, error) {
	return params.AsTime("before", defalt)
}

func (params *HttpQueryParams) GetCount(defalt *int) (*int, error) {
	return params.AsInt("count", defalt)
}

func (params *HttpQueryParams) GetOrder(defalt *PaginationOrder) (*PaginationOrder, error) {
	return params.AsOrder("order", defalt)
}

func (params *HttpQueryParams) GetPage(defalt *int) (*int, error) {
	return params.AsInt("page", defalt)
}

func (params *HttpQueryParams) GetPerPage(defalt *int) (*int, error) {
	return params.AsInt("perPage", defalt)
}

func (params *HttpQueryParams) GetSortBy(defalt *string) (*string, error) {
	return params.AsString("sortBy", defalt)
}

//---------------------------------------------------------------------

func GinReturnJson(c *gin.Context, resp *JsonResponse) {
	raw, err := json.MarshalIndent(resp, "", "  ")
	if err != nil {
		log.Fatalf("Internal Error: marshalling of %#v", resp)
	}
	log.Printf("%s", string(raw))
	c.Data(resp.StatusCode, ContentTypeJSON, raw)
}

// Get the Pz API key, in this order:
//
// (1) if $PZKEY present, use that
// (2) if ~/.pzkey exists, use that
// (3) error
//
// And no, we don't uspport Windows.
func GetApiKey() (string, error) {

	fileExists := func(s string) bool {
		if _, err := os.Stat(s); os.IsNotExist(err) {
			return false
		}
		return true
	}

	key := os.Getenv("PZKEY")
	if key != "" {
		key = strings.TrimSpace(key)
		return key, nil
	}

	home := os.Getenv("HOME")
	if home == "" {
		return "", errors.New("Unable read $HOME")
	}

	path := home + "/.pzkey"
	if !fileExists(path) {
		return "", errors.New("Unable to find env var $PZKEY or file $HOME/.pzkey")
	}

	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}

	key = strings.TrimSpace(string(raw))

	return key, nil
}
<commit_msg>remove logging<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package piazza

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

//----------------------------------------------------------

const (
	// ContentTypeJSON is the http content-type for JSON.
	ContentTypeJSON = "application/json"

	// ContentTypeText is the http content-type for plain text.
	ContentTypeText = "text/plain"
)

//----------------------------------------------------------

// Put, because there is no http.Put.
func HTTPPut(url string, contentType string, body io.Reader) (*http.Response, error) {
	req, err := http.NewRequest("PUT", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", contentType)
	client := &http.Client{}
	return client.Do(req)
}

// Delete, because there is no http.Delete.
func HTTPDelete(url string) (*http.Response, error) {
	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return nil, err
	}

	client := &http.Client{}
	return client.Do(req)
}

//----------------------------------------------------------
/*
// expects endpoint to return JSON
func HttpJsonGetObject(url string, output interface{}) (int, error) {
	h := &Http{}
	return h.Get(url, output)
}

// expects endpoint to take in and return JSON
func HttpJsonPostObject(url string, input interface{}, output interface{}) (int, error) {
	h := &Http{}
	return h.Post(url, input, output)
}

// expects endpoint to take in and return JSON
func HttpJsonPutObject(url string, input interface{}, output interface{}) (int, error) {
	h := &Http{}
	return h.Put(url, input, output)
}

// expects endpoint to return JSON (or empty)
func HttpJsonDeleteObject(url string) (int, error) {
	h := &Http{}
	return h.Delete(url)
}
*/
//----------------------------------------------------------

// We don't want to pass http.Request objects into the Services classes
// (and certainly not a gin.Context!), and we need this kind of information
// a lot, so we'll keep a special data structure which actually understands
// the semantics as well as the syntax.

type HttpQueryParams struct {
	raw map[string]string
}

func NewQueryParams(request *http.Request) *HttpQueryParams {
	params := HttpQueryParams{raw: make(map[string]string)}

	for k, v := range request.URL.Query() {
		params.raw[k] = v[0]
	}
	return &params
}

func (params *HttpQueryParams) AsInt(key string, defalt *int) (*int, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	i, err := strconv.Atoi(value)
	if err != nil {
		return nil, err
	}

	return &i, nil
}

func (params *HttpQueryParams) AsString(key string, defalt *string) (*string, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	s := value
	return &s, nil
}

func (params *HttpQueryParams) AsOrder(key string, defalt *PaginationOrder) (*PaginationOrder, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	var order PaginationOrder
	switch strings.ToLower(value) {
	case "desc":
		order = PaginationOrderDescending
	case "asc":
		order = PaginationOrderDescending
	default:
		s := fmt.Sprintf("query argument for '?%s' must be \"asc\" or \"desc\"", value)
		err := errors.New(s)
		return nil, err
	}

	return &order, nil
}

func (params *HttpQueryParams) AsTime(key string, defalt *time.Time) (*time.Time, error) {
	if key == "" {
		return defalt, nil
	}

	value, ok := params.raw[key]
	if !ok || value == "" {
		return defalt, nil
	}

	t, err := time.Parse(time.RFC3339, value)
	if err != nil {
		return nil, err
	}

	return &t, nil
}

func (params *HttpQueryParams) GetAfter(defalt *time.Time) (*time.Time, error) {
	return params.AsTime("after", defalt)
}

func (params *HttpQueryParams) GetBefore(defalt *time.Time) (*time.Time, error) {
	return params.AsTime("before", defalt)
}

func (params *HttpQueryParams) GetCount(defalt *int) (*int, error) {
	return params.AsInt("count", defalt)
}

func (params *HttpQueryParams) GetOrder(defalt *PaginationOrder) (*PaginationOrder, error) {
	return params.AsOrder("order", defalt)
}

func (params *HttpQueryParams) GetPage(defalt *int) (*int, error) {
	return params.AsInt("page", defalt)
}

func (params *HttpQueryParams) GetPerPage(defalt *int) (*int, error) {
	return params.AsInt("perPage", defalt)
}

func (params *HttpQueryParams) GetSortBy(defalt *string) (*string, error) {
	return params.AsString("sortBy", defalt)
}

//---------------------------------------------------------------------

func GinReturnJson(c *gin.Context, resp *JsonResponse) {
	raw, err := json.MarshalIndent(resp, "", "  ")
	if err != nil {
		log.Fatalf("Internal Error: marshalling of %#v", resp)
	}
	//log.Printf("%s", string(raw))
	c.Data(resp.StatusCode, ContentTypeJSON, raw)
}

// Get the Pz API key, in this order:
//
// (1) if $PZKEY present, use that
// (2) if ~/.pzkey exists, use that
// (3) error
//
// And no, we don't uspport Windows.
func GetApiKey() (string, error) {

	fileExists := func(s string) bool {
		if _, err := os.Stat(s); os.IsNotExist(err) {
			return false
		}
		return true
	}

	key := os.Getenv("PZKEY")
	if key != "" {
		key = strings.TrimSpace(key)
		return key, nil
	}

	home := os.Getenv("HOME")
	if home == "" {
		return "", errors.New("Unable read $HOME")
	}

	path := home + "/.pzkey"
	if !fileExists(path) {
		return "", errors.New("Unable to find env var $PZKEY or file $HOME/.pzkey")
	}

	raw, err := ioutil.ReadFile(path)
	if err != nil {
		return "", err
	}

	key = strings.TrimSpace(string(raw))

	return key, nil
}
<|endoftext|>


<commit_before>package xkcd

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"
)

const URL = "http://xkcd.com"
const INFO = "info.0.json"

// Info represents xkcd comic information.
type Info struct {
	Alt        string
	Day        string
	Image      string
	Link       string
	Month      string
	News       string
	Num        int
	SafeTitle  string `json:"safe_title"`
	Transcript string
	Year       string
}

// FetchInfo gets xkcd comic information as json and unpack one to info struct.
func FetchInfo(url string) (*Info, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	var info Info
	if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return nil, err
	}
	return &info, nil
}

// Cache represents cache of xkcd comic information.
type Cache struct {
	CheckedAt time.Time
	Comics    map[int]*Info
	LastNum   int
}

// Update get the latest xkcd comic from site and compares with the latest from
// the Cache. If new comics are appeared they well be added to Cache. Parameter
// 'force' allowes to reinitialize cache from the begginging.
func (c *Cache) Update(force bool) error {

	newLast, err := FetchInfo(URL + "/" + INFO)
	if err != nil {
		return fmt.Errorf("check last error: %s", err)
	}

	// TODO: think about moving update to function.
	if force || c.Comics == nil {
		// TODO: don't like this hack.
		c.LastNum = 0
		c.Comics = make(map[int]*Info, newLast.Num)
	}

	for i := c.LastNum + 1; i <= newLast.Num; i++ {

		if ok := c.Comics[i]; ok != nil {
			continue
		}
		url := fmt.Sprintf("%s/%d/%s", URL, i, INFO)
		info, err := FetchInfo(url)
		if err != nil {
			fmt.Fprintf(os.Stderr, "update error: %s, %s\n", url, err)
			continue
		}
		c.Comics[i] = info // no-thread safe
	}
	c.CheckedAt = time.Now()
	c.LastNum = newLast.Num
	return nil
}

// Load reads cache represented as JSON from Reader and unmarshal them to Cache.
func (c *Cache) Load(r io.Reader) error {
	if err := json.NewDecoder(r).Decode(c); err != nil {
		return err
	}
	return nil
}

// Dump writes Cache marshaled as JSON to Writer.
func (c *Cache) Dump(w io.Writer) error {
	if err := json.NewEncoder(w).Encode(c); err != nil {
		return err
	}
	return nil
}

// NewCache creates new pointer to Cache structure and initializes internal map.
func NewCache() *Cache {
	return &Cache{Comics: make(map[int]*Info)}
}
<commit_msg>Added http client and error handling for task 4.12<commit_after>package xkcd

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"time"
)

const URL = "http://xkcd.com"
const INFO = "info.0.json"

var httpClient = http.Client{Timeout: time.Duration(time.Second * 10)}

// Info represents xkcd comic information.
type Info struct {
	Alt        string
	Day        string
	Image      string
	Link       string
	Month      string
	News       string
	Num        int
	SafeTitle  string `json:"safe_title"`
	Transcript string
	Year       string
	URL        string
}

// FetchInfo gets xkcd comic information as json and unpack one to info struct.
func FetchInfo(url string) (*Info, error) {
	resp, err := httpClient.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("http status %d", resp.StatusCode)
	}

	var info Info
	if err := json.NewDecoder(resp.Body).Decode(&info); err != nil {
		return nil, err
	}
	return &info, nil
}

// Cache represents cache of xkcd comic information.
type Cache struct {
	CheckedAt time.Time
	Comics    map[int]*Info
	LastNum   int
}

// Update get the latest xkcd comic from site and compares with the latest from
// the Cache. If new comics are appeared they well be added to Cache. Parameter
// 'force' allowes to reinitialize cache from the begginging.
func (c *Cache) Update(force bool) error {

	newLast, err := FetchInfo(URL + "/" + INFO)
	if err != nil {
		return fmt.Errorf("check last error: %s", err)
	}

	// TODO: think about moving update to function.
	if force || c.Comics == nil {
		// TODO: don't like this hack.
		c.LastNum = 0
		c.Comics = make(map[int]*Info, newLast.Num)
	}

	for i := 400; i <= newLast.Num; i++ {

		if ok := c.Comics[i]; ok != nil {
			continue
		}

		comicURL := fmt.Sprintf("%s/%d/", URL, i)
		info, err := FetchInfo(comicURL + INFO)
		if err != nil {
			fmt.Fprintf(os.Stderr, "update error: %s, %s\n", comicURL, err)
			continue
		}

		// No-thread safe
		info.URL = comicURL
		c.Comics[i] = info
	}
	c.CheckedAt = time.Now()
	c.LastNum = newLast.Num
	return nil
}

// Load reads cache represented as JSON from Reader and unmarshal them to Cache.
func (c *Cache) Load(r io.Reader) error {
	if err := json.NewDecoder(r).Decode(c); err != nil {
		return err
	}
	return nil
}

// Dump writes Cache marshaled as JSON to Writer.
func (c *Cache) Dump(w io.Writer) error {
	if err := json.NewEncoder(w).Encode(c); err != nil {
		return err
	}
	return nil
}

// NewCache creates new pointer to Cache structure and initializes internal map.
func NewCache() *Cache {
	return &Cache{Comics: make(map[int]*Info)}
}
<|endoftext|>


<commit_before>package handler

import (
	"fmt"
	"html/template"
	"net/http"
)

type playgroundData struct {
	PlaygroundVersion    string
	Endpoint             string
	SubscriptionEndpoint string
	SetTitle             bool
}

// renderPlayground renders the Playground GUI
func renderPlayground(w http.ResponseWriter, r *http.Request) {
	t := template.New("Playground")
	t, err := t.Parse(graphcoolPlaygroundTemplate)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	d := playgroundData{
		PlaygroundVersion:    graphcoolPlaygroundVersion,
		Endpoint:             "/graphql",
		SubscriptionEndpoint: fmt.Sprintf("ws://%v/subscriptions", r.Host),
		SetTitle:             true,
	}
	err = t.ExecuteTemplate(w, "index", d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	return
}

const graphcoolPlaygroundVersion = "1.5.2"

const graphcoolPlaygroundTemplate = `
{{ define "index" }}

<!DOCTYPE html>

  <meta charset=utf-8/>
  <meta name="viewport" content="user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, minimal-ui">
  <title>GraphQL Playground</title>
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/css/index.css" />
  <link rel="shortcut icon" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/favicon.png" />

 Loading
      GraphQL Playground

{{ end }}
`
<commit_msg>Fixed the endpoint embedded in the playground's html template<commit_after>package handler

import (
	"fmt"
	"html/template"
	"net/http"
)

type playgroundData struct {
	PlaygroundVersion    string
	Endpoint             string
	SubscriptionEndpoint string
	SetTitle             bool
}

// renderPlayground renders the Playground GUI
func renderPlayground(w http.ResponseWriter, r *http.Request) {
	t := template.New("Playground")
	t, err := t.Parse(graphcoolPlaygroundTemplate)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	d := playgroundData{
		PlaygroundVersion:    graphcoolPlaygroundVersion,
		Endpoint:             r.URL.Path,
		SubscriptionEndpoint: fmt.Sprintf("ws://%v/subscriptions", r.Host),
		SetTitle:             true,
	}
	err = t.ExecuteTemplate(w, "index", d)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	return
}

const graphcoolPlaygroundVersion = "1.5.2"

const graphcoolPlaygroundTemplate = `
{{ define "index" }}

<!DOCTYPE html>

  <meta charset=utf-8/>
  <meta name="viewport" content="user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, minimal-ui">
  <title>GraphQL Playground</title>
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/static/css/index.css" />
  <link rel="shortcut icon" href="//cdn.jsdelivr.net/npm/graphql-playground-react/build/favicon.png" />

 Loading
      GraphQL Playground

{{ end }}
`
<|endoftext|>


<commit_before>package hal

// Response struct
type Response struct {
	Robot    *Robot
	Envelope *Envelope
	Message  *Message
	Match    []string
}

// Envelope contains metadata about the chat message.
type Envelope struct {
	Room string
	User *User
}

// NewResponseFromMessage returns a new Response object with an associated Message
func NewResponseFromMessage(robot *Robot, msg *Message) *Response {
	return &Response{
		Robot: robot,
		Envelope: &Envelope{
			Room: msg.Room,
			User: &msg.User,
		},
		Message: msg,
	}
}

// NewResponse returns a new Response object
func NewResponse(robot *Robot) *Response {
	return &Response{
		Robot:    robot,
		Envelope: &Envelope{},
	}
}

// UserID returns the id of the Message's User
func (res *Response) UserID() string {
	return res.Envelope.User.ID
}

// UserName returns the id of the Message's User
func (res *Response) UserName() string {
	return res.Envelope.User.Name
}

// UserRoles returns the roles of the Message's User
func (res *Response) UserRoles() []string {
	return res.Message.User.Roles
}

// Room is the room of the response's message
func (res *Response) Room() string {
	return res.Envelope.Room
}

// Text is the text of the response's message
func (res *Response) Text() string {
	return res.Message.Text
}

// Send posts a message back to the chat source
func (res *Response) Send(strings ...string) error {
	if err := res.Robot.Adapter.Send(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Reply posts a message mentioning the current user
func (res *Response) Reply(strings ...string) error {
	if err := res.Robot.Adapter.Reply(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Emote posts an emote back to the chat source
func (res *Response) Emote(strings ...string) error {
	if err := res.Robot.Adapter.Emote(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Topic posts a topic changing message
func (res *Response) Topic(strings ...string) error {
	if err := res.Robot.Adapter.Topic(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Play posts a sound message
func (res *Response) Play(strings ...string) error {
	if err := res.Robot.Adapter.Play(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}
<commit_msg>support custom envelope options<commit_after>package hal

// Response struct
type Response struct {
	Robot    *Robot
	Envelope *Envelope
	Message  *Message
	Match    []string
}

// Envelope contains metadata about the chat message.
type Envelope struct {
	Room    string
	User    *User
	Options map[string]interface{}
}

// Options type
type Options map[string]interface{}

// SetOptions sets the Envelope's Options
func (e *Envelope) SetOptions(opts map[string]interface{}) {
	e.Options = opts
}

// NewResponseFromMessage returns a new Response object with an associated Message
func NewResponseFromMessage(robot *Robot, msg *Message) *Response {
	return &Response{
		Robot: robot,
		Envelope: &Envelope{
			Room: msg.Room,
			User: &msg.User,
		},
		Message: msg,
	}
}

// NewResponse returns a new Response object
func NewResponse(robot *Robot) *Response {
	return &Response{
		Robot:    robot,
		Envelope: &Envelope{},
	}
}

// UserID returns the id of the Message's User
func (res *Response) UserID() string {
	return res.Envelope.User.ID
}

// UserName returns the id of the Message's User
func (res *Response) UserName() string {
	return res.Envelope.User.Name
}

// UserRoles returns the roles of the Message's User
func (res *Response) UserRoles() []string {
	return res.Message.User.Roles
}

// Room is the room of the response's message
func (res *Response) Room() string {
	return res.Envelope.Room
}

// Text is the text of the response's message
func (res *Response) Text() string {
	return res.Message.Text
}

// Send posts a message back to the chat source
func (res *Response) Send(strings ...string) error {
	if err := res.Robot.Adapter.Send(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Reply posts a message mentioning the current user
func (res *Response) Reply(strings ...string) error {
	if err := res.Robot.Adapter.Reply(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Emote posts an emote back to the chat source
func (res *Response) Emote(strings ...string) error {
	if err := res.Robot.Adapter.Emote(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Topic posts a topic changing message
func (res *Response) Topic(strings ...string) error {
	if err := res.Robot.Adapter.Topic(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}

// Play posts a sound message
func (res *Response) Play(strings ...string) error {
	if err := res.Robot.Adapter.Play(res, strings...); err != nil {
		Logger.Error(err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package go_mocket

import (
	"database/sql/driver"
	"fmt"
	"log"
	"reflect"
	"strings"
)

var Catcher *MockCatcher

// Global entity to save all mocks aka FakeResponses
type MockCatcher struct {
	Mocks                []*FakeResponse // Slice of all mocks
	Logging              bool            // Do we need to log what we catching?
	PanicOnEmptyResponse bool            // If not response matches - do we need to panic?
}

// Attach several mocks to MockCather. Could be useful to attach mocks from some factories of mocks
//
func (this *MockCatcher) Attach(fr []*FakeResponse) {
	this.Mocks = append(this.Mocks, fr...)
}

// Find suitable response by provided
func (this *MockCatcher) FindResponse(query string, args []driver.NamedValue) *FakeResponse {
	if this.Logging {
		log.Printf("mock_catcher: check query: %s", query)
	}

	for _, resp := range this.Mocks {
		if resp.IsMatch(query, args) {
			resp.MarkAsTriggered()
			return resp
		}
	}

	if this.PanicOnEmptyResponse {
		panic(fmt.Sprintf("No responses matches query %s ", query))
	}

	// Let's have always dummy version of response
	return &FakeResponse{
		Response:   make([]map[string]interface{}, 0),
		Exceptions: &Exceptions{},
	}
}

// Create new FakeResponse and return for chains of attachments
func (this *MockCatcher) NewMock() *FakeResponse {
	fr := &FakeResponse{Exceptions: &Exceptions{}, Response: make([]map[string]interface{}, 0)}
	this.Mocks = append(this.Mocks, fr)
	return fr
}

// Remove all Mocks to start process again
func (this *MockCatcher) Reset() *MockCatcher {
	this.Mocks = make([]*FakeResponse, 0)
	return this
}

// Possible exceptions during query executions
type Exceptions struct {
	HookQueryBadConnection func() bool
	HookExecBadConnection  func() bool
}

// Represents mock of response with holding all required values to return mocked response
type FakeResponse struct {
	Pattern      string                            // SQL query pattern to match with
	Args         []interface{}                     // List args to be matched with
	Response     []map[string]interface{}          // Array of rows to be parsed as result
	Once         bool                              // To trigger only once
	Triggered    bool                              // If it was triggered at least once
	Callback     func(string, []driver.NamedValue) // Callback to execute when response triggered
	RowsAffected int64                             // Defines affected rows count
	LastInsertId int64                             // ID to be returned for INSERT queries
	*Exceptions
}

// Return true either when nothing to compare or deep equal check passed
func (fr *FakeResponse) isArgsMatch(args []driver.NamedValue) bool {
	arguments := make([]interface{}, len(args))
	if len(args) > 0 {
		for index, arg := range args {
			arguments[index] = arg.Value
		}
	}
	return fr.Args == nil || reflect.DeepEqual(fr.Args, arguments)
}

func (fr *FakeResponse) isQueryMatch(query string) bool {
	return fr.Pattern == "" || strings.Contains(query, fr.Pattern)
}

func (fr *FakeResponse) IsMatch(query string, args []driver.NamedValue) bool {
	if fr.Once && fr.Triggered {
		return false
	}
	return fr.isQueryMatch(query) && fr.isArgsMatch(args)
}

// Mark Response as executed. For one time catches it will not make this possible to execute anymore
func (fr *FakeResponse) MarkAsTriggered() {
	fr.Triggered = true
}

// Add SQL query pattern to match for
func (fr *FakeResponse) WithQuery(query string) *FakeResponse {
	fr.Pattern = query
	return fr
}

// Attach Args check for prepared statements
func (fr *FakeResponse) WithArgs(vars ...interface{}) *FakeResponse {
	if len(vars) > 0 {
		fr.Args = make([]interface{}, len(vars))
		for index, v := range vars {
			fr.Args[index] = v
		}
	}
	return fr
}

// Methods to chain and assign some parts of response

func (fr *FakeResponse) WithReply(response []map[string]interface{}) *FakeResponse {
	fr.Response = response
	return fr
}

func (fr *FakeResponse) OneTime() *FakeResponse {
	fr.Once = true
	return fr
}

func (fr *FakeResponse) WithExecException() *FakeResponse {
	fr.Exceptions.HookExecBadConnection = func() bool {
		return true
	}
	return fr
}

func (fr *FakeResponse) WithQueryException() *FakeResponse {
	fr.Exceptions.HookQueryBadConnection = func() bool {
		return true
	}
	return fr
}

func (fr *FakeResponse) WithCallback(f func(string, []driver.NamedValue)) *FakeResponse {
	fr.Callback = f
	return fr
}

func (fr *FakeResponse) WithRowsNum(num int64) *FakeResponse {
	fr.RowsAffected = num
	return fr
}

func (fr *FakeResponse) WithId(id int64) *FakeResponse {
	fr.LastInsertId = id
	return fr
}

func init() {
	Catcher = &MockCatcher{}
}
<commit_msg>response comments according linter<commit_after>package go_mocket

import (
	"database/sql/driver"
	"fmt"
	"log"
	"reflect"
	"strings"
)

// Global instance of Catcher used for attaching all mocks to connection
var Catcher *MockCatcher

// Global entity to save all mocks aka FakeResponses
type MockCatcher struct {
	Mocks                []*FakeResponse // Slice of all mocks
	Logging              bool            // Do we need to log what we catching?
	PanicOnEmptyResponse bool            // If not response matches - do we need to panic?
}

// Attach several mocks to MockCather. Could be useful to attach mocks from some factories of mocks
func (mc *MockCatcher) Attach(fr []*FakeResponse) {
	mc.Mocks = append(mc.Mocks, fr...)
}

//FindResponse Finds suitable response by provided
func (mc *MockCatcher) FindResponse(query string, args []driver.NamedValue) *FakeResponse {
	if mc.Logging {
		log.Printf("mock_catcher: check query: %s", query)
	}

	for _, resp := range mc.Mocks {
		if resp.IsMatch(query, args) {
			resp.MarkAsTriggered()
			return resp
		}
	}

	if mc.PanicOnEmptyResponse {
		panic(fmt.Sprintf("No responses matches query %s ", query))
	}

	// Let's have always dummy version of response
	return &FakeResponse{
		Response:   make([]map[string]interface{}, 0),
		Exceptions: &Exceptions{},
	}
}

//NewMock Creates new FakeResponse and return for chains of attachments
func (mc *MockCatcher) NewMock() *FakeResponse {
	fr := &FakeResponse{Exceptions: &Exceptions{}, Response: make([]map[string]interface{}, 0)}
	mc.Mocks = append(mc.Mocks, fr)
	return fr
}

//Reset Remove all Mocks to start process again
func (mc *MockCatcher) Reset() *MockCatcher {
	mc.Mocks = make([]*FakeResponse, 0)
	return mc
}

// Possible exceptions during query executions
type Exceptions struct {
	HookQueryBadConnection func() bool
	HookExecBadConnection  func() bool
}

// Represents mock of response with holding all required values to return mocked response
type FakeResponse struct {
	Pattern      string                            // SQL query pattern to match with
	Args         []interface{}                     // List args to be matched with
	Response     []map[string]interface{}          // Array of rows to be parsed as result
	Once         bool                              // To trigger only once
	Triggered    bool                              // If it was triggered at least once
	Callback     func(string, []driver.NamedValue) // Callback to execute when response triggered
	RowsAffected int64                             // Defines affected rows count
	LastInsertId int64                             // ID to be returned for INSERT queries
	*Exceptions
}

// Return true either when nothing to compare or deep equal check passed
func (fr *FakeResponse) isArgsMatch(args []driver.NamedValue) bool {
	arguments := make([]interface{}, len(args))
	if len(args) > 0 {
		for index, arg := range args {
			arguments[index] = arg.Value
		}
	}
	return fr.Args == nil || reflect.DeepEqual(fr.Args, arguments)
}

func (fr *FakeResponse) isQueryMatch(query string) bool {
	return fr.Pattern == "" || strings.Contains(query, fr.Pattern)
}

// IsMatch Check if both query and args matcher's return true and if this is Once mock
func (fr *FakeResponse) IsMatch(query string, args []driver.NamedValue) bool {
	if fr.Once && fr.Triggered {
		return false
	}
	return fr.isQueryMatch(query) && fr.isArgsMatch(args)
}

//MarkAsTriggered Mark Response as executed. For one time catches it will not make this possible to execute anymore
func (fr *FakeResponse) MarkAsTriggered() {
	fr.Triggered = true
}

//WithQuery Add SQL query pattern to match for
func (fr *FakeResponse) WithQuery(query string) *FakeResponse {
	fr.Pattern = query
	return fr
}

// WithArgs attach Args check for prepared statements
func (fr *FakeResponse) WithArgs(vars ...interface{}) *FakeResponse {
	if len(vars) > 0 {
		fr.Args = make([]interface{}, len(vars))
		for index, v := range vars {
			fr.Args[index] = v
		}
	}
	return fr
}

// WithReply adds to chain and assign some parts of response
func (fr *FakeResponse) WithReply(response []map[string]interface{}) *FakeResponse {
	fr.Response = response
	return fr
}

// OneTime set current mock to be triggered only once
func (fr *FakeResponse) OneTime() *FakeResponse {
	fr.Once = true
	return fr
}

// WithExecException says that if mock attached to non-SELECT query we need to trigger error there
func (fr *FakeResponse) WithExecException() *FakeResponse {
	fr.Exceptions.HookExecBadConnection = func() bool {
		return true
	}
	return fr
}

// WithQueryException add to SELECT mocks triggering of error
func (fr *FakeResponse) WithQueryException() *FakeResponse {
	fr.Exceptions.HookQueryBadConnection = func() bool {
		return true
	}
	return fr
}

// WithCallback adds callback to be executed during matching
func (fr *FakeResponse) WithCallback(f func(string, []driver.NamedValue)) *FakeResponse {
	fr.Callback = f
	return fr
}

// WithRowsNum specify how many records to consider as affected
func (fr *FakeResponse) WithRowsNum(num int64) *FakeResponse {
	fr.RowsAffected = num
	return fr
}

// WithId set ID to be considered as insert ID for INSERT statements
func (fr *FakeResponse) WithId(id int64) *FakeResponse {
	fr.LastInsertId = id
	return fr
}

func init() {
	Catcher = &MockCatcher{}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"

	"io/ioutil"

	"bufio"
	"os"

	"strings"

	"os/exec"

	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/config"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"gopkg.in/olivere/elastic.v2"
)

var mappings string = `
{
	"mappings": {
	     "files": {
	         "properties": {
	              "project_id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "project": {
	              		"type": "string",
	              		"index": "not_analyzed"
	              },
	              "datadir_id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "usesid": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              }
	         }
	     },
	     "projects": {
	     	"properties": {
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             },
	             "datadir": {
	             	"type": "string",
	             	"index": "not_analyzed"
	             }
	     	}
	     },
	     "samples": {
	     	"properties":{
	     	     "id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             }
	        }
	     },
	     "processes": {
	     	"properties":{
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             }
	        }
	     },
		"users": {
			"properties":{
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             }
	        }
	    }
	}
}
`

var tikableMediaTypes map[string]bool = map[string]bool{
	"application/msword":                                                        true,
	"application/pdf":                                                           true,
	"application/rtf":                                                           true,
	"application/vnd.ms-excel":                                                  true,
	"application/vnd.ms-office":                                                 true,
	"application/vnd.ms-powerpoint":                                             true,
	"application/vnd.ms-powerpoint.presentation.macroEnabled.12":                true,
	"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":         true,
	"application/vnd.openxmlformats-officedocument.wordprocessingml.document":   true,
	"application/vnd.sealedmedia.softseal.pdf":                                  true,
	"text/plain; charset=utf-8":                                                 true,
}

//var onlyHeader map[string]bool = map[string]bool{
//	"application/vnd.ms-excel":                                          true,
//	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
//}

func main() {
	esurl := esURL()
	fmt.Println("Connecting to url:", esurl)
	client, err := elastic.NewClient(elastic.SetURL(esurl))
	if err != nil {
		panic("Unable to connect to elasticsearch")
	}

	session := db.RSessionMust()

	createIndex(client)
	loadFiles(client, session)
	loadUsers(client, session)
	loadProjects(client, session)

}

func esURL() string {
	if esURL := config.GetString("MC_ES_URL"); esURL != "" {
		return esURL
	}
	return "http://localhost:9200"
}

func createIndex(client *elastic.Client) {
	exists, err := client.IndexExists("mc").Do()
	if err != nil {
		panic("Failed checking index existence")
	}

	if exists {
		client.DeleteIndex("mc").Do()
	}

	createStatus, err := client.CreateIndex("mc").Body(mappings).Do()
	if err != nil {
		fmt.Println("Failed creating index: ", err)
		os.Exit(1)
	}
	if !createStatus.Acknowledged {
		fmt.Println("Index create not acknowledged")
	}
}

func loadFiles(client *elastic.Client, session *r.Session) {
	renameDirPath := func(row r.Term) interface{} {
		return row.Merge(map[string]interface{}{
			"right": map[string]interface{}{
				"path": row.Field("right").Field("name"),
			},
		})
	}

	var _ = renameDirPath

	res, err := r.Table("projects").Pluck("id").
		EqJoin("id", r.Table("project2datafile"), r.EqJoinOpts{Index: "project_id"}).Zip().
		EqJoin("datafile_id", r.Table("datadir2datafile"), r.EqJoinOpts{Index: "datafile_id"}).Zip().
		EqJoin("datadir_id", r.Table("datadirs")).
		Map(renameDirPath).
		Zip().
		EqJoin("datafile_id", r.Table("datafiles")).Zip().
		Run(session)
	if err != nil {
		panic(fmt.Sprintf("Unable to query database for files: %s", err))
	}
	defer res.Close()

	var df schema.File
	count := 0
	maxCount := 10
	bulkReq := client.Bulk()
	lastProject := ""
	for res.Next(&df) {
		readContents(&df)
		if df.ProjectID != lastProject {
			fmt.Println("Indexing files for project:", df.ProjectID)
			lastProject = df.ProjectID
		}
		if count < maxCount {
			indexReq := elastic.NewBulkIndexRequest().Index("mc").Type("files").Id(df.ID).Doc(df)
			bulkReq = bulkReq.Add(indexReq)
			count++
		} else {
			count = 0
			resp, err := bulkReq.Do()
			if err != nil {
				fmt.Printf("bulkreq failed: %s\n", err)
				fmt.Printf("%#v\n", resp)
				return
			}
		}
	}

	if count != 0 {
		bulkReq.Do()
	}
}

func loadUsers(client *elastic.Client, session *r.Session) {
	res, err := r.Table("users").Run(session)
	if err != nil {
		panic(fmt.Sprintf("Unable to query database for users: %s", err))
	}
	defer res.Close()

	var u schema.User
	count := 0
	maxCount := 1000
	bulkReq := client.Bulk()
	for res.Next(&u) {
		if count < maxCount {
			indexReq := elastic.NewBulkIndexRequest().Index("mc").Type("users").Id(u.ID).Doc(u)
			bulkReq = bulkReq.Add(indexReq)
			count++
		} else {
			count = 0
			resp, err := bulkReq.Do()
			if err != nil {
				fmt.Printf("bulkreq failed: %s\n", err)
				fmt.Printf("%#v\n", resp)
				return
			}
		}
	}

	if count != 0 {
		bulkReq.Do()
	}
}

func loadProjects(client *elastic.Client, session *r.Session) {
	res, err := r.Table("projects").Run(session)
	if err != nil {
		panic(fmt.Sprintf("Unable to query database for projects: %s", err))
	}
	defer res.Close()

	var p schema.Project
	count := 0
	maxCount := 100
	bulkReq := client.Bulk()
	for res.Next(&p) {
		if count < maxCount {
			indexReq := elastic.NewBulkIndexRequest().Index("mc").Type("projects").Id(p.ID).Doc(p)
			bulkReq = bulkReq.Add(indexReq)
			count++
		} else {
			count = 0
			resp, err := bulkReq.Do()
			if err != nil {
				fmt.Printf("bulkreq failed: %s\n", err)
				fmt.Printf("%#v\n", resp)
				return
			}
		}
	}

	if count != 0 {
		bulkReq.Do()
	}
}

const twoMeg = 2 * 1024 * 1024

func readContents(file *schema.File) {
	switch file.MediaType.Mime {
	case "text/csv":
		//fmt.Println("Reading csv file: ", file.ID, file.Name, file.Size)
		if contents, err := readCSVLines(file.ID); err == nil {
			file.Contents = string(contents)
		}
	case "text/plain":
		if file.Size > twoMeg {
			return
		}
		//fmt.Println("Reading text file: ", file.ID, file.Name, file.Size)
		if contents, err := ioutil.ReadFile(app.MCDir.FilePath(file.ID)); err == nil {
			file.Contents = string(contents)
		}
	default:
		if _, ok := tikableMediaTypes[file.MediaType.Mime]; ok {
			if contents := extractUsingTika(file); contents != "" {
				file.Contents = contents
			}
		}
	}
}

func readCSVLines(fileID string) (string, error) {
	if file, err := os.Open(app.MCDir.FilePath(fileID)); err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			text := scanner.Text()
			if text != "" && !strings.HasPrefix(text, "#") {
				return text, nil
			}
		}
		return "", errors.New("No data")
	} else {
		return "", err
	}
}

func extractUsingTika(file *schema.File) string {
	if file.Size > twoMeg {
		return ""
	}

	out, err := exec.Command("tika.sh", "--text", app.MCDir.FilePath(file.ID)).Output()
	if err != nil {
		fmt.Println("Tika failed for:", file.Name, file.ID, file.MediaType.Mime)
		fmt.Println("exec failed:", err)
		return ""
	}

	return string(out)
}
<commit_msg>Remove redundant code for indexing by consolidating into a single method and using reflection to get the type. Add samples index, and add tags to files.<commit_after>package main

import (
	"errors"
	"fmt"

	"io/ioutil"

	"bufio"
	"os"

	"strings"

	"os/exec"

	"time"

	"reflect"

	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/config"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/schema"
	"gopkg.in/olivere/elastic.v2"
)

var mappings string = `
{
	"mappings": {
	     "files": {
	         "properties": {
	              "project_id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "project": {
	              		"type": "string",
	              		"index": "not_analyzed"
	              },
	              "datadir_id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "id": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              },
	              "usesid": {
	                  "type": "string",
	                  "index": "not_analyzed"
	              }
	         }
	     },
	     "projects": {
	     	"properties": {
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             },
	             "datadir": {
	             	"type": "string",
	             	"index": "not_analyzed"
	             }
	     	}
	     },
	     "samples": {
	     	"properties":{
	     	    "id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	            },
	        	"project_id": {
	                "type": "string",
	                "index": "not_analyzed"
	        	},
	        	"sample_id": {
	        		"type": "string",
	        		"index": "not_analyzed"
	        	}
	        }
	     },
	     "processes": {
	     	"properties":{
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             }
	        }
	     },
		"users": {
			"properties":{
	     		"id": {
	            	"type": "string",
	             	"index": "not_analyzed"
	             }
	        }
	    }
	}
}
`

var tikableMediaTypes map[string]bool = map[string]bool{
	"application/msword":                                                        true,
	"application/pdf":                                                           true,
	"application/rtf":                                                           true,
	"application/vnd.ms-excel":                                                  true,
	"application/vnd.ms-office":                                                 true,
	"application/vnd.ms-powerpoint":                                             true,
	"application/vnd.ms-powerpoint.presentation.macroEnabled.12":                true,
	"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":         true,
	"application/vnd.openxmlformats-officedocument.wordprocessingml.document":   true,
	"application/vnd.sealedmedia.softseal.pdf":                                  true,
	"text/plain; charset=utf-8":                                                 true,
}

//var onlyHeader map[string]bool = map[string]bool{
//	"application/vnd.ms-excel":                                          true,
//	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
//}

func main() {
	esurl := esURL()
	fmt.Println("Elasticsearch URL:", esurl)
	client, err := elastic.NewClient(elastic.SetURL(esurl))
	if err != nil {
		panic("Unable to connect to elasticsearch")
	}

	session := db.RSessionMust()

	createIndex(client)
	loadFiles(client, session)
	loadUsers(client, session)
	loadProjects(client, session)
	loadSamples(client, session)
}

func esURL() string {
	if esURL := config.GetString("MC_ES_URL"); esURL != "" {
		return esURL
	}
	return "http://localhost:9200"
}

func createIndex(client *elastic.Client) {
	exists, err := client.IndexExists("mc").Do()
	if err != nil {
		panic("Failed checking index existence")
	}

	if exists {
		client.DeleteIndex("mc").Do()
	}

	createStatus, err := client.CreateIndex("mc").Body(mappings).Do()
	if err != nil {
		fmt.Println("Failed creating index: ", err)
		os.Exit(1)
	}
	if !createStatus.Acknowledged {
		fmt.Println("Index create not acknowledged")
	}
}

type TagID struct {
	TagID string `gorethink:"tag_id" json:"tag"`
}

type File struct {
	schema.File
	Tags      []TagID `gorethink:"tags" json:"tags"`
	DataDirID string  `gorethink:"datadir_id" json:"datadir_id"` // Directory file is located in
	ProjectID string  `gorethink:"project_id" json:"project_id"` // Project file is in
	Contents  string  `gorethink:"-" json:"contents"`            // Contents of the file (text only)
}

func loadFiles(client *elastic.Client, session *r.Session) {
	var df File
	renameDirPath := func(row r.Term) interface{} {
		return row.Merge(map[string]interface{}{
			"right": map[string]interface{}{
				"path": row.Field("right").Field("name"),
			},
		})
	}

	fileTags := func(row r.Term) interface{} {
		return map[string]interface{}{
			"tags": r.Table("tag2item").GetAllByIndex("item_id", row.Field("id")).
				Pluck("tag_id").CoerceTo("ARRAY"),
		}
	}

	rql := r.Table("projects").Pluck("id").
		EqJoin("id", r.Table("project2datafile"), r.EqJoinOpts{Index: "project_id"}).Zip().
		EqJoin("datafile_id", r.Table("datadir2datafile"), r.EqJoinOpts{Index: "datafile_id"}).Zip().
		EqJoin("datadir_id", r.Table("datadirs")).
		Map(renameDirPath).
		Zip().
		EqJoin("datafile_id", r.Table("datafiles")).Zip().
		Merge(fileTags)

	fileIndexer := &indexer{
		rql: rql,
		getID: func(item interface{}) string {
			dfile := item.(*File)
			return dfile.ID
		},
		apply: func(item interface{}) {
			dfile := item.(*File)
			readContents(dfile)
		},
		client:   client,
		session:  session,
		maxCount: 10,
	}
	fmt.Println("Indexing files...")
	fileIndexer.Do("files", df)
	fmt.Println("Done.")
}

func loadUsers(client *elastic.Client, session *r.Session) {
	var u schema.User
	rql := r.Table("users")

	userIndexer := &indexer{
		rql: rql,
		getID: func(item interface{}) string {
			user := item.(*schema.User)
			return user.ID
		},
		client:   client,
		session:  session,
		maxCount: 1000,
	}

	fmt.Println("Indexing users...")
	userIndexer.Do("users", u)
	fmt.Println("Done.")
}

func loadProjects(client *elastic.Client, session *r.Session) {
	var p schema.Project
	rql := r.Table("projects")
	projectIndexer := &indexer{
		rql: rql,
		getID: func(item interface{}) string {
			project := item.(*schema.Project)
			return project.ID
		},
		client:   client,
		session:  session,
		maxCount: 1000,
	}

	fmt.Println("Indexing projects...")
	projectIndexer.Do("projects", p)
	fmt.Println("Done.")
}

type Property struct {
	Attribute string `gorethink:"attribute" json:"attribute"`
	Name      string `gorethink:"name" json:"name"`
}

type Sample struct {
	ID          string     `gorethink:"id" json:"id"`
	Description string     `gorethink:"description" json:"description"`
	Birthtime   time.Time  `gorethink:"birthtime" json:"birthtime"`
	MTime       time.Time  `gorethink:"mtime" json:"mtime"`
	Owner       string     `gorethink:"owner" json:"owner"`
	Name        string     `gorethink:"name" json:"name"`
	ProjectID   string     `gorethink:"project_id" json:"project_id"`
	SampleID    string     `gorethink:"sample_id" json:"sample_id"`
	Properties  []Property `gorethink:"properties" json:"properties"`
}

func loadSamples(client *elastic.Client, session *r.Session) {
	var sample Sample
	getProperties := func(row r.Term) interface{} {
		return map[string]interface{}{
			"properties": r.Table("sample2propertyset").
				GetAllByIndex("sample_id", row.Field("sample_id")).
				EqJoin("property_set_id", r.Table("propertyset2property"), r.EqJoinOpts{Index: "property_set_id"}).
				Zip().
				EqJoin("property_id", r.Table("properties")).Zip().Pluck("attribute", "name").
				CoerceTo("ARRAY"),
		}
	}
	rql := r.Table("projects").Pluck("id").
		EqJoin("id", r.Table("project2sample"), r.EqJoinOpts{Index: "project_id"}).Zip().
		EqJoin("sample_id", r.Table("samples")).Zip().
		Merge(getProperties)

	sampleIndexer := &indexer{
		rql: rql,
		getID: func(item interface{}) string {
			s := item.(*Sample)
			return s.SampleID
		},
		client:   client,
		session:  session,
		maxCount: 1000,
	}

	fmt.Println("Indexing samples...")
	sampleIndexer.Do("samples", sample)
	fmt.Println("Done.")
}

type indexer struct {
	rql      r.Term
	getID    func(item interface{}) string
	apply    func(item interface{})
	client   *elastic.Client
	session  *r.Session
	maxCount int
}

func (i *indexer) Do(itype string, what interface{}) {
	res, err := i.rql.Run(i.session)
	if err != nil {
		fmt.Println("Failed to run query:", err)
		os.Exit(1)
	}
	defer res.Close()

	total := 0
	count := 0
	bulkReq := i.client.Bulk()
	elementType := reflect.TypeOf(what)
	result := reflect.New(elementType)
	for res.Next(result.Interface()) {
		if i.apply != nil {
			i.apply(result.Interface())
		}

		if count < i.maxCount {
			id := i.getID(result.Interface())
			indexReq := elastic.NewBulkIndexRequest().Index("mc").Type(itype).Id(id).Doc(result.Interface())
			bulkReq = bulkReq.Add(indexReq)
			count++
			total++
		} else {
			fmt.Printf("  Indexed %d...\n", total)
			count = 0
			resp, err := bulkReq.Do()
			if err != nil {
				fmt.Printf("bulkreq failed: %s\n", err)
				fmt.Printf("%#v\n", resp)
				return
			}
		}
		result = reflect.New(elementType)
	}

	if res.Err() != nil {
		fmt.Println("res err", err)
	}

	if count != 0 {
		fmt.Printf("  Indexed %d %s...\n", total, itype)
		bulkReq.Do()
	}
}

const twoMeg = 2 * 1024 * 1024

func readContents(file *File) {
	switch file.MediaType.Mime {
	case "text/csv":
		//fmt.Println("Reading csv file: ", file.ID, file.Name, file.Size)
		if contents, err := readCSVLines(file.ID); err == nil {
			file.Contents = string(contents)
		}
	case "text/plain":
		if file.Size > twoMeg {
			return
		}
		//fmt.Println("Reading text file: ", file.ID, file.Name, file.Size)
		if contents, err := ioutil.ReadFile(app.MCDir.FilePath(file.ID)); err == nil {
			file.Contents = string(contents)
		}
	default:
		if _, ok := tikableMediaTypes[file.MediaType.Mime]; ok {
			if contents := extractUsingTika(file); contents != "" {
				file.Contents = contents
			}
		}
	}
}

func readCSVLines(fileID string) (string, error) {
	if file, err := os.Open(app.MCDir.FilePath(fileID)); err == nil {
		defer file.Close()

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			text := scanner.Text()
			if text != "" && !strings.HasPrefix(text, "#") {
				return text, nil
			}
		}
		return "", errors.New("No data")
	} else {
		return "", err
	}
}

func extractUsingTika(file *File) string {
	if file.Size > twoMeg {
		return ""
	}

	out, err := exec.Command("tika.sh", "--text", app.MCDir.FilePath(file.ID)).Output()
	if err != nil {
		fmt.Println("Tika failed for:", file.Name, file.ID, file.MediaType.Mime)
		fmt.Println("exec failed:", err)
		return ""
	}

	return string(out)
}
<|endoftext|>