<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"errors"
	"strconv"
	"strings"

	"github.com/dhowden/numerus"
)

// parseFn is a type which represents a parse function, given a string it will return
// a uint, or non-nil error if the string could not be parsed.
type parseFn func(string) (uint, error)

// parseUInt attempts to parse the given string into a UInt, returns an error if the input
// is invalid.
func parseUInt(s string) (uint, error) {
	n, err := strconv.Atoi(s)
	return uint(n), err
}

// parseNumeral is a wrapper around numerus.Parse, returning an error if the input string
// is invalid (when empty, or if an error is returned from numerus.Parse).
func parseNumeral(x string) (uint, error) {
	if x == "" {
		// numerus will return 0 (non-error) here, but we want an error
		return 0, errors.New("invalid numeral: empty string")
	}
	n, err := numerus.Parse(strings.ToUpper(x))
	return uint(n), err
}

// parser is a pairing of parse functions with "style" description.
type parser struct {
	Fn    parseFn
	Style string
}

// parsers is an internal list of numerical representations that can be identified by this
// package.
var parsers = []parser{
	{
		Fn:    parseUInt,
		Style: "decimal",
	},
	{
		Fn:    parseNumeral,
		Style: "upper-roman",
	},
}

type isNext struct {
	fn parseFn
	n  uint
}

func (i *isNext) IsNext(s string) bool {
	x, err := i.fn(s)
	if err != nil {
		return false
	}
	if x == (i.n + 1) {
		i.n++
		return true
	}
	return false
}

// enumFieldSuffixes is a list of string suffixes which will be trimmed from "enumeration
// fields"
var enumFieldSuffixes = []string{".", ":", "-"}

// trimFieldEnumSuffix removes all enumSuffixes from the given string and returns the
// result along with the number of characters removed.
func trimEnumFieldSuffixes(s string) string {
	for {
		var trim bool
		for _, x := range enumFieldSuffixes {
			if strings.HasSuffix(s, x) {
				s = strings.TrimSuffix(s, x)
				trim = true
				break
			}
		}
		if !trim {
			return s
		}
	}
}

// enumWordSuffixes is a list of strings which should be removed from string resulting
// from the prefix removal.
var enumWordSuffixes = []string{"-"}

// prefixes of an enumeration element which might need to be removed.
var enumWordPrefixes = []string{"No."}

// trimPrefix removes at most one prefix from prefixes from the given string, returns the
// the result.
func trimPrefix(s string, prefixes []string) (string, int) {
	l := len(s)
	for _, x := range prefixes {
		if strings.HasPrefix(strings.ToLower(s), strings.ToLower(x)) {
			s = strings.TrimSpace(s[len(x):])
			return s, l - len(s)
		}
	}
	return s, 0
}

// enumPrefix returns the expected enumeration prefix for the given string, and the
// number of characters that were removed from the beginning of the string to retrieve
// the value.  If the resulting prefix is the whole string then we don't do anything.
func enumPrefix(s string) (string, int) {
	x := s
	x, r := trimPrefix(x, enumWordPrefixes)
	words := strings.SplitN(x, " ", 2)
	if len(words) == 2 {
		_, n := trimPrefix(words[1], enumWordSuffixes)
		r += n
	}
	w := words[0]
	r += len(w)

	w = trimEnumFieldSuffixes(w)
	if r == len(s) {
		return s, 0
	}
	return w, r
}

func trimEnumPrefix(field string, tracks []Track) ([]Track, string) {
	if len(tracks) == 0 {
		return tracks, ""
	}

	t0 := tracks[0]
	name := t0.GetString(field)
	ep, epl := enumPrefix(name)

	var p parser
	var n uint
	for _, x := range parsers {
		var err error
		n, err = x.Fn(ep)
		if err == nil {
			p = x
			break
		}
	}

	if p.Fn == nil {
		return tracks, ""
	}

	nexter := &isNext{
		fn: p.Fn,
		n:  n,
	}

	epls := make([]int, len(tracks))
	epls[0] = epl
	i := 1
	for i < len(tracks) {
		ep, epls[i] = enumPrefix(tracks[i].GetString(field))
		if !nexter.IsNext(ep) {
			break
		}
		i++
	}

	if i < len(tracks) {
		return tracks, ""
	}

	result := make([]Track, 0, len(tracks))
	for i, t := range tracks {
		result = append(result, pfxTrack{
			Track: t,
			field: field,
			pfx:   epls[i],
		})
	}
	return result, p.Style
}

// TrimEnumPrefix removes enumeratative prefixes from the Tracks in the Group. The
// resulting group will have a ListStyle field which will indicate the style of enumeration
// (if any).
func TrimEnumPrefix(g Group) Group {
	nt, style := trimEnumPrefix("Name", g.Tracks())
	return subGrpFlds{
		Group:  g,
		tracks: nt,
		flds: map[string]interface{}{
			"ListStyle": style,
		},
	}
}

// TransformFn is a type which represents a function which Transforms a Group into
// another.
type TransformFn func(Group) Group

// Transform applies the TransformFn to the Group and returns the result.
func Transform(g Group, fn TransformFn) Group { return fn(g) }

// SubTransform recursively applies the TransformFn to each Group in the Collection.
func SubTransform(c Collection, fn TransformFn) Collection {
	m := make(map[Key]Group, len(c.Keys()))
	col := subCol{c, m, make(map[string]interface{})}
	for _, k := range c.Keys() {
		g := c.Get(k)
		if cg, ok := g.(Collection); ok {
			m[k] = SubTransform(cg, fn)
			continue
		}
		m[k] = fn(g)
	}
	return col
}
<commit_msg>Small refactoring to enum routines.<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"errors"
	"strconv"
	"strings"

	"github.com/dhowden/numerus"
)

// parseFn is a type which represents a parse function, given a string it will return
// a uint, or non-nil error if the string could not be parsed.
type parseFn func(string) (uint, error)

// parseUInt attempts to parse the given string into a UInt, returns an error if the input
// is invalid.
func parseUInt(s string) (uint, error) {
	n, err := strconv.Atoi(s)
	return uint(n), err
}

// parseNumeral is a wrapper around numerus.Parse, returning an error if the input string
// is invalid (when empty, or if an error is returned from numerus.Parse).
func parseNumeral(x string) (uint, error) {
	if x == "" {
		// numerus will return 0 (non-error) here, but we want an error
		return 0, errors.New("invalid numeral: empty string")
	}
	n, err := numerus.Parse(strings.ToUpper(x))
	return uint(n), err
}

// parser is a pairing of parse functions with "style" description.
type parser struct {
	Fn    parseFn
	Style string
}

// parsers is an internal list of numerical representations that can be identified by this
// package.
var parsers = []parser{
	{
		Fn:    parseUInt,
		Style: "decimal",
	},
	{
		Fn:    parseNumeral,
		Style: "upper-roman",
	},
}

type isNext struct {
	fn parseFn
	n  uint
}

func (i *isNext) IsNext(s string) bool {
	x, err := i.fn(s)
	if err != nil {
		return false
	}
	if x == (i.n + 1) {
		i.n++
		return true
	}
	return false
}

// enumFieldSuffixes is a list of string suffixes which will be trimmed from "enumeration
// fields"
var enumFieldSuffixes = []string{".", ":", "-"}

// trimFieldEnumSuffix removes all enumSuffixes from the given string and returns the result.
func trimEnumFieldSuffixes(s string) string {
	for {
		done := true
		for _, x := range enumFieldSuffixes {
			if strings.HasSuffix(s, x) {
				s = s[:len(s)-len(x)]
				done = false
				break
			}
		}
		if done {
			return s
		}
	}
}

// enumWordSuffixes is a list of strings which should be removed from string resulting
// from the prefix removal.
var enumWordSuffixes = []string{"-"}

// prefixes of an enumeration element which might need to be removed.
var enumWordPrefixes = []string{"No."}

// trimPrefix removes at most one prefix from prefixes from the given string, returns the
// the result.
func trimPrefix(s string, prefixes []string) (string, int) {
	n := len(s)
	for _, x := range prefixes {
		if strings.HasPrefix(strings.ToLower(s), strings.ToLower(x)) {
			s = strings.TrimSpace(s[len(x):])
			return s, n - len(s)
		}
	}
	return s, 0
}

// enumPrefix returns the expected enumeration prefix for the given string, and the
// number of characters that were removed from the beginning of the string to retrieve
// the value.  If the resulting prefix is the whole string then we don't do anything.
func enumPrefix(s string) (string, int) {
	x := s
	x, r := trimPrefix(x, enumWordPrefixes)
	words := strings.SplitN(x, " ", 2)
	if len(words) == 2 {
		_, n := trimPrefix(words[1], enumWordSuffixes)
		r += n
	}
	w := words[0]
	r += len(w)

	w = trimEnumFieldSuffixes(w)
	if r == len(s) {
		return s, 0
	}
	return w, r
}

func trimEnumPrefix(field string, tracks []Track) ([]Track, string) {
	if len(tracks) == 0 {
		return tracks, ""
	}

	t0 := tracks[0]
	name := t0.GetString(field)
	ep, epl := enumPrefix(name)

	var p parser
	var n uint
	for _, x := range parsers {
		var err error
		n, err = x.Fn(ep)
		if err == nil {
			p = x
			break
		}
	}

	if p.Fn == nil {
		return tracks, ""
	}

	nexter := &isNext{
		fn: p.Fn,
		n:  n,
	}

	epls := make([]int, len(tracks))
	epls[0] = epl
	i := 1
	for i < len(tracks) {
		ep, epls[i] = enumPrefix(tracks[i].GetString(field))
		if !nexter.IsNext(ep) {
			break
		}
		i++
	}

	if i < len(tracks) {
		return tracks, ""
	}

	result := make([]Track, 0, len(tracks))
	for i, t := range tracks {
		result = append(result, pfxTrack{
			Track: t,
			field: field,
			pfx:   epls[i],
		})
	}
	return result, p.Style
}

// TrimEnumPrefix removes enumeratative prefixes from the Tracks in the Group. The
// resulting group will have a ListStyle field which will indicate the style of enumeration
// (if any).
func TrimEnumPrefix(g Group) Group {
	nt, style := trimEnumPrefix("Name", g.Tracks())
	return subGrpFlds{
		Group:  g,
		tracks: nt,
		flds: map[string]interface{}{
			"ListStyle": style,
		},
	}
}

// TransformFn is a type which represents a function which Transforms a Group into
// another.
type TransformFn func(Group) Group

// Transform applies the TransformFn to the Group and returns the result.
func Transform(g Group, fn TransformFn) Group { return fn(g) }

// SubTransform recursively applies the TransformFn to each Group in the Collection.
func SubTransform(c Collection, fn TransformFn) Collection {
	m := make(map[Key]Group, len(c.Keys()))
	col := subCol{c, m, make(map[string]interface{})}
	for _, k := range c.Keys() {
		g := c.Get(k)
		if cg, ok := g.(Collection); ok {
			m[k] = SubTransform(cg, fn)
			continue
		}
		m[k] = fn(g)
	}
	return col
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package bleve

import (
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/store"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/blevesearch/bleve/registry"
	"github.com/blevesearch/bleve/search"
)

type indexImpl struct {
	path  string
	meta  *indexMeta
	s     store.KVStore
	i     index.Index
	m     *IndexMapping
	mutex sync.RWMutex
	open  bool
}

const storePath = "store"

var mappingInternalKey = []byte("_mapping")

func indexStorePath(path string) string {
	return path + string(os.PathSeparator) + storePath
}

func newMemIndex(mapping *IndexMapping) (*indexImpl, error) {
	rv := indexImpl{
		path: "",
		m:    mapping,
		meta: newIndexMeta("mem"),
	}

	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}
	// now open the store
	var err error
	rv.s, err = storeConstructor(nil)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now persist the mapping
	mappingBytes, err := json.Marshal(mapping)
	if err != nil {
		return nil, err
	}
	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true
	return &rv, nil
}

func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) {
	// first validate the mapping
	err := mapping.validate()
	if err != nil {
		return nil, err
	}

	if path == "" {
		return newMemIndex(mapping)
	}

	rv := indexImpl{
		path: path,
		m:    mapping,
		meta: newIndexMeta(Config.DefaultKVStore),
	}
	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}
	// at this point there hope we can be successful, so save index meta
	err = rv.meta.Save(path)
	if err != nil {
		return nil, err
	}
	storeConfig := map[string]interface{}{
		"path":              indexStorePath(path),
		"create_if_missing": true,
		"error_if_exists":   true,
	}

	// now open the store
	rv.s, err = storeConstructor(storeConfig)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now persist the mapping
	mappingBytes, err := json.Marshal(mapping)
	if err != nil {
		return nil, err
	}
	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true
	return &rv, nil
}

func openIndex(path string) (*indexImpl, error) {

	rv := indexImpl{
		path: path,
	}
	var err error
	rv.meta, err = openIndexMeta(path)
	if err != nil {
		return nil, err
	}

	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}

	storeConfig := map[string]interface{}{
		"path":              indexStorePath(path),
		"create_if_missing": false,
		"error_if_exists":   false,
	}

	// now open the store
	rv.s, err = storeConstructor(storeConfig)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now load the mapping
	mappingBytes, err := rv.i.GetInternal(mappingInternalKey)
	if err != nil {
		return nil, err
	}

	var im IndexMapping
	err = json.Unmarshal(mappingBytes, &im)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true

	// validate the mapping
	err = im.validate()
	if err != nil {
		// note even if the mapping is invalid
		// we still return an open usable index
		return &rv, err
	}

	rv.m = &im
	return &rv, nil
}

func (i *indexImpl) Mapping() *IndexMapping {
	return i.m
}

func (i *indexImpl) Index(id string, data interface{}) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	doc := document.NewDocument(id)
	err := i.m.mapDocument(doc, data)
	if err != nil {
		return err
	}
	err = i.i.Update(doc)
	if err != nil {
		return err
	}
	return nil
}

func (i *indexImpl) Delete(id string) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	err := i.i.Delete(id)
	if err != nil {
		return err
	}
	return nil
}

func (i *indexImpl) Batch(b Batch) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	ib := make(index.Batch, len(b))
	for bk, bd := range b {
		if bd == nil {
			ib.Delete(bk)
		} else {
			doc := document.NewDocument(bk)
			err := i.m.mapDocument(doc, bd)
			if err != nil {
				return err
			}
			ib.Index(bk, doc)
		}
	}
	return i.i.Batch(ib)
}

func (i *indexImpl) Document(id string) (*document.Document, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}
	return i.i.Document(id)
}

func (i *indexImpl) DocCount() uint64 {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return 0
	}

	return i.i.DocCount()
}

func (i *indexImpl) Search(req *SearchRequest) (*SearchResult, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}

	collector := search.NewTopScorerSkipCollector(req.Size, req.From)
	searcher, err := req.Query.Searcher(i, req.Explain)
	if err != nil {
		return nil, err
	}
	defer searcher.Close()

	if req.Facets != nil {
		facetsBuilder := search.NewFacetsBuilder(i.i)
		for facetName, facetRequest := range req.Facets {
			if facetRequest.NumericRanges != nil {
				// build numeric range facet
				facetBuilder := search.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)
				for _, nr := range facetRequest.NumericRanges {
					facetBuilder.AddRange(nr.Name, nr.Min, nr.Max)
				}
				facetsBuilder.Add(facetName, facetBuilder)
			} else if facetRequest.DateTimeRanges != nil {
				// build date range facet
				facetBuilder := search.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
				dateTimeParser := i.m.dateTimeParserNamed(i.m.DefaultDateTimeParser)
				for _, dr := range facetRequest.DateTimeRanges {
					dr.ParseDates(dateTimeParser)
					facetBuilder.AddRange(dr.Name, dr.Start, dr.End)
				}
				facetsBuilder.Add(facetName, facetBuilder)
			} else {
				// build terms facet
				facetBuilder := search.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
				facetsBuilder.Add(facetName, facetBuilder)
			}
		}
		collector.SetFacetsBuilder(facetsBuilder)
	}

	err = collector.Collect(searcher)
	if err != nil {
		return nil, err
	}

	hits := collector.Results()

	if req.Highlight != nil {
		// get the right highlighter
		highlighter := Config.Highlight.Highlighters[*Config.DefaultHighlighter]
		if req.Highlight.Style != nil {
			highlighter = Config.Highlight.Highlighters[*req.Highlight.Style]
			if highlighter == nil {
				return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
			}
		}

		for _, hit := range hits {
			doc, err := i.Document(hit.ID)
			if err == nil {
				highlightFields := req.Highlight.Fields
				if highlightFields == nil {
					// add all fields with matches
					highlightFields = make([]string, 0, len(hit.Locations))
					for k, _ := range hit.Locations {
						highlightFields = append(highlightFields, k)
					}
				}

				for _, hf := range highlightFields {
					highlighter.BestFragmentsInField(hit, doc, hf, 1)
				}
			}
		}
	}

	if len(req.Fields) > 0 {
		for _, hit := range hits {
			// FIXME avoid loading doc second time
			// if we already loaded it for highlighting
			doc, err := i.Document(hit.ID)
			if err == nil {
				for _, f := range req.Fields {
					for _, docF := range doc.Fields {
						if docF.Name() == f {
							var value interface{}
							switch docF := docF.(type) {
							case *document.TextField:
								value = string(docF.Value())
							case *document.NumericField:
								num, err := docF.Number()
								if err == nil {
									value = num
								}
							case *document.DateTimeField:
								datetime, err := docF.DateTime()
								if err == nil {
									value = datetime.Format(time.RFC3339)
								}
							}
							if value != nil {
								hit.AddFieldValue(f, value)
							}
						}
					}
				}
			}
		}
	}

	return &SearchResult{
		Request:  req,
		Hits:     hits,
		Total:    collector.Total(),
		MaxScore: collector.MaxScore(),
		Took:     collector.Took(),
		Facets:   collector.FacetResults(),
	}, nil
}

func (i *indexImpl) DumpAll() chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}

	return i.i.DumpAll()
}

func (i *indexImpl) Fields() ([]string, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}
	return i.i.Fields()
}

func (i *indexImpl) DumpFields() chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}
	return i.i.DumpFields()
}

func (i *indexImpl) DumpDoc(id string) chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}
	return i.i.DumpDoc(id)
}

func (i *indexImpl) Close() {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	i.open = false
	i.i.Close()
}
<commit_msg>fix deadlock<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package bleve

import (
	"encoding/json"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/blevesearch/bleve/document"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/store"
	"github.com/blevesearch/bleve/index/upside_down"
	"github.com/blevesearch/bleve/registry"
	"github.com/blevesearch/bleve/search"
)

type indexImpl struct {
	path  string
	meta  *indexMeta
	s     store.KVStore
	i     index.Index
	m     *IndexMapping
	mutex sync.RWMutex
	open  bool
}

const storePath = "store"

var mappingInternalKey = []byte("_mapping")

func indexStorePath(path string) string {
	return path + string(os.PathSeparator) + storePath
}

func newMemIndex(mapping *IndexMapping) (*indexImpl, error) {
	rv := indexImpl{
		path: "",
		m:    mapping,
		meta: newIndexMeta("mem"),
	}

	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}
	// now open the store
	var err error
	rv.s, err = storeConstructor(nil)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now persist the mapping
	mappingBytes, err := json.Marshal(mapping)
	if err != nil {
		return nil, err
	}
	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true
	return &rv, nil
}

func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) {
	// first validate the mapping
	err := mapping.validate()
	if err != nil {
		return nil, err
	}

	if path == "" {
		return newMemIndex(mapping)
	}

	rv := indexImpl{
		path: path,
		m:    mapping,
		meta: newIndexMeta(Config.DefaultKVStore),
	}
	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}
	// at this point there hope we can be successful, so save index meta
	err = rv.meta.Save(path)
	if err != nil {
		return nil, err
	}
	storeConfig := map[string]interface{}{
		"path":              indexStorePath(path),
		"create_if_missing": true,
		"error_if_exists":   true,
	}

	// now open the store
	rv.s, err = storeConstructor(storeConfig)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now persist the mapping
	mappingBytes, err := json.Marshal(mapping)
	if err != nil {
		return nil, err
	}
	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true
	return &rv, nil
}

func openIndex(path string) (*indexImpl, error) {

	rv := indexImpl{
		path: path,
	}
	var err error
	rv.meta, err = openIndexMeta(path)
	if err != nil {
		return nil, err
	}

	storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
	if storeConstructor == nil {
		return nil, ERROR_UNKNOWN_STORAGE_TYPE
	}

	storeConfig := map[string]interface{}{
		"path":              indexStorePath(path),
		"create_if_missing": false,
		"error_if_exists":   false,
	}

	// now open the store
	rv.s, err = storeConstructor(storeConfig)
	if err != nil {
		return nil, err
	}

	// open open the index
	rv.i = upside_down.NewUpsideDownCouch(rv.s)
	err = rv.i.Open()
	if err != nil {
		return nil, err
	}

	// now load the mapping
	mappingBytes, err := rv.i.GetInternal(mappingInternalKey)
	if err != nil {
		return nil, err
	}

	var im IndexMapping
	err = json.Unmarshal(mappingBytes, &im)
	if err != nil {
		return nil, err
	}

	// mark the index as open
	rv.mutex.Lock()
	defer rv.mutex.Unlock()
	rv.open = true

	// validate the mapping
	err = im.validate()
	if err != nil {
		// note even if the mapping is invalid
		// we still return an open usable index
		return &rv, err
	}

	rv.m = &im
	return &rv, nil
}

func (i *indexImpl) Mapping() *IndexMapping {
	return i.m
}

func (i *indexImpl) Index(id string, data interface{}) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	doc := document.NewDocument(id)
	err := i.m.mapDocument(doc, data)
	if err != nil {
		return err
	}
	err = i.i.Update(doc)
	if err != nil {
		return err
	}
	return nil
}

func (i *indexImpl) Delete(id string) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	err := i.i.Delete(id)
	if err != nil {
		return err
	}
	return nil
}

func (i *indexImpl) Batch(b Batch) error {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	if !i.open {
		return ERROR_INDEX_CLOSED
	}

	ib := make(index.Batch, len(b))
	for bk, bd := range b {
		if bd == nil {
			ib.Delete(bk)
		} else {
			doc := document.NewDocument(bk)
			err := i.m.mapDocument(doc, bd)
			if err != nil {
				return err
			}
			ib.Index(bk, doc)
		}
	}
	return i.i.Batch(ib)
}

func (i *indexImpl) Document(id string) (*document.Document, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}
	return i.i.Document(id)
}

func (i *indexImpl) DocCount() uint64 {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return 0
	}

	return i.i.DocCount()
}

func (i *indexImpl) Search(req *SearchRequest) (*SearchResult, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}

	collector := search.NewTopScorerSkipCollector(req.Size, req.From)
	searcher, err := req.Query.Searcher(i, req.Explain)
	if err != nil {
		return nil, err
	}
	defer searcher.Close()

	if req.Facets != nil {
		facetsBuilder := search.NewFacetsBuilder(i.i)
		for facetName, facetRequest := range req.Facets {
			if facetRequest.NumericRanges != nil {
				// build numeric range facet
				facetBuilder := search.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)
				for _, nr := range facetRequest.NumericRanges {
					facetBuilder.AddRange(nr.Name, nr.Min, nr.Max)
				}
				facetsBuilder.Add(facetName, facetBuilder)
			} else if facetRequest.DateTimeRanges != nil {
				// build date range facet
				facetBuilder := search.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
				dateTimeParser := i.m.dateTimeParserNamed(i.m.DefaultDateTimeParser)
				for _, dr := range facetRequest.DateTimeRanges {
					dr.ParseDates(dateTimeParser)
					facetBuilder.AddRange(dr.Name, dr.Start, dr.End)
				}
				facetsBuilder.Add(facetName, facetBuilder)
			} else {
				// build terms facet
				facetBuilder := search.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
				facetsBuilder.Add(facetName, facetBuilder)
			}
		}
		collector.SetFacetsBuilder(facetsBuilder)
	}

	err = collector.Collect(searcher)
	if err != nil {
		return nil, err
	}

	hits := collector.Results()

	if req.Highlight != nil {
		// get the right highlighter
		highlighter := Config.Highlight.Highlighters[*Config.DefaultHighlighter]
		if req.Highlight.Style != nil {
			highlighter = Config.Highlight.Highlighters[*req.Highlight.Style]
			if highlighter == nil {
				return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
			}
		}

		for _, hit := range hits {
			doc, err := i.i.Document(hit.ID)
			if err == nil {
				highlightFields := req.Highlight.Fields
				if highlightFields == nil {
					// add all fields with matches
					highlightFields = make([]string, 0, len(hit.Locations))
					for k, _ := range hit.Locations {
						highlightFields = append(highlightFields, k)
					}
				}

				for _, hf := range highlightFields {
					highlighter.BestFragmentsInField(hit, doc, hf, 1)
				}
			}
		}
	}

	if len(req.Fields) > 0 {
		for _, hit := range hits {
			// FIXME avoid loading doc second time
			// if we already loaded it for highlighting
			doc, err := i.i.Document(hit.ID)
			if err == nil {
				for _, f := range req.Fields {
					for _, docF := range doc.Fields {
						if docF.Name() == f {
							var value interface{}
							switch docF := docF.(type) {
							case *document.TextField:
								value = string(docF.Value())
							case *document.NumericField:
								num, err := docF.Number()
								if err == nil {
									value = num
								}
							case *document.DateTimeField:
								datetime, err := docF.DateTime()
								if err == nil {
									value = datetime.Format(time.RFC3339)
								}
							}
							if value != nil {
								hit.AddFieldValue(f, value)
							}
						}
					}
				}
			}
		}
	}

	return &SearchResult{
		Request:  req,
		Hits:     hits,
		Total:    collector.Total(),
		MaxScore: collector.MaxScore(),
		Took:     collector.Took(),
		Facets:   collector.FacetResults(),
	}, nil
}

func (i *indexImpl) DumpAll() chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}

	return i.i.DumpAll()
}

func (i *indexImpl) Fields() ([]string, error) {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil, ERROR_INDEX_CLOSED
	}
	return i.i.Fields()
}

func (i *indexImpl) DumpFields() chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}
	return i.i.DumpFields()
}

func (i *indexImpl) DumpDoc(id string) chan interface{} {
	i.mutex.RLock()
	defer i.mutex.RUnlock()

	if !i.open {
		return nil
	}
	return i.i.DumpDoc(id)
}

func (i *indexImpl) Close() {
	i.mutex.Lock()
	defer i.mutex.Unlock()

	i.open = false
	i.i.Close()
}
<|endoftext|>


<commit_before>package git

import (
	"os"
	"runtime"
	"testing"
	"io/ioutil"
)

func TestCreateRepoAndStage(t *testing.T) {
	// figure out where we can create the test repo
	path, err := ioutil.TempDir("", "git2go")
	checkFatal(t, err)
	repo, err := Init(path, false)
	checkFatal(t, err)

	tmpfile := "README"
	err = ioutil.WriteFile(path + "/" + tmpfile, []byte("foo\n"), 0644)
	checkFatal(t, err)
	defer os.RemoveAll(path)

	idx, err := repo.Index()
	checkFatal(t, err)
	err = idx.AddByPath(tmpfile)
	checkFatal(t, err)
	treeId, err := idx.WriteTree()
	checkFatal(t, err)

	if treeId.String() != "b7119b11e8ef7a1a5a34d3ac87f5b075228ac81e" {
		t.Fatalf("%v", treeId.String())
	}
}

func checkFatal(t *testing.T, err error) {
	if err == nil {
		return
	}

	// The failure happens at wherever we were called, not here
	_, file, line, ok := runtime.Caller(1)
	if !ok {
		t.Fatal()
	}

	t.Fatalf("Fail at %v:%v", file, line)
}
<commit_msg>Fixed renamed func in index_test<commit_after>package git

import (
	"os"
	"runtime"
	"testing"
	"io/ioutil"
)

func TestCreateRepoAndStage(t *testing.T) {
	// figure out where we can create the test repo
	path, err := ioutil.TempDir("", "git2go")
	checkFatal(t, err)
	repo, err := InitRepository(path, false)
	checkFatal(t, err)

	tmpfile := "README"
	err = ioutil.WriteFile(path + "/" + tmpfile, []byte("foo\n"), 0644)
	checkFatal(t, err)
	defer os.RemoveAll(path)

	idx, err := repo.Index()
	checkFatal(t, err)
	err = idx.AddByPath(tmpfile)
	checkFatal(t, err)
	treeId, err := idx.WriteTree()
	checkFatal(t, err)

	if treeId.String() != "b7119b11e8ef7a1a5a34d3ac87f5b075228ac81e" {
		t.Fatalf("%v", treeId.String())
	}
}

func checkFatal(t *testing.T, err error) {
	if err == nil {
		return
	}

	// The failure happens at wherever we were called, not here
	_, file, line, ok := runtime.Caller(1)
	if !ok {
		t.Fatal()
	}

	t.Fatalf("Fail at %v:%v", file, line)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// +build libstemmer full
// +build icu full

package bleve

import (
	"io/ioutil"
	"os"
	"testing"
	"time"
)

type Address struct {
	Street string `json:"street"`
	City   string `json:"city"`
	State  string `json:"state"`
	Zip    string `json:"zip"`
}

type Person struct {
	Identifier string     `json:"id"`
	Name       string     `json:"name"`
	Age        float64    `json:"age"`
	Title      string     `json:"title"`
	Birthday   time.Time  `json:"birthday"`
	Address    *Address   `json:"address"`
	Hideouts   []*Address `json:"hideouts"`
	Tags       []string   `json:"tags"`
}

func (p *Person) Type() string {
	return "person"
}

var nameMapping = NewDocumentMapping().
	AddFieldMapping(NewFieldMapping("", "text", "en", true, true, true, true))

var tagsMapping = NewDocumentMapping().
	AddFieldMapping(NewFieldMapping("", "text", "standard", true, true, true, false))
var personMapping = NewDocumentMapping().
	AddSubDocumentMapping("name", nameMapping).
	AddSubDocumentMapping("id", NewDocumentDisabledMapping()).
	AddSubDocumentMapping("tags", tagsMapping)

var mapping = NewIndexMapping().
	AddDocumentMapping("person", personMapping)

var people = []*Person{
	&Person{
		Identifier: "a",
		Name:       "marty",
		Age:        19,
		// has no birthday set to test handling of zero time
		Title: "mista",
		Tags:  []string{"gopher", "belieber"},
	},
	&Person{
		Identifier: "b",
		Name:       "steve has a long name",
		Age:        27,
		Birthday:   time.Unix(1000000000, 0),
		Title:      "missess",
	},
	&Person{
		Identifier: "c",
		Name:       "bob walks home",
		Age:        64,
		Birthday:   time.Unix(1400000000, 0),
		Title:      "masta",
	},
	&Person{
		Identifier: "d",
		Name:       "bobbleheaded wings top the phone",
		Age:        72,
		Birthday:   time.Unix(1400000000, 0),
		Title:      "mizz",
	},
}

// FIXME needs more assertions
func TestIndex(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// index all the people
	for _, person := range people {
		err = index.Index(person.Identifier, person)
		if err != nil {
			t.Error(err)
		}
	}

	termQuery := NewTermQuery("marti").SetField("name")
	searchRequest := NewSearchRequest(termQuery)
	searchResult, err := index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "a" {
			t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
		}
	}

	termQuery = NewTermQuery("noone").SetField("name")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	matchPhraseQuery := NewMatchPhraseQuery("long name")
	searchRequest = NewSearchRequest(matchPhraseQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for phrase query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	termQuery = NewTermQuery("walking").SetField("name")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	matchQuery := NewMatchQuery("walking").SetField("name")
	searchRequest = NewSearchRequest(matchQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for match query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "c" {
			t.Errorf("expected top hit id 'c', got '%s'", searchResult.Hits[0].ID)
		}
	}

	prefixQuery := NewPrefixQuery("bobble").SetField("name")
	searchRequest = NewSearchRequest(prefixQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for prefix query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
	}

	syntaxQuery := NewSyntaxQuery("+name:phone")
	searchRequest = NewSearchRequest(syntaxQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for syntax query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
	}

	maxAge := 30.0
	numericRangeQuery := NewNumericRangeQuery(nil, &maxAge).SetField("age")
	searchRequest = NewSearchRequest(numericRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(2) {
		t.Errorf("expected 2 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
		if searchResult.Hits[1].ID != "a" {
			t.Errorf("expected next hit id 'a', got '%s'", searchResult.Hits[1].ID)
		}
	}

	startDate = "2010-01-01"
	dateRangeQuery := NewDateRangeQuery(&startDate, nil).SetField("birthday")
	searchRequest = NewSearchRequest(dateRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(2) {
		t.Errorf("expected 2 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
		if searchResult.Hits[1].ID != "c" {
			t.Errorf("expected next hit id 'c', got '%s'", searchResult.Hits[1].ID)
		}
	}

	// test that 0 time doesn't get indexed
	endDate = "2010-01-01"
	dateRangeQuery = NewDateRangeQuery(nil, &endDate).SetField("birthday")
	searchRequest = NewSearchRequest(dateRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	// test behavior of arrays
	// make sure we can successfully find by all elements in array
	termQuery = NewTermQuery("gopher").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	} else {
		if searchResult.Total != uint64(1) {
			t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
		} else {
			if searchResult.Hits[0].ID != "a" {
				t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
			}
		}
	}

	termQuery = NewTermQuery("belieber").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	} else {
		if searchResult.Total != uint64(1) {
			t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
		} else {
			if searchResult.Hits[0].ID != "a" {
				t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
			}
		}
	}

	termQuery = NewTermQuery("notintagsarray").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	// lookup document a
	// expect to find 2 values for field "tags"
	tagsCount := 0
	doc, err := index.Document("a")
	if err != nil {
		t.Error(err)
	} else {
		for _, f := range doc.Fields {
			if f.Name() == "tags" {
				tagsCount++
			}
		}
	}
	if tagsCount != 2 {
		t.Errorf("expected to find 2 values for tags")
	}
}

func TestIndexCreateNewOverExisting(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}
	index.Close()
	index, err = New("testidx", NewIndexMapping())
	if err != ERROR_INDEX_PATH_EXISTS {
		t.Fatalf("expected error index path exists, got %v", err)
	}
}

func TestIndexOpenNonExisting(t *testing.T) {
	_, err := Open("doesnotexist")
	if err != ERROR_INDEX_PATH_DOES_NOT_EXIST {
		t.Fatalf("expected error index path does not exist, got %v", err)
	}
}

func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}
	index.Close()

	// now intentionally corrupt the metadata
	ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666)

	index, err = Open("testidx")
	if err != ERROR_INDEX_META_CORRUPT {
		t.Fatalf("expected error index metadata corrupted, got %v", err)
	}

	// no intentionally remove the metadata
	os.Remove("testidx/index_meta.json")

	index, err = Open("testidx")
	if err != ERROR_INDEX_META_MISSING {
		t.Fatalf("expected error index metadata missing, got %v", err)
	}
}
<commit_msg>new test for numeric queries<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

// +build libstemmer full
// +build icu full

package bleve

import (
	"io/ioutil"
	"os"
	"testing"
	"time"
)

type Address struct {
	Street string `json:"street"`
	City   string `json:"city"`
	State  string `json:"state"`
	Zip    string `json:"zip"`
}

type Person struct {
	Identifier string     `json:"id"`
	Name       string     `json:"name"`
	Age        float64    `json:"age"`
	Title      string     `json:"title"`
	Birthday   time.Time  `json:"birthday"`
	Address    *Address   `json:"address"`
	Hideouts   []*Address `json:"hideouts"`
	Tags       []string   `json:"tags"`
}

func (p *Person) Type() string {
	return "person"
}

var nameMapping = NewDocumentMapping().
	AddFieldMapping(NewFieldMapping("", "text", "en", true, true, true, true))

var tagsMapping = NewDocumentMapping().
	AddFieldMapping(NewFieldMapping("", "text", "standard", true, true, true, false))
var personMapping = NewDocumentMapping().
	AddSubDocumentMapping("name", nameMapping).
	AddSubDocumentMapping("id", NewDocumentDisabledMapping()).
	AddSubDocumentMapping("tags", tagsMapping)

var mapping = NewIndexMapping().
	AddDocumentMapping("person", personMapping)

var people = []*Person{
	&Person{
		Identifier: "a",
		Name:       "marty",
		Age:        19,
		// has no birthday set to test handling of zero time
		Title: "mista",
		Tags:  []string{"gopher", "belieber"},
	},
	&Person{
		Identifier: "b",
		Name:       "steve has a long name",
		Age:        27,
		Birthday:   time.Unix(1000000000, 0),
		Title:      "missess",
	},
	&Person{
		Identifier: "c",
		Name:       "bob walks home",
		Age:        64,
		Birthday:   time.Unix(1400000000, 0),
		Title:      "masta",
	},
	&Person{
		Identifier: "d",
		Name:       "bobbleheaded wings top the phone",
		Age:        72,
		Birthday:   time.Unix(1400000000, 0),
		Title:      "mizz",
	},
}

// FIXME needs more assertions
func TestIndex(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", mapping)
	if err != nil {
		t.Fatal(err)
	}
	defer index.Close()

	// index all the people
	for _, person := range people {
		err = index.Index(person.Identifier, person)
		if err != nil {
			t.Error(err)
		}
	}

	termQuery := NewTermQuery("marti").SetField("name")
	searchRequest := NewSearchRequest(termQuery)
	searchResult, err := index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "a" {
			t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
		}
	}

	termQuery = NewTermQuery("noone").SetField("name")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	matchPhraseQuery := NewMatchPhraseQuery("long name")
	searchRequest = NewSearchRequest(matchPhraseQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for phrase query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	termQuery = NewTermQuery("walking").SetField("name")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	matchQuery := NewMatchQuery("walking").SetField("name")
	searchRequest = NewSearchRequest(matchQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for match query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "c" {
			t.Errorf("expected top hit id 'c', got '%s'", searchResult.Hits[0].ID)
		}
	}

	prefixQuery := NewPrefixQuery("bobble").SetField("name")
	searchRequest = NewSearchRequest(prefixQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for prefix query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
	}

	syntaxQuery := NewSyntaxQuery("+name:phone")
	searchRequest = NewSearchRequest(syntaxQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for syntax query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
	}

	maxAge := 30.0
	numericRangeQuery := NewNumericRangeQuery(nil, &maxAge).SetField("age")
	searchRequest = NewSearchRequest(numericRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(2) {
		t.Errorf("expected 2 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
		if searchResult.Hits[1].ID != "a" {
			t.Errorf("expected next hit id 'a', got '%s'", searchResult.Hits[1].ID)
		}
	}

	// test a numeric range with both endpoints
	minAge := 20.0
	numericRangeQuery = NewNumericRangeQuery(&minAge, &maxAge).SetField("age")
	searchRequest = NewSearchRequest(numericRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	// test the same query done as two
	// individual range queries and'd together
	q1 := NewNumericRangeQuery(&minAge, nil).SetField("age")
	q2 := NewNumericRangeQuery(nil, &maxAge).SetField("age")
	conQuery := NewConjunctionQuery([]Query{q1, q2})
	searchRequest = NewSearchRequest(conQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	startDate = "2010-01-01"
	dateRangeQuery := NewDateRangeQuery(&startDate, nil).SetField("birthday")
	searchRequest = NewSearchRequest(dateRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(2) {
		t.Errorf("expected 2 total hits for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "d" {
			t.Errorf("expected top hit id 'd', got '%s'", searchResult.Hits[0].ID)
		}
		if searchResult.Hits[1].ID != "c" {
			t.Errorf("expected next hit id 'c', got '%s'", searchResult.Hits[1].ID)
		}
	}

	// test that 0 time doesn't get indexed
	endDate = "2010-01-01"
	dateRangeQuery = NewDateRangeQuery(nil, &endDate).SetField("birthday")
	searchRequest = NewSearchRequest(dateRangeQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(1) {
		t.Errorf("expected 1 total hit for numeric range query, got %d", searchResult.Total)
	} else {
		if searchResult.Hits[0].ID != "b" {
			t.Errorf("expected top hit id 'b', got '%s'", searchResult.Hits[0].ID)
		}
	}

	// test behavior of arrays
	// make sure we can successfully find by all elements in array
	termQuery = NewTermQuery("gopher").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	} else {
		if searchResult.Total != uint64(1) {
			t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
		} else {
			if searchResult.Hits[0].ID != "a" {
				t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
			}
		}
	}

	termQuery = NewTermQuery("belieber").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	} else {
		if searchResult.Total != uint64(1) {
			t.Errorf("expected 1 total hit for term query, got %d", searchResult.Total)
		} else {
			if searchResult.Hits[0].ID != "a" {
				t.Errorf("expected top hit id 'a', got '%s'", searchResult.Hits[0].ID)
			}
		}
	}

	termQuery = NewTermQuery("notintagsarray").SetField("tags")
	searchRequest = NewSearchRequest(termQuery)
	searchResult, err = index.Search(searchRequest)
	if err != nil {
		t.Error(err)
	}
	if searchResult.Total != uint64(0) {
		t.Errorf("expected 0 total hits")
	}

	// lookup document a
	// expect to find 2 values for field "tags"
	tagsCount := 0
	doc, err := index.Document("a")
	if err != nil {
		t.Error(err)
	} else {
		for _, f := range doc.Fields {
			if f.Name() == "tags" {
				tagsCount++
			}
		}
	}
	if tagsCount != 2 {
		t.Errorf("expected to find 2 values for tags")
	}
}

func TestIndexCreateNewOverExisting(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}
	index.Close()
	index, err = New("testidx", NewIndexMapping())
	if err != ERROR_INDEX_PATH_EXISTS {
		t.Fatalf("expected error index path exists, got %v", err)
	}
}

func TestIndexOpenNonExisting(t *testing.T) {
	_, err := Open("doesnotexist")
	if err != ERROR_INDEX_PATH_DOES_NOT_EXIST {
		t.Fatalf("expected error index path does not exist, got %v", err)
	}
}

func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
	defer os.RemoveAll("testidx")

	index, err := New("testidx", NewIndexMapping())
	if err != nil {
		t.Fatal(err)
	}
	index.Close()

	// now intentionally corrupt the metadata
	ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666)

	index, err = Open("testidx")
	if err != ERROR_INDEX_META_CORRUPT {
		t.Fatalf("expected error index metadata corrupted, got %v", err)
	}

	// no intentionally remove the metadata
	os.Remove("testidx/index_meta.json")

	index, err = Open("testidx")
	if err != ERROR_INDEX_META_MISSING {
		t.Fatalf("expected error index metadata missing, got %v", err)
	}
}
<|endoftext|>


<commit_before>package pathers

import (
	"errors"
	"fmt"
)

// GroupName is the double-form of the name of a Group, which should be enough to find the group.
type GroupName struct {
	Group   string
	Account AccountName
}

// DefaultGroup is the default group name (just the group part - don't add dots!). Defaults to "default". Wow.
var DefaultGroup = "default"

func (g GroupName) defaultIfNeeded() {
	if g.Group == "" {
		g.Group = DefaultGroup
	}
}

func (g GroupName) String() string {
	g.defaultIfNeeded()
	if g.Account == "" {
		return g.Group
	}
	return g.Group + "." + string(g.Account)
}

// GroupPath returns a Brain URL for this group, or an error if the group is
// invalid
func (g GroupName) GroupPath() (string, error) {
	g.defaultIfNeeded()
	if g.Account == "" {
		return "", errors.New("no account specified - use GetDefaultAccount to set the Account if none is otherwise specified")
	}
	url := fmt.Sprintf("/accounts/%s/groups/%s", string(g.Account), g.Group)
	return url, nil
}
<commit_msg>implement AccountPath on GroupName<commit_after>package pathers

import (
	"fmt"
)

// GroupName is the double-form of the name of a Group, which should be enough to find the group.
type GroupName struct {
	Group   string
	Account AccountName
}

// DefaultGroup is the default group name (just the group part - don't add dots!). Defaults to "default". Wow.
var DefaultGroup = "default"

func (g GroupName) defaultIfNeeded() {
	if g.Group == "" {
		g.Group = DefaultGroup
	}
}

func (g GroupName) String() string {
	g.defaultIfNeeded()
	if g.Account == "" {
		return g.Group
	}
	return g.Group + "." + string(g.Account)
}

// GroupPath returns a Brain URL for this group, or an error if the group is
// invalid
func (g GroupName) GroupPath() (string, error) {
	g.defaultIfNeeded()
	base, err := g.AccountPath()
	return base + fmt.Sprintf("/groups/%s", g.Group)
}

// AccountPath returns a Brain URL for the account specified in this GroupName,
// or an error if it is blank
func (g GroupName) AccountPath() (string, error) {
	return g.Account.AccountPath()
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package language contains methods and information about the different
// programming languages the comment parser supports.
package language

import (
	"path/filepath"
	"strings"
)

// Language is the progamming language we're grabbing the comments from.
type Language int

// Languages we can retrieve comments from.
const (
	Unknown Language = iota
	AppleScript
	Assembly
	Batch
	C
	Clif
	Clojure
	CMake
	CSharp
	Dart
	Elixir
	Fortran
	GLSLF // OpenGL Shading Language
	Go
	Haskell
	HTML
	Flex
	Java
	JavaScript
	Kotlin
	Lisp
	Matlab
	Markdown
	MySQL
	NinjaBuild
	ObjectiveC
	Perl
	Python
	R
	Ruby
	Rust
	Shader
	Shell
	SQL
	Swift
	SWIG
	TypeScript
	Yacc
	Yaml
)

// style is the comment styles that a language uses.
type style int

// Comment styles.
const (
	unknown     style = iota
	applescript       // -- ... and (* ... *)
	batch             // @REM
	bcpl              // // ... and /* ... */
	cmake             // # ... and #[[ ... ]]
	fortran           // ! ...
	haskell           // -- ... and {- ... -}
	html              // <!-- ... -->
	lisp              // ;; ...
	matlab            // % ...
	mysql             // # ... and /* ... */
	ruby              // # ... and =begin ... =end
	shell             // # ... and %{ ... %}
	sql               // -- ... and /* ... */
)

// ClassifyLanguage determines what language the source code was written in. It
// does this by looking at the file's extension.
func ClassifyLanguage(filename string) Language {
	ext := strings.ToLower(filepath.Ext(filename))
	if len(ext) == 0 || ext[0] != '.' {
		return Unknown
	}

	switch ext[1:] { // Skip the '.'.
	case "applescript":
		return AppleScript
	case "bat":
		return Batch
	case "c", "cc", "cpp", "c++", "h", "hh", "hpp":
		return C
	case "clif":
		return Clif
	case "cmake":
		return CMake
	case "cs":
		return CSharp
	case "dart":
		return Dart
	case "ex", "exs":
		return Elixir
	case "f", "f90", "f95":
		return Fortran
	case "glslf":
		return GLSLF
	case "go":
		return Go
	case "hs":
		return Haskell
	case "html", "htm", "ng", "sgml":
		return HTML
	case "java":
		return Java
	case "js":
		return JavaScript
	case "kt":
		return Kotlin
	case "l":
		return Flex
	case "lisp", "el", "clj":
		return Lisp
	case "m", "mm":
		return ObjectiveC
	case "md":
		return Markdown
	case "gn":
		return NinjaBuild
	case "pl", "pm":
		return Perl
	case "py", "pi":
		return Python
	case "r":
		return R
	case "rb":
		return Ruby
	case "rs":
		return Rust
	case "s":
		return Assembly
	case "sh":
		return Shell
	case "shader":
		return Shader
	case "sql":
		return SQL
	case "swift":
		return Swift
	case "swig":
		return SWIG
	case "ts", "tsx":
		return TypeScript
	case "y":
		return Yacc
	case "yaml":
		return Yaml
	}
	return Unknown
}

// commentStyle returns the language's comment style.
func (lang Language) commentStyle() style {
	switch lang {
	case Assembly, C, CSharp, Dart, Flex, GLSLF, Go, Java, JavaScript, Kotlin, ObjectiveC, Rust, Shader, Swift, SWIG, TypeScript, Yacc:
		return bcpl
	case Batch:
		return batch
	case CMake:
		return cmake
	case Fortran:
		return fortran
	case Haskell:
		return haskell
	case HTML, Markdown:
		return html
	case Clojure, Lisp:
		return lisp
	case Ruby:
		return ruby
	case Clif, Elixir, NinjaBuild, Perl, Python, R, Shell, Yaml:
		return shell
	case Matlab:
		return matlab
	case MySQL:
		return mysql
	case SQL:
		return sql
	}
	return unknown
}

// SingleLineCommentStart returns the starting string of a single line comment
// for the given language. There is no equivalent "End" method, because it's
// the end of line.
func (lang Language) SingleLineCommentStart() string {
	switch lang.commentStyle() {
	case applescript, haskell, sql:
		return "--"
	case batch:
		return "@REM"
	case bcpl:
		return "//"
	case fortran:
		return "!"
	case lisp:
		return ";"
	case matlab:
		return "%"
	case shell, ruby, cmake, mysql:
		return "#"
	}
	return ""
}

// MultilineCommentStart returns the starting string of a multiline comment for
// the given language.
func (lang Language) MultilineCommentStart() string {
	switch lang.commentStyle() {
	case applescript:
		return "(*"
	case bcpl, mysql:
		if lang != Rust {
			return "/*"
		}
	case cmake:
		return "#[["
	case haskell:
		return "{-"
	case html:
		return "<!--"
	case matlab:
		return "%{"
	case ruby:
		return "=begin"
	}
	return ""
}

// MultilineCommentEnd returns the ending string of a multiline comment for the
// given language.
func (lang Language) MultilineCommentEnd() string {
	switch lang.commentStyle() {
	case applescript:
		return "*)"
	case bcpl, mysql:
		if lang != Rust {
			return "*/"
		}
	case cmake:
		return "]]"
	case haskell:
		return "-}"
	case html:
		return "-->"
	case matlab:
		return "%}"
	case ruby:
		return "=end"
	}
	return ""
}

// QuoteCharacter returns 'true' if the character is considered the beginning
// of a string in the given language. The second return value is true if the
// string allows for escaping.
func (lang Language) QuoteCharacter(quote rune) (ok bool, escape bool) {
	switch quote {
	case '"', '\'':
		return true, true
	case '`':
		if lang == Go {
			return true, false
		}
	}
	return false, false
}

// NestedComments returns true if the language allows for nested multiline comments.
func (lang Language) NestedComments() bool {
	return lang == Swift
}
<commit_msg>Adding a bunch of Hardware Description Language file formats.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package language contains methods and information about the different
// programming languages the comment parser supports.
package language

import (
	"path/filepath"
	"strings"
)

// Language is the progamming language we're grabbing the comments from.
type Language int

// Languages we can retrieve comments from.
const (
	Unknown Language = iota
	AppleScript
	Assembly
	BLIF // Berkley Logic Interface Format
	Batch
	C
	CMake
	CSharp
	Clif
	Clojure
	Dart
	EDIF // Electronic Design Interchange Format
	Elixir
	Flex
	Fortran
	GLSLF // OpenGL Shading Language
	Go
	HTML
	Haskell
	Java
	JavaScript
	Kotlin
	LEF // Library Exchange Format
	Lisp
	Markdown
	Matlab
	MySQL
	NinjaBuild
	ObjectiveC
	Perl
	Python
	R
	Ruby
	Rust
	SDC  // Synopsis Design Constraint
	SDF  // Standard Delay Format
	SPEF // Standard Parasitics Exchange Format
	SQL
	SWIG
	Shader
	Shell
	Swift
	SystemVerilog
	TCL
	TypeScript
	Verilog
	XDC // Xilinx Design Constraint files
	Yacc
	Yaml
)

// style is the comment styles that a language uses.
type style int

// Comment styles.
const (
	unknown     style = iota
	applescript       // -- ... and (* ... *)
	batch             // @REM
	bcpl              // // ... and /* ... */
	cmake             // # ... and #[[ ... ]]
	fortran           // ! ...
	hash              // # ...
	haskell           // -- ... and {- ... -}
	html              // <!-- ... -->
	lisp              // ;; ...
	matlab            // % ...
	mysql             // # ... and /* ... */
	ruby              // # ... and =begin ... =end
	shell             // # ... and %{ ... %}
	sql               // -- ... and /* ... */
)

// ClassifyLanguage determines what language the source code was written in. It
// does this by looking at the file's extension.
func ClassifyLanguage(filename string) Language {
	ext := strings.ToLower(filepath.Ext(filename))
	if len(ext) == 0 || ext[0] != '.' {
		return Unknown
	}

	switch ext[1:] { // Skip the '.'.
	case "applescript":
		return AppleScript
	case "bat":
		return Batch
	case "blif", "eblif":
		return BLIF
	case "c", "cc", "cpp", "c++", "h", "hh", "hpp":
		return C
	case "clif":
		return Clif
	case "cmake":
		return CMake
	case "cs":
		return CSharp
	case "dart":
		return Dart
	case "ex", "exs":
		return Elixir
	case "f", "f90", "f95":
		return Fortran
	case "glslf":
		return GLSLF
	case "go":
		return Go
	case "hs":
		return Haskell
	case "html", "htm", "ng", "sgml":
		return HTML
	case "java":
		return Java
	case "js":
		return JavaScript
	case "kt":
		return Kotlin
	case "l":
		return Flex
	case "lef":
		return LEF
	case "lisp", "el", "clj":
		return Lisp
	case "m", "mm":
		return ObjectiveC
	case "md":
		return Markdown
	case "gn":
		return NinjaBuild
	case "pl", "pm":
		return Perl
	case "py", "pi":
		return Python
	case "r":
		return R
	case "rb":
		return Ruby
	case "rs":
		return Rust
	case "s":
		return Assembly
	case "sdf":
		return SDF
	case "sh":
		return Shell
	case "shader":
		return Shader
	case "sql":
		return SQL
	case "swift":
		return Swift
	case "swig":
		return SWIG
	case "sv", "svh":
		return SystemVerilog
	case "tcl", "sdc", "xdc":
		return TCL
	case "ts", "tsx":
		return TypeScript
	case "v", "vh":
		return Verilog
	case "y":
		return Yacc
	case "yaml":
		return Yaml
	}
	return Unknown
}

// commentStyle returns the language's comment style.
func (lang Language) commentStyle() style {
	switch lang {
	case Assembly, C, CSharp, Dart, Flex, GLSLF, Go, Java, JavaScript, Kotlin, ObjectiveC, Rust, Shader, Swift, SWIG, TypeScript, Yacc, Verilog, SystemVerilog, SDF, SPEF:
		return bcpl
	case Batch:
		return batch
	case BLIF, TCL:
		return hash
	case CMake:
		return cmake
	case Fortran:
		return fortran
	case Haskell:
		return haskell
	case HTML, Markdown:
		return html
	case Clojure, Lisp:
		return lisp
	case Ruby:
		return ruby
	case Clif, Elixir, NinjaBuild, Perl, Python, R, Shell, Yaml:
		return shell
	case Matlab:
		return matlab
	case MySQL:
		return mysql
	case SQL:
		return sql
	}
	return unknown
}

// SingleLineCommentStart returns the starting string of a single line comment
// for the given language. There is no equivalent "End" method, because it's
// the end of line.
func (lang Language) SingleLineCommentStart() string {
	switch lang.commentStyle() {
	case applescript, haskell, sql:
		return "--"
	case batch:
		return "@REM"
	case bcpl:
		return "//"
	case fortran:
		return "!"
	case lisp:
		return ";"
	case matlab:
		return "%"
	case shell, ruby, cmake, mysql, hash:
		return "#"
	}
	return ""
}

// MultilineCommentStart returns the starting string of a multiline comment for
// the given language.
func (lang Language) MultilineCommentStart() string {
	switch lang.commentStyle() {
	case applescript:
		return "(*"
	case bcpl, mysql:
		if lang != Rust {
			return "/*"
		}
	case cmake:
		return "#[["
	case haskell:
		return "{-"
	case html:
		return "<!--"
	case matlab:
		return "%{"
	case ruby:
		return "=begin"
	}
	return ""
}

// MultilineCommentEnd returns the ending string of a multiline comment for the
// given language.
func (lang Language) MultilineCommentEnd() string {
	switch lang.commentStyle() {
	case applescript:
		return "*)"
	case bcpl, mysql:
		if lang != Rust {
			return "*/"
		}
	case cmake:
		return "]]"
	case haskell:
		return "-}"
	case html:
		return "-->"
	case matlab:
		return "%}"
	case ruby:
		return "=end"
	}
	return ""
}

// QuoteCharacter returns 'true' if the character is considered the beginning
// of a string in the given language. The second return value is true if the
// string allows for escaping.
func (lang Language) QuoteCharacter(quote rune) (ok bool, escape bool) {
	switch quote {
	case '"', '\'':
		return true, true
	case '`':
		if lang == Go {
			return true, false
		}
	}
	return false, false
}

// NestedComments returns true if the language allows for nested multiline comments.
func (lang Language) NestedComments() bool {
	return lang == Swift
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Erik St. Martin, Brian Ketelsen. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main

import (
	"crypto/sha1"
	"log"
	"os"
	"sync"
	"time"

	"github.com/miekg/dns"
)

const origTTL uint32 = 60

var (
	cache    *sigCache = newCache()
	inflight *single   = new(single)
)

// ParseKeyFile read a DNSSEC keyfile as generated by dnssec-keygen or other
// utilities. It add ".key" for the public key and ".private" for the private key.
func ParseKeyFile(file string) (*dns.DNSKEY, dns.PrivateKey, error) {
	f, e := os.Open(file + ".key")
	if e != nil {
		return nil, nil, e
	}
	k, e := dns.ReadRR(f, file+".key")
	if e != nil {
		return nil, nil, e
	}
	f, e = os.Open(file + ".private")
	if e != nil {
		return nil, nil, e
	}
	p, e := k.(*dns.DNSKEY).ReadPrivateKey(f, file+".private")
	if e != nil {
		return nil, nil, e
	}
	k.Header().Ttl = origTTL
	return k.(*dns.DNSKEY), p, nil
}

// NewNSEC creates (if needed) NSEC records that are included in the reply.
func (s *server) NewNSEC(m *dns.Msg) {
	if m.Rcode == dns.RcodeNameError {
		// qname nsec
		nsec1 := s.newNSEC(m.Question[0].Name)
		m.Ns = append(m.Ns, nsec1)
		// wildcard nsec
		idx := dns.Split(m.Question[0].Name)
		wildcard := "*." + m.Question[0].Name[idx[0]:]
		nsec2 := s.newNSEC(wildcard)
		if nsec1.Hdr.Name != nsec2.Hdr.Name || nsec1.NextDomain != nsec2.NextDomain {
			// different NSEC, add it
			m.Ns = append(m.Ns, nsec2)
		}
	}
	if m.Rcode == dns.RcodeSuccess && len(m.Ns) == 1 {
		if _, ok := m.Ns[0].(*dns.SOA); ok {
			m.Ns = append(m.Ns, s.newNSEC(m.Question[0].Name))
		}
	}
}

// sign signs a message m, it takes care of negative or nodata responses as
// well by synthesising NSEC records. It will also cache the signatures, using
// a hash of the signed data as a key.
// We also fake the origin TTL in the signature, because we don't want to
// throw away signatures when services decide to have longer TTL. So we just
// set the origTTL to 60.
func (s *server) sign(m *dns.Msg, bufsize uint16) {
	now := time.Now().UTC()
	incep := uint32(now.Add(-2 * time.Hour).Unix())     // 2 hours, be sure to catch daylight saving time and such
	expir := uint32(now.Add(7 * 24 * time.Hour).Unix()) // sign for a week

	// TODO(miek): repeating this two times?
	for _, r := range rrSets(m.Answer) {
		if r[0].Header().Rrtype == dns.TypeRRSIG {
			continue
		}
		key := cache.key(r)
		if s := cache.search(key); s != nil {
			if s.ValidityPeriod(now.Add(-24 * time.Hour)) {
				m.Answer = append(m.Answer, s)
				continue
			}
			cache.remove(key)
		}
		sig, err, shared := inflight.Do(key, func() (*dns.RRSIG, error) {
			sig1 := s.NewRRSIG(incep, expir)
			e := sig1.Sign(s.config.PrivKey, r)
			if e != nil {
				log.Printf("failed to sign: %s\n", e.Error())
			}
			return sig1, e
		})
		if err != nil {
			continue
		}
		if !shared {
			// is it possible to miss this, due the the c.dups > 0 in Do()? TODO(miek)
			cache.insert(key, sig)
		}
		m.Answer = append(m.Answer, dns.Copy(sig).(*dns.RRSIG))
	}
	for _, r := range rrSets(m.Ns) {
		if r[0].Header().Rrtype == dns.TypeRRSIG {
			continue
		}
		key := cache.key(r)
		if s := cache.search(key); s != nil {
			if s.ValidityPeriod(now.Add(-24 * time.Hour)) {
				m.Ns = append(m.Ns, s)
				continue
			}
			cache.remove(key)
		}
		sig, err, shared := inflight.Do(key, func() (*dns.RRSIG, error) {
			sig1 := s.NewRRSIG(incep, expir)
			e := sig1.Sign(s.config.PrivKey, r)
			if e != nil {
				log.Printf("failed to sign: %s\n", e.Error())
			}
			return sig1, e
		})
		if err != nil {
			continue
		}
		if !shared {
			// is it possible to miss this, due the the c.dups > 0 in Do()? TODO(miek)
			cache.insert(key, sig)
		}
		m.Ns = append(m.Ns, dns.Copy(sig).(*dns.RRSIG))
	}
	// TODO(miek): Forget the additional section for now
	if bufsize >= 512 || bufsize <= 4096 {
		m.Truncated = m.Len() > int(bufsize)
	}
	o := new(dns.OPT)
	o.Hdr.Name = "."
	o.Hdr.Rrtype = dns.TypeOPT
	o.SetDo()
	o.SetUDPSize(4096)
	m.Extra = append(m.Extra, o)
	return
}

func (s *server) NewRRSIG(incep, expir uint32) *dns.RRSIG {
	sig := new(dns.RRSIG)
	sig.Hdr.Rrtype = dns.TypeRRSIG
	sig.Hdr.Ttl = origTTL
	sig.OrigTtl = origTTL
	sig.Algorithm = s.config.PubKey.Algorithm
	sig.KeyTag = s.config.KeyTag
	sig.Inception = incep
	sig.Expiration = expir
	sig.SignerName = s.config.PubKey.Hdr.Name
	return sig
}

// newNSEC returns the NSEC record need to denial qname, or gives back a NODATA NSEC.
func (s *server) newNSEC(qname string) *dns.NSEC {
	qlabels := dns.SplitDomainName(qname)
	if len(qlabels) < s.domainLabels {
		// TODO(miek): can not happen...?
	}
	// Strip the last s.domainLabels, return up to 4 before
	// that. Four labels is the maximum qname we can handle.
	ls := len(qlabels) - s.domainLabels
	ls4 := ls - 4
	if ls4 < 0 {
		ls4 = 0
	}
	key := qlabels[ls4:ls]
	key = key // TODO(miek)
	// TODO etcd here
	//	prev, next := s.registry.GetNSEC(strings.Join(key, "."))
	prev, next := "", ""
	nsec := &dns.NSEC{Hdr: dns.RR_Header{Name: prev + s.config.Domain + ".", Rrtype: dns.TypeNSEC, Class: dns.ClassINET, Ttl: 60},
		NextDomain: next + s.config.Domain + "."}
	if prev == "" {
		nsec.TypeBitMap = []uint16{dns.TypeA, dns.TypeSOA, dns.TypeNS, dns.TypeAAAA, dns.TypeRRSIG, dns.TypeNSEC, dns.TypeDNSKEY}
	} else {
		nsec.TypeBitMap = []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeSRV, dns.TypeRRSIG, dns.TypeNSEC}
	}
	return nsec
}

type rrset struct {
	qname string
	qtype uint16
}

func rrSets(rrs []dns.RR) map[rrset][]dns.RR {
	m := make(map[rrset][]dns.RR)
	for _, r := range rrs {
		if s, ok := m[rrset{r.Header().Name, r.Header().Rrtype}]; ok {
			s = append(s, r)
			m[rrset{r.Header().Name, r.Header().Rrtype}] = s
		} else {
			s := make([]dns.RR, 1, 3)
			s[0] = r
			m[rrset{r.Header().Name, r.Header().Rrtype}] = s
		}
	}
	if len(m) > 0 {
		return m
	}
	return nil
}

type sigCache struct {
	sync.RWMutex
	m map[string]*dns.RRSIG
}

func newCache() *sigCache {
	c := new(sigCache)
	c.m = make(map[string]*dns.RRSIG)
	return c
}

func (c *sigCache) remove(s string) {
	delete(c.m, s)
}

func (c *sigCache) insert(s string, r *dns.RRSIG) {
	c.Lock()
	defer c.Unlock()
	if _, ok := c.m[s]; !ok {
		c.m[s] = r
	}
}

func (c *sigCache) search(s string) *dns.RRSIG {
	c.RLock()
	defer c.RUnlock()
	if s, ok := c.m[s]; ok {
		// we want to return a copy here, because if we didn't the RRSIG
		// could be removed by another goroutine before the packet containing
		// this signature is send out.
		log.Println("signature retrieved from cache")
		return dns.Copy(s).(*dns.RRSIG)
	}
	return nil
}

// key uses the name, type and rdata, which is serialized and then hashed as the
// key for the lookup
func (c *sigCache) key(rrs []dns.RR) string {
	h := sha1.New()
	i := []byte(rrs[0].Header().Name)
	i = append(i, packUint16(rrs[0].Header().Rrtype)...)
	for _, r := range rrs {
		switch t := r.(type) { // we only do a few type, serialize these manually
		case *dns.SOA:
			i = append(i, packUint32(t.Serial)...)
			// we only fiddle with the serial so store that
		case *dns.SRV:
			i = append(i, packUint16(t.Priority)...)
			i = append(i, packUint16(t.Weight)...)
			i = append(i, packUint16(t.Weight)...)
			i = append(i, []byte(t.Target)...)
		case *dns.A:
			i = append(i, []byte(t.A)...)
		case *dns.AAAA:
			i = append(i, []byte(t.AAAA)...)
		case *dns.DNSKEY:
			// Need nothing more, the rdata stays the same during a run
		case *dns.NSEC:
			i = append(i, []byte(t.NextDomain)...)
			// bitmap does not differentiate
		default:
			log.Printf("signature for unhandled type %T seen", t)
		}
	}
	return string(h.Sum(i))
}

// TODO(miek): prolly should use the stdlib ones
func packUint16(i uint16) []byte { return []byte{byte(i >> 8), byte(i)} }
func packUint32(i uint32) []byte { return []byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)} }

// Adapted from singleinflight.go from the original Go Code. Copyright 2013 The Go Authors.
type call struct {
	wg   sync.WaitGroup
	val  *dns.RRSIG
	err  error
	dups int
}

type single struct {
	sync.Mutex
	m map[string]*call
}

func (g *single) Do(key string, fn func() (*dns.RRSIG, error)) (*dns.RRSIG, error, bool) {
	g.Lock()
	if g.m == nil {
		g.m = make(map[string]*call)
	}
	if c, ok := g.m[key]; ok {
		c.dups++
		g.Unlock()
		c.wg.Wait()
		return c.val, c.err, true
	}
	c := new(call)
	c.wg.Add(1)
	g.m[key] = c
	g.Unlock()

	c.val, c.err = fn()
	c.wg.Done()

	g.Lock()
	delete(g.m, key)
	g.Unlock()

	return c.val, c.err, c.dups > 0
}
<commit_msg>comments on how DNSSEC is going to work<commit_after>// Copyright (c) 2013 Erik St. Martin, Brian Ketelsen. All rights reserved.
// Use of this source code is governed by The MIT License (MIT) that can be
// found in the LICENSE file.

package main

import (
	"crypto/sha1"
	"log"
	"os"
	"sync"
	"time"

	"github.com/miekg/dns"
)

// Do DNSSEC NXDOMAIN with NSEC3 whitelies: rfc 7129, appendix B.
// The closest encloser will always be server.config.Domain and we
// will deny the wildcard for *.server.config.Domain. This allows
// use to pre-compute those records. We then only need to compute
// the NSEC3 that covers the qname.

const origTTL uint32 = 60

var (
	cache    *sigCache = newCache()
	inflight *single   = new(single)
)

const SALT = "-"

// ParseKeyFile read a DNSSEC keyfile as generated by dnssec-keygen or other
// utilities. It add ".key" for the public key and ".private" for the private key.
func ParseKeyFile(file string) (*dns.DNSKEY, dns.PrivateKey, error) {
	f, e := os.Open(file + ".key")
	if e != nil {
		return nil, nil, e
	}
	k, e := dns.ReadRR(f, file+".key")
	if e != nil {
		return nil, nil, e
	}
	f, e = os.Open(file + ".private")
	if e != nil {
		return nil, nil, e
	}
	p, e := k.(*dns.DNSKEY).ReadPrivateKey(f, file+".private")
	if e != nil {
		return nil, nil, e
	}
	k.Header().Ttl = origTTL
	return k.(*dns.DNSKEY), p, nil
}

// NewNSEC creates (if needed) NSEC records that are included in the reply.
func (s *server) NewNSEC(m *dns.Msg) {
	if m.Rcode == dns.RcodeNameError {
		// qname nsec
		nsec1 := s.newNSEC(m.Question[0].Name)
		m.Ns = append(m.Ns, nsec1)
		// wildcard nsec
		idx := dns.Split(m.Question[0].Name)
		wildcard := "*." + m.Question[0].Name[idx[0]:]
		nsec2 := s.newNSEC(wildcard)
		if nsec1.Hdr.Name != nsec2.Hdr.Name || nsec1.NextDomain != nsec2.NextDomain {
			// different NSEC, add it
			m.Ns = append(m.Ns, nsec2)
		}
	}
	if m.Rcode == dns.RcodeSuccess && len(m.Ns) == 1 {
		if _, ok := m.Ns[0].(*dns.SOA); ok {
			m.Ns = append(m.Ns, s.newNSEC(m.Question[0].Name))
		}
	}
}

// sign signs a message m, it takes care of negative or nodata responses as
// well by synthesising NSEC records. It will also cache the signatures, using
// a hash of the signed data as a key.
// We also fake the origin TTL in the signature, because we don't want to
// throw away signatures when services decide to have longer TTL. So we just
// set the origTTL to 60.
func (s *server) sign(m *dns.Msg, bufsize uint16) {
	now := time.Now().UTC()
	incep := uint32(now.Add(-2 * time.Hour).Unix())     // 2 hours, be sure to catch daylight saving time and such
	expir := uint32(now.Add(7 * 24 * time.Hour).Unix()) // sign for a week

	// TODO(miek): repeating this two times?
	for _, r := range rrSets(m.Answer) {
		if r[0].Header().Rrtype == dns.TypeRRSIG {
			continue
		}
		key := cache.key(r)
		if s := cache.search(key); s != nil {
			if s.ValidityPeriod(now.Add(-24 * time.Hour)) {
				m.Answer = append(m.Answer, s)
				continue
			}
			cache.remove(key)
		}
		sig, err, shared := inflight.Do(key, func() (*dns.RRSIG, error) {
			sig1 := s.NewRRSIG(incep, expir)
			e := sig1.Sign(s.config.PrivKey, r)
			if e != nil {
				log.Printf("failed to sign: %s\n", e.Error())
			}
			return sig1, e
		})
		if err != nil {
			continue
		}
		if !shared {
			// is it possible to miss this, due the the c.dups > 0 in Do()? TODO(miek)
			cache.insert(key, sig)
		}
		m.Answer = append(m.Answer, dns.Copy(sig).(*dns.RRSIG))
	}
	for _, r := range rrSets(m.Ns) {
		if r[0].Header().Rrtype == dns.TypeRRSIG {
			continue
		}
		key := cache.key(r)
		if s := cache.search(key); s != nil {
			if s.ValidityPeriod(now.Add(-24 * time.Hour)) {
				m.Ns = append(m.Ns, s)
				continue
			}
			cache.remove(key)
		}
		sig, err, shared := inflight.Do(key, func() (*dns.RRSIG, error) {
			sig1 := s.NewRRSIG(incep, expir)
			e := sig1.Sign(s.config.PrivKey, r)
			if e != nil {
				log.Printf("failed to sign: %s\n", e.Error())
			}
			return sig1, e
		})
		if err != nil {
			continue
		}
		if !shared {
			// is it possible to miss this, due the the c.dups > 0 in Do()? TODO(miek)
			cache.insert(key, sig)
		}
		m.Ns = append(m.Ns, dns.Copy(sig).(*dns.RRSIG))
	}
	// TODO(miek): Forget the additional section for now
	if bufsize >= 512 || bufsize <= 4096 {
		m.Truncated = m.Len() > int(bufsize)
	}
	o := new(dns.OPT)
	o.Hdr.Name = "."
	o.Hdr.Rrtype = dns.TypeOPT
	o.SetDo()
	o.SetUDPSize(4096)
	m.Extra = append(m.Extra, o)
	return
}

func (s *server) NewRRSIG(incep, expir uint32) *dns.RRSIG {
	sig := new(dns.RRSIG)
	sig.Hdr.Rrtype = dns.TypeRRSIG
	sig.Hdr.Ttl = origTTL
	sig.OrigTtl = origTTL
	sig.Algorithm = s.config.PubKey.Algorithm
	sig.KeyTag = s.config.KeyTag
	sig.Inception = incep
	sig.Expiration = expir
	sig.SignerName = s.config.PubKey.Hdr.Name
	return sig
}

// newNSEC returns the NSEC record need to denial qname, or gives back a NODATA NSEC.
func (s *server) newNSEC(qname string) *dns.NSEC {
	qlabels := dns.SplitDomainName(qname)
	if len(qlabels) < s.domainLabels {
		// TODO(miek): can not happen...?
	}
	// Strip the last s.domainLabels, return up to 4 before
	// that. Four labels is the maximum qname we can handle.
	ls := len(qlabels) - s.domainLabels
	ls4 := ls - 4
	if ls4 < 0 {
		ls4 = 0
	}
	key := qlabels[ls4:ls]
	key = key // TODO(miek)
	// TODO etcd here
	//	prev, next := s.registry.GetNSEC(strings.Join(key, "."))
	prev, next := "", ""
	nsec := &dns.NSEC{Hdr: dns.RR_Header{Name: prev + s.config.Domain + ".", Rrtype: dns.TypeNSEC, Class: dns.ClassINET, Ttl: 60},
		NextDomain: next + s.config.Domain + "."}
	if prev == "" {
		nsec.TypeBitMap = []uint16{dns.TypeA, dns.TypeSOA, dns.TypeNS, dns.TypeAAAA, dns.TypeRRSIG, dns.TypeNSEC, dns.TypeDNSKEY}
	} else {
		nsec.TypeBitMap = []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeSRV, dns.TypeRRSIG, dns.TypeNSEC}
	}
	return nsec
}

type rrset struct {
	qname string
	qtype uint16
}

func rrSets(rrs []dns.RR) map[rrset][]dns.RR {
	m := make(map[rrset][]dns.RR)
	for _, r := range rrs {
		if s, ok := m[rrset{r.Header().Name, r.Header().Rrtype}]; ok {
			s = append(s, r)
			m[rrset{r.Header().Name, r.Header().Rrtype}] = s
		} else {
			s := make([]dns.RR, 1, 3)
			s[0] = r
			m[rrset{r.Header().Name, r.Header().Rrtype}] = s
		}
	}
	if len(m) > 0 {
		return m
	}
	return nil
}

type sigCache struct {
	sync.RWMutex
	m map[string]*dns.RRSIG
}

func newCache() *sigCache {
	c := new(sigCache)
	c.m = make(map[string]*dns.RRSIG)
	return c
}

func (c *sigCache) remove(s string) {
	delete(c.m, s)
}

func (c *sigCache) insert(s string, r *dns.RRSIG) {
	c.Lock()
	defer c.Unlock()
	if _, ok := c.m[s]; !ok {
		c.m[s] = r
	}
}

func (c *sigCache) search(s string) *dns.RRSIG {
	c.RLock()
	defer c.RUnlock()
	if s, ok := c.m[s]; ok {
		// we want to return a copy here, because if we didn't the RRSIG
		// could be removed by another goroutine before the packet containing
		// this signature is send out.
		log.Println("signature retrieved from cache")
		return dns.Copy(s).(*dns.RRSIG)
	}
	return nil
}

// key uses the name, type and rdata, which is serialized and then hashed as the
// key for the lookup
func (c *sigCache) key(rrs []dns.RR) string {
	h := sha1.New()
	i := []byte(rrs[0].Header().Name)
	i = append(i, packUint16(rrs[0].Header().Rrtype)...)
	for _, r := range rrs {
		switch t := r.(type) { // we only do a few type, serialize these manually
		case *dns.SOA:
			i = append(i, packUint32(t.Serial)...)
			// we only fiddle with the serial so store that
		case *dns.SRV:
			i = append(i, packUint16(t.Priority)...)
			i = append(i, packUint16(t.Weight)...)
			i = append(i, packUint16(t.Weight)...)
			i = append(i, []byte(t.Target)...)
		case *dns.A:
			i = append(i, []byte(t.A)...)
		case *dns.AAAA:
			i = append(i, []byte(t.AAAA)...)
		case *dns.DNSKEY:
			// Need nothing more, the rdata stays the same during a run
		case *dns.NSEC:
			i = append(i, []byte(t.NextDomain)...)
			// bitmap does not differentiate
		default:
			log.Printf("signature for unhandled type %T seen", t)
		}
	}
	return string(h.Sum(i))
}

// TODO(miek): prolly should use the stdlib ones
func packUint16(i uint16) []byte { return []byte{byte(i >> 8), byte(i)} }
func packUint32(i uint32) []byte { return []byte{byte(i >> 24), byte(i >> 16), byte(i >> 8), byte(i)} }

// Adapted from singleinflight.go from the original Go Code. Copyright 2013 The Go Authors.
type call struct {
	wg   sync.WaitGroup
	val  *dns.RRSIG
	err  error
	dups int
}

type single struct {
	sync.Mutex
	m map[string]*call
}

func (g *single) Do(key string, fn func() (*dns.RRSIG, error)) (*dns.RRSIG, error, bool) {
	g.Lock()
	if g.m == nil {
		g.m = make(map[string]*call)
	}
	if c, ok := g.m[key]; ok {
		c.dups++
		g.Unlock()
		c.wg.Wait()
		return c.val, c.err, true
	}
	c := new(call)
	c.wg.Add(1)
	g.m[key] = c
	g.Unlock()

	c.val, c.err = fn()
	c.wg.Done()

	g.Lock()
	delete(g.m, key)
	g.Unlock()

	return c.val, c.err, c.dups > 0
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/fsouza/go-dockerclient"
)

func main() {
	endpoint := flag.String("endpoint", "unix:///var/run/docker.sock", "Docker socket")

	client, err := docker.NewClient(*endpoint)
	if err != nil {
		fmt.Printf("Error (%s) %s\n", *endpoint, err.Error())
		os.Exit(1)
	}

	events := make(chan *docker.APIEvents)
	client.AddEventListener(events)
	fmt.Println("Listening...")

	for {
		select {
		case event := <-events:
			switch event.Status {
			case "start":
				fmt.Printf("> START %s: %s\n", event.From, event.ID)
			case "stop":
				fmt.Printf("> STOP %s: %s\n", event.From, event.ID)
			case "die":
				fmt.Printf("> DIE %s: %s\n", event.From, event.ID)
			}
		}
	}
}
<commit_msg>Run healtcheck<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/fsouza/go-dockerclient"
)

func healtcheck(client *docker.Client, container *docker.Container) error {
	cmd := []string{"/bin/sh", "-c"}
	for _, value := range container.Config.Healthcheck.Test[1:len(container.Config.Healthcheck.Test)] {
		cmd = append(cmd, value)
	}
	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStderr: true,
		AttachStdin:  true,
		AttachStdout: true,
		Tty:          false,
		Cmd:          cmd,
		Container:    container.ID,
	})
	if err != nil {
		return err
	}

	var stdout, stderr bytes.Buffer
	var reader = strings.NewReader("")

	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: &stdout,
		ErrorStream:  &stderr,
		InputStream:  reader,
		RawTerminal:  true,
	})
	if err != nil {
		return err
	}

	fmt.Printf("stdout: '%s'\n", stdout.String())
	return nil
}

func main() {
	endpoint := flag.String("endpoint", "unix:///var/run/docker.sock", "Docker socket")

	client, err := docker.NewClient(*endpoint)
	if err != nil {
		fmt.Printf("Error (%s) %s\n", *endpoint, err.Error())
		os.Exit(1)
	}

	events := make(chan *docker.APIEvents)
	client.AddEventListener(events)
	fmt.Println("Listening...")

	for {
		select {
		case event := <-events:
			switch event.Status {
			case "start":
				fmt.Printf("> START %s: %s\n", event.From, event.ID)

				container, err := client.InspectContainer(event.ID)
				if err != nil {
					fmt.Printf("  %s\n", err.Error())
					break
				}

				fmt.Printf("  %+v\n", container.Config.Healthcheck)
				if container.Config.Healthcheck == nil || len(container.Config.Healthcheck.Test) == 0 {
					fmt.Printf("  No Healthcheck, assuming container started.")
					break
				}

				healtcheck(client, container)

			case "stop":
				fmt.Printf("> STOP %s: %s\n", event.From, event.ID)
			case "die":
				fmt.Printf("> DIE %s: %s\n", event.From, event.ID)
			}
		}
	}
}
<|endoftext|>


<commit_before>package isvcs

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type ISvc struct {
	Name	   string
	Dockerfile string
	Tag		   string
	Ports	   []int
}

func (s *ISvc) exists() (bool, error) {

	cmd := exec.Command("docker", "images", s.Tag)
	output, err := cmd.Output()
	if err != nil {
		return false, err
	}
	return strings.Contains(string(output), s.Tag), nil
}

func (s *ISvc) create() error {
	exists, err := s.exists()
	if err != nil || exists {
		return err
	}
	glog.V(1).Infof("Creating temp directory for building image: %s", s.Tag)
	tdir, err := ioutil.TempDir("", "isvc_")
	if err != nil {
		return err
	}
	dockerfile := tdir + "/Dockerfile"
	ioutil.WriteFile(dockerfile, []byte(s.Dockerfile), 0660)
	glog.V(0).Infof("building %s with dockerfile in %s", s.Tag, dockerfile)
	cmd := exec.Command("docker", "build", "-t", s.Tag, tdir)
	output, returnErr := cmd.CombinedOutput()
	if returnErr != nil {
		glog.Errorf("Problem running docker build: %s", string(output))
	}
	err = os.RemoveAll(tdir)
	if err != nil {
		glog.Warningf("Failed to cleanup directory :%s ", err)
	}
	return returnErr
}

func (s *ISvc) Running() (bool, error) {
	cmd := exec.Command("docker", "ps")
	output, err := cmd.Output()
	if err != nil {
		return false, err
	}
	return strings.Contains(string(output), s.Tag+":latest"), nil
}

func (s *ISvc) Run() error {

	err := s.create()
	if err != nil {
		return err
	}

	running, err := s.Running()
	if err != nil || running {
		return err
	}

	containerId, err := s.getContainerId()
	if err != nil && !IsSvcNotFoundErr(err) {
		return err
	}

	var cmd *exec.Cmd
	if containerId != "" {
		cmd = exec.Command("docker", "start", containerId)
	} else {
		args := make([]string, (len(s.Ports)) * 2 + 5)
		glog.V(1).Info("About to build.")
		args[0] = "run"
		args[1] = "-d"
		glog.V(1).Info("Ports %v", s.Ports)
		// add the ports to the arg list
		for i, port := range s.Ports {
			args[2+i*2] = "-p"
			args[2+i*2+1] = fmt.Sprintf("%d:%d", port, port)
		}
		servicedHome := serviced.ServiceDHome()
		resourcesDir := servicedHome + "/resources"
		// bind mount the resources directory, always make it /usr/local/serviced to simplify the dockerfile commands
		containerServiceDResources := "/usr/local/serviced/resources"
		args[(len(s.Ports))*2+2] = "-v"
		args[(len(s.Ports))*2+3] = fmt.Sprintf("%s:%s", resourcesDir, containerServiceDResources)

		// specify the image
		args[(len(s.Ports))*2+4] = s.Tag
		cmd = exec.Command("docker", args...)
	}
	glog.V(0).Info("Running docker cmd: ", cmd)
	return	cmd.Run()
}

func (s *ISvc) Stop() error {
	containerId, err := s.getContainerId()
	if err != nil {
		return err
	}
	cmd := exec.Command("docker", "stop", containerId)
	return cmd.Run()
}

func (s *ISvc) Kill() error {
	containerId, err := s.getContainerId()
	if err != nil {
		return err
	}
	cmd := exec.Command("docker", "kill", containerId)
	return cmd.Run()
}

var SvcNotFoundErr error
var IsSvcNotFoundErr func(error) bool

const msgSvcNotFoundErr = "svc not found"

func init() {
	SvcNotFoundErr = fmt.Errorf(msgSvcNotFoundErr)
	IsSvcNotFoundErr = func(err error) bool {
		if err == nil {
			return false
		}
		return err.Error() == msgSvcNotFoundErr
	}
}

func (s *ISvc) getContainerId() (string, error) {
	cmd := exec.Command("docker", "ps", "-a")
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, s.Tag+":latest") {
			fields := strings.Fields(line)
			return fields[0], nil
		}
	}
	return "", SvcNotFoundErr
}
<commit_msg>go fmt<commit_after>package isvcs

import (
	"fmt"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type ISvc struct {
	Name       string
	Dockerfile string
	Tag        string
	Ports      []int
}

func (s *ISvc) exists() (bool, error) {

	cmd := exec.Command("docker", "images", s.Tag)
	output, err := cmd.Output()
	if err != nil {
		return false, err
	}
	return strings.Contains(string(output), s.Tag), nil
}

func (s *ISvc) create() error {
	exists, err := s.exists()
	if err != nil || exists {
		return err
	}
	glog.V(1).Infof("Creating temp directory for building image: %s", s.Tag)
	tdir, err := ioutil.TempDir("", "isvc_")
	if err != nil {
		return err
	}
	dockerfile := tdir + "/Dockerfile"
	ioutil.WriteFile(dockerfile, []byte(s.Dockerfile), 0660)
	glog.V(0).Infof("building %s with dockerfile in %s", s.Tag, dockerfile)
	cmd := exec.Command("docker", "build", "-t", s.Tag, tdir)
	output, returnErr := cmd.CombinedOutput()
	if returnErr != nil {
		glog.Errorf("Problem running docker build: %s", string(output))
	}
	err = os.RemoveAll(tdir)
	if err != nil {
		glog.Warningf("Failed to cleanup directory :%s ", err)
	}
	return returnErr
}

func (s *ISvc) Running() (bool, error) {
	cmd := exec.Command("docker", "ps")
	output, err := cmd.Output()
	if err != nil {
		return false, err
	}
	return strings.Contains(string(output), s.Tag+":latest"), nil
}

func (s *ISvc) Run() error {

	err := s.create()
	if err != nil {
		return err
	}

	running, err := s.Running()
	if err != nil || running {
		return err
	}

	containerId, err := s.getContainerId()
	if err != nil && !IsSvcNotFoundErr(err) {
		return err
	}

	var cmd *exec.Cmd
	if containerId != "" {
		cmd = exec.Command("docker", "start", containerId)
	} else {
		args := make([]string, (len(s.Ports))*2+5)
		glog.V(1).Info("About to build.")
		args[0] = "run"
		args[1] = "-d"
		glog.V(1).Info("Ports %v", s.Ports)
		// add the ports to the arg list
		for i, port := range s.Ports {
			args[2+i*2] = "-p"
			args[2+i*2+1] = fmt.Sprintf("%d:%d", port, port)
		}
		servicedHome := serviced.ServiceDHome()
		resourcesDir := servicedHome + "/resources"
		// bind mount the resources directory, always make it /usr/local/serviced to simplify the dockerfile commands
		containerServiceDResources := "/usr/local/serviced/resources"
		args[(len(s.Ports))*2+2] = "-v"
		args[(len(s.Ports))*2+3] = fmt.Sprintf("%s:%s", resourcesDir, containerServiceDResources)

		// specify the image
		args[(len(s.Ports))*2+4] = s.Tag
		cmd = exec.Command("docker", args...)
	}
	glog.V(0).Info("Running docker cmd: ", cmd)
	return cmd.Run()
}

func (s *ISvc) Stop() error {
	containerId, err := s.getContainerId()
	if err != nil {
		return err
	}
	cmd := exec.Command("docker", "stop", containerId)
	return cmd.Run()
}

func (s *ISvc) Kill() error {
	containerId, err := s.getContainerId()
	if err != nil {
		return err
	}
	cmd := exec.Command("docker", "kill", containerId)
	return cmd.Run()
}

var SvcNotFoundErr error
var IsSvcNotFoundErr func(error) bool

const msgSvcNotFoundErr = "svc not found"

func init() {
	SvcNotFoundErr = fmt.Errorf(msgSvcNotFoundErr)
	IsSvcNotFoundErr = func(err error) bool {
		if err == nil {
			return false
		}
		return err.Error() == msgSvcNotFoundErr
	}
}

func (s *ISvc) getContainerId() (string, error) {
	cmd := exec.Command("docker", "ps", "-a")
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	for _, line := range strings.Split(string(output), "\n") {
		if strings.Contains(line, s.Tag+":latest") {
			fields := strings.Fields(line)
			return fields[0], nil
		}
	}
	return "", SvcNotFoundErr
}
<|endoftext|>