<commit_before>package organisations

import (
	"errors"
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (organisation Organisation, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instantiate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (x) RETURN ID(x) LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	log.Debugf("CheckConnectivity results:%+v  err: %+v", results, err)
	return err
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	O struct {
		ID        string
		Types     []string
		LEICode   string
		PrefLabel string
		Labels    []string
	}
	Parent struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Ind struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Sub []struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	PM []struct {
		M struct {
			ID           string
			Types        []string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		P struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (organisation Organisation, found bool, err error) {
	organisation = Organisation{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
				MATCH (o:Organisation{uuid:{uuid}})
	   OPTIONAL MATCH (o)<-[:HAS_ORGANISATION]-(m:Membership)
	   OPTIONAL MATCH (m)-[:HAS_MEMBER]->(p:Person)
	   OPTIONAL MATCH (p)<-[:MENTIONS]-(poc:Content)-[:MENTIONS]->(o)
	   WITH    o,
	   { id:p.uuid, types:labels(p), prefLabel:p.prefLabel} as p,
	   { id:m.uuid, prefLabel:m.prefLabel, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}], annCount:COUNT(poc) } as m ORDER BY m.annCount DESC LIMIT 1000
	   WITH o, collect({m:m, p:p}) as pm
	   OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:IndustryClassification)
	   WITH o, pm,
	   { id:ind.uuid, types:labels(ind), prefLabel:ind.prefLabel} as ind
	   WITH o, pm, ind
	   OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(parent:Organisation)
	   OPTIONAL MATCH (o)<-[:SUB_ORGANISATION_OF]-(sub:Organisation)
	   OPTIONAL MATCH (soc:Content)-[ms:MENTIONS]->(sub)
	   WITH o, pm, ind,
	   { id:parent.uuid, types:labels(parent), prefLabel:parent.prefLabel} as parent,
	   { id:sub.uuid, types:labels(sub), prefLabel:sub.prefLabel, annCount:COUNT(ms) } as sub ORDER BY sub.annCount DESC
	   WITH o, pm, ind, parent, collect(sub) as sub
	      WITH pm, ind, parent, sub, { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.aliases, leicode:o.leiCode} as o
	   return collect ({o:o, parent:parent, ind:ind, sub:sub, pm:pm}) as rs
							`,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Organisation{}, false, fmt.Errorf("Error accessing Organisation datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadOrganisation for uuid: %s was: %+v", uuid, results)
	if (len(results)) == 0 || len(results[0].Rs) == 0 {
		return Organisation{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple organisations found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Organisation{}, true, errors.New(errMsg)
	}
	organisation = neoReadStructToOrganisation(results[0].Rs[0])
	log.Debugf("Returning %v", organisation)
	return organisation, true, nil
}

func neoReadStructToOrganisation(neo neoReadStruct) Organisation {
	//TODO find out why we only get two memberships here compared to 17 off PROD graphDB... also, performance of e.g. Barclays
	public := Organisation{}
	public.Thing = &Thing{}
	public.ID = mapper.IDURL(neo.O.ID)
	public.APIURL = mapper.APIURL(neo.O.ID, neo.O.Types)
	public.Types = mapper.TypeURIs(neo.O.Types)
	public.LEICode = neo.O.LEICode
	public.PrefLabel = neo.O.PrefLabel
	if len(neo.O.Labels) > 0 {
		public.Labels = &neo.O.Labels
	}

	if neo.Ind.ID != "" {
		public.IndustryClassification = &IndustryClassification{}
		public.IndustryClassification.Thing = &Thing{}
		public.IndustryClassification.ID = mapper.IDURL(neo.Ind.ID)
		public.IndustryClassification.APIURL = mapper.APIURL(neo.Ind.ID, neo.Ind.Types)
		public.IndustryClassification.PrefLabel = neo.Ind.PrefLabel
	}

	if neo.Parent.ID != "" {
		public.Parent = &Parent{}
		public.Parent.Thing = &Thing{}
		public.Parent.ID = mapper.IDURL(neo.Parent.ID)
		public.Parent.APIURL = mapper.APIURL(neo.Parent.ID, neo.Parent.Types)
		public.Parent.Types = mapper.TypeURIs(neo.Parent.Types)
		public.Parent.PrefLabel = neo.Parent.PrefLabel
	}

	if len(neo.Sub) == 1 && neo.Sub[0].ID == "" {
		public.Subsidiaries = make([]Subsidiary, 0, 0)
	} else {
		public.Subsidiaries = make([]Subsidiary, len(neo.Sub))
		for idx, neoSub := range neo.Sub {
			subsidiary := Subsidiary{}
			subsidiary.Thing = &Thing{}
			subsidiary.ID = mapper.IDURL(neoSub.ID)
			subsidiary.APIURL = mapper.APIURL(neoSub.ID, neoSub.Types)
			subsidiary.Types = mapper.TypeURIs(neoSub.Types)
			subsidiary.PrefLabel = neoSub.PrefLabel
			public.Subsidiaries[idx] = subsidiary
		}
	}

	if len(neo.PM) == 1 && (neo.PM[0].M.ID == "") {
		public.Memberships = make([]Membership, 0, 0)
	} else {
		public.Memberships = make([]Membership, len(neo.PM))
		for mIdx, neoMem := range neo.PM {
			membership := Membership{}
			membership.Title = neoMem.M.PrefLabel
			membership.Person = Person{}
			membership.Person.Thing = &Thing{}
			membership.Person.ID = mapper.IDURL(neoMem.P.ID)
			membership.Person.APIURL = mapper.APIURL(neoMem.P.ID, neoMem.P.Types)
			membership.Person.Types = mapper.TypeURIs(neoMem.P.Types)
			membership.Person.PrefLabel = neoMem.P.PrefLabel
			if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
				membership.ChangeEvents = b
			}
			public.Memberships[mIdx] = membership
		}
	}
	log.Debugf("neoReadStructToOrganisation neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
	var results []ChangeEvent
	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, &results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	log.Debugf("changeEvent converted: %+v result:%+v", neoChgEvts, results)
	return true, &results
}
<commit_msg>Changed the change events to only show seconds not ms<commit_after>package organisations

import (
	"errors"
	"fmt"
	"time"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (organisation Organisation, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instantiate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (x) RETURN ID(x) LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	log.Debugf("CheckConnectivity results:%+v  err: %+v", results, err)
	return err
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	O struct {
		ID        string
		Types     []string
		LEICode   string
		PrefLabel string
		Labels    []string
	}
	Parent struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Ind struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	Sub []struct {
		ID        string
		Types     []string
		PrefLabel string
	}
	PM []struct {
		M struct {
			ID           string
			Types        []string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		P struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (organisation Organisation, found bool, err error) {
	organisation = Organisation{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
				MATCH (o:Organisation{uuid:{uuid}})
	   OPTIONAL MATCH (o)<-[:HAS_ORGANISATION]-(m:Membership)
	   OPTIONAL MATCH (m)-[:HAS_MEMBER]->(p:Person)
	   OPTIONAL MATCH (p)<-[:MENTIONS]-(poc:Content)-[:MENTIONS]->(o)
	   WITH    o,
	   { id:p.uuid, types:labels(p), prefLabel:p.prefLabel} as p,
	   { id:m.uuid, prefLabel:m.prefLabel, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}], annCount:COUNT(poc) } as m ORDER BY m.annCount DESC LIMIT 1000
	   WITH o, collect({m:m, p:p}) as pm
	   OPTIONAL MATCH (o)-[:HAS_CLASSIFICATION]->(ind:IndustryClassification)
	   WITH o, pm,
	   { id:ind.uuid, types:labels(ind), prefLabel:ind.prefLabel} as ind
	   WITH o, pm, ind
	   OPTIONAL MATCH (o)-[:SUB_ORGANISATION_OF]->(parent:Organisation)
	   OPTIONAL MATCH (o)<-[:SUB_ORGANISATION_OF]-(sub:Organisation)
	   OPTIONAL MATCH (soc:Content)-[ms:MENTIONS]->(sub)
	   WITH o, pm, ind,
	   { id:parent.uuid, types:labels(parent), prefLabel:parent.prefLabel} as parent,
	   { id:sub.uuid, types:labels(sub), prefLabel:sub.prefLabel, annCount:COUNT(ms) } as sub ORDER BY sub.annCount DESC
	   WITH o, pm, ind, parent, collect(sub) as sub
	      WITH pm, ind, parent, sub, { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.aliases, leicode:o.leiCode} as o
	   return collect ({o:o, parent:parent, ind:ind, sub:sub, pm:pm}) as rs
							`,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Organisation{}, false, fmt.Errorf("Error accessing Organisation datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadOrganisation for uuid: %s was: %+v", uuid, results)
	if (len(results)) == 0 || len(results[0].Rs) == 0 {
		return Organisation{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple organisations found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Organisation{}, true, errors.New(errMsg)
	}
	organisation = neoReadStructToOrganisation(results[0].Rs[0])
	log.Debugf("Returning %v", organisation)
	return organisation, true, nil
}

func neoReadStructToOrganisation(neo neoReadStruct) Organisation {
	//TODO find out why we only get two memberships here compared to 17 off PROD graphDB... also, performance of e.g. Barclays
	public := Organisation{}
	public.Thing = &Thing{}
	public.ID = mapper.IDURL(neo.O.ID)
	public.APIURL = mapper.APIURL(neo.O.ID, neo.O.Types)
	public.Types = mapper.TypeURIs(neo.O.Types)
	public.LEICode = neo.O.LEICode
	public.PrefLabel = neo.O.PrefLabel
	if len(neo.O.Labels) > 0 {
		public.Labels = &neo.O.Labels
	}

	if neo.Ind.ID != "" {
		public.IndustryClassification = &IndustryClassification{}
		public.IndustryClassification.Thing = &Thing{}
		public.IndustryClassification.ID = mapper.IDURL(neo.Ind.ID)
		public.IndustryClassification.APIURL = mapper.APIURL(neo.Ind.ID, neo.Ind.Types)
		public.IndustryClassification.PrefLabel = neo.Ind.PrefLabel
	}

	if neo.Parent.ID != "" {
		public.Parent = &Parent{}
		public.Parent.Thing = &Thing{}
		public.Parent.ID = mapper.IDURL(neo.Parent.ID)
		public.Parent.APIURL = mapper.APIURL(neo.Parent.ID, neo.Parent.Types)
		public.Parent.Types = mapper.TypeURIs(neo.Parent.Types)
		public.Parent.PrefLabel = neo.Parent.PrefLabel
	}

	if len(neo.Sub) == 1 && neo.Sub[0].ID == "" {
		public.Subsidiaries = make([]Subsidiary, 0, 0)
	} else {
		public.Subsidiaries = make([]Subsidiary, len(neo.Sub))
		for idx, neoSub := range neo.Sub {
			subsidiary := Subsidiary{}
			subsidiary.Thing = &Thing{}
			subsidiary.ID = mapper.IDURL(neoSub.ID)
			subsidiary.APIURL = mapper.APIURL(neoSub.ID, neoSub.Types)
			subsidiary.Types = mapper.TypeURIs(neoSub.Types)
			subsidiary.PrefLabel = neoSub.PrefLabel
			public.Subsidiaries[idx] = subsidiary
		}
	}

	if len(neo.PM) == 1 && (neo.PM[0].M.ID == "") {
		public.Memberships = make([]Membership, 0, 0)
	} else {
		public.Memberships = make([]Membership, len(neo.PM))
		for mIdx, neoMem := range neo.PM {
			membership := Membership{}
			membership.Title = neoMem.M.PrefLabel
			membership.Person = Person{}
			membership.Person.Thing = &Thing{}
			membership.Person.ID = mapper.IDURL(neoMem.P.ID)
			membership.Person.APIURL = mapper.APIURL(neoMem.P.ID, neoMem.P.Types)
			membership.Person.Types = mapper.TypeURIs(neoMem.P.Types)
			membership.Person.PrefLabel = neoMem.P.PrefLabel
			if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
				membership.ChangeEvents = b
			}
			public.Memberships[mIdx] = membership
		}
	}
	log.Debugf("neoReadStructToOrganisation neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, *[]ChangeEvent) {
	var results []ChangeEvent
	layout := "2000-01-01T00:00:00Z"

	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, &results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			t, _ := time.Parse(layout, neoChgEvt.StartedAt)
			results = append(results, ChangeEvent{StartedAt: t.Format(layout)})
		}
		if neoChgEvt.EndedAt != "" {
			t, _ := time.Parse(layout, neoChgEvt.EndedAt)
			results = append(results, ChangeEvent{EndedAt: t.Format(layout)})
		}
	}
	log.Debugf("changeEvent converted: %+v result:%+v", neoChgEvts, results)
	return true, &results
}
<|endoftext|>


<commit_before>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package param

import (
	"strconv"
	"strings"
)

type StringSlice []string

func (p StringSlice) String() []string {
	return []string(p)
}

func (p StringSlice) GetByIndex(i int, defaults ...string) string {
	if len(p) > i {
		return p[i]
	}
	if len(defaults) > 0 {
		return defaults[0]
	}
	return ``
}

func (p StringSlice) Unique() StringSlice {
	record := map[string]struct{}{}
	result := StringSlice{}
	for _, s := range p {
		if _, ok := record[s]; !ok {
			record[s] = struct{}{}
			result = append(result, s)
		}
	}
	return result
}

func defaultStringFilterFunc(s *string) bool {
	if s == nil {
		return false
	}
	return len(*s) > 0
}

func (p StringSlice) Filter(filterFuncs ...func(s *string) bool) StringSlice {
	filterFunc := defaultStringFilterFunc
	if len(filterFuncs) > 0 {
		filterFunc = filterFuncs[0]
		if filterFunc == nil {
			return p
		}
	}
	result := StringSlice{}
	for _, s := range p {
		if filterFunc(&s) {
			result = append(result, s)
		}
	}
	return result
}

func (p StringSlice) HasValue(v interface{}) bool {
	expected := AsString(v)
	for _, val := range p {
		if val == expected {
			return true
		}
	}
	return false
}

func (p StringSlice) Size() int {
	return len(p)
}

func (p StringSlice) Join(sep string) string {
	return strings.Join([]string(p), sep)
}

func (p StringSlice) Interface(filters ...func(int, string) bool) []interface{} {
	var filter func(int, string) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []interface{}
	for idx, id := range p {
		if filter == nil || filter(idx, id) {
			ids = append(ids, interface{}(id))
		}
	}
	return ids
}

func (p StringSlice) Int(filters ...func(int, int) bool) []int {
	var filter func(int, int) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int
	for idx, id := range p {
		i, _ := strconv.Atoi(strings.TrimSpace(id))
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Int64(filters ...func(int, int64) bool) []int64 {
	var filter func(int, int64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int64
	for idx, id := range p {
		i, _ := strconv.ParseInt(strings.TrimSpace(id), 10, 64)
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Int32(filters ...func(int, int32) bool) []int32 {
	var filter func(int, int32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int32
	for idx, id := range p {
		i, _ := strconv.ParseInt(strings.TrimSpace(id), 10, 32)
		iv := int32(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Uint(filters ...func(int, uint) bool) []uint {
	var filter func(int, uint) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
		iv := uint(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Uint64(filters ...func(int, uint64) bool) []uint64 {
	var filter func(int, uint64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint64
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Uint32(filters ...func(int, uint32) bool) []uint32 {
	var filter func(int, uint32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint32
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 32)
		iv := uint32(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Float32(filters ...func(int, float32) bool) []float32 {
	var filter func(int, float32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []float32
	for idx, v := range p {
		i, _ := strconv.ParseFloat(strings.TrimSpace(v), 32)
		iv := float32(i)
		if filter == nil || filter(idx, iv) {
			values = append(values, iv)
		}
	}
	return values
}

func (p StringSlice) Float64(filters ...func(int, float64) bool) []float64 {
	var filter func(int, float64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []float64
	for idx, v := range p {
		i, _ := strconv.ParseFloat(strings.TrimSpace(v), 64)
		if filter == nil || filter(idx, i) {
			values = append(values, i)
		}
	}
	return values
}

func (p StringSlice) Bool(filters ...func(int, bool) bool) []bool {
	var filter func(int, bool) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []bool
	for idx, v := range p {
		i, e := strconv.ParseBool(strings.TrimSpace(v))
		if e != nil {
			continue
		}
		if filter == nil || filter(idx, i) {
			values = append(values, i)
		}
	}
	return values
}

func GetByIndex(v interface{}, i int, defaults ...interface{}) interface{} {
	switch p := v.(type) {
	case []string:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return ``
	case []interface{}:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return nil
	case []int:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []int32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []int64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []float32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []float64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []bool:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return false
	default:
		if len(defaults) > 0 {
			return defaults[0]
		}
		return nil
	}
}
<commit_msg>update<commit_after>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package param

import (
	"strconv"
	"strings"
)

type StringSlice []string

func (p StringSlice) String() []string {
	return []string(p)
}

func (p StringSlice) GetByIndex(i int, defaults ...string) string {
	if len(p) > i {
		return p[i]
	}
	if len(defaults) > 0 {
		return defaults[0]
	}
	return ``
}

func (p StringSlice) Unique() StringSlice {
	record := map[string]struct{}{}
	result := StringSlice{}
	for _, s := range p {
		if _, ok := record[s]; !ok {
			record[s] = struct{}{}
			result = append(result, s)
		}
	}
	return result
}

func (p StringSlice) Split(sep string, limit ...int) StringSlice {
	result := StringSlice{}
	for _, s := range p {
		if len(s) == 0 {
			continue
		}
		var sl []string
		if len(limit) > 0 {
			sl = strings.SplitN(s, sep, limit[0])
		} else {
			sl = strings.Split(s, sep)
		}
		result = append(result, sl...)
	}
	return result
}

func (p StringSlice) SplitAny(sep string, limit ...int) StringSlice {
	result := StringSlice{}
	for _, s := range p {
		var sl []string
		if len(limit) > 0 {
			sl = strings.SplitN(s, sep, limit[0])
		} else {
			sl = strings.Split(s, sep)
		}
		result = append(result, sl...)
	}
	return result
}

func defaultStringFilterFunc(s *string) bool {
	if s == nil {
		return false
	}
	return len(*s) > 0
}

func (p StringSlice) Filter(filterFuncs ...func(s *string) bool) StringSlice {
	filterFunc := defaultStringFilterFunc
	if len(filterFuncs) > 0 {
		filterFunc = filterFuncs[0]
		if filterFunc == nil {
			return p
		}
	}
	result := StringSlice{}
	for _, s := range p {
		if filterFunc(&s) {
			result = append(result, s)
		}
	}
	return result
}

func (p StringSlice) HasValue(v interface{}) bool {
	expected := AsString(v)
	for _, val := range p {
		if val == expected {
			return true
		}
	}
	return false
}

func (p StringSlice) Size() int {
	return len(p)
}

func (p StringSlice) Join(sep string) string {
	return strings.Join([]string(p), sep)
}

func (p StringSlice) Interface(filters ...func(int, string) bool) []interface{} {
	var filter func(int, string) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []interface{}
	for idx, id := range p {
		if filter == nil || filter(idx, id) {
			ids = append(ids, interface{}(id))
		}
	}
	return ids
}

func (p StringSlice) Int(filters ...func(int, int) bool) []int {
	var filter func(int, int) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int
	for idx, id := range p {
		i, _ := strconv.Atoi(strings.TrimSpace(id))
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Int64(filters ...func(int, int64) bool) []int64 {
	var filter func(int, int64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int64
	for idx, id := range p {
		i, _ := strconv.ParseInt(strings.TrimSpace(id), 10, 64)
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Int32(filters ...func(int, int32) bool) []int32 {
	var filter func(int, int32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []int32
	for idx, id := range p {
		i, _ := strconv.ParseInt(strings.TrimSpace(id), 10, 32)
		iv := int32(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Uint(filters ...func(int, uint) bool) []uint {
	var filter func(int, uint) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
		iv := uint(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Uint64(filters ...func(int, uint64) bool) []uint64 {
	var filter func(int, uint64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint64
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 64)
		if filter == nil || filter(idx, i) {
			ids = append(ids, i)
		}
	}
	return ids
}

func (p StringSlice) Uint32(filters ...func(int, uint32) bool) []uint32 {
	var filter func(int, uint32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var ids []uint32
	for idx, id := range p {
		i, _ := strconv.ParseUint(strings.TrimSpace(id), 10, 32)
		iv := uint32(i)
		if filter == nil || filter(idx, iv) {
			ids = append(ids, iv)
		}
	}
	return ids
}

func (p StringSlice) Float32(filters ...func(int, float32) bool) []float32 {
	var filter func(int, float32) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []float32
	for idx, v := range p {
		i, _ := strconv.ParseFloat(strings.TrimSpace(v), 32)
		iv := float32(i)
		if filter == nil || filter(idx, iv) {
			values = append(values, iv)
		}
	}
	return values
}

func (p StringSlice) Float64(filters ...func(int, float64) bool) []float64 {
	var filter func(int, float64) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []float64
	for idx, v := range p {
		i, _ := strconv.ParseFloat(strings.TrimSpace(v), 64)
		if filter == nil || filter(idx, i) {
			values = append(values, i)
		}
	}
	return values
}

func (p StringSlice) Bool(filters ...func(int, bool) bool) []bool {
	var filter func(int, bool) bool
	if len(filters) > 0 {
		filter = filters[0]
	}
	var values []bool
	for idx, v := range p {
		i, e := strconv.ParseBool(strings.TrimSpace(v))
		if e != nil {
			continue
		}
		if filter == nil || filter(idx, i) {
			values = append(values, i)
		}
	}
	return values
}

func GetByIndex(v interface{}, i int, defaults ...interface{}) interface{} {
	switch p := v.(type) {
	case []string:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return ``
	case []interface{}:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return nil
	case []int:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []int32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []int64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []uint64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []float32:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []float64:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return 0
	case []bool:
		if len(p) > i {
			return p[i]
		}
		if len(defaults) > 0 {
			return defaults[0]
		}
		return false
	default:
		if len(defaults) > 0 {
			return defaults[0]
		}
		return nil
	}
}
<|endoftext|>


<commit_before>package pilosa

import (
	"errors"
	"regexp"

	"github.com/pilosa/pilosa/internal"
)

// System errors.
var (
	ErrHostRequired = errors.New("host required")

	ErrDatabaseRequired = errors.New("database required")
	ErrDatabaseExists   = errors.New("database already exists")
	ErrDatabaseNotFound = errors.New("database not found")

	// ErrFrameRequired is returned when no frame is specified.
	ErrFrameRequired = errors.New("frame required")
	ErrFrameExists   = errors.New("frame already exists")
	ErrFrameNotFound = errors.New("frame not found")

	ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]")

	// ErrFragmentNotFound is returned when a fragment does not exist.
	ErrFragmentNotFound = errors.New("fragment not found")
	ErrQueryRequired    = errors.New("query required")
)

// Regular expression to valuate db and frame's name
// Todo: remove . when frame doesn't require . for topN
var nameRegexp = regexp.MustCompile(`^([^.][a-z0-9._-]{1,64}$)`)

// Profile represents vertical column in a database.
// A profile can have a set of attributes attached to it.
type Profile struct {
	ID    uint64                 `json:"id"`
	Attrs map[string]interface{} `json:"attrs,omitempty"`
}

// encodeProfiles converts a into its internal representation.
func encodeProfiles(a []*Profile) []*internal.Profile {
	other := make([]*internal.Profile, len(a))
	for i := range a {
		other[i] = encodeProfile(a[i])
	}
	return other
}

// decodeProfiles converts a from its internal representation.
func decodeProfiles(a []*internal.Profile) []*Profile {
	other := make([]*Profile, len(a))
	for i := range a {
		other[i] = decodeProfile(a[i])
	}
	return other
}

// encodeProfile converts p into its internal representation.
func encodeProfile(p *Profile) *internal.Profile {
	return &internal.Profile{
		ID:    p.ID,
		Attrs: encodeAttrs(p.Attrs),
	}
}

// decodeProfile converts b from its internal representation.
func decodeProfile(pb *internal.Profile) *Profile {
	p := &Profile{
		ID: pb.ID,
	}

	if len(pb.Attrs) > 0 {
		p.Attrs = make(map[string]interface{}, len(pb.Attrs))
		for _, attr := range pb.Attrs {
			k, v := decodeAttr(attr)
			p.Attrs[k] = v
		}
	}

	return p
}

// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"

// Restrict name using regex
func ValidateName(name string) error {
	validName := nameRegexp.Match([]byte(name))
	if validName == false {
		return ErrName
	}
	return nil
}
<commit_msg>invalid name start with dot<commit_after>package pilosa

import (
	"errors"
	"regexp"

	"github.com/pilosa/pilosa/internal"
)

// System errors.
var (
	ErrHostRequired = errors.New("host required")

	ErrDatabaseRequired = errors.New("database required")
	ErrDatabaseExists   = errors.New("database already exists")
	ErrDatabaseNotFound = errors.New("database not found")

	// ErrFrameRequired is returned when no frame is specified.
	ErrFrameRequired = errors.New("frame required")
	ErrFrameExists   = errors.New("frame already exists")
	ErrFrameNotFound = errors.New("frame not found")

	ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]")

	// ErrFragmentNotFound is returned when a fragment does not exist.
	ErrFragmentNotFound = errors.New("fragment not found")
	ErrQueryRequired    = errors.New("query required")
)

// Regular expression to valuate db and frame's name
// Todo: remove . when frame doesn't require . for topN
var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`)

// Profile represents vertical column in a database.
// A profile can have a set of attributes attached to it.
type Profile struct {
	ID    uint64                 `json:"id"`
	Attrs map[string]interface{} `json:"attrs,omitempty"`
}

// encodeProfiles converts a into its internal representation.
func encodeProfiles(a []*Profile) []*internal.Profile {
	other := make([]*internal.Profile, len(a))
	for i := range a {
		other[i] = encodeProfile(a[i])
	}
	return other
}

// decodeProfiles converts a from its internal representation.
func decodeProfiles(a []*internal.Profile) []*Profile {
	other := make([]*Profile, len(a))
	for i := range a {
		other[i] = decodeProfile(a[i])
	}
	return other
}

// encodeProfile converts p into its internal representation.
func encodeProfile(p *Profile) *internal.Profile {
	return &internal.Profile{
		ID:    p.ID,
		Attrs: encodeAttrs(p.Attrs),
	}
}

// decodeProfile converts b from its internal representation.
func decodeProfile(pb *internal.Profile) *Profile {
	p := &Profile{
		ID: pb.ID,
	}

	if len(pb.Attrs) > 0 {
		p.Attrs = make(map[string]interface{}, len(pb.Attrs))
		for _, attr := range pb.Attrs {
			k, v := decodeAttr(attr)
			p.Attrs[k] = v
		}
	}

	return p
}

// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"

// Restrict name using regex
func ValidateName(name string) error {
	validName := nameRegexp.Match([]byte(name))
	if validName == false {
		return ErrName
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"./cinemex"
	"./cinepolis"
)

func main() {
	os.Remove("./foo.db")

	db, err := sql.Open("sqlite3", "./foo.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := `
	create table funciones (id integer not null primary key autoincrement, cine text, edo text, col text, cineId text, cineName text, title text, img text, rating text, language text, roomType text, date text, time text);
	`
	_, err = db.Exec(sql)
	if err != nil {
		log.Printf("%q: %s\n", err, sql)
		return
	}

	tx, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare("insert into funciones(cine , edo , col , cineId, cineName , title , img, rating , language , roomType , date , time) values(? , ? , ? , ?, ? , ? , ? , ? , ? , ? , ?, ?)")
		if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	for _,screening := range cinemex.Screenings() {
		fmt.Print("mx.")
		bla,err := stmt.Exec(screening["cine"], screening["edo"] , screening["col"] , screening["cineId"], screening["cineName"] , screening["title"], screening["img"] , screening["rating"] , screening["language"] , screening["roomType"] , screening["date"] , screening["time"])
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%#v",bla)
	}

	for _,screening := range cinepolis.Screenings() {
		fmt.Print("pl.")
		bla,err := stmt.Exec(screening["cine"], screening["edo"] , screening["col"] , screening["cineId"], screening["cineName"] , screening["title"] , screening["img"], screening["rating"] , screening["language"] , screening["roomType"] , screening["date"] , screening["time"])
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%#v",bla)
	}

	tx.Commit()

/*	rows, err := db.Query("select id, name from foo")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		fmt.Println(id, name)
	}
	rows.Close()

	stmt, err = db.Prepare("select name from foo where id = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var name string
	err = stmt.QueryRow("3").Scan(&name)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(name)

	_, err = db.Exec("delete from foo")
	if err != nil {
		log.Fatal(err)
	}

	_, err = db.Exec("insert into foo(id, name) values(1, 'foo'), (2, 'bar'), (3, 'baz')")
	if err != nil {
		log.Fatal(err)
	}

	rows, err = db.Query("select id, name from foo")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		fmt.Println(id, name)
	}
*/
}
<commit_msg>refs #4: it creates de foo.db in the current directory<commit_after>package main

import (
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"os"
	"./cinemex"
	"./cinepolis"
)

func main() {
	os.Remove("foo.db")

	db, err := sql.Open("sqlite3", "foo.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	sql := `
	create table funciones (id integer not null primary key autoincrement, cine text, edo text, col text, cineId text, cineName text, title text, img text, rating text, language text, roomType text, date text, time text);
	`
	_, err = db.Exec(sql)
	if err != nil {
		log.Printf("%q: %s\n", err, sql)
		return
	}

	tx, err := db.Begin()
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := tx.Prepare("insert into funciones(cine , edo , col , cineId, cineName , title , img, rating , language , roomType , date , time) values(? , ? , ? , ?, ? , ? , ? , ? , ? , ? , ?, ?)")
		if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	for _,screening := range cinemex.Screenings() {
		fmt.Print("mx.")
		bla,err := stmt.Exec(screening["cine"], screening["edo"] , screening["col"] , screening["cineId"], screening["cineName"] , screening["title"], screening["img"] , screening["rating"] , screening["language"] , screening["roomType"] , screening["date"] , screening["time"])
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%#v",bla)
	}

	for _,screening := range cinepolis.Screenings() {
		fmt.Print("pl.")
		bla,err := stmt.Exec(screening["cine"], screening["edo"] , screening["col"] , screening["cineId"], screening["cineName"] , screening["title"] , screening["img"], screening["rating"] , screening["language"] , screening["roomType"] , screening["date"] , screening["time"])
		if err != nil {
			fmt.Println(err)
		}
		fmt.Printf("%#v",bla)
	}

	tx.Commit()

/*	rows, err := db.Query("select id, name from foo")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		fmt.Println(id, name)
	}
	rows.Close()

	stmt, err = db.Prepare("select name from foo where id = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()
	var name string
	err = stmt.QueryRow("3").Scan(&name)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(name)

	_, err = db.Exec("delete from foo")
	if err != nil {
		log.Fatal(err)
	}

	_, err = db.Exec("insert into foo(id, name) values(1, 'foo'), (2, 'bar'), (3, 'baz')")
	if err != nil {
		log.Fatal(err)
	}

	rows, err = db.Query("select id, name from foo")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var name string
		rows.Scan(&id, &name)
		fmt.Println(id, name)
	}
*/
}
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

// For ease of implementation, our database's interface matches Ethereum's
// database implementation. This was to allow use to use Geth code as is for the
// EVM chain.

package database

// Batch is a write-only database that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
type Batch interface {
	KeyValueWriter

	// Size retrieves the amount of data queued up for writing, this includes
	// the keys, values, and deleted keys.
	Size() int

	// Write flushes any accumulated data to disk.
	Write() error

	// Reset resets the batch for reuse.
	Reset()

	// Replay replays the batch contents.
	Replay(w KeyValueWriter) error

	// Inner returns a Batch writing to the inner database, if one exists. If
	// this batch is already writing to the base DB, then itself should be
	// returned.
	Inner() Batch
}

// Batcher wraps the NewBatch method of a backing data store.
type Batcher interface {
	// NewBatch creates a write-only database that buffers changes to its host db
	// until a final write is called.
	NewBatch() Batch
}
<commit_msg>Improve comment on Batch interface<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

// For ease of implementation, our database's interface matches Ethereum's
// database implementation. This was to allow use to use Geth code as is for the
// EVM chain.

package database

// Batch is a write-only database that commits changes to its host database
// when Write is called. A batch cannot be used concurrently.
type Batch interface {
	KeyValueWriter

	// Size retrieves the amount of data queued up for writing, this includes
	// the keys, values, and deleted keys.
	Size() int

	// Write flushes any accumulated data to disk.
	Write() error

	// Reset resets the batch for reuse.
	Reset()

	// Replay replays the batch contents in the same order they were written
	// to the batch.
	Replay(w KeyValueWriter) error

	// Inner returns a Batch writing to the inner database, if one exists. If
	// this batch is already writing to the base DB, then itself should be
	// returned.
	Inner() Batch
}

// Batcher wraps the NewBatch method of a backing data store.
type Batcher interface {
	// NewBatch creates a write-only database that buffers changes to its host db
	// until a final write is called.
	NewBatch() Batch
}
<|endoftext|>


<commit_before>package database

import (
	"log"
	"strconv"

	"github.com/blevesearch/bleve"

	"github.com/bobinette/papernet/models"
)

type bleveSearch struct {
	index bleve.Index
}

func NewBleveSearch(searchpath string) (Search, error) {
	index, err := bleve.Open(searchpath)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Print("Creating index...")
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(searchpath, mapping)
		log.Println("Done")
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}

	return &bleveSearch{
		index: index,
	}, nil
}

func (s *bleveSearch) Find(q string) ([]int, error) {
	query := bleve.NewMatchQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := s.index.Search(search)
	if err != nil {
		return nil, err
	}

	ids := make([]int, searchResults.Total)
	for i, hit := range searchResults.Hits {
		ids[i], err = strconv.Atoi(hit.ID)
		if err != nil {
			return nil, err
		}
	}
	return ids, nil
}

func (s *bleveSearch) Index(p *models.Paper) error {
	data := map[string]interface{}{
		"title": string(p.Title),
		"tags":  p.Tags,
	}

	err := s.index.Index(strconv.Itoa(p.ID), data)
	if err != nil {
		return err
	}
	return nil
}

func (s *bleveSearch) Close() error {
	return s.index.Close()
}
<commit_msg>Use prefix query in search - connects to #3<commit_after>package database

import (
	"log"
	"strconv"

	"github.com/blevesearch/bleve"

	"github.com/bobinette/papernet/models"
)

type bleveSearch struct {
	index bleve.Index
}

func NewBleveSearch(searchpath string) (Search, error) {
	index, err := bleve.Open(searchpath)
	if err == bleve.ErrorIndexPathDoesNotExist {
		log.Print("Creating index...")
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(searchpath, mapping)
		log.Println("Done")
		if err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}

	return &bleveSearch{
		index: index,
	}, nil
}

func (s *bleveSearch) Find(q string) ([]int, error) {
	query := bleve.NewPrefixQuery(q)
	search := bleve.NewSearchRequest(query)
	searchResults, err := s.index.Search(search)
	if err != nil {
		return nil, err
	}

	ids := make([]int, searchResults.Total)
	for i, hit := range searchResults.Hits {
		ids[i], err = strconv.Atoi(hit.ID)
		if err != nil {
			return nil, err
		}
	}
	return ids, nil
}

func (s *bleveSearch) Index(p *models.Paper) error {
	data := map[string]interface{}{
		"title": string(p.Title),
		"tags":  p.Tags,
	}

	err := s.index.Index(strconv.Itoa(p.ID), data)
	if err != nil {
		return err
	}
	return nil
}

func (s *bleveSearch) Close() error {
	return s.index.Close()
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"io/ioutil"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	gorums "github.com/relab/raft/raftgorums/gorumspb"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

// Node ties an instance of Raft to the network.
type Node struct {
	id uint64

	Raft    *Raft
	storage Storage

	grpcServer *grpc.Server

	lookup  map[uint64]int
	peers   []string
	cluster []uint64

	match map[uint32]chan uint64

	mgr  *gorums.Manager
	conf *gorums.Configuration

	logger logrus.FieldLogger
}

// NewNode returns a Node with an instance of Raft given the configuration.
func NewNode(server *grpc.Server, sm raft.StateMachine, cfg *Config) *Node {
	peers := make([]string, len(cfg.Servers))
	// We don't want to mutate cfg.Servers.
	copy(peers, cfg.Servers)

	id := cfg.ID
	// Exclude self.
	peers = append(peers[:id-1], peers[id:]...)

	var pos int
	lookup := make(map[uint64]int)

	for i := 1; i <= len(cfg.Servers); i++ {
		if uint64(i) == id {
			continue
		}

		lookup[uint64(i)] = pos
		pos++
	}

	if cfg.Logger == nil {
		l := logrus.New()
		l.Out = ioutil.Discard
		cfg.Logger = l
	}

	cfg.Logger = cfg.Logger.WithField("nodeid", cfg.ID)

	n := &Node{
		id:         id,
		Raft:       NewRaft(sm, cfg),
		storage:    cfg.Storage,
		grpcServer: server,
		lookup:     lookup,
		peers:      peers,
		cluster:    cfg.InitialCluster,
		match:      make(map[uint32]chan uint64),
		logger:     cfg.Logger,
	}

	return n
}

// Run start listening for incoming messages, and delivers outgoing messages.
func (n *Node) Run() error {
	opts := []gorums.ManagerOption{
		gorums.WithGrpcDialOptions(
			grpc.WithBlock(),
			grpc.WithInsecure(),
			grpc.WithTimeout(TCPConnect*time.Millisecond)),
	}

	mgr, err := gorums.NewManager(n.peers, opts...)

	if err != nil {
		return err
	}

	gorums.RegisterRaftServer(n.grpcServer, n)

	n.mgr = mgr

	var clusterIDs []uint32

	for _, id := range n.cluster {
		if n.id == id {
			// Exclude self.
			continue
		}
		clusterIDs = append(clusterIDs, n.getNodeID(id))
	}

	n.conf, err = mgr.NewConfiguration(clusterIDs, NewQuorumSpec(len(clusterIDs)+1))

	if err != nil {
		return err
	}

	for _, nodeID := range n.mgr.NodeIDs() {
		n.match[nodeID] = make(chan uint64, 1)
	}

	go n.Raft.Run()

	// January 1, 1970 UTC.
	var lastCuReq time.Time

	for {
		select {
		case err := <-n.conf.SubError():
			// TODO If a node becomes unavailable and there is a
			// backup available in the same or an alternate region,
			// instantiate reconfiguration. TODO How many errors
			// before a node is considered unavailable? If there is
			// no backup node available, don't do anything, but
			// schedule the reconfiguration.
			n.logger.WithField("nodeid", err.NodeID).Warnln("Node unavailable")
		case req := <-n.Raft.cureqout:
			// TODO Use config.
			if time.Since(lastCuReq) < 100*time.Millisecond {
				continue
			}
			lastCuReq = time.Now()

			n.logger.WithField("matchindex", req.matchIndex).Warnln("Sending catch-up")
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			leader, _ := n.mgr.Node(n.getNodeID(req.leaderID))
			_, err := leader.RaftClient.CatchMeUp(ctx, &pb.CatchMeUpRequest{
				FollowerID: n.id,
				NextIndex:  req.matchIndex + 1,
			})
			cancel()

			if err != nil {
				n.logger.WithError(err).Warnln("CatchMeUp failed")
			}
		case req := <-n.Raft.rvreqout:
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := n.conf.RequestVote(ctx, req)
			cancel()

			if err != nil {
				n.logger.WithError(err).Warnln("RequestVote failed")
			}

			if res == nil {
				continue
			}

			n.Raft.HandleRequestVoteResponse(res)

		case req := <-n.Raft.aereqout:
			next := make(map[uint32]uint64)
			nextIndex := req.PrevLogIndex + 1

			for nodeID, ch := range n.match {
				select {
				case index := <-ch:
					// TODO Acessing maxAppendEntries, safe but needs fix.
					atLeastMaxEntries := req.PrevLogIndex+1 > n.Raft.maxAppendEntries
					lessThenMaxEntriesBehind := index < req.PrevLogIndex+1-n.Raft.maxAppendEntries

					if atLeastMaxEntries && lessThenMaxEntriesBehind {
						n.logger.WithField("gorumsid", nodeID).Warnln("Server too far behind")
						index = req.PrevLogIndex + 1
					}
					next[nodeID] = index
					if index < nextIndex {
						nextIndex = index
					}
				default:
				}
			}

			// TODO This should be safe as it only accesses storage
			// which uses transactions. TODO It accesses
			// maxAppendEntries but this on does not change after
			// startup.
			entries := n.Raft.getNextEntries(nextIndex)
			e := uint64(len(entries))
			maxIndex := nextIndex + e - 1

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := n.conf.AppendEntries(ctx, req,
				// These functions will be executed concurrently.
				func(req pb.AppendEntriesRequest, nodeID uint32) *pb.AppendEntriesRequest {
					if index, ok := next[nodeID]; ok {
						req.PrevLogIndex = index - 1
						// TODO This should be safe as
						// it only accesses storage
						// which uses transactions.
						req.PrevLogTerm = n.Raft.logTerm(index - 1)
					}

					need := maxIndex - req.PrevLogIndex
					req.Entries = entries[e-need:]

					n.logger.WithFields(logrus.Fields{
						"prevlogindex": req.PrevLogIndex,
						"prevlogterm":  req.PrevLogTerm,
						"commitindex":  req.CommitIndex,
						"currentterm":  req.Term,
						"lenentries":   len(req.Entries),
						"gorumsid":     nodeID,
					}).Infoln("Sending AppendEntries")

					return &req
				},
			)

			if err != nil {
				n.logger.WithError(err).Warnln("AppendEntries failed")
			}

			if res == nil {
				continue
			}

			// Cancel on abort.
			if !res.Success {
				cancel()
			}

			n.Raft.HandleAppendEntriesResponse(res, res.Replies)
		}
	}
}

// RequestVote implements gorums.RaftServer.
func (n *Node) RequestVote(ctx context.Context, req *pb.RequestVoteRequest) (*pb.RequestVoteResponse, error) {
	return n.Raft.HandleRequestVoteRequest(req), nil
}

// AppendEntries implements gorums.RaftServer.
func (n *Node) AppendEntries(ctx context.Context, req *pb.AppendEntriesRequest) (*pb.AppendEntriesResponse, error) {
	return n.Raft.HandleAppendEntriesRequest(req), nil
}

// InstallSnapshot implements gorums.RaftServer.
func (n *Node) InstallSnapshot(ctx context.Context, snapshot *commonpb.Snapshot) (*pb.InstallSnapshotResponse, error) {
	return n.Raft.HandleInstallSnapshotRequest(snapshot), nil
}

// CatchMeUp implements gorums.RaftServer.
func (n *Node) CatchMeUp(ctx context.Context, req *pb.CatchMeUpRequest) (res *pb.Empty, err error) {
	res = &pb.Empty{}
	n.match[n.getNodeID(req.FollowerID)] <- req.NextIndex
	return
}

func (n *Node) getNodeID(raftID uint64) uint32 {
	return n.mgr.NodeIDs()[n.lookup[raftID]]
}
<commit_msg>raftgorums/node.go: Terminate server not part of cluster<commit_after>package raftgorums

import (
	"errors"
	"io/ioutil"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/grpc"

	"github.com/Sirupsen/logrus"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	gorums "github.com/relab/raft/raftgorums/gorumspb"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

// Node ties an instance of Raft to the network.
type Node struct {
	id uint64

	Raft    *Raft
	storage Storage

	grpcServer *grpc.Server

	lookup  map[uint64]int
	peers   []string
	cluster []uint64

	match map[uint32]chan uint64

	mgr  *gorums.Manager
	conf *gorums.Configuration

	logger logrus.FieldLogger
}

// NewNode returns a Node with an instance of Raft given the configuration.
func NewNode(server *grpc.Server, sm raft.StateMachine, cfg *Config) *Node {
	peers := make([]string, len(cfg.Servers))
	// We don't want to mutate cfg.Servers.
	copy(peers, cfg.Servers)

	id := cfg.ID
	// Exclude self.
	peers = append(peers[:id-1], peers[id:]...)

	var pos int
	lookup := make(map[uint64]int)

	for i := 1; i <= len(cfg.Servers); i++ {
		if uint64(i) == id {
			continue
		}

		lookup[uint64(i)] = pos
		pos++
	}

	if cfg.Logger == nil {
		l := logrus.New()
		l.Out = ioutil.Discard
		cfg.Logger = l
	}

	cfg.Logger = cfg.Logger.WithField("nodeid", cfg.ID)

	n := &Node{
		id:         id,
		Raft:       NewRaft(sm, cfg),
		storage:    cfg.Storage,
		grpcServer: server,
		lookup:     lookup,
		peers:      peers,
		cluster:    cfg.InitialCluster,
		match:      make(map[uint32]chan uint64),
		logger:     cfg.Logger,
	}

	return n
}

// Run start listening for incoming messages, and delivers outgoing messages.
func (n *Node) Run() error {
	opts := []gorums.ManagerOption{
		gorums.WithGrpcDialOptions(
			grpc.WithBlock(),
			grpc.WithInsecure(),
			grpc.WithTimeout(TCPConnect*time.Millisecond)),
	}

	mgr, err := gorums.NewManager(n.peers, opts...)

	if err != nil {
		return err
	}

	gorums.RegisterRaftServer(n.grpcServer, n)

	n.mgr = mgr

	var clusterIDs []uint32

	var active bool

	for _, id := range n.cluster {
		if n.id == id {
			// Exclude self.
			active = true
			continue
		}
		n.logger.WithField("serverid", id).Warnln("Added to cluster")
		clusterIDs = append(clusterIDs, n.getNodeID(id))
	}

	if !active {
		// TODO We terminate this server after 10s as it is not part of
		// the initial cluster. We need to change this so that the
		// server stays dormant until a leader invokes a catch-up
		// request on it.
		time.Sleep(10 * time.Second)
		return errors.New("not part of cluster")
	}

	n.conf, err = mgr.NewConfiguration(clusterIDs, NewQuorumSpec(len(clusterIDs)+1))

	if err != nil {
		return err
	}

	for _, nodeID := range n.mgr.NodeIDs() {
		n.match[nodeID] = make(chan uint64, 1)
	}

	go n.Raft.Run()

	// January 1, 1970 UTC.
	var lastCuReq time.Time

	for {
		select {
		case err := <-n.conf.SubError():
			// TODO If a node becomes unavailable and there is a
			// backup available in the same or an alternate region,
			// instantiate reconfiguration. TODO How many errors
			// before a node is considered unavailable? If there is
			// no backup node available, don't do anything, but
			// schedule the reconfiguration.
			n.logger.WithField("nodeid", err.NodeID).Warnln("Node unavailable")
		case req := <-n.Raft.cureqout:
			// TODO Use config.
			if time.Since(lastCuReq) < 100*time.Millisecond {
				continue
			}
			lastCuReq = time.Now()

			n.logger.WithField("matchindex", req.matchIndex).Warnln("Sending catch-up")
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			leader, _ := n.mgr.Node(n.getNodeID(req.leaderID))
			_, err := leader.RaftClient.CatchMeUp(ctx, &pb.CatchMeUpRequest{
				FollowerID: n.id,
				NextIndex:  req.matchIndex + 1,
			})
			cancel()

			if err != nil {
				n.logger.WithError(err).Warnln("CatchMeUp failed")
			}
		case req := <-n.Raft.rvreqout:
			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := n.conf.RequestVote(ctx, req)
			cancel()

			if err != nil {
				n.logger.WithError(err).Warnln("RequestVote failed")
			}

			if res == nil {
				continue
			}

			n.Raft.HandleRequestVoteResponse(res)

		case req := <-n.Raft.aereqout:
			next := make(map[uint32]uint64)
			nextIndex := req.PrevLogIndex + 1

			for nodeID, ch := range n.match {
				select {
				case index := <-ch:
					// TODO Acessing maxAppendEntries, safe but needs fix.
					atLeastMaxEntries := req.PrevLogIndex+1 > n.Raft.maxAppendEntries
					lessThenMaxEntriesBehind := index < req.PrevLogIndex+1-n.Raft.maxAppendEntries

					if atLeastMaxEntries && lessThenMaxEntriesBehind {
						n.logger.WithField("gorumsid", nodeID).Warnln("Server too far behind")
						index = req.PrevLogIndex + 1
					}
					next[nodeID] = index
					if index < nextIndex {
						nextIndex = index
					}
				default:
				}
			}

			// TODO This should be safe as it only accesses storage
			// which uses transactions. TODO It accesses
			// maxAppendEntries but this on does not change after
			// startup.
			entries := n.Raft.getNextEntries(nextIndex)
			e := uint64(len(entries))
			maxIndex := nextIndex + e - 1

			ctx, cancel := context.WithTimeout(context.Background(), TCPHeartbeat*time.Millisecond)
			res, err := n.conf.AppendEntries(ctx, req,
				// These functions will be executed concurrently.
				func(req pb.AppendEntriesRequest, nodeID uint32) *pb.AppendEntriesRequest {
					if index, ok := next[nodeID]; ok {
						req.PrevLogIndex = index - 1
						// TODO This should be safe as
						// it only accesses storage
						// which uses transactions.
						req.PrevLogTerm = n.Raft.logTerm(index - 1)
					}

					need := maxIndex - req.PrevLogIndex
					req.Entries = entries[e-need:]

					n.logger.WithFields(logrus.Fields{
						"prevlogindex": req.PrevLogIndex,
						"prevlogterm":  req.PrevLogTerm,
						"commitindex":  req.CommitIndex,
						"currentterm":  req.Term,
						"lenentries":   len(req.Entries),
						"gorumsid":     nodeID,
					}).Infoln("Sending AppendEntries")

					return &req
				},
			)

			if err != nil {
				n.logger.WithError(err).Warnln("AppendEntries failed")
			}

			if res == nil {
				continue
			}

			// Cancel on abort.
			if !res.Success {
				cancel()
			}

			n.Raft.HandleAppendEntriesResponse(res, res.Replies)
		}
	}
}

// RequestVote implements gorums.RaftServer.
func (n *Node) RequestVote(ctx context.Context, req *pb.RequestVoteRequest) (*pb.RequestVoteResponse, error) {
	return n.Raft.HandleRequestVoteRequest(req), nil
}

// AppendEntries implements gorums.RaftServer.
func (n *Node) AppendEntries(ctx context.Context, req *pb.AppendEntriesRequest) (*pb.AppendEntriesResponse, error) {
	return n.Raft.HandleAppendEntriesRequest(req), nil
}

// InstallSnapshot implements gorums.RaftServer.
func (n *Node) InstallSnapshot(ctx context.Context, snapshot *commonpb.Snapshot) (*pb.InstallSnapshotResponse, error) {
	return n.Raft.HandleInstallSnapshotRequest(snapshot), nil
}

// CatchMeUp implements gorums.RaftServer.
func (n *Node) CatchMeUp(ctx context.Context, req *pb.CatchMeUpRequest) (res *pb.Empty, err error) {
	res = &pb.Empty{}
	n.match[n.getNodeID(req.FollowerID)] <- req.NextIndex
	return
}

func (n *Node) getNodeID(raftID uint64) uint32 {
	return n.mgr.NodeIDs()[n.lookup[raftID]]
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"net/http"
	"net/http/httputil"
	"net/url"

	"github.com/Unknwon/macaron"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
)

func InitApiPluginRoutes(r *macaron.Macaron) {
	for _, plugin := range plugins.ApiPlugins {
		log.Info("Plugin: Adding proxy routes for api plugin")
		for _, route := range plugin.Routes {
			url := util.JoinUrlFragments("/api/plugin-proxy/", route.Path)
			handlers := make([]macaron.Handler, 0)
			if route.ReqSignedIn {
				handlers = append(handlers, middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true}))
			}
			if route.ReqGrafanaAdmin {
				handlers = append(handlers, middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}))
			}
			if route.ReqSignedIn && route.ReqRole != "" {
				if route.ReqRole == m.ROLE_ADMIN {
					handlers = append(handlers, middleware.RoleAuth(m.ROLE_ADMIN))
				} else if route.ReqRole == m.ROLE_EDITOR {
					handlers = append(handlers, middleware.RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN))
				}
			}
			handlers = append(handlers, ApiPlugin(route.Url))
			r.Route(url, route.Method, handlers...)
			log.Info("Plugin: Adding route %s", url)
		}
	}
}

func ApiPlugin(routeUrl string) macaron.Handler {
	return func(c *middleware.Context) {
		path := c.Params("*")

		//Create a HTTP header with the context in it.
		ctx, err := json.Marshal(c.SignedInUser)
		if err != nil {
			c.JsonApiErr(500, "failed to marshal context to json.", err)
			return
		}
		targetUrl, _ := url.Parse(routeUrl)
		proxy := NewApiPluginProxy(string(ctx), path, targetUrl)
		proxy.Transport = dataProxyTransport
		proxy.ServeHTTP(c.RW(), c.Req.Request)
	}
}

func NewApiPluginProxy(ctx string, proxyPath string, targetUrl *url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		req.URL.Scheme = targetUrl.Scheme
		req.URL.Host = targetUrl.Host
		req.Host = targetUrl.Host

		req.URL.Path = util.JoinUrlFragments(targetUrl.Path, proxyPath)

		// clear cookie headers
		req.Header.Del("Cookie")
		req.Header.Del("Set-Cookie")
		req.Header.Add("Grafana-Context", ctx)
	}

	return &httputil.ReverseProxy{Director: director}
}
<commit_msg>fix(merge): minor merge fix<commit_after>package api

import (
	"encoding/json"
	"net/http"
	"net/http/httputil"
	"net/url"

	"gopkg.in/macaron.v1"

	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
)

func InitApiPluginRoutes(r *macaron.Macaron) {
	for _, plugin := range plugins.ApiPlugins {
		log.Info("Plugin: Adding proxy routes for api plugin")
		for _, route := range plugin.Routes {
			url := util.JoinUrlFragments("/api/plugin-proxy/", route.Path)
			handlers := make([]macaron.Handler, 0)
			if route.ReqSignedIn {
				handlers = append(handlers, middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true}))
			}
			if route.ReqGrafanaAdmin {
				handlers = append(handlers, middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}))
			}
			if route.ReqSignedIn && route.ReqRole != "" {
				if route.ReqRole == m.ROLE_ADMIN {
					handlers = append(handlers, middleware.RoleAuth(m.ROLE_ADMIN))
				} else if route.ReqRole == m.ROLE_EDITOR {
					handlers = append(handlers, middleware.RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN))
				}
			}
			handlers = append(handlers, ApiPlugin(route.Url))
			r.Route(url, route.Method, handlers...)
			log.Info("Plugin: Adding route %s", url)
		}
	}
}

func ApiPlugin(routeUrl string) macaron.Handler {
	return func(c *middleware.Context) {
		path := c.Params("*")

		//Create a HTTP header with the context in it.
		ctx, err := json.Marshal(c.SignedInUser)
		if err != nil {
			c.JsonApiErr(500, "failed to marshal context to json.", err)
			return
		}
		targetUrl, _ := url.Parse(routeUrl)
		proxy := NewApiPluginProxy(string(ctx), path, targetUrl)
		proxy.Transport = dataProxyTransport
		proxy.ServeHTTP(c.Resp, c.Req.Request)
	}
}

func NewApiPluginProxy(ctx string, proxyPath string, targetUrl *url.URL) *httputil.ReverseProxy {
	director := func(req *http.Request) {
		req.URL.Scheme = targetUrl.Scheme
		req.URL.Host = targetUrl.Host
		req.Host = targetUrl.Host

		req.URL.Path = util.JoinUrlFragments(targetUrl.Path, proxyPath)

		// clear cookie headers
		req.Header.Del("Cookie")
		req.Header.Del("Set-Cookie")
		req.Header.Add("Grafana-Context", ctx)
	}

	return &httputil.ReverseProxy{Director: director}
}
<|endoftext|>


<commit_before>package apparmor

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

const (
	DefaultProfilePath = "/etc/apparmor.d/docker"
)

const DefaultProfile = `
#include <tunables/global>
profile docker-default flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>
  network,
  capability,
  file,
  umount,

  mount fstype=tmpfs,
  mount fstype=mqueue,
  mount fstype=fuse.*,
  mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/,
  mount fstype=efivarfs -> /sys/firmware/efi/efivars/,
  mount fstype=fusectl -> /sys/fs/fuse/connections/,
  mount fstype=securityfs -> /sys/kernel/security/,
  mount fstype=debugfs -> /sys/kernel/debug/,
  mount fstype=proc -> /proc/,
  mount fstype=sysfs -> /sys/,

  deny @{PROC}/sys/fs/** wklx,
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
  deny @{PROC}/sys/kernel/*/** wklx,

  deny mount options=(ro, remount) -> /,
  deny mount fstype=debugfs -> /var/lib/ureadahead/debugfs/,
  deny mount fstype=devpts,

  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/efi/efivars/** rwklx,
  deny /sys/kernel/security/** rwklx,
}
`

func InstallDefaultProfile(backupPath string) error {
	if !IsEnabled() {
		return nil
	}

	// If the profile already exists, check if we already have a backup
	// if not, do the backup and override it. (docker 0.10 upgrade changed the apparmor profile)
	// see gh#5049, apparmor blocks signals in ubuntu 14.04
	if _, err := os.Stat(DefaultProfilePath); err == nil {
		if _, err := os.Stat(backupPath); err == nil {
			// If both the profile and the backup are present, do nothing
			return nil
		}
		// Make sure the directory exists
		if err := os.MkdirAll(path.Dir(backupPath), 0755); err != nil {
			return err
		}

		// Create the backup file
		f, err := os.Create(backupPath)
		if err != nil {
			return err
		}
		defer f.Close()

		src, err := os.Open(DefaultProfilePath)
		if err != nil {
			return err
		}
		defer src.Close()

		if _, err := io.Copy(f, src); err != nil {
			return err
		}
	}

	// Make sure /etc/apparmor.d exists
	if err := os.MkdirAll(path.Dir(DefaultProfilePath), 0755); err != nil {
		return err
	}

	if err := ioutil.WriteFile(DefaultProfilePath, []byte(DefaultProfile), 0644); err != nil {
		return err
	}

	// the current functionality of the load script is the exit 0 if the parser does not exist.
	// we think we should fail loudly if you have apparmor enabled but not the parser to load
	// the profile for use.
	output, err := exec.Command("/sbin/apparmor_parser", "-r", "-W", "docker").CombinedOutput()
	if err != nil {
		return fmt.Errorf("Error loading docker profile: %s (%s)", err, output)
	}
	return nil
}
<commit_msg>Ignore is not exist error Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)<commit_after>package apparmor

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
)

const (
	DefaultProfilePath = "/etc/apparmor.d/docker"
)

const DefaultProfile = `
#include <tunables/global>
profile docker-default flags=(attach_disconnected,mediate_deleted) {
  #include <abstractions/base>
  network,
  capability,
  file,
  umount,

  mount fstype=tmpfs,
  mount fstype=mqueue,
  mount fstype=fuse.*,
  mount fstype=binfmt_misc -> /proc/sys/fs/binfmt_misc/,
  mount fstype=efivarfs -> /sys/firmware/efi/efivars/,
  mount fstype=fusectl -> /sys/fs/fuse/connections/,
  mount fstype=securityfs -> /sys/kernel/security/,
  mount fstype=debugfs -> /sys/kernel/debug/,
  mount fstype=proc -> /proc/,
  mount fstype=sysfs -> /sys/,

  deny @{PROC}/sys/fs/** wklx,
  deny @{PROC}/sysrq-trigger rwklx,
  deny @{PROC}/mem rwklx,
  deny @{PROC}/kmem rwklx,
  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
  deny @{PROC}/sys/kernel/*/** wklx,

  deny mount options=(ro, remount) -> /,
  deny mount fstype=debugfs -> /var/lib/ureadahead/debugfs/,
  deny mount fstype=devpts,

  deny /sys/[^f]*/** wklx,
  deny /sys/f[^s]*/** wklx,
  deny /sys/fs/[^c]*/** wklx,
  deny /sys/fs/c[^g]*/** wklx,
  deny /sys/fs/cg[^r]*/** wklx,
  deny /sys/firmware/efi/efivars/** rwklx,
  deny /sys/kernel/security/** rwklx,
}
`

func InstallDefaultProfile(backupPath string) error {
	if !IsEnabled() {
		return nil
	}

	// If the profile already exists, check if we already have a backup
	// if not, do the backup and override it. (docker 0.10 upgrade changed the apparmor profile)
	// see gh#5049, apparmor blocks signals in ubuntu 14.04
	if _, err := os.Stat(DefaultProfilePath); err == nil {
		if _, err := os.Stat(backupPath); err == nil {
			// If both the profile and the backup are present, do nothing
			return nil
		}
		// Make sure the directory exists
		if err := os.MkdirAll(path.Dir(backupPath), 0755); err != nil {
			return err
		}

		// Create the backup file
		f, err := os.Create(backupPath)
		if err != nil {
			return err
		}
		defer f.Close()

		src, err := os.Open(DefaultProfilePath)
		if err != nil {
			return err
		}
		defer src.Close()

		if _, err := io.Copy(f, src); err != nil {
			return err
		}
	}

	// Make sure /etc/apparmor.d exists
	if err := os.MkdirAll(path.Dir(DefaultProfilePath), 0755); err != nil {
		return err
	}

	if err := ioutil.WriteFile(DefaultProfilePath, []byte(DefaultProfile), 0644); err != nil {
		return err
	}

	output, err := exec.Command("/sbin/apparmor_parser", "-r", "-W", "docker").CombinedOutput()
	if err != nil && !os.IsNotExist(err) {
		if e, ok := err.(*exec.Error); ok {
			// keeping with the current profile load code, if the parser does not exist then
			// just return
			if e.Err == exec.ErrNotFound || os.IsNotExist(e.Err) {
				return nil
			}
		}
		return fmt.Errorf("Error loading docker profile: %s (%s)", err, output)
	}
	return nil
}
<|endoftext|>


<commit_before>package date

import (
	"testing"
	"time"
)

func TestGetDiffString(t *testing.T) {
	location, _ := time.LoadLocation("America/Los_Angeles")

	tests := []struct {
		start   time.Time
		end     time.Time
		output  string
		options *DiffDisplayOptions
	}{
		// alwaysShowYear
		{
			time.Date(2014, time.January, 1, 12, 0, 0, 0, location),
			time.Date(2014, time.January, 1, 13, 0, 0, 0, location),
			"0 years, 1 hour, and 0 minutes",
			&DiffDisplayOptions{
				AlwaysShowYear: true,
			},
		},
		// Leap day check - not a leap year
		{
			time.Date(2015, time.February, 26, 0, 0, 0, 0, location),
			time.Date(2015, time.March, 3, 0, 0, 0, 0, location),
			"5 days, 0 hours, and 0 minutes",
			nil,
		},
		// Leap day check - a leap year
		{
			time.Date(2016, time.February, 26, 0, 0, 0, 0, location),
			time.Date(2016, time.March, 3, 0, 0, 0, 0, location),
			"6 days, 0 hours, and 0 minutes",
			nil,
		},
	}

	for _, test := range tests {
		// TODO unit test GetDiff and Diff.Display() instead
		result, err := GetDiffString(test.start, test.end, test.options)
		if err != nil {
			t.Errorf("Unexpected error '%s'", err.Error())
		}
		if result != test.output {
			t.Errorf("Expected '%s', got '%s'", test.output, result)
		}
	}
}
<commit_msg>date.Diff tests: improve test cases for Diff.Display()<commit_after>package date

import (
	"testing"
	"time"
)

func TestGetDiff(t *testing.T) {
	location, _ := time.LoadLocation("America/Los_Angeles")

	tests := []struct {
		start  time.Time
		end    time.Time
		output Diff
	}{
		// alwaysShowYear
		{
			time.Date(2014, time.January, 1, 12, 0, 0, 0, location),
			time.Date(2014, time.January, 1, 13, 0, 0, 0, location),
			Diff{
				Hours: 1,
			},
		},
		// Leap day check - not a leap year
		{
			time.Date(2015, time.February, 26, 0, 0, 0, 0, location),
			time.Date(2015, time.March, 3, 0, 0, 0, 0, location),
			Diff{
				Days: 5,
			},
		},
		// Leap day check - a leap year
		{
			time.Date(2016, time.February, 26, 0, 0, 0, 0, location),
			time.Date(2016, time.March, 3, 0, 0, 0, 0, location),
			Diff{
				Days: 6,
			},
		},
	}

	for _, test := range tests {
		result, err := GetDiff(test.start, test.end)
		if err != nil {
			t.Errorf("Unexpected error '%s'", err.Error())
		}
		if result != test.output {
			t.Errorf("Expected '%s', got '%s'", test.output, result)
		}
	}
}

func TestDiffDisplay(t *testing.T) {
	tests := []struct {
		diff    Diff
		options *DiffDisplayOptions
		output  string
	}{
		{
			Diff{
				Days: 5,
			},
			nil,
			"5 days, 0 hours, and 0 minutes",
		},
		// alwaysShowYear
		{
			Diff{
				Hours: 1,
			},
			&DiffDisplayOptions{
				AlwaysShowYear: true,
			},
			"0 years, 1 hour, and 0 minutes",
		},
		// Granularity
		{
			Diff{
				Months:  2,
				Days:    10,
				Hours:   12,
				Minutes: 12,
			},
			&DiffDisplayOptions{
				Granularity: Hour,
			},
			"2 months, 10 days, and 12 hours",
		},
		// TODO add test cases for when it displays 2 numbers or 1
	}

	for _, test := range tests {
		result, err := test.diff.Display(test.options)
		if err != nil {
			t.Errorf("Unexpected error '%s'", err.Error())
		}
		if result != test.output {
			t.Errorf("Expected '%s', got '%s'", test.output, result)
		}
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

type DocCache struct {
	docs  map[string]*template.Template //map of documents in 'path', indexed by filename
	size  int64                         //Size of the document in bytes
	count int                           //Amount of documents in the cache
	path  string                        //Path to the document directory
}

var Docs *DocCache // Represents the global document cache

//Allocated and builds a new cache from the directory 'path'
func InitCache(path string) error {
	if Docs = NewDocCache(path); Docs == nil {
		return errors.New("Failed to get document cache")
	} else {
		return Docs.BuildCache()
	}
}

//Allocated a new DocCache with path 'path'
func NewDocCache(path string) *DocCache {
	return &DocCache{make(map[string]*template.Template), 0, 0, path}
}

//Builds a cache of documents from the files in the 'docDir' path
func (cache *DocCache) BuildCache() error {
	log.Println("Building document cache from", cache.path)

	list, err := ioutil.ReadDir(cache.path)
	if err != nil {
		return err
	}

	for _, file := range list {
		log.Println("\t++ Cache:", file.Name())
		cache.CacheDoc(file.Name())
	}

	log.Println("\t!! Cached", cache.count, "file(s) (", cache.size, " bytes) in", cache.path)
	return nil
}

//Adds the document 'name' to the cache
func (cache *DocCache) CacheDoc(name string) error {
	if strings.HasSuffix(name, ".html") == false {
		log.Println("\t!! Ignore:", name)
		return errors.New("Refusing to cache non .html file")
	}
	data, err := ioutil.ReadFile(cache.path + name)
	if err != nil {
		return err
	}

	cache.docs[name], err = template.New(name).Parse(string(data))
	if err != nil {
		return err
	}
	stat, _ := os.Stat(cache.path + name)
	cache.size += stat.Size()
	cache.count++
	return nil
}

//Returns true if 'name' is in cache, false otherwise
func (cache *DocCache) IsCached(name string) bool {
	_, ok := cache.docs[name]
	return ok
}

//Returns true if doc 'name' is on disk, false otherwise
func (cache *DocCache) IsOnDisk(name string) bool {
	_, err := os.Stat(name)
	return err != nil
}

//Looks for the document 'name' in the cache, then on the disk, then gives up and returns 404
// If the document is found in the cache it returns the document immediately
// If the document isn't in the cache, but on the disk, the document is read and
//	added to the cache and returned
// If the document isn't in the cache or on the disk, return 404
func (cache *DocCache) GetDoc(name string) *template.Template {

	log.Println("\t", name, "?? Querying cache")
	//File is in cache, simply return it
	if cache.IsCached(name) == true {
		log.Println("\t", name, "++ Cached/Found")
		return cache.docs[name]
		//Doc isn't in cache, see if it's on disk
		//cache and return it if it is
	} else if cache.IsOnDisk(name) == true {
		log.Println("\t", name, "!! Not in cache")
		log.Println("\t", name, "++ On disk/Found")
		log.Println("\t", name, "!! Caching")

		cache.CacheDoc(name)
		return cache.docs[name]
	} else {
		//No luck in cache or on disk, return 404
		log.Println("\t", name, "-- Not cached/Not on disk")
		return nil
	}
	return nil
}
<commit_msg>Fixed typo in cache/cache.go:IsOnDisk()<commit_after>package cache

import (
	"errors"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

type DocCache struct {
	docs  map[string]*template.Template //map of documents in 'path', indexed by filename
	size  int64                         //Size of the document in bytes
	count int                           //Amount of documents in the cache
	path  string                        //Path to the document directory
}

var Docs *DocCache // Represents the global document cache

//Allocated and builds a new cache from the directory 'path'
func InitCache(path string) error {
	if Docs = NewDocCache(path); Docs == nil {
		return errors.New("Failed to get document cache")
	} else {
		return Docs.BuildCache()
	}
}

//Allocated a new DocCache with path 'path'
func NewDocCache(path string) *DocCache {
	return &DocCache{make(map[string]*template.Template), 0, 0, path}
}

//Builds a cache of documents from the files in the 'docDir' path
func (cache *DocCache) BuildCache() error {
	log.Println("Building document cache from", cache.path)

	list, err := ioutil.ReadDir(cache.path)
	if err != nil {
		return err
	}

	for _, file := range list {
		log.Println("\t++ Cache:", file.Name())
		cache.CacheDoc(file.Name())
	}

	log.Println("\t!! Cached", cache.count, "file(s) (", cache.size, " bytes) in", cache.path)
	return nil
}

//Adds the document 'name' to the cache
func (cache *DocCache) CacheDoc(name string) error {
	if strings.HasSuffix(name, ".html") == false {
		log.Println("\t!! Ignore:", name)
		return errors.New("Refusing to cache non .html file")
	}
	data, err := ioutil.ReadFile(cache.path + name)
	if err != nil {
		return err
	}

	cache.docs[name], err = template.New(name).Parse(string(data))
	if err != nil {
		return err
	}
	stat, _ := os.Stat(cache.path + name)
	cache.size += stat.Size()
	cache.count++
	return nil
}

//Returns true if 'name' is in cache, false otherwise
func (cache *DocCache) IsCached(name string) bool {
	_, ok := cache.docs[name]
	return ok
}

//Returns true if doc 'name' is on disk, false otherwise
func (cache *DocCache) IsOnDisk(name string) bool {
	_, err := os.Stat(name)
	return err == nil
}

//Looks for the document 'name' in the cache, then on the disk, then gives up and returns 404
// If the document is found in the cache it returns the document immediately
// If the document isn't in the cache, but on the disk, the document is read and
//	added to the cache and returned
// If the document isn't in the cache or on the disk, return 404
func (cache *DocCache) GetDoc(name string) *template.Template {

	log.Println("\t", name, "?? Querying cache")
	//File is in cache, simply return it
	if cache.IsCached(name) == true {
		log.Println("\t", name, "++ Cached/Found")
		return cache.docs[name]
		//Doc isn't in cache, see if it's on disk
		//cache and return it if it is
	} else if cache.IsOnDisk(name) == true {
		log.Println("\t", name, "!! Not in cache")
		log.Println("\t", name, "++ On disk/Found")
		log.Println("\t", name, "!! Caching")

		cache.CacheDoc(name)
		return cache.docs[name]
	} else {
		//No luck in cache or on disk, return 404
		log.Println("\t", name, "-- Not cached/Not on disk")
		return nil
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "github.com/StefanSchroeder/Golang-Ellipsoid/ellipsoid"

func main() {
	lat1, lon1 := 37.619002, -122.374843 //SFO
	lat2, lon2 := 33.942536, -118.408074 //LAX

	// Create Ellipsoid object with WGS84-ellipsoid,
	// angle units are degrees, distance units are meter.
	geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.Longitude_is_symmetric, ellipsoid.Bearing_is_symmetric)

	// Calculate the distance and bearing from SFO to LAX.
	distance, bearing := geo1.To(lat1, lon1, lat2, lon2)
	fmt.Printf("----- Testing To\n")
	fmt.Printf("Distance = %v Bearing = %v\n", distance, bearing)

	// Calculate where you are when going from SFO in
	// direction 45.0 deg. for 20000 meters.
	fmt.Printf("----- Testing At\n")
	lat3, lon3 := geo1.At(lat1, lon1, 20000.0, 45.0)
	fmt.Printf("lat3 = %v lon3 = %v\n", lat3, lon3)

	// Convert Lat-Lon-Alt to ECEF.
	lat4, lon4, alt4 := 39.197807, -77.108574, 55.0 // Some location near Baltimore
	// that the author of the Perl module geo-ecef used. I reused the coords of the tests.
	fmt.Printf("----- Testing ToECEF\n")
	x, y, z := geo1.ToECEF(lat4, lon4, alt4)
	fmt.Printf("x = %v \ny = %v \nz = %v\n", x, y, z)

	// Convert ECEF to Lat-Lon-Alt.
	x1, y1, z1 := 1.1042590709397183e+06, -4.824765955871677e+06, 4.0093940281868847e+06
	fmt.Printf("----- Testing ToLLA\n")
	lat5, lon5, alt5 := geo1.ToLLA(x1, y1, z1)
	fmt.Printf("lat5 = %v lon5 = %v alt3 = %v\n", lat5, lon5, alt5)

	// Displacement. Use only for small distances!
	l1, l2 := 41.978744444444, 272.096858333333
	m1, m2 := 42.005419444444, 272.073286111111
	dx, dy := geo1.Displacement(l1, l2, m1, m2)
	fmt.Printf("----- Testing Displacement\n")
	fmt.Printf("x,y=      %v %v\n", dx, dy)
	fmt.Printf("Expected: -1952.8108885261 2963.14446772882\n")

	fmt.Printf("----- Done ---\n")
}
<commit_msg>Change underscores to CamelCase.<commit_after>package main

import "fmt"
import "github.com/StefanSchroeder/Golang-Ellipsoid/ellipsoid"

func main() {
	lat1, lon1 := 37.619002, -122.374843 //SFO
	lat2, lon2 := 33.942536, -118.408074 //LAX

	// Create Ellipsoid object with WGS84-ellipsoid,
	// angle units are degrees, distance units are meter.
	geo1 := ellipsoid.Init("WGS84", ellipsoid.Degrees, ellipsoid.Meter, ellipsoid.LongitudeIsSymmetric, ellipsoid.BearingIsSymmetric)

	// Calculate the distance and bearing from SFO to LAX.
	distance, bearing := geo1.To(lat1, lon1, lat2, lon2)
	fmt.Printf("----- Testing To\n")
	fmt.Printf("Distance = %v Bearing = %v\n", distance, bearing)

	// Calculate where you are when going from SFO in
	// direction 45.0 deg. for 20000 meters.
	fmt.Printf("----- Testing At\n")
	lat3, lon3 := geo1.At(lat1, lon1, 20000.0, 45.0)
	fmt.Printf("lat3 = %v lon3 = %v\n", lat3, lon3)

	// Convert Lat-Lon-Alt to ECEF.
	lat4, lon4, alt4 := 39.197807, -77.108574, 55.0 // Some location near Baltimore
	// that the author of the Perl module geo-ecef used. I reused the coords of the tests.
	fmt.Printf("----- Testing ToECEF\n")
	x, y, z := geo1.ToECEF(lat4, lon4, alt4)
	fmt.Printf("x = %v \ny = %v \nz = %v\n", x, y, z)

	// Convert ECEF to Lat-Lon-Alt.
	x1, y1, z1 := 1.1042590709397183e+06, -4.824765955871677e+06, 4.0093940281868847e+06
	fmt.Printf("----- Testing ToLLA\n")
	lat5, lon5, alt5 := geo1.ToLLA(x1, y1, z1)
	fmt.Printf("lat5 = %v lon5 = %v alt3 = %v\n", lat5, lon5, alt5)

	// Displacement. Use only for small distances!
	l1, l2 := 41.978744444444, 272.096858333333
	m1, m2 := 42.005419444444, 272.073286111111
	dx, dy := geo1.Displacement(l1, l2, m1, m2)
	fmt.Printf("----- Testing Displacement\n")
	fmt.Printf("x,y=      %v %v\n", dx, dy)
	fmt.Printf("Expected: -1952.8108885261 2963.14446772882\n")

	fmt.Printf("----- Done ---\n")
}
<|endoftext|>