<commit_before>package people

import (
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"errors"
	"fmt"
	"strings"
)

const (
	messageServiceNotHealthy = "Service is NOT healthy"
	messageServiceHealthy    = "Service is healthy"
)

// Driver interface
type Driver interface {
	Read(id string, transactionID string) (person Person, found bool, err error)
	Healthchecks() []fthealth.Check
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) Driver {
	return &CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw *CypherDriver) CheckConnectivity() (string, error) {
	err := neoutils.Check(pcw.conn)
	if err != nil {
		return messageServiceNotHealthy, err
	}
	return messageServiceHealthy, nil
}

func (pcw *CypherDriver) Healthchecks() []fthealth.Check {
	checks := []fthealth.Check{fthealth.Check{
		Name:             "Neo4j Connectivity",
		BusinessImpact:   "Unable to retrieve People from Neo4j",
		PanicGuide:       "https://dewey.ft.com/public-people-api.html",
		Severity:         2,
		TechnicalSummary: "Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.",
		Checker:          pcw.CheckConnectivity,
	},
	}
	return checks
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	P struct {
		ID              string
		Types           []string
		DirectType      string
		PrefLabel       string
		Labels          []string
		Salutation      string
		BirthYear       int
		EmailAddress    string
		TwitterHandle   string
		FacebookProfile string
		LinkedinProfile string
		Description     string
		DescriptionXML  string
		ImageURL        string
	}
	M []struct {
		M struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		O struct {
			ID         string
			Types      []string
			DirectType string
			PrefLabel  string
			Labels     []string
		}
		R []struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			ChangeEvents []neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string, transactionID string) (Person, bool, error) {
	person := Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(pp:Person)-[:EQUIVALENT_TO]->(canonical:Person)
						MATCH (canonical)<-[:EQUIVALENT_TO]-(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    canonical,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m,
                                { id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate}, {endedAt:rr.terminationDate}] } as r
                        WITH canonical, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH canonical, collect({m:m, o:o, r:r}) as m
                        WITH m, { ID:canonical.prefUUID, types:labels(canonical), prefLabel:canonical.prefLabel, labels:canonical.aliases,
								birthYear:canonical.birthYear, salutation:canonical.salutation, emailAddress:canonical.emailAddress,
								twitterHandle:canonical.twitterHandle, facebookProfile:canonical.facebookProfile, linkedinProfile:canonical.linkedinProfile,
								imageUrl:canonical.imageUrl, Description:canonical.description, descriptionXML:canonical.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err := pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Error Querying Neo4J for a Person")
		return Person{}, true, err
	}

	if len(results) == 0 || (len(results[0].Rs) == 0 || results[0].Rs[0].P.ID == "") {
		p, f, e := pcw.ReadOldConcordanceModel(uuid, transactionID)
		return p, f, e
	} else if len(results) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid: %s", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func (pcw CypherDriver) ReadOldConcordanceModel(uuid string, transactionID string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m,
                                { id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate}, {endedAt:rr.terminationDate}] } as r
                        WITH p, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH p, collect({m:m, o:o, r:r}) as m
                        WITH m, { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.aliases,
												     birthYear:p.birthYear, salutation:p.salutation, emailAddress:p.emailAddress,
														 twitterHandle:p.twitterHandle, facebookProfile:p.facebookProfile, linkedinProfile:p.linkedinProfile,
														 imageUrl:p.imageUrl, Description:p.description, descriptionXML:p.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Query execution failed")
		return Person{}, false, err
	} else if len(results) == 0 || len(results[0].Rs) == 0 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Person not found")
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid:%s !", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct, env string) Person {
	public := Person{}
	public.Thing = Thing{}
	public.ID = mapper.IDURL(neo.P.ID)
	public.APIURL = mapper.APIURL(neo.P.ID, neo.P.Types, env)
	public.Types = mapper.TypeURIs(neo.P.Types)
	public.DirectType = filterToMostSpecificType(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = neo.P.Labels
	}
	public.BirthYear = neo.P.BirthYear
	public.Salutation = neo.P.Salutation
	public.Description = neo.P.Description
	public.DescriptionXML = neo.P.DescriptionXML
	public.EmailAddress = neo.P.EmailAddress
	public.TwitterHandle = neo.P.TwitterHandle
	public.FacebookProfile = neo.P.FacebookProfile
	public.ImageURL = neo.P.ImageURL

	if len(neo.M) > 0 {
		memberships := []Membership{}
		for _, neoMem := range neo.M {
			if neoMem.M.ID != "" && neoMem.O.ID != "" && len(neoMem.R) > 0 {
				membership := Membership{}
				membership.Title = neoMem.M.PrefLabel
				membership.Types = mapper.TypeURIs(neoMem.M.Types)
				membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
				membership.Organisation = Organisation{}
				membership.Organisation.Thing = Thing{}
				membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
				membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
				membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
				membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
				membership.Organisation.PrefLabel = neoMem.O.PrefLabel
				if len(neoMem.O.Labels) > 0 {
					membership.Organisation.Labels = neoMem.O.Labels
				}
				if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
					membership.ChangeEvents = b
				}

				roles := []Role{}
				for _, neoRole := range neoMem.R {
					if neoRole.ID != "" {
						role := Role{}
						role.Thing = Thing{}
						role.ID = mapper.IDURL(neoRole.ID)
						role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
						role.Types = mapper.TypeURIs(neoRole.Types)
						role.DirectType = filterToMostSpecificType(neoRole.Types)
						role.PrefLabel = neoRole.PrefLabel
						if a, b := changeEvent(neoRole.ChangeEvents); a == true {
							role.ChangeEvents = b
						}
						roles = append(roles, role)
					}
				}
				if len(roles) > 0 {
					membership.Roles = roles
					memberships = append(memberships, membership)
				}
			}
			public.Memberships = memberships
		}
	}

	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, []ChangeEvent) {
	var results []ChangeEvent

	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	return true, results
}

func filterToMostSpecificType(unfilteredTypes []string) string {
	mostSpecificType, err := mapper.MostSpecificType(unfilteredTypes)
	if err != nil {
		return ""
	}
	fullURI := mapper.TypeURIs([]string{mostSpecificType})
	return fullURI[0]
}

func handleEmptyError(e error, defaultMessage string) error {

	if e.Error() != "" {
		return e
	}

	neoError, ok := e.(neoism.NeoError)

	if !ok {
		return errors.New(defaultMessage)
	}

	if neoError.Exception != "" {
		neoError.Message = neoError.Exception
		return neoError
	}

	if neoError.Cause != nil {
		cause := fmt.Sprint(neoError.Cause)

		if cause != "" {
			neoError.Message = "Cause: " + cause
			return neoError
		}
	}

	if len(neoError.Stacktrace) > 0 {
		neoError.Message = strings.Join(neoError.Stacktrace, ", ")
		return neoError
	}

	neoError.Message = defaultMessage

	return neoError
}<commit_msg>Memberships and roles are order by termination date, then inception date with the most recent at the top of the list<commit_after>package people

import (
	fthealth "github.com/Financial-Times/go-fthealth/v1_1"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
	"errors"
	"fmt"
	"strings"
)

const (
	messageServiceNotHealthy = "Service is NOT healthy"
	messageServiceHealthy    = "Service is healthy"
)

// Driver interface
type Driver interface {
	Read(id string, transactionID string) (person Person, found bool, err error)
	Healthchecks() []fthealth.Check
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) Driver {
	return &CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw *CypherDriver) CheckConnectivity() (string, error) {
	err := neoutils.Check(pcw.conn)
	if err != nil {
		return messageServiceNotHealthy, err
	}
	return messageServiceHealthy, nil
}

func (pcw *CypherDriver) Healthchecks() []fthealth.Check {
	checks := []fthealth.Check{fthealth.Check{
		Name:             "Neo4j Connectivity",
		BusinessImpact:   "Unable to retrieve People from Neo4j",
		PanicGuide:       "https://dewey.ft.com/public-people-api.html",
		Severity:         2,
		TechnicalSummary: "Cannot connect to Neo4j. If this check fails, check that the Neo4J cluster is responding.",
		Checker:          pcw.CheckConnectivity,
	},
	}
	return checks
}

type neoChangeEvent struct {
	StartedAt string
	EndedAt   string
}

type neoReadStruct struct {
	P struct {
		ID              string
		Types           []string
		DirectType      string
		PrefLabel       string
		Labels          []string
		Salutation      string
		BirthYear       int
		EmailAddress    string
		TwitterHandle   string
		FacebookProfile string
		LinkedinProfile string
		Description     string
		DescriptionXML  string
		ImageURL        string
	}
	M []struct {
		M struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			Title        string
			ChangeEvents []neoChangeEvent
		}
		O struct {
			ID         string
			Types      []string
			DirectType string
			PrefLabel  string
			Labels     []string
		}
		R []struct {
			ID           string
			Types        []string
			DirectType   string
			PrefLabel    string
			ChangeEvents []neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string, transactionID string) (Person, bool, error) {
	person := Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(pp:Person)-[:EQUIVALENT_TO]->(canonical:Person)
						MATCH (canonical)<-[:EQUIVALENT_TO]-(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    canonical,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch DESC, m.inceptionDateEpoch DESC, r, rr
						WITH 	m, o,canonical,
                        		{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch DESC, rr.inceptionDateEpoch DESC
                        WITH canonical, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH canonical, collect({m:m, o:o, r:r}) as m
                        WITH m, { ID:canonical.prefUUID, types:labels(canonical), prefLabel:canonical.prefLabel, labels:canonical.aliases,
								birthYear:canonical.birthYear, salutation:canonical.salutation, emailAddress:canonical.emailAddress,
								twitterHandle:canonical.twitterHandle, facebookProfile:canonical.facebookProfile, linkedinProfile:canonical.linkedinProfile,
								imageUrl:canonical.imageUrl, Description:canonical.description, descriptionXML:canonical.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err := pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Error Querying Neo4J for a Person")
		return Person{}, true, err
	}

	if len(results) == 0 || (len(results[0].Rs) == 0 || results[0].Rs[0].P.ID == "") {
		p, f, e := pcw.ReadOldConcordanceModel(uuid, transactionID)
		return p, f, e
	} else if len(results) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid: %s", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func (pcw CypherDriver) ReadOldConcordanceModel(uuid string, transactionID string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}

	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (identifier:UPPIdentifier{value:{uuid}})
                        MATCH (identifier)-[:IDENTIFIES]->(p:Person)
                        OPTIONAL MATCH (p)<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:MembershipRole)
                        WITH    p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel} as o,
								{ id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvents:[{startedAt:m.inceptionDate}, {endedAt:m.terminationDate}] } as m ORDER BY m.terminationDateEpoch DESC, m.inceptionDateEpoch DESC, r, rr
						WITH   	p, m, o,
								{ id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvents:[{startedAt:rr.inceptionDate},{endedAt:rr.terminationDate}] } as r ORDER BY rr.terminationDateEpoch DESC, rr.inceptionDateEpoch DESC
                        WITH p, m, o, collect(r) as r ORDER BY o.uuid DESC
                        WITH p, collect({m:m, o:o, r:r}) as m
                        WITH m, { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.aliases,
												     birthYear:p.birthYear, salutation:p.salutation, emailAddress:p.emailAddress,
														 twitterHandle:p.twitterHandle, facebookProfile:p.facebookProfile, linkedinProfile:p.linkedinProfile,
														 imageUrl:p.imageUrl, Description:p.description, descriptionXML:p.descriptionXML} as p
                        RETURN collect ({p:p, m:m}) as rs
                        `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Query execution failed")
		return Person{}, false, err
	} else if len(results) == 0 || len(results[0].Rs) == 0 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Error("Person not found")
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		logger.WithTransactionID(transactionID).WithField("UUID", uuid).Errorf("Multiple people found with the same uuid:%s !", uuid)
		return Person{}, true, err
	}

	person = neoReadStructToPerson(results[0].Rs[0], pcw.env)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct, env string) Person {
	public := Person{}
	public.Thing = Thing{}
	public.ID = mapper.IDURL(neo.P.ID)
	public.APIURL = mapper.APIURL(neo.P.ID, neo.P.Types, env)
	public.Types = mapper.TypeURIs(neo.P.Types)
	public.DirectType = filterToMostSpecificType(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = neo.P.Labels
	}
	public.BirthYear = neo.P.BirthYear
	public.Salutation = neo.P.Salutation
	public.Description = neo.P.Description
	public.DescriptionXML = neo.P.DescriptionXML
	public.EmailAddress = neo.P.EmailAddress
	public.TwitterHandle = neo.P.TwitterHandle
	public.FacebookProfile = neo.P.FacebookProfile
	public.ImageURL = neo.P.ImageURL

	if len(neo.M) > 0 {
		memberships := []Membership{}
		for _, neoMem := range neo.M {
			if neoMem.M.ID != "" && neoMem.O.ID != "" && len(neoMem.R) > 0 {
				membership := Membership{}
				membership.Title = neoMem.M.PrefLabel
				membership.Types = mapper.TypeURIs(neoMem.M.Types)
				membership.DirectType = filterToMostSpecificType(neoMem.M.Types)
				membership.Organisation = Organisation{}
				membership.Organisation.Thing = Thing{}
				membership.Organisation.ID = mapper.IDURL(neoMem.O.ID)
				membership.Organisation.APIURL = mapper.APIURL(neoMem.O.ID, neoMem.O.Types, env)
				membership.Organisation.Types = mapper.TypeURIs(neoMem.O.Types)
				membership.Organisation.DirectType = filterToMostSpecificType(neoMem.O.Types)
				membership.Organisation.PrefLabel = neoMem.O.PrefLabel
				if len(neoMem.O.Labels) > 0 {
					membership.Organisation.Labels = neoMem.O.Labels
				}
				if a, b := changeEvent(neoMem.M.ChangeEvents); a == true {
					membership.ChangeEvents = b
				}

				roles := []Role{}
				for _, neoRole := range neoMem.R {
					if neoRole.ID != "" {
						role := Role{}
						role.Thing = Thing{}
						role.ID = mapper.IDURL(neoRole.ID)
						role.APIURL = mapper.APIURL(neoRole.ID, neoRole.Types, env)
						role.Types = mapper.TypeURIs(neoRole.Types)
						role.DirectType = filterToMostSpecificType(neoRole.Types)
						role.PrefLabel = neoRole.PrefLabel
						if a, b := changeEvent(neoRole.ChangeEvents); a == true {
							role.ChangeEvents = b
						}
						roles = append(roles, role)
					}
				}
				if len(roles) > 0 {
					membership.Roles = roles
					memberships = append(memberships, membership)
				}
			}
			public.Memberships = memberships
		}
	}

	return public
}

func changeEvent(neoChgEvts []neoChangeEvent) (bool, []ChangeEvent) {
	var results []ChangeEvent

	if neoChgEvts[0].StartedAt == "" && neoChgEvts[1].EndedAt == "" {
		results = make([]ChangeEvent, 0, 0)
		return false, results
	}
	for _, neoChgEvt := range neoChgEvts {
		if neoChgEvt.StartedAt != "" {
			results = append(results, ChangeEvent{StartedAt: neoChgEvt.StartedAt})
		}
		if neoChgEvt.EndedAt != "" {
			results = append(results, ChangeEvent{EndedAt: neoChgEvt.EndedAt})
		}
	}
	return true, results
}

func filterToMostSpecificType(unfilteredTypes []string) string {
	mostSpecificType, err := mapper.MostSpecificType(unfilteredTypes)
	if err != nil {
		return ""
	}
	fullURI := mapper.TypeURIs([]string{mostSpecificType})
	return fullURI[0]
}

func handleEmptyError(e error, defaultMessage string) error {

	if e.Error() != "" {
		return e
	}

	neoError, ok := e.(neoism.NeoError)

	if !ok {
		return errors.New(defaultMessage)
	}

	if neoError.Exception != "" {
		neoError.Message = neoError.Exception
		return neoError
	}

	if neoError.Cause != nil {
		cause := fmt.Sprint(neoError.Cause)

		if cause != "" {
			neoError.Message = "Cause: " + cause
			return neoError
		}
	}

	if len(neoError.Stacktrace) > 0 {
		neoError.Message = strings.Join(neoError.Stacktrace, ", ")
		return neoError
	}

	neoError.Message = defaultMessage

	return neoError
}<|endoftext|>


<commit_before>package news_getter

import (
	"encoding/json"
	"fmt"
	"time"

	"web_apps/news_aggregator/modules/database"
)

var (
	loop_counter_delay   = 5
	hacker_news_provider = "https://news.ycombinator.com"
	hacker_news_name     = "HackerNews"
)

type HackerNewsTopStoriesId []int

func StartHackerNews() {
	fmt.Println("starting hn")
	content_out := make(chan jsonNewsBody)
	time_profiler := make(chan string)

	go func() {
		for t := range time.Tick(time.Duration(loop_counter_delay) * time.Second) {
			top_stories_ids, err := topStoriesId()
			if err != nil {
				return
			}
			fmt.Println("running the loop")
			_ = t
			for _, id := range top_stories_ids {
				go func(id int, content_out chan jsonNewsBody, time_profiler chan string) {
					start := time.Now()
					news_content := hackerNewsReader(id)
					content_out <- news_content
					time_profiler <- fmt.Sprintf("%v", time.Since(start))
				}(id, content_out, time_profiler)
			}
		}
	}()

	go func() {
		for {
			content_out_msg := <-content_out
			time_profiler_out := <-time_profiler

			time_f := content_out_msg.Time
			content_out_msg.CreatedAt = fmt.Sprintf("%v", time.Now().Local())
			content_out_msg.ProviderUrl = hacker_news_provider
			content_out_msg.ProviderName = hacker_news_name

			_ = time_f

			// check if can save
			// then save
			can_save := database.HackerNewsFindIfExist(content_out_msg.Title)
			if can_save {
				database.HackerNewsInsert(content_out_msg)
			} else {
				//fmt.Println("did not save!")
			}
			_ = time_profiler_out
			//fmt.Println(time_profiler_out)
			//fmt.Println("----------------------------")
		}
	}()

}

func topStoriesId() ([]int, error) {
	var top_stories_id_url string = "https://hacker-news.firebaseio.com/v0/topstories.json"
	var id_containers HackerNewsTopStoriesId
	response, err := httpGet(top_stories_id_url)
	if err != nil {
		var x []int
		return x, err
	}

	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &id_containers); err != nil {
		return id_containers, nil
	}
	fmt.Println(id_containers)

	// make error handler
	return id_containers, nil
}

func hackerNewsReader(id int) jsonNewsBody {
	news_url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
	var news_content jsonNewsBody
	response, err := httpGet(news_url)
	if err != nil {
		fmt.Println(err)
		return news_content
	}
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &news_content); err != nil {
		return news_content
	}
	return news_content
}
<commit_msg>revert delay<commit_after>package news_getter

import (
	"encoding/json"
	"fmt"
	"time"

	"web_apps/news_aggregator/modules/database"
)

var (
	loop_counter_delay   = 300
	hacker_news_provider = "https://news.ycombinator.com"
	hacker_news_name     = "HackerNews"
)

type HackerNewsTopStoriesId []int

func StartHackerNews() {
	fmt.Println("starting hn")
	content_out := make(chan jsonNewsBody)
	time_profiler := make(chan string)

	go func() {
		for t := range time.Tick(time.Duration(loop_counter_delay) * time.Second) {
			top_stories_ids, err := topStoriesId()
			if err != nil {
				return
			}
			fmt.Println("running the loop")
			_ = t
			for _, id := range top_stories_ids {
				go func(id int, content_out chan jsonNewsBody, time_profiler chan string) {
					start := time.Now()
					news_content := hackerNewsReader(id)
					content_out <- news_content
					time_profiler <- fmt.Sprintf("%v", time.Since(start))
				}(id, content_out, time_profiler)
			}
		}
	}()

	go func() {
		for {
			content_out_msg := <-content_out
			time_profiler_out := <-time_profiler

			time_f := content_out_msg.Time
			content_out_msg.CreatedAt = fmt.Sprintf("%v", time.Now().Local())
			content_out_msg.ProviderUrl = hacker_news_provider
			content_out_msg.ProviderName = hacker_news_name

			_ = time_f

			// check if can save
			// then save
			can_save := database.HackerNewsFindIfExist(content_out_msg.Title)
			if can_save {
				database.HackerNewsInsert(content_out_msg)
			} else {
				//fmt.Println("did not save!")
			}
			_ = time_profiler_out
			//fmt.Println(time_profiler_out)
			//fmt.Println("----------------------------")
		}
	}()

}

func topStoriesId() ([]int, error) {
	var top_stories_id_url string = "https://hacker-news.firebaseio.com/v0/topstories.json"
	var id_containers HackerNewsTopStoriesId
	response, err := httpGet(top_stories_id_url)
	if err != nil {
		var x []int
		return x, err
	}

	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &id_containers); err != nil {
		return id_containers, nil
	}
	fmt.Println(id_containers)

	// make error handler
	return id_containers, nil
}

func hackerNewsReader(id int) jsonNewsBody {
	news_url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
	var news_content jsonNewsBody
	response, err := httpGet(news_url)
	if err != nil {
		fmt.Println(err)
		return news_content
	}
	defer response.Body.Close()

	contents, _ := responseReader(response)
	if err := json.Unmarshal(contents, &news_content); err != nil {
		return news_content
	}
	return news_content
}
<|endoftext|>


<commit_before>package fs

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/fs"
	"time"

	"github.com/ungerik/go-fs/fsimpl"
)

// Ensure MemFile implements interfaces
var (
	_ FileReader     = new(MemFile)
	_ io.Writer      = new(MemFile)
	_ io.WriterTo    = new(MemFile)
	_ io.ReaderAt    = new(MemFile)
	_ gob.GobEncoder = new(MemFile)
	_ gob.GobDecoder = new(MemFile)
	_ fmt.Stringer   = new(MemFile)
)

// MemFile implements FileReader with a filename and an in memory byte slice.
// It exposes FileName and FileData as exported struct fields to emphasize
// its simple nature as just an wrapper of a name and some bytes.
//
// MemFile implements the following interfaces:
//   - FileReader
//   - io.Writer
//   - io.WriterTo
//   - io.ReaderAt
//   - gob.GobEncoder
//   - gob.GobDecoder
//   - fmt.Stringer
type MemFile struct {
	FileName string `json:"name"`
	FileData []byte `json:"data"`
}

// NewMemFile returns a new MemFile
func NewMemFile(name string, data []byte) *MemFile {
	return &MemFile{FileName: name, FileData: data}
}

// MemFileFromFileReader tries to cast and return the passed FileReader
// as *MemFile or else returns the result of ReadMemFile.
func MemFileFromFileReader(fileReader FileReader) (*MemFile, error) {
	if memFile, ok := fileReader.(*MemFile); ok {
		return memFile, nil
	}
	return ReadMemFile(fileReader)
}

// ReadMemFile returns a new MemFile with
// the name from fileReader.Name() and
// the data from fileReader.ReadAll()
func ReadMemFile(fileReader FileReader) (*MemFile, error) {
	data, err := fileReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("ReadMemFile: error reading from FileReader: %w", err)
	}
	return &MemFile{FileName: fileReader.Name(), FileData: data}, nil
}

// ReadMemFileRename returns a new MemFile with the data
// from fileReader.ReadAll() and the passed name.
func ReadMemFileRename(fileReader FileReader, name string) (*MemFile, error) {
	data, err := fileReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("ReadMemFileRename: error reading from FileReader: %w", err)
	}
	return &MemFile{FileName: name, FileData: data}, nil
}

// ReadAllMemFile returns a new MemFile with the data
// from io.ReadAll(r) and the passed name.
func ReadAllMemFile(r io.Reader, name string) (*MemFile, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return nil, fmt.Errorf("ReadAllMemFile: error reading from io.Reader: %w", err)
	}
	return &MemFile{FileName: name, FileData: data}, nil
}

// String returns the name and meta information for the FileReader.
// String implements the fmt.Stringer interface.
func (f *MemFile) String() string {
	return fmt.Sprintf("MemFile{name: %q, size: %d}", f.FileName, len(f.FileData))
}

// Name returns the name of the file
func (f *MemFile) Name() string {
	return f.FileName
}

// Ext returns the extension of file name including the point, or an empty string.
func (f *MemFile) Ext() string {
	return fsimpl.Ext(f.FileName, "")
}

// LocalPath always returns an empty string for a MemFile.
func (f *MemFile) LocalPath() string {
	return ""
}

// Size returns the size of the file
func (f *MemFile) Size() int64 {
	return int64(len(f.FileData))
}

// Exists returns true if the MemFile has non empty FileName.
// It's valid to call this method on a nil pointer,
// will return false in this case.
func (f *MemFile) Exists() bool {
	return f != nil && f.FileName != ""
}

// CheckExists return an ErrDoesNotExist error
// if the file does not exist.
func (f *MemFile) CheckExists() error {
	if !f.Exists() {
		return NewErrDoesNotExistFileReader(f)
	}
	return nil
}

// ContentHash returns a Dropbox compatible content hash for the file.
// See https://www.dropbox.com/developers/reference/content-hash
func (f *MemFile) ContentHash() (string, error) {
	return fsimpl.ContentHashBytes(f.FileData), nil
}

// Write appends the passed bytes to the FileData,
// implementing the io.Writer interface.
func (f *MemFile) Write(b []byte) (int, error) {
	f.FileData = append(f.FileData, b...)
	return len(b), nil
}

// ReadAll copies all bytes of the file
func (f *MemFile) ReadAll() (data []byte, err error) {
	return append([]byte(nil), f.FileData...), nil
}

// ReadAllContentHash copies all bytes of the file
// together with a Dropbox compatible content hash.
// See https://www.dropbox.com/developers/reference/content-hash
func (f *MemFile) ReadAllContentHash() (data []byte, hash string, err error) {
	return append([]byte(nil), f.FileData...), fsimpl.ContentHashBytes(f.FileData), nil
}

// ReadAllString reads the complete file and returns the content as string.
func (f *MemFile) ReadAllString() (string, error) {
	return string(f.FileData), nil
}

// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying input source. It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
//
// When ReadAt returns n < len(p), it returns a non-nil error
// explaining why more bytes were not returned. In this respect,
// ReadAt is stricter than Read.
//
// If the n = len(p) bytes returned by ReadAt are at the end of the
// input source, ReadAt returns err == nil.
//
// Clients of ReadAt can execute parallel ReadAt calls on the
// same input source.
//
// ReadAt implements the interface io.ReaderAt.
func (f *MemFile) ReadAt(p []byte, off int64) (n int, err error) {
	if off >= int64(len(f.FileData)) {
		return 0, io.EOF
	}
	n = copy(p, f.FileData[off:])
	if n < len(p) {
		return n, fmt.Errorf("could only read %d of %d requested bytes", n, len(p))
	}
	return n, nil
}

// WriteTo implements the io.WriterTo interface
func (f *MemFile) WriteTo(writer io.Writer) (n int64, err error) {
	i, err := writer.Write(f.FileData)
	return int64(i), err
}

// OpenReader opens the file and returns a os/fs.File that has be closed after reading
func (f *MemFile) OpenReader() (fs.File, error) {
	return fsimpl.NewReadonlyFileBuffer(f.FileData, memFileInfo{f}), nil
}

// OpenReadSeeker opens the file and returns a ReadSeekCloser.
// Use OpenReader if seeking is not necessary because implementations
// may need additional buffering to support seeking or not support it at all.
func (f *MemFile) OpenReadSeeker() (ReadSeekCloser, error) {
	return fsimpl.NewReadonlyFileBuffer(f.FileData, memFileInfo{f}), nil
}

// ReadJSON reads and unmarshalles the JSON content of the file to output.
func (f *MemFile) ReadJSON(output interface{}) error {
	return json.Unmarshal(f.FileData, output)
}

// ReadXML reads and unmarshalles the XML content of the file to output.
func (f *MemFile) ReadXML(output interface{}) error {
	return xml.Unmarshal(f.FileData, output)
}

// GobEncode gob encodes the file name and content,
// implementing encoding/gob.GobEncoder.
func (f *MemFile) GobEncode() ([]byte, error) {
	buf := bytes.NewBuffer(make([]byte, 0, 16+len(f.FileName)+len(f.FileData)))
	enc := gob.NewEncoder(buf)
	err := enc.Encode(f.FileName)
	if err != nil {
		return nil, fmt.Errorf("MemFile.GobEncode: error encoding FileName: %w", err)
	}
	err = enc.Encode(f.FileData)
	if err != nil {
		return nil, fmt.Errorf("MemFile.GobEncode: error encoding FileData: %w", err)
	}
	return buf.Bytes(), nil
}

// GobDecode decodes gobBytes file name and content,
// implementing encoding/gob.GobDecoder.
func (f *MemFile) GobDecode(gobBytes []byte) error {
	dec := gob.NewDecoder(bytes.NewReader(gobBytes))
	err := dec.Decode(&f.FileName)
	if err != nil {
		return fmt.Errorf("MemFile.GobDecode: error decoding FileName: %w", err)
	}
	err = dec.Decode(&f.FileData)
	if err != nil {
		return fmt.Errorf("MemFile.GobDecode: error decoding FileData: %w", err)
	}
	return nil
}

// Stat returns a io/fs.FileInfo describing the MemFile.
func (f *MemFile) Stat() (fs.FileInfo, error) {
	return memFileInfo{f}, nil
}

type memFileInfo struct {
	*MemFile
}

func (i memFileInfo) Mode() fs.FileMode  { return 0666 }
func (i memFileInfo) ModTime() time.Time { return time.Now() }
func (i memFileInfo) IsDir() bool        { return false }
func (i memFileInfo) Sys() interface{}   { return nil }
<commit_msg>MemFile.MarshalJSON and UnmarshalJSON<commit_after>package fs

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io"
	"io/fs"
	"time"

	"github.com/ungerik/go-fs/fsimpl"
)

// Ensure MemFile implements interfaces
var (
	_ FileReader       = new(MemFile)
	_ io.Writer        = new(MemFile)
	_ io.WriterTo      = new(MemFile)
	_ io.ReaderAt      = new(MemFile)
	_ json.Marshaler   = new(MemFile)
	_ json.Unmarshaler = new(MemFile)
	_ gob.GobEncoder   = new(MemFile)
	_ gob.GobDecoder   = new(MemFile)
	_ fmt.Stringer     = new(MemFile)
)

// MemFile implements FileReader with a filename and an in memory byte slice.
// It exposes FileName and FileData as exported struct fields to emphasize
// its simple nature as just an wrapper of a name and some bytes.
//
// MemFile implements the following interfaces:
//   - FileReader
//   - io.Writer
//   - io.WriterTo
//   - io.ReaderAt
//   - json.Marshaler
//   - json.Unmarshaler
//   - gob.GobEncoder
//   - gob.GobDecoder
//   - fmt.Stringer
type MemFile struct {
	FileName string
	FileData []byte
}

// NewMemFile returns a new MemFile
func NewMemFile(name string, data []byte) *MemFile {
	return &MemFile{FileName: name, FileData: data}
}

// MemFileFromFileReader tries to cast and return the passed FileReader
// as *MemFile or else returns the result of ReadMemFile.
func MemFileFromFileReader(fileReader FileReader) (*MemFile, error) {
	if memFile, ok := fileReader.(*MemFile); ok {
		return memFile, nil
	}
	return ReadMemFile(fileReader)
}

// ReadMemFile returns a new MemFile with
// the name from fileReader.Name() and
// the data from fileReader.ReadAll()
func ReadMemFile(fileReader FileReader) (*MemFile, error) {
	data, err := fileReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("ReadMemFile: error reading from FileReader: %w", err)
	}
	return &MemFile{FileName: fileReader.Name(), FileData: data}, nil
}

// ReadMemFileRename returns a new MemFile with the data
// from fileReader.ReadAll() and the passed name.
func ReadMemFileRename(fileReader FileReader, name string) (*MemFile, error) {
	data, err := fileReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("ReadMemFileRename: error reading from FileReader: %w", err)
	}
	return &MemFile{FileName: name, FileData: data}, nil
}

// ReadAllMemFile returns a new MemFile with the data
// from io.ReadAll(r) and the passed name.
func ReadAllMemFile(r io.Reader, name string) (*MemFile, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return nil, fmt.Errorf("ReadAllMemFile: error reading from io.Reader: %w", err)
	}
	return &MemFile{FileName: name, FileData: data}, nil
}

// String returns the name and meta information for the FileReader.
// String implements the fmt.Stringer interface.
func (f *MemFile) String() string {
	return fmt.Sprintf("MemFile{name: %q, size: %d}", f.FileName, len(f.FileData))
}

// Name returns the name of the file
func (f *MemFile) Name() string {
	return f.FileName
}

// Ext returns the extension of file name including the point, or an empty string.
func (f *MemFile) Ext() string {
	return fsimpl.Ext(f.FileName, "")
}

// LocalPath always returns an empty string for a MemFile.
func (f *MemFile) LocalPath() string {
	return ""
}

// Size returns the size of the file
func (f *MemFile) Size() int64 {
	return int64(len(f.FileData))
}

// Exists returns true if the MemFile has non empty FileName.
// It's valid to call this method on a nil pointer,
// will return false in this case.
func (f *MemFile) Exists() bool {
	return f != nil && f.FileName != ""
}

// CheckExists return an ErrDoesNotExist error
// if the file does not exist.
func (f *MemFile) CheckExists() error {
	if !f.Exists() {
		return NewErrDoesNotExistFileReader(f)
	}
	return nil
}

// ContentHash returns a Dropbox compatible content hash for the file.
// See https://www.dropbox.com/developers/reference/content-hash
func (f *MemFile) ContentHash() (string, error) {
	return fsimpl.ContentHashBytes(f.FileData), nil
}

// Write appends the passed bytes to the FileData,
// implementing the io.Writer interface.
func (f *MemFile) Write(b []byte) (int, error) {
	f.FileData = append(f.FileData, b...)
	return len(b), nil
}

// ReadAll copies all bytes of the file
func (f *MemFile) ReadAll() (data []byte, err error) {
	return append([]byte(nil), f.FileData...), nil
}

// ReadAllContentHash copies all bytes of the file
// together with a Dropbox compatible content hash.
// See https://www.dropbox.com/developers/reference/content-hash
func (f *MemFile) ReadAllContentHash() (data []byte, hash string, err error) {
	return append([]byte(nil), f.FileData...), fsimpl.ContentHashBytes(f.FileData), nil
}

// ReadAllString reads the complete file and returns the content as string.
func (f *MemFile) ReadAllString() (string, error) {
	return string(f.FileData), nil
}

// ReadAt reads len(p) bytes into p starting at offset off in the
// underlying input source. It returns the number of bytes
// read (0 <= n <= len(p)) and any error encountered.
//
// When ReadAt returns n < len(p), it returns a non-nil error
// explaining why more bytes were not returned. In this respect,
// ReadAt is stricter than Read.
//
// If the n = len(p) bytes returned by ReadAt are at the end of the
// input source, ReadAt returns err == nil.
//
// Clients of ReadAt can execute parallel ReadAt calls on the
// same input source.
//
// ReadAt implements the interface io.ReaderAt.
func (f *MemFile) ReadAt(p []byte, off int64) (n int, err error) {
	if off >= int64(len(f.FileData)) {
		return 0, io.EOF
	}
	n = copy(p, f.FileData[off:])
	if n < len(p) {
		return n, fmt.Errorf("could only read %d of %d requested bytes", n, len(p))
	}
	return n, nil
}

// WriteTo implements the io.WriterTo interface
func (f *MemFile) WriteTo(writer io.Writer) (n int64, err error) {
	i, err := writer.Write(f.FileData)
	return int64(i), err
}

// OpenReader opens the file and returns a os/fs.File that has be closed after reading
func (f *MemFile) OpenReader() (fs.File, error) {
	return fsimpl.NewReadonlyFileBuffer(f.FileData, memFileInfo{f}), nil
}

// OpenReadSeeker opens the file and returns a ReadSeekCloser.
// Use OpenReader if seeking is not necessary because implementations
// may need additional buffering to support seeking or not support it at all.
func (f *MemFile) OpenReadSeeker() (ReadSeekCloser, error) {
	return fsimpl.NewReadonlyFileBuffer(f.FileData, memFileInfo{f}), nil
}

// ReadJSON reads and unmarshalles the JSON content of the file to output.
func (f *MemFile) ReadJSON(output interface{}) error {
	return json.Unmarshal(f.FileData, output)
}

// ReadXML reads and unmarshalles the XML content of the file to output.
func (f *MemFile) ReadXML(output interface{}) error {
	return xml.Unmarshal(f.FileData, output)
}

// MarshalJSON implements the json.Marshaler interface
func (f *MemFile) MarshalJSON() ([]byte, error) {
	encodedData := base64.RawURLEncoding.EncodeToString(f.FileData)
	return json.Marshal(map[string]string{f.FileName: encodedData})
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (f *MemFile) UnmarshalJSON(j []byte) error {
	m := make(map[string]string, 1)
	err := json.Unmarshal(j, &m)
	if err != nil {
		return fmt.Errorf("can't unmarshal JSON as MemFile: %w", err)
	}
	if len(m) != 1 {
		return fmt.Errorf("can't unmarshal JSON as MemFile: %d object keys", len(m))
	}
	for fileName, encodedData := range m {
		fileData, err := base64.RawURLEncoding.DecodeString(encodedData)
		if err != nil {
			return fmt.Errorf("can't decode base64 JSON data of MemFile: %w", err)
		}
		f.FileName = fileName
		f.FileData = fileData
	}
	return nil
}

// GobEncode gob encodes the file name and content,
// implementing encoding/gob.GobEncoder.
func (f *MemFile) GobEncode() ([]byte, error) {
	buf := bytes.NewBuffer(make([]byte, 0, 16+len(f.FileName)+len(f.FileData)))
	enc := gob.NewEncoder(buf)
	err := enc.Encode(f.FileName)
	if err != nil {
		return nil, fmt.Errorf("MemFile.GobEncode: error encoding FileName: %w", err)
	}
	err = enc.Encode(f.FileData)
	if err != nil {
		return nil, fmt.Errorf("MemFile.GobEncode: error encoding FileData: %w", err)
	}
	return buf.Bytes(), nil
}

// GobDecode decodes gobBytes file name and content,
// implementing encoding/gob.GobDecoder.
func (f *MemFile) GobDecode(gobBytes []byte) error {
	dec := gob.NewDecoder(bytes.NewReader(gobBytes))
	err := dec.Decode(&f.FileName)
	if err != nil {
		return fmt.Errorf("MemFile.GobDecode: error decoding FileName: %w", err)
	}
	err = dec.Decode(&f.FileData)
	if err != nil {
		return fmt.Errorf("MemFile.GobDecode: error decoding FileData: %w", err)
	}
	return nil
}

// Stat returns a io/fs.FileInfo describing the MemFile.
func (f *MemFile) Stat() (fs.FileInfo, error) {
	return memFileInfo{f}, nil
}

type memFileInfo struct {
	*MemFile
}

func (i memFileInfo) Mode() fs.FileMode  { return 0666 }
func (i memFileInfo) ModTime() time.Time { return time.Now() }
func (i memFileInfo) IsDir() bool        { return false }
func (i memFileInfo) Sys() interface{}   { return nil }
<|endoftext|>


<commit_before>package gomol

import (
	"fmt"
	"math"
	"strings"
	"time"
)

type LogLevel int

const (
	LevelDebug   LogLevel = 7
	LevelInfo    LogLevel = 6
	LevelWarning LogLevel = 4
	LevelError   LogLevel = 3
	LevelFatal   LogLevel = 2
	LevelNone    LogLevel = math.MinInt64
)

// ToLogLevel will take a string and return the appropriate log level for
// the string if known.  If the string is not recognized it will return
// LevelUnknown and ErrUnknownLevel.
func ToLogLevel(level string) (LogLevel, error) {
	lowLevel := strings.ToLower(level)

	switch lowLevel {
	case "dbg":
		fallthrough
	case "debug":
		return LevelDebug, nil
	case "info":
		return LevelInfo, nil
	case "warn":
		fallthrough
	case "warning":
		return LevelWarning, nil
	case "err":
		fallthrough
	case "error":
		return LevelError, nil
	case "fatal":
		return LevelFatal, nil
	case "none":
		return LevelNone, nil
	}

	return 0, ErrUnknownLevel
}

func (ll LogLevel) String() string {
	return getLevelName(ll)
}

func getLevelName(level LogLevel) string {
	switch level {
	case LevelNone:
		return "none"
	case LevelDebug:
		return "debug"
	case LevelInfo:
		return "info"
	case LevelWarning:
		return "warn"
	case LevelError:
		return "error"
	case LevelFatal:
		return "fatal"
	default:
		return "unknown"
	}
}

type message struct {
	Base      *Base
	Level     LogLevel
	Timestamp time.Time
	Attrs     *Attrs
	Msg       string
}

func newMessage(timestamp time.Time,
	base *Base,
	level LogLevel,
	msgAttrs *Attrs,
	format string, va ...interface{}) *message {

	msgStr := format
	if len(va) > 0 {
		msgStr = fmt.Sprintf(format, va...)
	}

	var attrs *Attrs
	if msgAttrs != nil {
		attrs = msgAttrs
	} else {
		attrs = NewAttrs()
	}

	nm := &message{
		Base:      base,
		Level:     level,
		Timestamp: timestamp,
		Attrs:     attrs,
		Msg:       msgStr,
	}

	return nm
}
<commit_msg>Fix incorrect doc for ToLogLevel<commit_after>package gomol

import (
	"fmt"
	"math"
	"strings"
	"time"
)

type LogLevel int

const (
	LevelDebug   LogLevel = 7
	LevelInfo    LogLevel = 6
	LevelWarning LogLevel = 4
	LevelError   LogLevel = 3
	LevelFatal   LogLevel = 2
	LevelNone    LogLevel = math.MinInt64
)

// ToLogLevel will take a string and return the appropriate log level for
// the string if known.  If the string is not recognized it will return
// an ErrUnknownLevel error.
func ToLogLevel(level string) (LogLevel, error) {
	lowLevel := strings.ToLower(level)

	switch lowLevel {
	case "dbg":
		fallthrough
	case "debug":
		return LevelDebug, nil
	case "info":
		return LevelInfo, nil
	case "warn":
		fallthrough
	case "warning":
		return LevelWarning, nil
	case "err":
		fallthrough
	case "error":
		return LevelError, nil
	case "fatal":
		return LevelFatal, nil
	case "none":
		return LevelNone, nil
	}

	return 0, ErrUnknownLevel
}

func (ll LogLevel) String() string {
	return getLevelName(ll)
}

func getLevelName(level LogLevel) string {
	switch level {
	case LevelNone:
		return "none"
	case LevelDebug:
		return "debug"
	case LevelInfo:
		return "info"
	case LevelWarning:
		return "warn"
	case LevelError:
		return "error"
	case LevelFatal:
		return "fatal"
	default:
		return "unknown"
	}
}

type message struct {
	Base      *Base
	Level     LogLevel
	Timestamp time.Time
	Attrs     *Attrs
	Msg       string
}

func newMessage(timestamp time.Time,
	base *Base,
	level LogLevel,
	msgAttrs *Attrs,
	format string, va ...interface{}) *message {

	msgStr := format
	if len(va) > 0 {
		msgStr = fmt.Sprintf(format, va...)
	}

	var attrs *Attrs
	if msgAttrs != nil {
		attrs = msgAttrs
	} else {
		attrs = NewAttrs()
	}

	nm := &message{
		Base:      base,
		Level:     level,
		Timestamp: timestamp,
		Attrs:     attrs,
		Msg:       msgStr,
	}

	return nm
}
<|endoftext|>


<commit_before>package golevel7

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"strings"

	"golang.org/x/net/html/charset"
)

// Message is an HL7 message
type Message struct {
	Segments   []Segment
	Value      []rune
	Delimeters Delimeters
}

// NewMessage returns a new message with the v byte value
func NewMessage(v []byte) *Message {
	var utf8V []byte
	if len(v) != 0 {
		reader, err := charset.NewReader(bytes.NewReader(v), "text/plain")
		if err != nil {
			return nil
		}
		utf8V, err = ioutil.ReadAll(reader)
	} else {
		utf8V = v
	}
	newMessage := &Message{
		Value:      []rune(string(utf8V)),
		Delimeters: *NewDelimeters(),
	}
	if err := newMessage.parse(); err != nil {
		log.Fatal(fmt.Sprintf("Parse Error: %+v", err))
	}
	return newMessage
}

func (m *Message) String() string {
	var str string
	for _, s := range m.Segments {
		str += "Message Segment: " + string(s.Value) + "\n"
		// str += s.String() // removed, way too verbose for now
	}
	return str
}

// Segment returns the first matching segmane with name s
func (m *Message) Segment(s string) (*Segment, error) {
	for i, seg := range m.Segments {
		fld, err := seg.Field(0)
		if err != nil {
			continue
		}
		if string(fld.Value) == s {
			return &m.Segments[i], nil
		}
	}
	return nil, fmt.Errorf("Segment not found")
}

// AllSegments returns the first matching segmane with name s
func (m *Message) AllSegments(s string) ([]*Segment, error) {
	segs := []*Segment{}
	for i, seg := range m.Segments {
		fld, err := seg.Field(0)
		if err != nil {
			continue
		}
		if string(fld.Value) == s {
			segs = append(segs, &m.Segments[i])
		}
	}
	if len(segs) == 0 {
		return segs, fmt.Errorf("Segment not found")
	}
	return segs, nil
}

// Find gets a value from a message using location syntax
// finds the first occurence of the segment and first of repeating fields
// if the loc is not valid an error is returned
func (m *Message) Find(loc string) (string, error) {
	return m.Get(NewLocation(loc))
}

// FindAll gets all values from a message using location syntax
// finds all occurences of the segments and all repeating fields
// if the loc is not valid an error is returned
func (m *Message) FindAll(loc string) ([]string, error) {
	return m.GetAll(NewLocation(loc))
}

// Get returns the first value specified by the Location
func (m *Message) Get(l *Location) (string, error) {
	if l.Segment == "" {
		return string(m.Value), nil
	}
	seg, err := m.Segment(l.Segment)
	if err != nil {
		return "", err
	}
	return seg.Get(l)
}

// GetAll returns all values specified by the Location
func (m *Message) GetAll(l *Location) ([]string, error) {
	vals := []string{}
	if l.Segment == "" {
		vals = append(vals, string(m.Value))
		return vals, nil
	}
	segs, err := m.AllSegments(l.Segment)
	if err != nil {
		return vals, err
	}
	for _, s := range segs {
		vs, err := s.GetAll(l)
		if err != nil {
			return vals, err
		}
		vals = append(vals, vs...)
	}
	return vals, nil
}

// Set will insert a value into a message at Location
func (m *Message) Set(l *Location, val string) error {
	if l.Segment == "" {
		return errors.New("Segment is required")
	}
	seg, err := m.Segment(l.Segment)
	if err != nil {
		s := Segment{}
		s.forceField([]rune(l.Segment), 0)
		s.Set(l, val, &m.Delimeters)
		m.Segments = append(m.Segments, s)
	} else {
		seg.Set(l, val, &m.Delimeters)
	}
	m.Value = m.encode()
	return nil
}

func (m *Message) parse() error {
	m.Value = []rune(strings.Trim(string(m.Value), "\n\r\x1c\x0b"))
	if err := m.parseSep(); err != nil {
		return err
	}
	r := strings.NewReader(string(m.Value))
	i := 0
	ii := 0
	for {
		ch, _, err := r.ReadRune()
		if err != nil {
			ch = eof
		}
		ii++
		switch {
		case ch == eof || (ch == endMsg && m.Delimeters.LFTermMsg):
			//just for safety: cannot reproduce this on windows
			safeii := map[bool]int{true: len(m.Value), false: ii}[ii > len(m.Value)]
			v := m.Value[i:safeii]
			if len(v) > 4 { // seg name + field sep
				seg := Segment{Value: v}
				seg.parse(&m.Delimeters)
				m.Segments = append(m.Segments, seg)
			}
			return nil
		case ch == segTerm:
			seg := Segment{Value: m.Value[i : ii-1]}
			seg.parse(&m.Delimeters)
			m.Segments = append(m.Segments, seg)
			i = ii
		case ch == m.Delimeters.Escape:
			ii++
			r.ReadRune()
		}
	}
}

func (m *Message) parseSep() error {
	if len(m.Value) < 8 {
		return errors.New("Invalid message length less than 8 bytes")
	}
	if string(m.Value[:3]) != "MSH" {
		return errors.New("Invalid message: Missing MSH segment")
	}

	r := bytes.NewReader([]byte(string(m.Value)))
	for i := 0; i < 8; i++ {
		ch, _, _ := r.ReadRune()
		if ch == eof {
			return fmt.Errorf("Invalid message: eof while parsing MSH")
		}
		switch i {
		case 3:
			m.Delimeters.Field = ch
		case 4:
			m.Delimeters.DelimeterField = string(ch)
			m.Delimeters.Component = ch
		case 5:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.Repetition = ch
		case 6:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.Escape = ch
		case 7:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.SubComponent = ch
		}
	}
	return nil
}

func (m *Message) encode() []rune {
	buf := [][]byte{}
	for _, s := range m.Segments {
		buf = append(buf, []byte(string(s.Value)))
	}
	return []rune(string(bytes.Join(buf, []byte(string(segTerm)))))
}

// IsValid checks a message for validity based on a set of criteria
// it returns valid and any failed validation rules
func (m *Message) IsValid(val []Validation) (bool, []Validation) {
	failures := []Validation{}
	valid := true
	for _, v := range val {
		values, err := m.FindAll(v.Location)
		if err != nil || len(values) == 0 {
			valid = false
			failures = append(failures, v)
		}
		for _, value := range values {
			if value == "" || (v.VCheck == SpecificValue && v.Value != value) {
				valid = false
				failures = append(failures, v)
			}
		}
	}

	return valid, failures
}

// Unmarshal fills a structure from an HL7 message
// It will panic if interface{} is not a pointer to a struct
// Unmarshal will decode the entire message before trying to set values
// it will set the first matching segment / first matching field
// repeating segments and fields is not well suited to this
// for the moment all unmarshal target fields must be strings
func (m *Message) Unmarshal(it interface{}) error {
	st := reflect.ValueOf(it).Elem()
	stt := st.Type()
	for i := 0; i < st.NumField(); i++ {
		fld := stt.Field(i)
		r := fld.Tag.Get("hl7")
		if r != "" {
			if val, _ := m.Find(r); val != "" {
				if st.Field(i).CanSet() {
					// TODO support fields other than string
					//fldT := st.Field(i).Type()
					st.Field(i).SetString(strings.TrimSpace(val))
				}
			}
		}
	}

	return nil
}

// Info returns the MsgInfo for the message
func (m *Message) Info() (MsgInfo, error) {
	mi := MsgInfo{}
	err := m.Unmarshal(&mi)
	return mi, err
}

func (m *Message) ScanSegments() bool {

	return false
}
<commit_msg>chore: better error message<commit_after>package golevel7

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"strings"

	"golang.org/x/net/html/charset"
)

// Message is an HL7 message
type Message struct {
	Segments   []Segment
	Value      []rune
	Delimeters Delimeters
}

// NewMessage returns a new message with the v byte value
func NewMessage(v []byte) *Message {
	var utf8V []byte
	if len(v) != 0 {
		reader, err := charset.NewReader(bytes.NewReader(v), "text/plain")
		if err != nil {
			return nil
		}
		utf8V, err = ioutil.ReadAll(reader)
	} else {
		utf8V = v
	}
	newMessage := &Message{
		Value:      []rune(string(utf8V)),
		Delimeters: *NewDelimeters(),
	}
	if err := newMessage.parse(); err != nil {
		log.Fatal(fmt.Sprintf("Parse Error: %+v", err))
	}
	return newMessage
}

func (m *Message) String() string {
	var str string
	for _, s := range m.Segments {
		str += "Message Segment: " + string(s.Value) + "\n"
		// str += s.String() // removed, way too verbose for now
	}
	return str
}

// Segment returns the first matching segmane with name s
func (m *Message) Segment(s string) (*Segment, error) {
	for i, seg := range m.Segments {
		fld, err := seg.Field(0)
		if err != nil {
			continue
		}
		if string(fld.Value) == s {
			return &m.Segments[i], nil
		}
	}
	return nil, fmt.Errorf("Segment not found")
}

// AllSegments returns the first matching segmane with name s
func (m *Message) AllSegments(s string) ([]*Segment, error) {
	segs := []*Segment{}
	for i, seg := range m.Segments {
		fld, err := seg.Field(0)
		if err != nil {
			continue
		}
		if string(fld.Value) == s {
			segs = append(segs, &m.Segments[i])
		}
	}
	if len(segs) == 0 {
		return segs, fmt.Errorf("Segment not found")
	}
	return segs, nil
}

// Find gets a value from a message using location syntax
// finds the first occurence of the segment and first of repeating fields
// if the loc is not valid an error is returned
func (m *Message) Find(loc string) (string, error) {
	return m.Get(NewLocation(loc))
}

// FindAll gets all values from a message using location syntax
// finds all occurences of the segments and all repeating fields
// if the loc is not valid an error is returned
func (m *Message) FindAll(loc string) ([]string, error) {
	return m.GetAll(NewLocation(loc))
}

// Get returns the first value specified by the Location
func (m *Message) Get(l *Location) (string, error) {
	if l.Segment == "" {
		return string(m.Value), nil
	}
	seg, err := m.Segment(l.Segment)
	if err != nil {
		return "", err
	}
	return seg.Get(l)
}

// GetAll returns all values specified by the Location
func (m *Message) GetAll(l *Location) ([]string, error) {
	vals := []string{}
	if l.Segment == "" {
		vals = append(vals, string(m.Value))
		return vals, nil
	}
	segs, err := m.AllSegments(l.Segment)
	if err != nil {
		return vals, err
	}
	for _, s := range segs {
		vs, err := s.GetAll(l)
		if err != nil {
			return vals, err
		}
		vals = append(vals, vs...)
	}
	return vals, nil
}

// Set will insert a value into a message at Location
func (m *Message) Set(l *Location, val string) error {
	if l.Segment == "" {
		return errors.New("Segment is required")
	}
	seg, err := m.Segment(l.Segment)
	if err != nil {
		s := Segment{}
		s.forceField([]rune(l.Segment), 0)
		s.Set(l, val, &m.Delimeters)
		m.Segments = append(m.Segments, s)
	} else {
		seg.Set(l, val, &m.Delimeters)
	}
	m.Value = m.encode()
	return nil
}

func (m *Message) parse() error {
	m.Value = []rune(strings.Trim(string(m.Value), "\n\r\x1c\x0b"))
	if err := m.parseSep(); err != nil {
		return err
	}
	r := strings.NewReader(string(m.Value))
	i := 0
	ii := 0
	for {
		ch, _, err := r.ReadRune()
		if err != nil {
			ch = eof
		}
		ii++
		switch {
		case ch == eof || (ch == endMsg && m.Delimeters.LFTermMsg):
			//just for safety: cannot reproduce this on windows
			safeii := map[bool]int{true: len(m.Value), false: ii}[ii > len(m.Value)]
			v := m.Value[i:safeii]
			if len(v) > 4 { // seg name + field sep
				seg := Segment{Value: v}
				seg.parse(&m.Delimeters)
				m.Segments = append(m.Segments, seg)
			}
			return nil
		case ch == segTerm:
			seg := Segment{Value: m.Value[i : ii-1]}
			seg.parse(&m.Delimeters)
			m.Segments = append(m.Segments, seg)
			i = ii
		case ch == m.Delimeters.Escape:
			ii++
			r.ReadRune()
		}
	}
}

func (m *Message) parseSep() error {
	if len(m.Value) < 8 {
		return errors.New("Invalid message length less than 8 bytes")
	}
	if string(m.Value[:3]) != "MSH" {
		return fmt.Errorf("Invalid message: Missing MSH segment -> %v", m.Value[:3])
	}

	r := bytes.NewReader([]byte(string(m.Value)))
	for i := 0; i < 8; i++ {
		ch, _, _ := r.ReadRune()
		if ch == eof {
			return fmt.Errorf("Invalid message: eof while parsing MSH")
		}
		switch i {
		case 3:
			m.Delimeters.Field = ch
		case 4:
			m.Delimeters.DelimeterField = string(ch)
			m.Delimeters.Component = ch
		case 5:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.Repetition = ch
		case 6:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.Escape = ch
		case 7:
			m.Delimeters.DelimeterField += string(ch)
			m.Delimeters.SubComponent = ch
		}
	}
	return nil
}

func (m *Message) encode() []rune {
	buf := [][]byte{}
	for _, s := range m.Segments {
		buf = append(buf, []byte(string(s.Value)))
	}
	return []rune(string(bytes.Join(buf, []byte(string(segTerm)))))
}

// IsValid checks a message for validity based on a set of criteria
// it returns valid and any failed validation rules
func (m *Message) IsValid(val []Validation) (bool, []Validation) {
	failures := []Validation{}
	valid := true
	for _, v := range val {
		values, err := m.FindAll(v.Location)
		if err != nil || len(values) == 0 {
			valid = false
			failures = append(failures, v)
		}
		for _, value := range values {
			if value == "" || (v.VCheck == SpecificValue && v.Value != value) {
				valid = false
				failures = append(failures, v)
			}
		}
	}

	return valid, failures
}

// Unmarshal fills a structure from an HL7 message
// It will panic if interface{} is not a pointer to a struct
// Unmarshal will decode the entire message before trying to set values
// it will set the first matching segment / first matching field
// repeating segments and fields is not well suited to this
// for the moment all unmarshal target fields must be strings
func (m *Message) Unmarshal(it interface{}) error {
	st := reflect.ValueOf(it).Elem()
	stt := st.Type()
	for i := 0; i < st.NumField(); i++ {
		fld := stt.Field(i)
		r := fld.Tag.Get("hl7")
		if r != "" {
			if val, _ := m.Find(r); val != "" {
				if st.Field(i).CanSet() {
					// TODO support fields other than string
					//fldT := st.Field(i).Type()
					st.Field(i).SetString(strings.TrimSpace(val))
				}
			}
		}
	}

	return nil
}

// Info returns the MsgInfo for the message
func (m *Message) Info() (MsgInfo, error) {
	mi := MsgInfo{}
	err := m.Unmarshal(&mi)
	return mi, err
}

func (m *Message) ScanSegments() bool {

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/ActiveState/log"
	"logyard"
	"logyard/clients/common"
	"logyard/clients/sieve"
	"logyard/clients/systail"
	"time"
)

func main() {
	LoadConfig()

	parser := sieve.NewStackatoParser(getConfig().Events)
	parser.DeleteSamples()

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()
	sub := logyard.Broker.Subscribe("systail")
	defer sub.Stop()

	log.Info("Watching the systail stream on this node")
	for message := range sub.Ch {
		var record systail.Message
		err := json.Unmarshal([]byte(message.Value), &record)
		if err != nil {
			log.Warnf("failed to parse json: %s; ignoring record: %s",
				err, message.Value)
			continue
		}

		event, err := parser.Parse(record.Name, record.Text)
		if err != nil {
			log.Warnf(
				"failed to parse event from %s: %s -- source: %s", record.Name, err, record.Text)
			continue
		}
		if event != nil {
			event.MessageCommon = common.NewMessageCommon(event.Desc, time.Unix(record.UnixTime, 0), record.NodeID)
			event.MustPublish(pub)
		}

	}
}
<commit_msg>log something when starting sieve<commit_after>package main

import (
	"encoding/json"
	"github.com/ActiveState/log"
	"github.com/alecthomas/gozmq"
	"logyard"
	"logyard/clients/common"
	"logyard/clients/sieve"
	"logyard/clients/systail"
	"time"
)

func main() {
	major, minor, patch := gozmq.Version()
	log.Infof("Starting logyard_sieve (zeromq %d.%d.%d)", major, minor, patch)

	LoadConfig()

	parser := sieve.NewStackatoParser(getConfig().Events)
	parser.DeleteSamples()

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()
	sub := logyard.Broker.Subscribe("systail")
	defer sub.Stop()

	log.Info("Watching the systail stream on this node")
	for message := range sub.Ch {
		var record systail.Message
		err := json.Unmarshal([]byte(message.Value), &record)
		if err != nil {
			log.Warnf("failed to parse json: %s; ignoring record: %s",
				err, message.Value)
			continue
		}

		event, err := parser.Parse(record.Name, record.Text)
		if err != nil {
			log.Warnf(
				"failed to parse event from %s: %s -- source: %s", record.Name, err, record.Text)
			continue
		}
		if event != nil {
			event.MessageCommon = common.NewMessageCommon(event.Desc, time.Unix(record.UnixTime, 0), record.NodeID)
			event.MustPublish(pub)
		}

	}
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/json"
	"errors"
	"fmt"

	"code.cloudfoundry.org/lager"
	"github.com/jinzhu/gorm"

	"cred-alert/sniff"
)

//go:generate counterfeiter . RepositoryRepository

type RepositoryRepository interface {
	Create(*Repository) error

	Find(owner string, name string) (Repository, bool, error)
	MustFind(owner string, name string) (Repository, error)

	All() ([]Repository, error)
	Active() ([]Repository, error)
	AllForOrganization(string) ([]Repository, error)
	NotScannedWithVersion(int) ([]Repository, error)

	MarkAsCloned(string, string, string) error
	RegisterFailedFetch(lager.Logger, *Repository) error
	UpdateCredentialCount(*Repository, map[string]uint) error
}

type repositoryRepository struct {
	db *gorm.DB
}

func NewRepositoryRepository(db *gorm.DB) *repositoryRepository {
	return &repositoryRepository{
		db: db,
	}
}

func (r *repositoryRepository) Find(owner, name string) (Repository, bool, error) {
	repo, err := r.MustFind(owner, name)

	if err == gorm.ErrRecordNotFound {
		return Repository{}, false, nil
	} else if err != nil {
		return Repository{}, false, err
	}

	return repo, true, nil
}

func (r *repositoryRepository) MustFind(owner string, name string) (Repository, error) {
	var repo Repository
	err := r.db.Where("owner = ? AND name = ?", owner, name).First(&repo).Error

	if err != nil {
		return Repository{}, err
	}

	return repo, nil
}

func (r *repositoryRepository) Create(repository *Repository) error {
	return r.db.Create(repository).Error
}

func (r *repositoryRepository) All() ([]Repository, error) {
	var existingRepositories []Repository
	err := r.db.Find(&existingRepositories).Error
	if err != nil {
		return nil, err
	}

	return existingRepositories, nil
}

func (r *repositoryRepository) Active() ([]Repository, error) {
	var repos []Repository
	err := r.db.Where("disabled = ? AND cloned = ?", false, true).Find(&repos).Error
	if err != nil {
		return nil, err
	}

	return repos, nil
}

func (r *repositoryRepository) AllForOrganization(owner string) ([]Repository, error) {
	var repositories []Repository
	err := r.db.Where("owner = ?", owner).Find(&repositories).Error
	if err != nil {
		return nil, err
	}

	return repositories, nil
}

func (r *repositoryRepository) MarkAsCloned(owner, name, path string) error {
	return r.db.Model(&Repository{}).Where(
		Repository{Name: name, Owner: owner},
	).Updates(
		map[string]interface{}{"cloned": true, "path": path},
	).Error
}

func (r *repositoryRepository) NotScannedWithVersion(version int) ([]Repository, error) {
	rows, err := r.db.Raw(`
    SELECT r.id
    FROM   scans s
           JOIN repositories r
             ON r.id = s.repository_id
           JOIN (SELECT repository_id      AS r_id,
                        MAX(rules_version) AS rules_version
                 FROM   scans
                 GROUP  BY repository_id
                ) latest_scans
             ON s.rules_version = latest_scans.rules_version
                AND s.repository_id = latest_scans.r_id
    WHERE  r.cloned = true
      AND  latest_scans.rules_version != ?`, sniff.RulesVersion).Rows()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var ids []int
	for rows.Next() {
		var id int
		scanErr := rows.Scan(&id)
		if scanErr != nil {
			return nil, scanErr
		}
		ids = append(ids, id)
	}

	var repositories []Repository
	err = r.db.Model(&Repository{}).Where("id IN (?)", ids).Find(&repositories).Error
	if err != nil {
		return nil, err
	}

	return repositories, nil
}

const FailedFetchThreshold = 3

func (r *repositoryRepository) RegisterFailedFetch(
	logger lager.Logger,
	repo *Repository,
) error {
	logger = logger.Session("register-failed-fetch", lager.Data{
		"ID": repo.ID,
	})

	tx, err := r.db.DB().Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	result, err := tx.Exec(`
		UPDATE repositories
		SET failed_fetches = failed_fetches + 1
		WHERE id = ?
	`, repo.ID)
	if err != nil {
		return err
	}

	rows, err := result.RowsAffected()
	if err != nil {
		return err
	}

	if rows == 0 {
		err := errors.New("repository could not be found")
		logger.Error("repository-not-found", err)
		return err
	}

	result, err = tx.Exec(`
		UPDATE repositories
		SET disabled = true
		WHERE id = ?
		AND failed_fetches >= ?
	`, repo.ID, FailedFetchThreshold)
	if err != nil {
		return err
	}

	rows, err = result.RowsAffected()
	if err != nil {
		return err
	}

	if rows > 0 {
		e := errors.New(fmt.Sprintf("failed to fetch %d times", FailedFetchThreshold))
		logger.Error("repository-disabled", e)
	}

	return tx.Commit()
}

func (r *repositoryRepository) UpdateCredentialCount(repository *Repository, credentialCounts map[string]uint) error {
	credentialCountJSON, err := json.Marshal(credentialCounts)
	if err != nil {
		return err
	}

	_, err = r.db.DB().Exec(`
		UPDATE repositories
		SET credential_counts = ?
		WHERE id = ?
	`, credentialCountJSON, repository.ID)

	return err
}
<commit_msg>Log disabled repositories as Info instead of Error<commit_after>package db

import (
	"encoding/json"
	"errors"

	"code.cloudfoundry.org/lager"
	"github.com/jinzhu/gorm"

	"cred-alert/sniff"
)

//go:generate counterfeiter . RepositoryRepository

type RepositoryRepository interface {
	Create(*Repository) error

	Find(owner string, name string) (Repository, bool, error)
	MustFind(owner string, name string) (Repository, error)

	All() ([]Repository, error)
	Active() ([]Repository, error)
	AllForOrganization(string) ([]Repository, error)
	NotScannedWithVersion(int) ([]Repository, error)

	MarkAsCloned(string, string, string) error
	RegisterFailedFetch(lager.Logger, *Repository) error
	UpdateCredentialCount(*Repository, map[string]uint) error
}

type repositoryRepository struct {
	db *gorm.DB
}

func NewRepositoryRepository(db *gorm.DB) *repositoryRepository {
	return &repositoryRepository{
		db: db,
	}
}

func (r *repositoryRepository) Find(owner, name string) (Repository, bool, error) {
	repo, err := r.MustFind(owner, name)

	if err == gorm.ErrRecordNotFound {
		return Repository{}, false, nil
	} else if err != nil {
		return Repository{}, false, err
	}

	return repo, true, nil
}

func (r *repositoryRepository) MustFind(owner string, name string) (Repository, error) {
	var repo Repository
	err := r.db.Where("owner = ? AND name = ?", owner, name).First(&repo).Error

	if err != nil {
		return Repository{}, err
	}

	return repo, nil
}

func (r *repositoryRepository) Create(repository *Repository) error {
	return r.db.Create(repository).Error
}

func (r *repositoryRepository) All() ([]Repository, error) {
	var existingRepositories []Repository
	err := r.db.Find(&existingRepositories).Error
	if err != nil {
		return nil, err
	}

	return existingRepositories, nil
}

func (r *repositoryRepository) Active() ([]Repository, error) {
	var repos []Repository
	err := r.db.Where("disabled = ? AND cloned = ?", false, true).Find(&repos).Error
	if err != nil {
		return nil, err
	}

	return repos, nil
}

func (r *repositoryRepository) AllForOrganization(owner string) ([]Repository, error) {
	var repositories []Repository
	err := r.db.Where("owner = ?", owner).Find(&repositories).Error
	if err != nil {
		return nil, err
	}

	return repositories, nil
}

func (r *repositoryRepository) MarkAsCloned(owner, name, path string) error {
	return r.db.Model(&Repository{}).Where(
		Repository{Name: name, Owner: owner},
	).Updates(
		map[string]interface{}{"cloned": true, "path": path},
	).Error
}

func (r *repositoryRepository) NotScannedWithVersion(version int) ([]Repository, error) {
	rows, err := r.db.Raw(`
    SELECT r.id
    FROM   scans s
           JOIN repositories r
             ON r.id = s.repository_id
           JOIN (SELECT repository_id      AS r_id,
                        MAX(rules_version) AS rules_version
                 FROM   scans
                 GROUP  BY repository_id
                ) latest_scans
             ON s.rules_version = latest_scans.rules_version
                AND s.repository_id = latest_scans.r_id
    WHERE  r.cloned = true
      AND  latest_scans.rules_version != ?`, sniff.RulesVersion).Rows()
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var ids []int
	for rows.Next() {
		var id int
		scanErr := rows.Scan(&id)
		if scanErr != nil {
			return nil, scanErr
		}
		ids = append(ids, id)
	}

	var repositories []Repository
	err = r.db.Model(&Repository{}).Where("id IN (?)", ids).Find(&repositories).Error
	if err != nil {
		return nil, err
	}

	return repositories, nil
}

const FailedFetchThreshold = 3

func (r *repositoryRepository) RegisterFailedFetch(
	logger lager.Logger,
	repo *Repository,
) error {
	logger = logger.Session("register-failed-fetch", lager.Data{
		"ID": repo.ID,
	})

	tx, err := r.db.DB().Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	result, err := tx.Exec(`
		UPDATE repositories
		SET failed_fetches = failed_fetches + 1
		WHERE id = ?
	`, repo.ID)
	if err != nil {
		return err
	}

	rows, err := result.RowsAffected()
	if err != nil {
		return err
	}

	if rows == 0 {
		err := errors.New("repository could not be found")
		logger.Error("repository-not-found", err)
		return err
	}

	result, err = tx.Exec(`
		UPDATE repositories
		SET disabled = true
		WHERE id = ?
		AND failed_fetches >= ?
	`, repo.ID, FailedFetchThreshold)
	if err != nil {
		return err
	}

	rows, err = result.RowsAffected()
	if err != nil {
		return err
	}

	if rows > 0 {
		logger.Info("repository-disabled", lager.Data{
			"fetch-attempts": FailedFetchThreshold,
		})
	}

	return tx.Commit()
}

func (r *repositoryRepository) UpdateCredentialCount(repository *Repository, credentialCounts map[string]uint) error {
	credentialCountJSON, err := json.Marshal(credentialCounts)
	if err != nil {
		return err
	}

	_, err = r.db.DB().Exec(`
		UPDATE repositories
		SET credential_counts = ?
		WHERE id = ?
	`, credentialCountJSON, repository.ID)

	return err
}
<|endoftext|>


<commit_before>package main

import "github.com/prometheus/client_golang/prometheus"

const (
	namespace = "rabbitmq"
)

var (
	queueLabels    = []string{"vhost", "queue"}
	exchangeLabels = []string{"vhost", "exchange"}
	nodeLabels     = []string{"vhost", "node"}

	upMetricDescription = newGauge("up", "Was the last scrape of rabbitmq successful.")

	overviewMetricDescription = map[string]prometheus.Gauge{
		"object_totals.channels":    newGauge("channelsTotal", "Total number of open channels."),
		"object_totals.connections": newGauge("connectionsTotal", "Total number of open connections."),
		"object_totals.consumers":   newGauge("consumersTotal", "Total number of message consumers."),
		"object_totals.queues":      newGauge("queuesTotal", "Total number of queues in use."),
		"object_totals.exchanges":   newGauge("exchangesTotal", "Total number of exchanges in use."),
	}

	queueGaugeVec = map[string]*prometheus.GaugeVec{
		"messages_ready":               newGaugeVec("queue_messages_ready", "Number of messages ready to be delivered to clients.", queueLabels),
		"messages_unacknowledged":      newGaugeVec("queue_messages_unacknowledged", "Number of messages delivered to clients but not yet acknowledged.", queueLabels),
		"messages":                     newGaugeVec("queue_messages", "Sum of ready and unacknowledged messages (queue depth).", queueLabels),
		"messages_ready_ram":           newGaugeVec("queue_messages_ready_ram", "Number of messages from messages_ready which are resident in ram.", queueLabels),
		"messages_unacknowledged_ram":  newGaugeVec("queue_messages_unacknowledged_ram", "Number of messages from messages_unacknowledged which are resident in ram.", queueLabels),
		"messages_ram":                 newGaugeVec("queue_messages_ram", "Total number of messages which are resident in ram.", queueLabels),
		"messages_persistent":          newGaugeVec("queue_messages_persistent", "Total number of persistent messages in the queue (will always be 0 for transient queues).", queueLabels),
		"message_bytes":                newGaugeVec("queue_message_bytes", "Sum of the size of all message bodies in the queue. This does not include the message properties (including headers) or any overhead.", queueLabels),
		"message_bytes_ready":          newGaugeVec("queue_message_bytes_ready", "Like message_bytes but counting only those messages ready to be delivered to clients.", queueLabels),
		"message_bytes_unacknowledged": newGaugeVec("queue_message_bytes_unacknowledged", "Like message_bytes but counting only those messages delivered to clients but not yet acknowledged.", queueLabels),
		"message_bytes_ram":            newGaugeVec("queue_message_bytes_ram", "Like message_bytes but counting only those messages which are in RAM.", queueLabels),
		"message_bytes_persistent":     newGaugeVec("queue_message_bytes_persistent", "Like message_bytes but counting only those messages which are persistent.", queueLabels),
		"consumers":                    newGaugeVec("queue_consumers", "Number of consumers.", queueLabels),
		"consumer_utilisation":         newGaugeVec("queue_consumer_utilisation", "Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.", queueLabels),
		"memory":                       newGaugeVec("queue_memory", "Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures.", queueLabels),
		"head_message_timestamp":       newGaugeVec("queue_head_message_timestamp", "The timestamp property of the first message in the queue, if present. Timestamps of messages only appear when they are in the paged-in state.", queueLabels), //https://github.com/rabbitmq/rabbitmq-server/pull/54
	}

	queueCounterVec = map[string]*prometheus.Desc{
		"disk_reads":                  newDesc("queue_disk_reads", "Total number of times messages have been read from disk by this queue since it started.", queueLabels),
		"disk_writes":                 newDesc("queue_disk_writes", "Total number of times messages have been written to disk by this queue since it started.", queueLabels),
		"message_stats.publish":       newDesc("queue_messages_published_total", "Count of messages published.", queueLabels),
		"message_stats.confirm":       newDesc("queue_messages_confirmed_total", "Count of messages confirmed. ", queueLabels),
		"message_stats.deliver":       newDesc("queue_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", queueLabels),
		"message_stats.deliver_noack": newDesc("queue_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", queueLabels),
		"message_stats.get":           newDesc("queue_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.get_noack":     newDesc("queue_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.redeliver":     newDesc("queue_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", queueLabels),
		"message_stats.return":        newDesc("queue_messages_returned_total", "Count of messages returned to publisher as unroutable.", queueLabels),
	}

	nodeCounterVec = map[string]*prometheus.Desc{
		"running":         newDesc("running", "test", nodeLabels),
		"mem_used":        newDesc("node_mem_used", "Memory used in bytes", nodeLabels),
		"mem_limit":       newDesc("node_mem_limit", "Point at which the memory alarm will go off", nodeLabels),
		"mem_alarm":       newDesc("node_mem_alarm", "Whether the memory alarm has gone off", nodeLabels),
		"disk_free":       newDesc("node_disk_free", "Disk free space in bytes.", nodeLabels),
		"disk_free_alarm": newDesc("node_disk_free_alarm", "Whether the disk alarm has gone off.", nodeLabels),
		"disk_free_limit": newDesc("node_disk_free_limit", "Point at which the disk alarm will go off.", nodeLabels),
		"fd_used":         newDesc("fd_used", "Used File descriptors", nodeLabels),
		"fd_limit":        newDesc("fd_limit", "File descriptors available", nodeLabels),
		"socket_used":     newDesc("socket_used", "File descriptors used as sockets.", nodeLabels),
		"socket_limit":    newDesc("socket_limit", "File descriptors available for use as sockets", nodeLabels),
	}

	exchangeCounterVec = map[string]*prometheus.Desc{
		"message_stats.publish":           newDesc("exchange_messages_published_total", "Count of messages published.", exchangeLabels),
		"message_stats.publish_in":        newDesc("exchange_messages_published_in_total", "Count of messages published in to an exchange, i.e. not taking account of routing.", exchangeLabels),
		"message_stats.publish_out":       newDesc("exchange_messages_published_out_total", "Count of messages published out of an exchange, i.e. taking account of routing.", exchangeLabels),
		"message_stats.confirm":           newDesc("exchange_messages_confirmed_total", "Count of messages confirmed. ", exchangeLabels),
		"message_stats.deliver":           newDesc("exchange_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", exchangeLabels),
		"message_stats.deliver_noack":     newDesc("exchange_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", exchangeLabels),
		"message_stats.get":               newDesc("exchange_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.get_noack":         newDesc("exchange_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.ack":               newDesc("exchange_messages_ack_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.redeliver":         newDesc("exchange_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", exchangeLabels),
		"message_stats.return_unroutable": newDesc("exchange_messages_returned_total", "Count of messages returned to publisher as unroutable.", exchangeLabels),
	}
)

func newGaugeVec(metricName string, docString string, labels []string) *prometheus.GaugeVec {
	return prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      metricName,
			Help:      docString,
		},
		labels,
	)
}

func newGauge(metricName string, docString string) prometheus.Gauge {
	return prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      metricName,
			Help:      docString,
		},
	)
}

func newDesc(metricName string, docString string, labels []string) *prometheus.Desc {
	return prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", metricName),
		docString,
		labels,
		nil)
}
<commit_msg>Fix field names of node (#27)<commit_after>package main

import "github.com/prometheus/client_golang/prometheus"

const (
	namespace = "rabbitmq"
)

var (
	queueLabels    = []string{"vhost", "queue"}
	exchangeLabels = []string{"vhost", "exchange"}
	nodeLabels     = []string{"vhost", "node"}

	upMetricDescription = newGauge("up", "Was the last scrape of rabbitmq successful.")

	overviewMetricDescription = map[string]prometheus.Gauge{
		"object_totals.channels":    newGauge("channelsTotal", "Total number of open channels."),
		"object_totals.connections": newGauge("connectionsTotal", "Total number of open connections."),
		"object_totals.consumers":   newGauge("consumersTotal", "Total number of message consumers."),
		"object_totals.queues":      newGauge("queuesTotal", "Total number of queues in use."),
		"object_totals.exchanges":   newGauge("exchangesTotal", "Total number of exchanges in use."),
	}

	queueGaugeVec = map[string]*prometheus.GaugeVec{
		"messages_ready":               newGaugeVec("queue_messages_ready", "Number of messages ready to be delivered to clients.", queueLabels),
		"messages_unacknowledged":      newGaugeVec("queue_messages_unacknowledged", "Number of messages delivered to clients but not yet acknowledged.", queueLabels),
		"messages":                     newGaugeVec("queue_messages", "Sum of ready and unacknowledged messages (queue depth).", queueLabels),
		"messages_ready_ram":           newGaugeVec("queue_messages_ready_ram", "Number of messages from messages_ready which are resident in ram.", queueLabels),
		"messages_unacknowledged_ram":  newGaugeVec("queue_messages_unacknowledged_ram", "Number of messages from messages_unacknowledged which are resident in ram.", queueLabels),
		"messages_ram":                 newGaugeVec("queue_messages_ram", "Total number of messages which are resident in ram.", queueLabels),
		"messages_persistent":          newGaugeVec("queue_messages_persistent", "Total number of persistent messages in the queue (will always be 0 for transient queues).", queueLabels),
		"message_bytes":                newGaugeVec("queue_message_bytes", "Sum of the size of all message bodies in the queue. This does not include the message properties (including headers) or any overhead.", queueLabels),
		"message_bytes_ready":          newGaugeVec("queue_message_bytes_ready", "Like message_bytes but counting only those messages ready to be delivered to clients.", queueLabels),
		"message_bytes_unacknowledged": newGaugeVec("queue_message_bytes_unacknowledged", "Like message_bytes but counting only those messages delivered to clients but not yet acknowledged.", queueLabels),
		"message_bytes_ram":            newGaugeVec("queue_message_bytes_ram", "Like message_bytes but counting only those messages which are in RAM.", queueLabels),
		"message_bytes_persistent":     newGaugeVec("queue_message_bytes_persistent", "Like message_bytes but counting only those messages which are persistent.", queueLabels),
		"consumers":                    newGaugeVec("queue_consumers", "Number of consumers.", queueLabels),
		"consumer_utilisation":         newGaugeVec("queue_consumer_utilisation", "Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.", queueLabels),
		"memory":                       newGaugeVec("queue_memory", "Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures.", queueLabels),
		"head_message_timestamp":       newGaugeVec("queue_head_message_timestamp", "The timestamp property of the first message in the queue, if present. Timestamps of messages only appear when they are in the paged-in state.", queueLabels), //https://github.com/rabbitmq/rabbitmq-server/pull/54
	}

	queueCounterVec = map[string]*prometheus.Desc{
		"disk_reads":                  newDesc("queue_disk_reads", "Total number of times messages have been read from disk by this queue since it started.", queueLabels),
		"disk_writes":                 newDesc("queue_disk_writes", "Total number of times messages have been written to disk by this queue since it started.", queueLabels),
		"message_stats.publish":       newDesc("queue_messages_published_total", "Count of messages published.", queueLabels),
		"message_stats.confirm":       newDesc("queue_messages_confirmed_total", "Count of messages confirmed. ", queueLabels),
		"message_stats.deliver":       newDesc("queue_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", queueLabels),
		"message_stats.deliver_noack": newDesc("queue_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", queueLabels),
		"message_stats.get":           newDesc("queue_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.get_noack":     newDesc("queue_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.redeliver":     newDesc("queue_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", queueLabels),
		"message_stats.return":        newDesc("queue_messages_returned_total", "Count of messages returned to publisher as unroutable.", queueLabels),
	}

	nodeCounterVec = map[string]*prometheus.Desc{
		"running":         newDesc("running", "test", nodeLabels),
		"mem_used":        newDesc("node_mem_used", "Memory used in bytes", nodeLabels),
		"mem_limit":       newDesc("node_mem_limit", "Point at which the memory alarm will go off", nodeLabels),
		"mem_alarm":       newDesc("node_mem_alarm", "Whether the memory alarm has gone off", nodeLabels),
		"disk_free":       newDesc("node_disk_free", "Disk free space in bytes.", nodeLabels),
		"disk_free_alarm": newDesc("node_disk_free_alarm", "Whether the disk alarm has gone off.", nodeLabels),
		"disk_free_limit": newDesc("node_disk_free_limit", "Point at which the disk alarm will go off.", nodeLabels),
		"fd_used":         newDesc("fd_used", "Used File descriptors", nodeLabels),
		"fd_total":        newDesc("fd_total", "File descriptors available", nodeLabels),
		"sockets_used":    newDesc("sockets_used", "File descriptors used as sockets.", nodeLabels),
		"sockets_total":   newDesc("sockets_total", "File descriptors available for use as sockets", nodeLabels),
	}

	exchangeCounterVec = map[string]*prometheus.Desc{
		"message_stats.publish":           newDesc("exchange_messages_published_total", "Count of messages published.", exchangeLabels),
		"message_stats.publish_in":        newDesc("exchange_messages_published_in_total", "Count of messages published in to an exchange, i.e. not taking account of routing.", exchangeLabels),
		"message_stats.publish_out":       newDesc("exchange_messages_published_out_total", "Count of messages published out of an exchange, i.e. taking account of routing.", exchangeLabels),
		"message_stats.confirm":           newDesc("exchange_messages_confirmed_total", "Count of messages confirmed. ", exchangeLabels),
		"message_stats.deliver":           newDesc("exchange_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", exchangeLabels),
		"message_stats.deliver_noack":     newDesc("exchange_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", exchangeLabels),
		"message_stats.get":               newDesc("exchange_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.get_noack":         newDesc("exchange_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.ack":               newDesc("exchange_messages_ack_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", exchangeLabels),
		"message_stats.redeliver":         newDesc("exchange_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", exchangeLabels),
		"message_stats.return_unroutable": newDesc("exchange_messages_returned_total", "Count of messages returned to publisher as unroutable.", exchangeLabels),
	}
)

func newGaugeVec(metricName string, docString string, labels []string) *prometheus.GaugeVec {
	return prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      metricName,
			Help:      docString,
		},
		labels,
	)
}

func newGauge(metricName string, docString string) prometheus.Gauge {
	return prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Name:      metricName,
			Help:      docString,
		},
	)
}

func newDesc(metricName string, docString string, labels []string) *prometheus.Desc {
	return prometheus.NewDesc(
		prometheus.BuildFQName(namespace, "", metricName),
		docString,
		labels,
		nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/valyala/fasthttp"
)

var bytesSlash = []byte("/") // heap optimization

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	HostString string
	host       []byte
	uri        []byte
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		HostString: host,
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		host:  []byte(host),
		uri:   []byte{}, // heap optimization
		debug: debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	w.uri = w.uri[:0]
	w.uri = append(w.uri, w.host...)
	w.uri = append(w.uri, bytesSlash...)
	w.uri = append(w.uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(w.uri)
	req.SetBody(q.Body)

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// Assumes the response is JSON! This holds for Influx
			// and Elastic.

			var pretty bytes.Buffer
			prefix := fmt.Sprintf("ID %d: ", q.ID)
			err = json.Indent(&pretty, resp.Body(), prefix, "  ")
			if err != nil {
				return
			}

			_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty.Bytes())
			if err != nil {
				return
			}
		}
	}

	return lag, err
}
<commit_msg>ES 6.x API requires content-type application/json<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/valyala/fasthttp"
)

var bytesSlash = []byte("/") // heap optimization
var applicationJson = []byte("application/json")

// HTTPClient is a reusable HTTP Client.
type HTTPClient struct {
	client     fasthttp.Client
	HostString string
	host       []byte
	uri        []byte
	debug      int
}

// HTTPClientDoOptions wraps options uses when calling `Do`.
type HTTPClientDoOptions struct {
	Debug                int
	PrettyPrintResponses bool
}

// NewHTTPClient creates a new HTTPClient.
func NewHTTPClient(host string, debug int) *HTTPClient {
	return &HTTPClient{
		HostString: host,
		client: fasthttp.Client{
			Name: "query_benchmarker",
		},
		host:  []byte(host),
		uri:   []byte{}, // heap optimization
		debug: debug,
	}
}

// Do performs the action specified by the given Query. It uses fasthttp, and
// tries to minimize heap allocations.
func (w *HTTPClient) Do(q *Query, opts *HTTPClientDoOptions) (lag float64, err error) {
	// populate uri from the reusable byte slice:
	w.uri = w.uri[:0]
	w.uri = append(w.uri, w.host...)
	w.uri = append(w.uri, bytesSlash...)
	w.uri = append(w.uri, q.Path...)

	// populate a request with data from the Query:
	req := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(req)

	req.Header.SetMethodBytes(q.Method)
	req.Header.SetRequestURIBytes(w.uri)
	req.Header.SetContentTypeBytes(applicationJson)
	req.SetBody(q.Body)

	// Perform the request while tracking latency:
	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	start := time.Now()
	err = w.client.Do(req, resp)
	lag = float64(time.Since(start).Nanoseconds()) / 1e6 // milliseconds

	// Check that the status code was 200 OK:
	if err == nil {
		sc := resp.StatusCode()
		if sc != fasthttp.StatusOK {
			err = fmt.Errorf("Invalid write response (status %d): %s", sc, resp.Body())
			return
		}
	}

	if opts != nil {
		// Print debug messages, if applicable:
		switch opts.Debug {
		case 1:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms\n", q.HumanLabel, lag)
		case 2:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
		case 3:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
		case 4:
			fmt.Fprintf(os.Stderr, "debug: %s in %7.2fms -- %s\n", q.HumanLabel, lag, q.HumanDescription)
			fmt.Fprintf(os.Stderr, "debug:   request: %s\n", string(q.String()))
			fmt.Fprintf(os.Stderr, "debug:   response: %s\n", string(resp.Body()))
		default:
		}

		// Pretty print JSON responses, if applicable:
		if opts.PrettyPrintResponses {
			// Assumes the response is JSON! This holds for Influx
			// and Elastic.

			var pretty bytes.Buffer
			prefix := fmt.Sprintf("ID %d: ", q.ID)
			err = json.Indent(&pretty, resp.Body(), prefix, "  ")
			if err != nil {
				return
			}

			_, err = fmt.Fprintf(os.Stderr, "%s%s\n", prefix, pretty.Bytes())
			if err != nil {
				return
			}
		}
	}

	return lag, err
}
<|endoftext|>