<commit_before>package mapper

import (
	"log"
	"testing"

	"github.com/stretchr/testify/assert"
)

func allLabelsFor(label string) []string {
	allLabels := []string{}
	for t := label; t != ""; t = ParentType(t) {
		allLabels = append(allLabels, t)
	}
	return allLabels
}

var (
	thingLabels                  = allLabelsFor("Thing")
	conceptLabels                = allLabelsFor("Concept")
	roleLabels                   = allLabelsFor("Role")
	boardRoleLabels              = allLabelsFor("BoardRole")
	membershipRoleLabels         = allLabelsFor("MembershipRole")
	brandLabels                  = allLabelsFor("Brand")
	personLabels                 = allLabelsFor("Person")
	organisationLabels           = allLabelsFor("Organisation")
	membershipLabels             = allLabelsFor("Membership")
	companyLabels                = allLabelsFor("Company")
	publicCompanyLabels          = allLabelsFor("PublicCompany")
	privateCompanyLabels         = allLabelsFor("PrivateCompany")
	classificationLabels         = allLabelsFor("Classification")
	industryClassificationLabels = allLabelsFor("IndustryClassification")
	subjectLabels                = allLabelsFor("Subject")
	sectionLabels                = allLabelsFor("Section")
	genreLabels                  = allLabelsFor("Genre")
	topicLabels                  = allLabelsFor("Topic")
	locationLabels               = allLabelsFor("Location")
	specialReportLabels          = allLabelsFor("SpecialReport")
	alphavilleSeriesLabels       = allLabelsFor("AlphavilleSeries")
	financialInstrumentsLabels   = allLabelsFor("FinancialInstrument")

	thingURI                  = "http://www.ft.com/ontology/core/Thing"
	conceptURI                = "http://www.ft.com/ontology/concept/Concept"
	roleURI                   = "http://www.ft.com/ontology/organisation/Role"
	boardRoleURI              = "http://www.ft.com/ontology/organisation/BoardRole"
	membershipRoleURI         = "http://www.ft.com/ontology/MembershipRole"
	classificationURI         = "http://www.ft.com/ontology/classification/Classification"
	industryClassificationURI = "http://www.ft.com/ontology/industry/IndustryClassification"
	personURI                 = "http://www.ft.com/ontology/person/Person"
	brandURI                  = "http://www.ft.com/ontology/product/Brand"
	organisationURI           = "http://www.ft.com/ontology/organisation/Organisation"
	membershipURI             = "http://www.ft.com/ontology/organisation/Membership"
	companyURI                = "http://www.ft.com/ontology/company/Company"
	publicCompanyURI          = "http://www.ft.com/ontology/company/PublicCompany"
	privateCompanyURI         = "http://www.ft.com/ontology/company/PrivateCompany"
	subjectURI                = "http://www.ft.com/ontology/Subject"
	sectionURI                = "http://www.ft.com/ontology/Section"
	genreURI                  = "http://www.ft.com/ontology/Genre"
	topicURI                  = "http://www.ft.com/ontology/Topic"
	locationURI               = "http://www.ft.com/ontology/Location"
	specialReportURI          = "http://www.ft.com/ontology/SpecialReport"
	alphavilleSeriesURI       = "http://www.ft.com/ontology/AlphavilleSeries"
	financialInstrumentURI    = "http://www.ft.com/ontology/FinancialInstrument"

	uuid = "92f4ec09-436d-4092-a88c-96f54e34007c"

	env     = "prod"
	envTest = "test"

	baseAPIURL         = "http://api.ft.com/"
	thingAPIURL        = baseAPIURL + "things/" + uuid
	personAPIURL       = baseAPIURL + "people/" + uuid
	organisationAPIURL = baseAPIURL + "organisations/" + uuid
	contentAPIURL      = baseAPIURL + "content/" + uuid
	brandAPIURL        = baseAPIURL + "brands/" + uuid
)

func TestTypeURIsForPeople(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, personURI}, TypeURIs(personLabels))
}

func TestTypeURIsForOrganisations(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI}, TypeURIs(organisationLabels))
}

func TestTypeURIsForMemberships(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, membershipURI}, TypeURIs(membershipLabels))
}

func TestTypeURIsForRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, roleURI}, TypeURIs(roleLabels))
}

func TestTypeURIsForMembershipBoardRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, roleURI, membershipRoleURI}, TypeURIs(membershipRoleLabels))
}

func TestTypeURIsForBoardRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, roleURI, boardRoleURI}, TypeURIs(boardRoleLabels))
}

func TestTypeURIsForCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI}, TypeURIs(companyLabels))
}

func TestTypeURIsForPublicCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI, publicCompanyURI}, TypeURIs(publicCompanyLabels))
}

func TestTypeURIsForPrivateCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI, privateCompanyURI}, TypeURIs(privateCompanyLabels))
}

func TestTypeURIsForBrand(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, brandURI}, TypeURIs(brandLabels))
}

func TestTypeURIsForSubject(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, subjectURI}, TypeURIs(subjectLabels))
}

func TestTypeURIsForSection(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, sectionURI}, TypeURIs(sectionLabels))
}

func TestTypeURIsForGenre(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, genreURI}, TypeURIs(genreLabels))
}

func TestTypeURIsForTopic(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, topicURI}, TypeURIs(topicLabels))
}

func TestTypeURIsForLocation(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, locationURI}, TypeURIs(locationLabels))
}

func TestTypeURIsForSpecialReport(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, specialReportURI}, TypeURIs(specialReportLabels))
}

func TestTypeURIsForIndustryClassification(t *testing.T) {
	log.Printf("labels =%v", industryClassificationLabels)
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, industryClassificationURI}, TypeURIs(industryClassificationLabels))
}

func TestTypeURIsForAlphavilleSeries(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, alphavilleSeriesURI}, TypeURIs(alphavilleSeriesLabels))
}

func TestTypeURIForFinancialInstruments(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, financialInstrumentURI}, TypeURIs(financialInstrumentsLabels))
}

func TestContentAPIURLs(t *testing.T) {
	neoLabels := []string{"Content"}
	assert.New(t).EqualValues(contentAPIURL, APIURL(uuid, neoLabels, env))
}

func TestCompanyAPIURLs(t *testing.T) {
	neoLabels := []string{"PublicCompany", "Organisation", "Company"}
	assert.New(t).EqualValues(organisationAPIURL, APIURL(uuid, neoLabels, env))
}

func TestOrganisationAPIURLs(t *testing.T) {
	neoLabels := []string{"Organisation", "Concept", "Thing"}
	assert.New(t).EqualValues(organisationAPIURL, APIURL(uuid, neoLabels, env))
}

func TestPeopleAPIURLs(t *testing.T) {
	neoLabels := []string{"Person"}
	assert.New(t).EqualValues(personAPIURL, APIURL(uuid, neoLabels, env))

}

func TestBrandAPIURLs(t *testing.T) {
	neoLabels := []string{"Brand"}
	assert.New(t).EqualValues(brandAPIURL, APIURL(uuid, neoLabels, env))

}

func TestDefaultAPIURLs(t *testing.T) {
	neoLabels := []string{"thing", "relationship", "otherPrivateType"}
	assert.New(t).EqualValues(thingAPIURL, APIURL(uuid, neoLabels, env))

}

func TestMostSpecific(t *testing.T) {
	assert := assert.New(t)

	for _, t := range []struct {
		input    []string
		expected string
		err      error
	}{
		{
			[]string{"Organisation"},
			"Organisation",
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "Company"},
			"PublicCompany",
			nil,
		}, {
			[]string{"PublicCompany", "Organisation"},
			"PublicCompany",
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			"",
			ErrNotHierarchy,
		}, {
			[]string{"zzzzzz", "yyyyyy"},
			"",
			ErrNotHierarchy,
		},
	} {
		ms, err := MostSpecificType(t.input)
		assert.Equal(t.expected, ms)
		assert.Equal(t.err, err)
	}
}

func TestTypeSorter(t *testing.T) {
	assert := assert.New(t)

	for _, t := range []struct {
		input    []string
		expected []string
		err      error
	}{
		{
			[]string{"Organisation"},
			[]string{"Organisation"},
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "Company"},
			[]string{"Organisation", "Company", "PublicCompany"},
			nil,
		}, {
			[]string{"PublicCompany", "Organisation"},
			[]string{"Organisation", "PublicCompany"},
			nil,
		}, {
			[]string{"Content", "Thing"},
			[]string{"Thing", "Content"},
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			ErrNotHierarchy,
		}, {
			[]string{"zzzzzz", "yyyyyy"},
			[]string{"zzzzzz", "yyyyyy"},
			ErrNotHierarchy,
		},
	} {
		sorted, err := SortTypes(t.input)
		assert.Equal(t.expected, sorted)
		assert.Equal(t.err, err)
	}
}
<commit_msg>Missing ROLE uri<commit_after>package mapper

import (
	"log"
	"testing"

	"github.com/stretchr/testify/assert"
)

func allLabelsFor(label string) []string {
	allLabels := []string{}
	for t := label; t != ""; t = ParentType(t) {
		allLabels = append(allLabels, t)
	}
	return allLabels
}

var (
	thingLabels                  = allLabelsFor("Thing")
	conceptLabels                = allLabelsFor("Concept")
	roleLabels                   = allLabelsFor("Role")
	boardRoleLabels              = allLabelsFor("BoardRole")
	membershipRoleLabels         = allLabelsFor("MembershipRole")
	brandLabels                  = allLabelsFor("Brand")
	personLabels                 = allLabelsFor("Person")
	organisationLabels           = allLabelsFor("Organisation")
	membershipLabels             = allLabelsFor("Membership")
	companyLabels                = allLabelsFor("Company")
	publicCompanyLabels          = allLabelsFor("PublicCompany")
	privateCompanyLabels         = allLabelsFor("PrivateCompany")
	classificationLabels         = allLabelsFor("Classification")
	industryClassificationLabels = allLabelsFor("IndustryClassification")
	subjectLabels                = allLabelsFor("Subject")
	sectionLabels                = allLabelsFor("Section")
	genreLabels                  = allLabelsFor("Genre")
	topicLabels                  = allLabelsFor("Topic")
	locationLabels               = allLabelsFor("Location")
	specialReportLabels          = allLabelsFor("SpecialReport")
	alphavilleSeriesLabels       = allLabelsFor("AlphavilleSeries")
	financialInstrumentsLabels   = allLabelsFor("FinancialInstrument")

	thingURI                  = "http://www.ft.com/ontology/core/Thing"
	conceptURI                = "http://www.ft.com/ontology/concept/Concept"
	roleURI                   = "http://www.ft.com/ontology/organisation/Role"
	boardRoleURI              = "http://www.ft.com/ontology/organisation/BoardRole"
	membershipRoleURI         = "http://www.ft.com/ontology/MembershipRole"
	classificationURI         = "http://www.ft.com/ontology/classification/Classification"
	industryClassificationURI = "http://www.ft.com/ontology/industry/IndustryClassification"
	personURI                 = "http://www.ft.com/ontology/person/Person"
	brandURI                  = "http://www.ft.com/ontology/product/Brand"
	organisationURI           = "http://www.ft.com/ontology/organisation/Organisation"
	membershipURI             = "http://www.ft.com/ontology/organisation/Membership"
	companyURI                = "http://www.ft.com/ontology/company/Company"
	publicCompanyURI          = "http://www.ft.com/ontology/company/PublicCompany"
	privateCompanyURI         = "http://www.ft.com/ontology/company/PrivateCompany"
	subjectURI                = "http://www.ft.com/ontology/Subject"
	sectionURI                = "http://www.ft.com/ontology/Section"
	genreURI                  = "http://www.ft.com/ontology/Genre"
	topicURI                  = "http://www.ft.com/ontology/Topic"
	locationURI               = "http://www.ft.com/ontology/Location"
	specialReportURI          = "http://www.ft.com/ontology/SpecialReport"
	alphavilleSeriesURI       = "http://www.ft.com/ontology/AlphavilleSeries"
	financialInstrumentURI    = "http://www.ft.com/ontology/FinancialInstrument"

	uuid = "92f4ec09-436d-4092-a88c-96f54e34007c"

	env     = "prod"
	envTest = "test"

	baseAPIURL         = "http://api.ft.com/"
	thingAPIURL        = baseAPIURL + "things/" + uuid
	personAPIURL       = baseAPIURL + "people/" + uuid
	organisationAPIURL = baseAPIURL + "organisations/" + uuid
	contentAPIURL      = baseAPIURL + "content/" + uuid
	brandAPIURL        = baseAPIURL + "brands/" + uuid
)

func TestTypeURIsForPeople(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, personURI}, TypeURIs(personLabels))
}

func TestTypeURIsForOrganisations(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI}, TypeURIs(organisationLabels))
}

func TestTypeURIsForMemberships(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, membershipURI}, TypeURIs(membershipLabels))
}

func TestTypeURIsForRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, roleURI}, TypeURIs(roleLabels))
}

func TestTypeURIsForMembershipBoardRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, membershipRoleURI}, TypeURIs(membershipRoleLabels))
}

func TestTypeURIsForBoardRoles(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, membershipRoleURI, boardRoleURI}, TypeURIs(boardRoleLabels))
}

func TestTypeURIsForCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI}, TypeURIs(companyLabels))
}

func TestTypeURIsForPublicCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI, publicCompanyURI}, TypeURIs(publicCompanyLabels))
}

func TestTypeURIsForPrivateCompany(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, organisationURI, companyURI, privateCompanyURI}, TypeURIs(privateCompanyLabels))
}

func TestTypeURIsForBrand(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, brandURI}, TypeURIs(brandLabels))
}

func TestTypeURIsForSubject(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, subjectURI}, TypeURIs(subjectLabels))
}

func TestTypeURIsForSection(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, sectionURI}, TypeURIs(sectionLabels))
}

func TestTypeURIsForGenre(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, genreURI}, TypeURIs(genreLabels))
}

func TestTypeURIsForTopic(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, topicURI}, TypeURIs(topicLabels))
}

func TestTypeURIsForLocation(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, locationURI}, TypeURIs(locationLabels))
}

func TestTypeURIsForSpecialReport(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, specialReportURI}, TypeURIs(specialReportLabels))
}

func TestTypeURIsForIndustryClassification(t *testing.T) {
	log.Printf("labels =%v", industryClassificationLabels)
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, industryClassificationURI}, TypeURIs(industryClassificationLabels))
}

func TestTypeURIsForAlphavilleSeries(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, classificationURI, alphavilleSeriesURI}, TypeURIs(alphavilleSeriesLabels))
}

func TestTypeURIForFinancialInstruments(t *testing.T) {
	assert.New(t).EqualValues([]string{thingURI, conceptURI, financialInstrumentURI}, TypeURIs(financialInstrumentsLabels))
}

func TestContentAPIURLs(t *testing.T) {
	neoLabels := []string{"Content"}
	assert.New(t).EqualValues(contentAPIURL, APIURL(uuid, neoLabels, env))
}

func TestCompanyAPIURLs(t *testing.T) {
	neoLabels := []string{"PublicCompany", "Organisation", "Company"}
	assert.New(t).EqualValues(organisationAPIURL, APIURL(uuid, neoLabels, env))
}

func TestOrganisationAPIURLs(t *testing.T) {
	neoLabels := []string{"Organisation", "Concept", "Thing"}
	assert.New(t).EqualValues(organisationAPIURL, APIURL(uuid, neoLabels, env))
}

func TestPeopleAPIURLs(t *testing.T) {
	neoLabels := []string{"Person"}
	assert.New(t).EqualValues(personAPIURL, APIURL(uuid, neoLabels, env))

}

func TestBrandAPIURLs(t *testing.T) {
	neoLabels := []string{"Brand"}
	assert.New(t).EqualValues(brandAPIURL, APIURL(uuid, neoLabels, env))

}

func TestDefaultAPIURLs(t *testing.T) {
	neoLabels := []string{"thing", "relationship", "otherPrivateType"}
	assert.New(t).EqualValues(thingAPIURL, APIURL(uuid, neoLabels, env))

}

func TestMostSpecific(t *testing.T) {
	assert := assert.New(t)

	for _, t := range []struct {
		input    []string
		expected string
		err      error
	}{
		{
			[]string{"Organisation"},
			"Organisation",
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "Company"},
			"PublicCompany",
			nil,
		}, {
			[]string{"PublicCompany", "Organisation"},
			"PublicCompany",
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			"",
			ErrNotHierarchy,
		}, {
			[]string{"zzzzzz", "yyyyyy"},
			"",
			ErrNotHierarchy,
		},
	} {
		ms, err := MostSpecificType(t.input)
		assert.Equal(t.expected, ms)
		assert.Equal(t.err, err)
	}
}

func TestTypeSorter(t *testing.T) {
	assert := assert.New(t)

	for _, t := range []struct {
		input    []string
		expected []string
		err      error
	}{
		{
			[]string{"Organisation"},
			[]string{"Organisation"},
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "Company"},
			[]string{"Organisation", "Company", "PublicCompany"},
			nil,
		}, {
			[]string{"PublicCompany", "Organisation"},
			[]string{"Organisation", "PublicCompany"},
			nil,
		}, {
			[]string{"Content", "Thing"},
			[]string{"Thing", "Content"},
			nil,
		}, {
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			[]string{"Organisation", "PublicCompany", "PrivateCompany", "Company"},
			ErrNotHierarchy,
		}, {
			[]string{"zzzzzz", "yyyyyy"},
			[]string{"zzzzzz", "yyyyyy"},
			ErrNotHierarchy,
		},
	} {
		sorted, err := SortTypes(t.input)
		assert.Equal(t.expected, sorted)
		assert.Equal(t.err, err)
	}
}
<|endoftext|>


<commit_before>package kepub

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
	"github.com/beevik/etree"
)

func cleanFiles(basepath string) error {
	_ = os.Remove(filepath.Join(basepath, "META-INF", "calibre_bookmarks.txt"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "iTunesMetadata.plist"))
	_ = os.Remove(filepath.Join(basepath, "iTunesMetadata.plist"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "iTunesArtwork.plist"))
	_ = os.Remove(filepath.Join(basepath, "iTunesArtwork.plist"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", ".DS_STORE"))
	_ = os.Remove(filepath.Join(basepath, ".DS_STORE"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "thumbs.db"))
	_ = os.Remove(filepath.Join(basepath, "thumbs.db"))
	return nil
}

// cleanOPF cleans up extra calibre metadata from the content.opf file
func cleanOPF(opftext *string) error {
	calibreTimestampRe := regexp.MustCompile(`<meta\s+name="calibre:timestamp"\s+content=".+"\s*\/>`)
	*opftext = calibreTimestampRe.ReplaceAllString(*opftext, "")

	calibreContributorRe := regexp.MustCompile(`<dc:contributor\s+opf:role="bkp"\s*>calibre .+<\/dc:contributor>`)
	*opftext = calibreContributorRe.ReplaceAllString(*opftext, "")

	opf := etree.NewDocument()
	err := opf.ReadFromString(*opftext)
	if err != nil {
		return err
	}

	for _, e := range opf.FindElements("//meta[@name='cover']") {
		coverid := e.SelectAttrValue("content", "")
		if coverid != "" {
			*opftext = strings.Replace(*opftext, `id="`+coverid+`"`, `id="`+coverid+`" properties="cover-image"`, -1)
		} else {
			*opftext = strings.Replace(*opftext, `id="cover"`, `id="cover" properties="cover-image"`, -1)
		}
	}
	return nil
}

// addDivs adds kobo divs.
func addDivs(doc *goquery.Document) error {
	if len(doc.Find("div").Nodes) > len(doc.Find("p").Nodes) {
		// If there are more divs than ps, divs are probably being used as paragraphs, and adding the kobo divs will most likely break the book.
		return nil
	}
	doc.Find("body>*").WrapAllHtml(`<div class="book-inner"></div>`)
	doc.Find("body>*").WrapAllHtml(`<div class="book-columns"></div>`)
	return nil
}

// addSpansToNode is a recursive helper function for addSpans.
func addSpansToNode(node *html.Node, paragraph *int, segment *int) {
	sentencere := regexp.MustCompile(`((?m).*?[\.\!\?\:]['"”’“…]?\s*)`)

	// Part 2 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards
	newAttr := []html.Attribute{}
	for _, a := range node.Attr {
		a.Key = html.EscapeString(a.Key)
		a.Val = html.EscapeString(a.Val)
		newAttr = append(newAttr, a)
	}
	node.Attr = newAttr

	if node.Type == html.TextNode {
		if node.Parent.Data == "pre" {
			// Do not add spans to pre elements
			return
		}
		*segment++

		sentencesindexes := sentencere.FindAllStringIndex(node.Data, -1)
		sentences := []string{}
		lasti := []int{0, 0}
		for _, i := range sentencesindexes {
			if lasti[1] != i[0] {
				// If gap in regex matches, add the gap to the sentence list to avoid losing text
				sentences = append(sentences, node.Data[lasti[1]:i[0]])
			}
			sentences = append(sentences, node.Data[i[0]:i[1]])
			lasti = i
		}
		if lasti[1] != len(node.Data) {
			// If gap in regex matches, add the gap to the sentence list to avoid losing text
			sentences = append(sentences, node.Data[lasti[1]:len(node.Data)])
		}

		var newhtml bytes.Buffer

		for _, sentence := range sentences {
			if strings.TrimSpace(sentence) != "" {
				newhtml.WriteString(fmt.Sprintf(`<span class="koboSpan" id="kobo.%v.%v">%s</span>`, *paragraph, *segment, html.EscapeString(sentence)))
				*segment++
			}
		}

		// Part 1 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards
		node.Data = newhtml.String()

		return
	}
	if node.Type != html.ElementNode {
		return
	}
	if node.Data == "img" {
		return
	}
	if node.Data == "p" || node.Data == "ol" || node.Data == "ul" {
		*segment = 0
		*paragraph++
	}
	for c := node.FirstChild; c != nil; c = c.NextSibling {
		addSpansToNode(c, paragraph, segment)
	}
}

// addSpans adds kobo spans.
func addSpans(doc *goquery.Document) error {
	alreadyHasSpans := false
	doc.Find("span").Each(func(i int, s *goquery.Selection) {
		if val, _ := s.Attr("class"); strings.Contains(val, "koboSpan") {
			alreadyHasSpans = true
		}
	})
	if alreadyHasSpans {
		return nil
	}

	paragraph := 0
	segment := 0

	for _, n := range doc.Find("body").Nodes {
		addSpansToNode(n, &paragraph, &segment)
	}

	return nil
}

// openSelfClosingPs opens self-closing p tags.
func openSelfClosingPs(html *string) error {
	re := regexp.MustCompile(`<p[^>/]*/>`)
	*html = re.ReplaceAllString(*html, `<p></p>`)
	return nil
}

// smartenPunctuation smartens punctuation in html code. It must be run last.
func smartenPunctuation(html *string) error {
	// em and en dashes
	*html = strings.Replace(*html, "---", " – ", -1)
	*html = strings.Replace(*html, "--", " — ", -1)

	// TODO: smart quotes

	// Fix comments
	*html = strings.Replace(*html, "<! — ", "", -1)
	return nil
}

// cleanHTML cleans up html for a kobo epub.
func cleanHTML(html *string) error {
	emptyHeadingRe := regexp.MustCompile(`<h\d+>\s*</h\d+>`)
	*html = emptyHeadingRe.ReplaceAllString(*html, "")

	msPRe := regexp.MustCompile(`\s*<o:p>\s*<\/o:p>`)
	*html = msPRe.ReplaceAllString(*html, " ")

	msStRe := regexp.MustCompile(`<\/?st1:\w+>`)
	*html = msStRe.ReplaceAllString(*html, "")

	// unicode replacement chars
	*html = strings.Replace(*html, "�", "", -1)

	// ADEPT drm tags
	adeptRe := regexp.MustCompile(`(<meta\s+content=".+"\s+name="Adept.expected.resource"\s+\/>)`)
	*html = adeptRe.ReplaceAllString(*html, "")

	return nil
}

// process processes the html of a content file in an ordinary epub and converts it into a kobo epub by adding kobo divs, kobo spans, smartening punctuation, and cleaning html.
func process(content *string) error {
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(*content))
	if err != nil {
		return err
	}

	if err := addDivs(doc); err != nil {
		return err
	}

	if err := addSpans(doc); err != nil {
		return err
	}

	h, err := doc.Html()
	if err != nil {
		return err
	}

	// Part 3 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards. Must be done before further html processing
	h = html.UnescapeString(h)

	if err := openSelfClosingPs(&h); err != nil {
		return err
	}

	if err := cleanHTML(&h); err != nil {
		return err
	}

	if err := smartenPunctuation(&h); err != nil {
		return err
	}

	// Kobo style fixes
	h = strings.Replace(h, "", "", 1)

	*content = h

	return nil
}
<commit_msg>Fix kobo css<commit_after>package kepub

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
	"github.com/beevik/etree"
)

func cleanFiles(basepath string) error {
	_ = os.Remove(filepath.Join(basepath, "META-INF", "calibre_bookmarks.txt"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "iTunesMetadata.plist"))
	_ = os.Remove(filepath.Join(basepath, "iTunesMetadata.plist"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "iTunesArtwork.plist"))
	_ = os.Remove(filepath.Join(basepath, "iTunesArtwork.plist"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", ".DS_STORE"))
	_ = os.Remove(filepath.Join(basepath, ".DS_STORE"))
	_ = os.Remove(filepath.Join(basepath, "META-INF", "thumbs.db"))
	_ = os.Remove(filepath.Join(basepath, "thumbs.db"))
	return nil
}

// cleanOPF cleans up extra calibre metadata from the content.opf file
func cleanOPF(opftext *string) error {
	calibreTimestampRe := regexp.MustCompile(`<meta\s+name="calibre:timestamp"\s+content=".+"\s*\/>`)
	*opftext = calibreTimestampRe.ReplaceAllString(*opftext, "")

	calibreContributorRe := regexp.MustCompile(`<dc:contributor\s+opf:role="bkp"\s*>calibre .+<\/dc:contributor>`)
	*opftext = calibreContributorRe.ReplaceAllString(*opftext, "")

	opf := etree.NewDocument()
	err := opf.ReadFromString(*opftext)
	if err != nil {
		return err
	}

	for _, e := range opf.FindElements("//meta[@name='cover']") {
		coverid := e.SelectAttrValue("content", "")
		if coverid != "" {
			*opftext = strings.Replace(*opftext, `id="`+coverid+`"`, `id="`+coverid+`" properties="cover-image"`, -1)
		} else {
			*opftext = strings.Replace(*opftext, `id="cover"`, `id="cover" properties="cover-image"`, -1)
		}
	}
	return nil
}

// addDivs adds kobo divs.
func addDivs(doc *goquery.Document) error {
	if len(doc.Find("div").Nodes) > len(doc.Find("p").Nodes) {
		// If there are more divs than ps, divs are probably being used as paragraphs, and adding the kobo divs will most likely break the book.
		return nil
	}
	doc.Find("body>*").WrapAllHtml(`<div class="book-inner"></div>`)
	doc.Find("body>*").WrapAllHtml(`<div class="book-columns"></div>`)
	return nil
}

// addSpansToNode is a recursive helper function for addSpans.
func addSpansToNode(node *html.Node, paragraph *int, segment *int) {
	sentencere := regexp.MustCompile(`((?m).*?[\.\!\?\:]['"”’“…]?\s*)`)

	// Part 2 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards
	newAttr := []html.Attribute{}
	for _, a := range node.Attr {
		a.Key = html.EscapeString(a.Key)
		a.Val = html.EscapeString(a.Val)
		newAttr = append(newAttr, a)
	}
	node.Attr = newAttr

	if node.Type == html.TextNode {
		if node.Parent.Data == "pre" {
			// Do not add spans to pre elements
			return
		}
		*segment++

		sentencesindexes := sentencere.FindAllStringIndex(node.Data, -1)
		sentences := []string{}
		lasti := []int{0, 0}
		for _, i := range sentencesindexes {
			if lasti[1] != i[0] {
				// If gap in regex matches, add the gap to the sentence list to avoid losing text
				sentences = append(sentences, node.Data[lasti[1]:i[0]])
			}
			sentences = append(sentences, node.Data[i[0]:i[1]])
			lasti = i
		}
		if lasti[1] != len(node.Data) {
			// If gap in regex matches, add the gap to the sentence list to avoid losing text
			sentences = append(sentences, node.Data[lasti[1]:len(node.Data)])
		}

		var newhtml bytes.Buffer

		for _, sentence := range sentences {
			if strings.TrimSpace(sentence) != "" {
				newhtml.WriteString(fmt.Sprintf(`<span class="koboSpan" id="kobo.%v.%v">%s</span>`, *paragraph, *segment, html.EscapeString(sentence)))
				*segment++
			}
		}

		// Part 1 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards
		node.Data = newhtml.String()

		return
	}
	if node.Type != html.ElementNode {
		return
	}
	if node.Data == "img" {
		return
	}
	if node.Data == "p" || node.Data == "ol" || node.Data == "ul" {
		*segment = 0
		*paragraph++
	}
	for c := node.FirstChild; c != nil; c = c.NextSibling {
		addSpansToNode(c, paragraph, segment)
	}
}

// addSpans adds kobo spans.
func addSpans(doc *goquery.Document) error {
	alreadyHasSpans := false
	doc.Find("span").Each(func(i int, s *goquery.Selection) {
		if val, _ := s.Attr("class"); strings.Contains(val, "koboSpan") {
			alreadyHasSpans = true
		}
	})
	if alreadyHasSpans {
		return nil
	}

	paragraph := 0
	segment := 0

	for _, n := range doc.Find("body").Nodes {
		addSpansToNode(n, &paragraph, &segment)
	}

	return nil
}

// openSelfClosingPs opens self-closing p tags.
func openSelfClosingPs(html *string) error {
	re := regexp.MustCompile(`<p[^>/]*/>`)
	*html = re.ReplaceAllString(*html, `<p></p>`)
	return nil
}

// smartenPunctuation smartens punctuation in html code. It must be run last.
func smartenPunctuation(html *string) error {
	// em and en dashes
	*html = strings.Replace(*html, "---", " – ", -1)
	*html = strings.Replace(*html, "--", " — ", -1)

	// TODO: smart quotes

	// Fix comments
	*html = strings.Replace(*html, "<! — ", "", -1)
	return nil
}

// cleanHTML cleans up html for a kobo epub.
func cleanHTML(html *string) error {
	emptyHeadingRe := regexp.MustCompile(`<h\d+>\s*</h\d+>`)
	*html = emptyHeadingRe.ReplaceAllString(*html, "")

	msPRe := regexp.MustCompile(`\s*<o:p>\s*<\/o:p>`)
	*html = msPRe.ReplaceAllString(*html, " ")

	msStRe := regexp.MustCompile(`<\/?st1:\w+>`)
	*html = msStRe.ReplaceAllString(*html, "")

	// unicode replacement chars
	*html = strings.Replace(*html, "�", "", -1)

	// ADEPT drm tags
	adeptRe := regexp.MustCompile(`(<meta\s+content=".+"\s+name="Adept.expected.resource"\s+\/>)`)
	*html = adeptRe.ReplaceAllString(*html, "")

	return nil
}

// process processes the html of a content file in an ordinary epub and converts it into a kobo epub by adding kobo divs, kobo spans, smartening punctuation, and cleaning html.
func process(content *string) error {
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(*content))
	if err != nil {
		return err
	}

	if err := addDivs(doc); err != nil {
		return err
	}

	if err := addSpans(doc); err != nil {
		return err
	}

	h, err := doc.Html()
	if err != nil {
		return err
	}

	// Part 3 of hacky way of setting innerhtml of a textnode by double escaping everything, and deescaping once afterwards. Must be done before further html processing
	h = html.UnescapeString(h)

	if err := openSelfClosingPs(&h); err != nil {
		return err
	}

	if err := cleanHTML(&h); err != nil {
		return err
	}

	if err := smartenPunctuation(&h); err != nil {
		return err
	}

	// Kobo style fixes
	h = strings.Replace(h, "", "", 1)

	*content = h

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level           Level
	name            string
	seqid           uint64
	levelno         int
	levelname       string
	pathname        string
	filename        string
	module          string
	lineno          int
	funcName        string
	created         int64
	asctime         string
	msecs           int64
	relativeCreated int64
	thread          int
	threadName      string
	process         int
	message         string
	timestamp       int64
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"seqid":           (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = fshort
		l.filename = path.Base(file)
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = 0
	}
}

func (logger *logging) lname(l *log) interface{} {
	if l.name == "" {
		l.name = logger.name
	}
	return l.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	if l.seqid == 0 {
		l.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return l.seqid
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime
}

func (logger *logging) asctime(l *log) interface{} {
	if l.asctime == "" {
		l.asctime = time.Now().String()
	}
	return l.asctime
}

func (logger *logging) msecs(l *log) interface{} {
	if l.msecs == 0 {
		l.msecs = logger.startTime % 1000
	}
	return l.msecs
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.timestamp == 0 {
		l.timestamp = time.Now().UnixNano()
	}
	return l.timestamp
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.relativeCreated == 0 {
		l.relativeCreated = time.Now().UnixNano() - logger.startTime
	}
	return l.relativeCreated
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<commit_msg>if genRuntime fails, return -1 to lineno rather than 0<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

type log struct {
	level           Level
	name            string
	seqid           uint64
	levelno         int
	levelname       string
	pathname        string
	filename        string
	module          string
	lineno          int
	funcName        string
	created         int64
	asctime         string
	msecs           int64
	relativeCreated int64
	thread          int
	threadName      string
	process         int
	message         string
	timestamp       int64
}

type field func(*logging, *log) interface{}

var fields = map[string]field{
	"name":            (*logging).lname,
	"seqid":           (*logging).nextSeqid,
	"levelno":         (*logging).levelno,
	"levelname":       (*logging).levelname,
	"pathname":        (*logging).pathname,
	"filename":        (*logging).filename,
	"module":          (*logging).module,
	"lineno":          (*logging).lineno,
	"funcName":        (*logging).funcName,
	"created":         (*logging).created,
	"asctime":         (*logging).asctime,
	"msecs":           (*logging).msecs,
	"relativeCreated": (*logging).relativeCreated,
	"thread":          (*logging).thread,
	"threadName":      (*logging).threadName,
	"process":         (*logging).process,
	"message":         (*logging).message,
	"timestamp":       (*logging).timestamp,
}

const errString = "???"

// GetGoId returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoId() int32

// generate the runtime information, including pathname, function name,
// filename, line number.
func genRuntime(l *log) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		l.pathname = file
		l.funcName = fshort
		l.filename = path.Base(file)
		l.lineno = line
	} else {
		l.pathname = errString
		l.funcName = errString
		l.filename = errString
		l.lineno = -1
	}
}

func (logger *logging) lname(l *log) interface{} {
	if l.name == "" {
		l.name = logger.name
	}
	return l.name
}

func (logger *logging) nextSeqid(l *log) interface{} {
	if l.seqid == 0 {
		l.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return l.seqid
}

func (logger *logging) levelno(l *log) interface{} {
	return int(l.level)
}

func (logger *logging) levelname(l *log) interface{} {
	return levelNames[l.level]
}

func (logger *logging) pathname(l *log) interface{} {
	if l.pathname == "" {
		genRuntime(l)
	}
	return l.pathname
}

func (logger *logging) filename(l *log) interface{} {
	if l.filename == "" {
		genRuntime(l)
	}
	return l.filename
}

func (logger *logging) module(l *log) interface{} {
	return ""
}

func (logger *logging) lineno(l *log) interface{} {
	if l.lineno == 0 {
		genRuntime(l)
	}
	return l.lineno
}

func (logger *logging) funcName(l *log) interface{} {
	if l.funcName == "" {
		genRuntime(l)
	}
	return l.funcName
}

func (logger *logging) created(l *log) interface{} {
	return logger.startTime
}

func (logger *logging) asctime(l *log) interface{} {
	if l.asctime == "" {
		l.asctime = time.Now().String()
	}
	return l.asctime
}

func (logger *logging) msecs(l *log) interface{} {
	if l.msecs == 0 {
		l.msecs = logger.startTime % 1000
	}
	return l.msecs
}

func (logger *logging) timestamp(l *log) interface{} {
	if l.timestamp == 0 {
		l.timestamp = time.Now().UnixNano()
	}
	return l.timestamp
}

func (logger *logging) relativeCreated(l *log) interface{} {
	if l.relativeCreated == 0 {
		l.relativeCreated = time.Now().UnixNano() - logger.startTime
	}
	return l.relativeCreated
}

func (logger *logging) thread(l *log) interface{} {
	if l.thread == 0 {
		l.thread = int(GetGoId())
	}
	return l.thread
}

func (logger *logging) threadName(l *log) interface{} {
	if l.threadName == "" {
		l.threadName = fmt.Sprintf("Thread-%d", GetGoId())
	}
	return l.threadName
}

// Process ID
func (logger *logging) process(l *log) interface{} {
	if l.process == 0 {
		l.process = os.Getpid()
	}
	return l.process
}

func (logger *logging) message(l *log) interface{} {
	return l.message
}
<|endoftext|>


<commit_before>package crypto

import (
	"fmt"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/go-wire"
)

// Signature is a part of Txs and consensus Votes.
type Signature interface {
	Bytes() []byte
	IsZero() bool
	String() string
}

// Types of Signature implementations
const (
	SignatureTypeEd25519   = byte(0x01)
	SignatureTypeSecp256k1 = byte(0x02)
)

// for wire.readReflect
var _ = wire.RegisterInterface(
	struct{ Signature }{},
	wire.ConcreteType{SignatureEd25519{}, SignatureTypeEd25519},
	wire.ConcreteType{SignatureSecp256k1{}, SignatureTypeSecp256k1},
)

//-------------------------------------

// Implements Signature
type SignatureEd25519 [64]byte

func (sig SignatureEd25519) Bytes() []byte {
	return wire.BinaryBytes(struct{ Signature }{sig})
}

func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 }

func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }

//-------------------------------------

// Implements Signature
type SignatureSecp256k1 []byte

func (sig SignatureSecp256k1) Bytes() []byte {
	return wire.BinaryBytes(struct{ Signature }{sig})
}

func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 }

func (sig SignatureSecp256k1) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }
<commit_msg>ReadSignature*<commit_after>package crypto

import (
	"fmt"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/go-wire"
)

// Signature is a part of Txs and consensus Votes.
type Signature interface {
	Bytes() []byte
	IsZero() bool
	String() string
}

// Types of Signature implementations
const (
	SignatureTypeEd25519   = byte(0x01)
	SignatureTypeSecp256k1 = byte(0x02)
)

// for wire.readReflect
var _ = wire.RegisterInterface(
	struct{ Signature }{},
	wire.ConcreteType{SignatureEd25519{}, SignatureTypeEd25519},
	wire.ConcreteType{SignatureSecp256k1{}, SignatureTypeSecp256k1},
)

//-------------------------------------

// Implements Signature
type SignatureEd25519 [64]byte

func (sig SignatureEd25519) Bytes() []byte {
	return wire.BinaryBytes(struct{ Signature }{sig})
}

func (sig SignatureEd25519) IsZero() bool { return len(sig) == 0 }

func (sig SignatureEd25519) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }

func ReadSignatureEd25519(bz []byte) (SignatureEd25519, error) {
	sig := struct{ Signature }{}
	err := wire.ReadBinaryBytes(bz, &sig)
	if err != nil {
		return SignatureEd25519{}, err
	}
	return sig.Signature.(SignatureEd25519), nil
}

//-------------------------------------

// Implements Signature
type SignatureSecp256k1 []byte

func (sig SignatureSecp256k1) Bytes() []byte {
	return wire.BinaryBytes(struct{ Signature }{sig})
}

func (sig SignatureSecp256k1) IsZero() bool { return len(sig) == 0 }

func (sig SignatureSecp256k1) String() string { return fmt.Sprintf("/%X.../", Fingerprint(sig[:])) }

func ReadSignatureSecp256k1(bz []byte) (SignatureSecp256k1, error) {
	sig := struct{ Signature }{}
	err := wire.ReadBinaryBytes(bz, &sig)
	if err != nil {
		return nil, err
	}
	return sig.Signature.(SignatureSecp256k1), nil
}
<|endoftext|>


<commit_before>package apigee

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/gofrs/uuid"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/zambien/go-apigee-edge"
)

func resourceApiProxyDeployment() *schema.Resource {
	return &schema.Resource{
		Create: resourceApiProxyDeploymentCreate,
		Read:   resourceApiProxyDeploymentRead,
		Update: resourceApiProxyDeploymentUpdate,
		Delete: resourceApiProxyDeploymentDelete,
		Importer: &schema.ResourceImporter{
			State: resourceApiProxyDeploymentImport,
		},

		Schema: map[string]*schema.Schema{
			"proxy_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"org": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"env": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"revision": {
				Type:     schema.TypeString,
				Required: true,
			},
			"delay": {
				Type:     schema.TypeInt,
				Optional: true,
				Default:  0,
			},
			"override": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
		},
	}
}

func resourceApiProxyDeploymentImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	log.Print("[DEBUG] resourceApiProxyDeploymentImport START")
	client := meta.(*apigee.EdgeClient)

	splits := strings.Split(d.Id(), "_")
	if len(splits) < 2 {
		return []*schema.ResourceData{}, fmt.Errorf("[ERR] Wrong format of resource: %s. Please follow '{name}_{env}_deployment'", d.Id())
	}
	nameOffset := len(splits[len(splits)-1]) + len(splits[len(splits)-2])
	envOffset := len(splits[len(splits)-1])
	name := d.Id()[:(len(d.Id())-nameOffset)-2]
	IDEnv := d.Id()[len(name)+1 : (len(d.Id())-envOffset)-1]
	deployment, _, err := client.Proxies.GetDeployments(name)
	if err != nil {
		log.Printf("[DEBUG] resourceApiProxyDeploymentImport. Error getting deployment api: %v", err)
		return nil, nil
	}
	d.Set("org", deployment.Organization)
	d.Set("proxy_name", deployment.Name)
	d.Set("env", IDEnv)

	return []*schema.ResourceData{d}, nil
}

func resourceApiProxyDeploymentRead(d *schema.ResourceData, meta interface{}) (e error) {
	log.Print("[DEBUG] resourceApiProxyDeploymentRead START")
	log.Printf("[DEBUG] resourceApiProxyDeploymentRead proxy_name: %#v", d.Get("proxy_name").(string))

	client := meta.(*apigee.EdgeClient)

	found := false
	matchedRevision := "0"

	if deployments, _, err := client.Proxies.GetDeployments(d.Get("proxy_name").(string)); err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentRead error getting deployments: %s", err.Error())
		if strings.Contains(err.Error(), "404 ") {
			log.Printf("[DEBUG] resourceApiProxyDeploymentRead 404 encountered.  Removing state for deployment proxy_name: %#v", d.Get("proxy_name").(string))
			d.SetId("")
			return nil
		} else {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentRead error reading deployments: %s", err.Error())
		}
	} else {
		log.Printf("[DEBUG] resourceApiProxyDeploymentRead deployments call fired for proxy_name: %#v", d.Get("proxy_name").(string))
		for _, environment := range deployments.Environments {
			log.Printf("[DEBUG] resourceApiProxyDeploymentRead checking revisions in deployed environment: %#v for expected environment: %#v\n", environment.Name, d.Get("env").(string))
			if environment.Name == d.Get("env").(string) {
				//We don't break.  Always get the last one if there are multiple deployments.
				for _, revision := range environment.Revision {
					log.Printf("[DEBUG] resourceApiProxyDeploymentRead checking deployed revision: %#v for expected revision: %#v\n", revision.Number.String(), d.Get("revision").(string))
					if (d.Get("revision").(string) != "latest" && d.Get("revision").(string) == revision.Number.String()) {
						matchedRevision = revision.Number.String()
						found = true
						break
					} else {
						matchedRevision = revision.Number.String()
					}
					found = true
				}
			}
		}
	}

	if found {
		log.Printf("[DEBUG] resourceApiProxyDeploymentRead - deployment found. Revision is: %#v", matchedRevision)
		d.Set("revision", matchedRevision)
	} else {
		log.Print("[DEBUG] resourceApiProxyDeploymentRead - no deployment found")
		d.SetId("")
	}
	return nil
}

func resourceApiProxyDeploymentCreate(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentCreate START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)
	delay := int(d.Get("delay").(int))
	override := bool(d.Get("override").(bool))

	if d.Get("revision").(string) == "latest" {
		// deploy latest
		rev, err := getLatestRevision(client, proxy_name)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error getting latest revision: %v", err)
		}
		_, _, err = client.Proxies.Deploy(proxy_name, env, apigee.Revision(rev), delay, override)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error deploying: %v", err)
		}
		log.Printf("[DEBUG] resourceApiProxyDeploymentUpdate Deployed revision %d of %s", rev, proxy_name)
		return resourceApiProxyDeploymentRead(d, meta)
	}

	proxyDep, _, err := client.Proxies.Deploy(proxy_name, env, rev, delay, override)

	if err != nil {

		if strings.Contains(err.Error(), "conflicts with existing deployment path") {
			//create, fail, update
			log.Printf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
			log.Print("[DEBUG] resourceApiProxyDeploymentCreate something got out of sync... maybe someone messing around in apigee directly.  Terraform OVERRIDE!!!")
			resourceApiProxyDeploymentUpdate(d, meta)
		} else {
			log.Printf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
		}
	}

	id, _ := uuid.NewV4()
	d.SetId(id.String())
	d.Set("revision", proxyDep.Revision.String())

	return resourceApiProxyDeploymentRead(d, meta)
}

func resourceApiProxyDeploymentUpdate(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentUpdate START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	delay := int(d.Get("delay").(int))
	override := bool(d.Get("override").(bool))

	//We must set delay and override here if not set.
	if delay == 0 {
		delay = 15 //seconds
	}
	if override == false {
		override = true
	}

	if d.Get("revision").(string) == "latest" {
		// deploy latest
		rev, err := getLatestRevision(client, proxy_name)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error getting latest revision: %v", err)
		}
		_, _, err = client.Proxies.ReDeploy(proxy_name, env, apigee.Revision(rev), delay, override)
		if err != nil {
			if strings.Contains(err.Error(), " is already deployed ") {
				return resourceApiProxyDeploymentRead(d, meta)
			}
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error deploying: %v", err)
		}
		log.Printf("[DEBUG] resourceApiProxyDeploymentUpdate Deployed revision %d of %s", rev, proxy_name)
		return resourceApiProxyDeploymentRead(d, meta)
	}

	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)
	_, _, err := client.Proxies.ReDeploy(proxy_name, env, rev, delay, override)

	if err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentUpdate error redeploying: %s", err.Error())
		if strings.Contains(err.Error(), " is already deployed into environment ") {
			return resourceApiProxyDeploymentRead(d, meta)
		}
		return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error redeploying: %s", err.Error())
	}

	return resourceApiProxyDeploymentRead(d, meta)
}

func resourceApiProxyDeploymentDelete(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentDelete START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)

	_, _, err := client.Proxies.Undeploy(proxy_name, env, rev)
	if err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentDelete error undeploying: %s", err.Error())
		return fmt.Errorf("[ERROR] resourceApiProxyDeploymentDelete error undeploying: %s", err.Error())
	}

	return nil
}

func getLatestRevision(client *apigee.EdgeClient, proxyName string) (int, error) {
	proxy, _, err := client.Proxies.Get(proxyName)
	if err != nil {
		return -1, fmt.Errorf("[ERROR] resourceApiProxyRead error reading proxies: %s", err.Error())
	}
	return len(proxy.Revisions), nil
}
<commit_msg>cleanup using go fmt<commit_after>package apigee

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/gofrs/uuid"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/zambien/go-apigee-edge"
)

func resourceApiProxyDeployment() *schema.Resource {
	return &schema.Resource{
		Create: resourceApiProxyDeploymentCreate,
		Read:   resourceApiProxyDeploymentRead,
		Update: resourceApiProxyDeploymentUpdate,
		Delete: resourceApiProxyDeploymentDelete,
		Importer: &schema.ResourceImporter{
			State: resourceApiProxyDeploymentImport,
		},

		Schema: map[string]*schema.Schema{
			"proxy_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"org": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"env": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"revision": {
				Type:     schema.TypeString,
				Required: true,
			},
			"delay": {
				Type:     schema.TypeInt,
				Optional: true,
				Default:  0,
			},
			"override": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			},
		},
	}
}

func resourceApiProxyDeploymentImport(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceData, error) {
	log.Print("[DEBUG] resourceApiProxyDeploymentImport START")
	client := meta.(*apigee.EdgeClient)

	splits := strings.Split(d.Id(), "_")
	if len(splits) < 2 {
		return []*schema.ResourceData{}, fmt.Errorf("[ERR] Wrong format of resource: %s. Please follow '{name}_{env}_deployment'", d.Id())
	}
	nameOffset := len(splits[len(splits)-1]) + len(splits[len(splits)-2])
	envOffset := len(splits[len(splits)-1])
	name := d.Id()[:(len(d.Id())-nameOffset)-2]
	IDEnv := d.Id()[len(name)+1 : (len(d.Id())-envOffset)-1]
	deployment, _, err := client.Proxies.GetDeployments(name)
	if err != nil {
		log.Printf("[DEBUG] resourceApiProxyDeploymentImport. Error getting deployment api: %v", err)
		return nil, nil
	}
	d.Set("org", deployment.Organization)
	d.Set("proxy_name", deployment.Name)
	d.Set("env", IDEnv)

	return []*schema.ResourceData{d}, nil
}

func resourceApiProxyDeploymentRead(d *schema.ResourceData, meta interface{}) (e error) {
	log.Print("[DEBUG] resourceApiProxyDeploymentRead START")
	log.Printf("[DEBUG] resourceApiProxyDeploymentRead proxy_name: %#v", d.Get("proxy_name").(string))

	client := meta.(*apigee.EdgeClient)

	found := false
	matchedRevision := "0"

	if deployments, _, err := client.Proxies.GetDeployments(d.Get("proxy_name").(string)); err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentRead error getting deployments: %s", err.Error())
		if strings.Contains(err.Error(), "404 ") {
			log.Printf("[DEBUG] resourceApiProxyDeploymentRead 404 encountered.  Removing state for deployment proxy_name: %#v", d.Get("proxy_name").(string))
			d.SetId("")
			return nil
		} else {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentRead error reading deployments: %s", err.Error())
		}
	} else {
		log.Printf("[DEBUG] resourceApiProxyDeploymentRead deployments call fired for proxy_name: %#v", d.Get("proxy_name").(string))
		for _, environment := range deployments.Environments {
			log.Printf("[DEBUG] resourceApiProxyDeploymentRead checking revisions in deployed environment: %#v for expected environment: %#v\n", environment.Name, d.Get("env").(string))
			if environment.Name == d.Get("env").(string) {
				//We don't break.  Always get the last one if there are multiple deployments.
				for _, revision := range environment.Revision {
					log.Printf("[DEBUG] resourceApiProxyDeploymentRead checking deployed revision: %#v for expected revision: %#v\n", revision.Number.String(), d.Get("revision").(string))
					if d.Get("revision").(string) != "latest" && d.Get("revision").(string) == revision.Number.String() {
						matchedRevision = revision.Number.String()
						found = true
						break
					} else {
						matchedRevision = revision.Number.String()
					}
					found = true
				}
			}
		}
	}

	if found {
		log.Printf("[DEBUG] resourceApiProxyDeploymentRead - deployment found. Revision is: %#v", matchedRevision)
		d.Set("revision", matchedRevision)
	} else {
		log.Print("[DEBUG] resourceApiProxyDeploymentRead - no deployment found")
		d.SetId("")
	}
	return nil
}

func resourceApiProxyDeploymentCreate(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentCreate START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)
	delay := int(d.Get("delay").(int))
	override := bool(d.Get("override").(bool))

	if d.Get("revision").(string) == "latest" {
		// deploy latest
		rev, err := getLatestRevision(client, proxy_name)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error getting latest revision: %v", err)
		}
		_, _, err = client.Proxies.Deploy(proxy_name, env, apigee.Revision(rev), delay, override)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error deploying: %v", err)
		}
		log.Printf("[DEBUG] resourceApiProxyDeploymentUpdate Deployed revision %d of %s", rev, proxy_name)
		return resourceApiProxyDeploymentRead(d, meta)
	}

	proxyDep, _, err := client.Proxies.Deploy(proxy_name, env, rev, delay, override)

	if err != nil {

		if strings.Contains(err.Error(), "conflicts with existing deployment path") {
			//create, fail, update
			log.Printf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
			log.Print("[DEBUG] resourceApiProxyDeploymentCreate something got out of sync... maybe someone messing around in apigee directly.  Terraform OVERRIDE!!!")
			resourceApiProxyDeploymentUpdate(d, meta)
		} else {
			log.Printf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentCreate error deploying: %s", err.Error())
		}
	}

	id, _ := uuid.NewV4()
	d.SetId(id.String())
	d.Set("revision", proxyDep.Revision.String())

	return resourceApiProxyDeploymentRead(d, meta)
}

func resourceApiProxyDeploymentUpdate(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentUpdate START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	delay := int(d.Get("delay").(int))
	override := bool(d.Get("override").(bool))

	//We must set delay and override here if not set.
	if delay == 0 {
		delay = 15 //seconds
	}
	if override == false {
		override = true
	}

	if d.Get("revision").(string) == "latest" {
		// deploy latest
		rev, err := getLatestRevision(client, proxy_name)
		if err != nil {
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error getting latest revision: %v", err)
		}
		_, _, err = client.Proxies.ReDeploy(proxy_name, env, apigee.Revision(rev), delay, override)
		if err != nil {
			if strings.Contains(err.Error(), " is already deployed ") {
				return resourceApiProxyDeploymentRead(d, meta)
			}
			return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error deploying: %v", err)
		}
		log.Printf("[DEBUG] resourceApiProxyDeploymentUpdate Deployed revision %d of %s", rev, proxy_name)
		return resourceApiProxyDeploymentRead(d, meta)
	}

	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)
	_, _, err := client.Proxies.ReDeploy(proxy_name, env, rev, delay, override)

	if err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentUpdate error redeploying: %s", err.Error())
		if strings.Contains(err.Error(), " is already deployed into environment ") {
			return resourceApiProxyDeploymentRead(d, meta)
		}
		return fmt.Errorf("[ERROR] resourceApiProxyDeploymentUpdate error redeploying: %s", err.Error())
	}

	return resourceApiProxyDeploymentRead(d, meta)
}

func resourceApiProxyDeploymentDelete(d *schema.ResourceData, meta interface{}) error {

	log.Print("[DEBUG] resourceApiProxyDeploymentDelete START")

	client := meta.(*apigee.EdgeClient)

	proxy_name := d.Get("proxy_name").(string)
	env := d.Get("env").(string)
	rev_int, _ := strconv.Atoi(d.Get("revision").(string))
	rev := apigee.Revision(rev_int)

	_, _, err := client.Proxies.Undeploy(proxy_name, env, rev)
	if err != nil {
		log.Printf("[ERROR] resourceApiProxyDeploymentDelete error undeploying: %s", err.Error())
		return fmt.Errorf("[ERROR] resourceApiProxyDeploymentDelete error undeploying: %s", err.Error())
	}

	return nil
}

func getLatestRevision(client *apigee.EdgeClient, proxyName string) (int, error) {
	proxy, _, err := client.Proxies.Get(proxyName)
	if err != nil {
		return -1, fmt.Errorf("[ERROR] resourceApiProxyRead error reading proxies: %s", err.Error())
	}
	return len(proxy.Revisions), nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"errors"
	"fmt"
	"time"

	etcdclient "github.com/m3db/m3/src/cluster/client/etcd"
	"github.com/m3db/m3/src/cmd/services/m3coordinator/downsample"
	ingestm3msg "github.com/m3db/m3/src/cmd/services/m3coordinator/ingest/m3msg"
	"github.com/m3db/m3/src/cmd/services/m3coordinator/server/m3msg"
	"github.com/m3db/m3/src/metrics/aggregation"
	"github.com/m3db/m3/src/query/graphite/graphite"
	"github.com/m3db/m3/src/query/models"
	"github.com/m3db/m3/src/query/storage"
	"github.com/m3db/m3/src/query/storage/m3"
	xconfig "github.com/m3db/m3/src/x/config"
	"github.com/m3db/m3/src/x/config/listenaddress"
	"github.com/m3db/m3/src/x/cost"
	xdocs "github.com/m3db/m3/src/x/docs"
	"github.com/m3db/m3/src/x/instrument"
	"github.com/m3db/m3/src/x/opentracing"
)

// BackendStorageType is an enum for different backends.
type BackendStorageType string

const (
	// GRPCStorageType is for backends which only support grpc endpoints.
	GRPCStorageType BackendStorageType = "grpc"
	// M3DBStorageType is for m3db backend.
	M3DBStorageType BackendStorageType = "m3db"

	defaultCarbonIngesterListenAddress = "0.0.0.0:7204"
	errNoIDGenerationScheme            = "error: a recent breaking change means that an ID " +
		"generation scheme is required in coordinator configuration settings. " +
		"More information is available here: %s"
)

var (
	// 5m is the default lookback in Prometheus
	defaultLookbackDuration = 5 * time.Minute

	defaultCarbonIngesterAggregationType = aggregation.Mean
)

// Configuration is the configuration for the query service.
type Configuration struct {
	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// Tracing configures opentracing. If not provided, tracing is disabled.
	Tracing opentracing.TracingConfiguration `yaml:"tracing"`

	// Clusters is the DB cluster configurations for read, write and
	// query endpoints.
	Clusters m3.ClustersStaticConfiguration `yaml:"clusters"`

	// LocalConfiguration is the local embedded configuration if running
	// coordinator embedded in the DB.
	Local *LocalConfiguration `yaml:"local"`

	// ClusterManagement for placemement, namespaces and database management
	// endpoints (optional).
	ClusterManagement *ClusterManagementConfiguration `yaml:"clusterManagement"`

	// ListenAddress is the server listen address.
	ListenAddress *listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"`

	// Filter is the read/write/complete tags filter configuration.
	Filter FilterConfiguration `yaml:"filter"`

	// RPC is the RPC configuration.
	RPC *RPCConfiguration `yaml:"rpc"`

	// Backend is the backend store for query service. We currently support grpc and m3db (default).
	Backend BackendStorageType `yaml:"backend"`

	// TagOptions is the tag configuration options.
	TagOptions TagOptionsConfiguration `yaml:"tagOptions"`

	// ReadWorkerPool is the worker pool policy for read requests.
	ReadWorkerPool xconfig.WorkerPoolPolicy `yaml:"readWorkerPoolPolicy"`

	// WriteWorkerPool is the worker pool policy for write requests.
	WriteWorkerPool xconfig.WorkerPoolPolicy `yaml:"writeWorkerPoolPolicy"`

	// Downsample configurates how the metrics should be downsampled.
	Downsample downsample.Configuration `yaml:"downsample"`

	// Ingest is the ingest server.
	Ingest *IngestConfiguration `yaml:"ingest"`

	// Carbon is the carbon configuration.
	Carbon *CarbonConfiguration `yaml:"carbon"`

	// Limits specifies limits on per-query resource usage.
	Limits LimitsConfiguration `yaml:"limits"`

	// LookbackDuration determines the lookback duration for queries
	LookbackDuration *time.Duration `yaml:"lookbackDuration"`

	// ResultOptions are the results options for query.
	ResultOptions ResultOptions `yaml:"resultOptions"`
}

// Filter is a query filter type.
type Filter string

const (
	// FilterLocalOnly is a filter that specifies local only storage should be used.
	FilterLocalOnly Filter = "local_only"
	// FilterRemoteOnly is a filter that specifies remote only storage should be used.
	FilterRemoteOnly Filter = "remote_only"
	// FilterAllowAll is a filter that specifies all storages should be used.
	FilterAllowAll Filter = "allow_all"
	// FilterAllowNone is a filter that specifies no storages should be used.
	FilterAllowNone Filter = "allow_none"
)

// FilterConfiguration is the filters for write/read/complete tags storage filters.
type FilterConfiguration struct {
	Read         Filter `yaml:"read"`
	Write        Filter `yaml:"write"`
	CompleteTags Filter `yaml:"completeTags"`
}

// ResultOptions are the result options for query.
type ResultOptions struct {
	// KeepNans keeps NaNs before returning query results.
	// The default is false, which matches Prometheus
	KeepNans bool `yaml:"keepNans"`
}

// LimitsConfiguration represents limitations on resource usage in the query instance. Limits are split between per-query
// and global limits.
type LimitsConfiguration struct {
	// deprecated: use PerQuery.MaxComputedDatapoints instead.
	DeprecatedMaxComputedDatapoints int64 `yaml:"maxComputedDatapoints"`

	// Global configures limits which apply across all queries running on this
	// instance.
	Global GlobalLimitsConfiguration `yaml:"global"`

	// PerQuery configures limits which apply to each query individually.
	PerQuery PerQueryLimitsConfiguration `yaml:"perQuery"`
}

// MaxComputedDatapoints is a getter providing backwards compatibility between
// LimitsConfiguration.DeprecatedMaxComputedDatapoints and
// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints. See
// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints for a comment on
// the semantics.
func (lc *LimitsConfiguration) MaxComputedDatapoints() int64 {
	if lc.PerQuery.PrivateMaxComputedDatapoints != 0 {
		return lc.PerQuery.PrivateMaxComputedDatapoints
	}

	return lc.DeprecatedMaxComputedDatapoints
}

// GlobalLimitsConfiguration represents limits on resource usage across a query instance. Zero or negative values imply no limit.
type GlobalLimitsConfiguration struct {
	// MaxFetchedDatapoints limits the total number of datapoints actually fetched by all queries at any given time.
	MaxFetchedDatapoints int64 `yaml:"maxFetchedDatapoints"`
}

// AsLimitManagerOptions converts this configuration to cost.LimitManagerOptions for MaxFetchedDatapoints.
func (l *GlobalLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions {
	return toLimitManagerOptions(l.MaxFetchedDatapoints)
}

// PerQueryLimitsConfiguration represents limits on resource usage within a single query. Zero or negative values imply no limit.
type PerQueryLimitsConfiguration struct {
	// PrivateMaxComputedDatapoints limits the number of datapoints that can be
	// returned by a query. It's determined purely
	// from the size of the time range and the step size (end - start / step).
	//
	// N.B.: the hacky "Private" prefix is to indicate that callers should use
	// LimitsConfiguration.MaxComputedDatapoints() instead of accessing
	// this field directly.
	PrivateMaxComputedDatapoints int64 `yaml:"maxComputedDatapoints"`

	// MaxFetchedDatapoints limits the number of datapoints actually used by a given query.
	MaxFetchedDatapoints int64 `yaml:"maxFetchedDatapoints"`
}

// AsLimitManagerOptions converts this configuration to cost.LimitManagerOptions for MaxFetchedDatapoints.
func (l *PerQueryLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions {
	return toLimitManagerOptions(l.MaxFetchedDatapoints)
}

func toLimitManagerOptions(limit int64) cost.LimitManagerOptions {
	return cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{
		Threshold: cost.Cost(limit),
		Enabled:   limit > 0,
	})
}

// IngestConfiguration is the configuration for ingestion server.
type IngestConfiguration struct {
	// Ingester is the configuration for storage based ingester.
	Ingester ingestm3msg.Configuration `yaml:"ingester"`

	// M3Msg is the configuration for m3msg server.
	M3Msg m3msg.Configuration `yaml:"m3msg"`
}

// CarbonConfiguration is the configuration for the carbon server.
type CarbonConfiguration struct {
	Ingester *CarbonIngesterConfiguration `yaml:"ingester"`
}

// CarbonIngesterConfiguration is the configuration struct for carbon ingestion.
type CarbonIngesterConfiguration struct {
	Debug          bool                              `yaml:"debug"`
	ListenAddress  string                            `yaml:"listenAddress"`
	MaxConcurrency int                               `yaml:"maxConcurrency"`
	Rules          []CarbonIngesterRuleConfiguration `yaml:"rules"`
}

// LookbackDurationOrDefault validates the LookbackDuration
func (c Configuration) LookbackDurationOrDefault() (time.Duration, error) {
	if c.LookbackDuration == nil {
		return defaultLookbackDuration, nil
	}

	v := *c.LookbackDuration
	if v < 0 {
		return 0, errors.New("lookbackDuration must be > 0")
	}

	return v, nil
}

// ListenAddressOrDefault returns the specified carbon ingester listen address if provided, or the
// default value if not.
func (c *CarbonIngesterConfiguration) ListenAddressOrDefault() string {
	if c.ListenAddress != "" {
		return c.ListenAddress
	}

	return defaultCarbonIngesterListenAddress
}

// RulesOrDefault returns the specified carbon ingester rules if provided, or generates reasonable
// defaults using the provided aggregated namespaces if not.
func (c *CarbonIngesterConfiguration) RulesOrDefault(namespaces m3.ClusterNamespaces) []CarbonIngesterRuleConfiguration {
	if len(c.Rules) > 0 {
		return c.Rules
	}

	if namespaces.NumAggregatedClusterNamespaces() == 0 {
		return nil
	}

	// Default to fanning out writes for all metrics to all aggregated namespaces if any exists.
	policies := []CarbonIngesterStoragePolicyConfiguration{}
	for _, ns := range namespaces {
		if ns.Options().Attributes().MetricsType == storage.AggregatedMetricsType {
			policies = append(policies, CarbonIngesterStoragePolicyConfiguration{
				Resolution: ns.Options().Attributes().Resolution,
				Retention:  ns.Options().Attributes().Retention,
			})
		}
	}

	if len(policies) == 0 {
		return nil
	}

	// Create a single catch-all rule with a policy for each of the aggregated namespaces we
	// enumerated above.
	aggregationEnabled := true
	return []CarbonIngesterRuleConfiguration{
		{
			Pattern: graphite.MatchAllPattern,
			Aggregation: CarbonIngesterAggregationConfiguration{
				Enabled: &aggregationEnabled,
				Type:    &defaultCarbonIngesterAggregationType,
			},
			Policies: policies,
		},
	}
}

// CarbonIngesterRuleConfiguration is the configuration struct for a carbon
// ingestion rule.
type CarbonIngesterRuleConfiguration struct {
	Pattern     string                                     `yaml:"pattern"`
	Aggregation CarbonIngesterAggregationConfiguration     `yaml:"aggregation"`
	Policies    []CarbonIngesterStoragePolicyConfiguration `yaml:"policies"`
}

// CarbonIngesterAggregationConfiguration is the configuration struct
// for the aggregation for a carbon ingest rule's storage policy.
type CarbonIngesterAggregationConfiguration struct {
	Enabled *bool             `yaml:"enabled"`
	Type    *aggregation.Type `yaml:"type"`
}

// EnabledOrDefault returns whether aggregation should be enabled based on the provided configuration,
// or a default value otherwise.
func (c *CarbonIngesterAggregationConfiguration) EnabledOrDefault() bool {
	if c.Enabled != nil {
		return *c.Enabled
	}

	return true
}

// TypeOrDefault returns the aggregation type that should be used based on the provided configuration,
// or a default value otherwise.
func (c *CarbonIngesterAggregationConfiguration) TypeOrDefault() aggregation.Type {
	if c.Type != nil {
		return *c.Type
	}

	return defaultCarbonIngesterAggregationType
}

// CarbonIngesterStoragePolicyConfiguration is the configuration struct for
// a carbon rule's storage policies.
type CarbonIngesterStoragePolicyConfiguration struct {
	Resolution time.Duration `yaml:"resolution" validate:"nonzero"`
	Retention  time.Duration `yaml:"retention" validate:"nonzero"`
}

// LocalConfiguration is the local embedded configuration if running
// coordinator embedded in the DB.
type LocalConfiguration struct {
	// Namespaces is the list of namespaces that the local embedded DB has.
	Namespaces []m3.ClusterStaticNamespaceConfiguration `yaml:"namespaces"`
}

// ClusterManagementConfiguration is configuration for the placemement,
// namespaces and database management endpoints (optional).
type ClusterManagementConfiguration struct {
	// Etcd is the client configuration for etcd.
	Etcd etcdclient.Configuration `yaml:"etcd"`
}

// RPCConfiguration is the RPC configuration for the coordinator for
// the GRPC server used for remote coordinator to coordinator calls.
type RPCConfiguration struct {
	// Enabled determines if coordinator RPC is enabled for remote calls.
	Enabled bool `yaml:"enabled"`

	// ListenAddress is the RPC server listen address.
	ListenAddress string `yaml:"listenAddress"`

	// RemoteListenAddresses is the remote listen addresses to call for remote
	// coordinator calls.
	RemoteListenAddresses []string `yaml:"remoteListenAddresses"`
}

// TagOptionsConfiguration is the configuration for shared tag options
// Currently only name, but can expand to cover deduplication settings, or other
// relevant options.
type TagOptionsConfiguration struct {
	// MetricName specifies the tag name that corresponds to the metric's name tag
	// If not provided, defaults to `__name__`.
	MetricName string `yaml:"metricName"`

	// BucketName specifies the tag name that corresponds to the metric's bucket.
	// If not provided, defaults to `le`.
	BucketName string `yaml:"bucketName"`

	// Scheme determines the default ID generation scheme. Defaults to TypeLegacy.
	Scheme models.IDSchemeType `yaml:"idScheme"`
}

// TagOptionsFromConfig translates tag option configuration into tag options.
func TagOptionsFromConfig(cfg TagOptionsConfiguration) (models.TagOptions, error) {
	opts := models.NewTagOptions()
	name := cfg.MetricName
	if name != "" {
		opts = opts.SetMetricName([]byte(name))
	}

	bucket := cfg.BucketName
	if bucket != "" {
		opts = opts.SetBucketName([]byte(bucket))
	}

	if cfg.Scheme == models.TypeDefault {
		// If no config has been set, error.
		docLink := xdocs.Path("how_to/query#migration")
		return nil, fmt.Errorf(errNoIDGenerationScheme, docLink)
	}

	opts = opts.SetIDSchemeType(cfg.Scheme)
	if err := opts.Validate(); err != nil {
		return nil, err
	}

	return opts, nil
}
<commit_msg>[query] Deprecate query conversion cache (#1548)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package config

import (
	"errors"
	"fmt"
	"time"

	etcdclient "github.com/m3db/m3/src/cluster/client/etcd"
	"github.com/m3db/m3/src/cmd/services/m3coordinator/downsample"
	ingestm3msg "github.com/m3db/m3/src/cmd/services/m3coordinator/ingest/m3msg"
	"github.com/m3db/m3/src/cmd/services/m3coordinator/server/m3msg"
	"github.com/m3db/m3/src/metrics/aggregation"
	"github.com/m3db/m3/src/query/graphite/graphite"
	"github.com/m3db/m3/src/query/models"
	"github.com/m3db/m3/src/query/storage"
	"github.com/m3db/m3/src/query/storage/m3"
	xconfig "github.com/m3db/m3/src/x/config"
	"github.com/m3db/m3/src/x/config/listenaddress"
	"github.com/m3db/m3/src/x/cost"
	xdocs "github.com/m3db/m3/src/x/docs"
	"github.com/m3db/m3/src/x/instrument"
	"github.com/m3db/m3/src/x/opentracing"
)

// BackendStorageType is an enum for different backends.
type BackendStorageType string

const (
	// GRPCStorageType is for backends which only support grpc endpoints.
	GRPCStorageType BackendStorageType = "grpc"
	// M3DBStorageType is for m3db backend.
	M3DBStorageType BackendStorageType = "m3db"

	defaultCarbonIngesterListenAddress = "0.0.0.0:7204"
	errNoIDGenerationScheme            = "error: a recent breaking change means that an ID " +
		"generation scheme is required in coordinator configuration settings. " +
		"More information is available here: %s"
)

var (
	// 5m is the default lookback in Prometheus
	defaultLookbackDuration = 5 * time.Minute

	defaultCarbonIngesterAggregationType = aggregation.Mean
)

// Configuration is the configuration for the query service.
type Configuration struct {
	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// Tracing configures opentracing. If not provided, tracing is disabled.
	Tracing opentracing.TracingConfiguration `yaml:"tracing"`

	// Clusters is the DB cluster configurations for read, write and
	// query endpoints.
	Clusters m3.ClustersStaticConfiguration `yaml:"clusters"`

	// LocalConfiguration is the local embedded configuration if running
	// coordinator embedded in the DB.
	Local *LocalConfiguration `yaml:"local"`

	// ClusterManagement for placemement, namespaces and database management
	// endpoints (optional).
	ClusterManagement *ClusterManagementConfiguration `yaml:"clusterManagement"`

	// ListenAddress is the server listen address.
	ListenAddress *listenaddress.Configuration `yaml:"listenAddress" validate:"nonzero"`

	// Filter is the read/write/complete tags filter configuration.
	Filter FilterConfiguration `yaml:"filter"`

	// RPC is the RPC configuration.
	RPC *RPCConfiguration `yaml:"rpc"`

	// Backend is the backend store for query service. We currently support grpc and m3db (default).
	Backend BackendStorageType `yaml:"backend"`

	// TagOptions is the tag configuration options.
	TagOptions TagOptionsConfiguration `yaml:"tagOptions"`

	// ReadWorkerPool is the worker pool policy for read requests.
	ReadWorkerPool xconfig.WorkerPoolPolicy `yaml:"readWorkerPoolPolicy"`

	// WriteWorkerPool is the worker pool policy for write requests.
	WriteWorkerPool xconfig.WorkerPoolPolicy `yaml:"writeWorkerPoolPolicy"`

	// Downsample configurates how the metrics should be downsampled.
	Downsample downsample.Configuration `yaml:"downsample"`

	// Ingest is the ingest server.
	Ingest *IngestConfiguration `yaml:"ingest"`

	// Carbon is the carbon configuration.
	Carbon *CarbonConfiguration `yaml:"carbon"`

	// Limits specifies limits on per-query resource usage.
	Limits LimitsConfiguration `yaml:"limits"`

	// LookbackDuration determines the lookback duration for queries
	LookbackDuration *time.Duration `yaml:"lookbackDuration"`

	// ResultOptions are the results options for query.
	ResultOptions ResultOptions `yaml:"resultOptions"`

	// Cache configurations.
	//
	// Deprecated: cache configurations are no longer supported. Remove from file.
	DeprecatedCache CacheConfiguration `yaml:"cache"`
}

// Filter is a query filter type.
type Filter string

const (
	// FilterLocalOnly is a filter that specifies local only storage should be used.
	FilterLocalOnly Filter = "local_only"
	// FilterRemoteOnly is a filter that specifies remote only storage should be used.
	FilterRemoteOnly Filter = "remote_only"
	// FilterAllowAll is a filter that specifies all storages should be used.
	FilterAllowAll Filter = "allow_all"
	// FilterAllowNone is a filter that specifies no storages should be used.
	FilterAllowNone Filter = "allow_none"
)

// FilterConfiguration is the filters for write/read/complete tags storage filters.
type FilterConfiguration struct {
	Read         Filter `yaml:"read"`
	Write        Filter `yaml:"write"`
	CompleteTags Filter `yaml:"completeTags"`
}

// CacheConfiguration contains the cache configurations.
type CacheConfiguration struct {
	// Deprecated: remove from config.
	DeprecatedQueryConversion *DeprecatedQueryConversionCacheConfiguration `yaml:"queryConversion"`
}

// DeprecatedQueryConversionCacheConfiguration is deprecated: remove from config.
type DeprecatedQueryConversionCacheConfiguration struct {
	Size *int `yaml:"size"`
}

// ResultOptions are the result options for query.
type ResultOptions struct {
	// KeepNans keeps NaNs before returning query results.
	// The default is false, which matches Prometheus
	KeepNans bool `yaml:"keepNans"`
}

// LimitsConfiguration represents limitations on resource usage in the query instance. Limits are split between per-query
// and global limits.
type LimitsConfiguration struct {
	// deprecated: use PerQuery.MaxComputedDatapoints instead.
	DeprecatedMaxComputedDatapoints int64 `yaml:"maxComputedDatapoints"`

	// Global configures limits which apply across all queries running on this
	// instance.
	Global GlobalLimitsConfiguration `yaml:"global"`

	// PerQuery configures limits which apply to each query individually.
	PerQuery PerQueryLimitsConfiguration `yaml:"perQuery"`
}

// MaxComputedDatapoints is a getter providing backwards compatibility between
// LimitsConfiguration.DeprecatedMaxComputedDatapoints and
// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints. See
// LimitsConfiguration.PerQuery.PrivateMaxComputedDatapoints for a comment on
// the semantics.
func (lc *LimitsConfiguration) MaxComputedDatapoints() int64 {
	if lc.PerQuery.PrivateMaxComputedDatapoints != 0 {
		return lc.PerQuery.PrivateMaxComputedDatapoints
	}

	return lc.DeprecatedMaxComputedDatapoints
}

// GlobalLimitsConfiguration represents limits on resource usage across a query instance. Zero or negative values imply no limit.
type GlobalLimitsConfiguration struct {
	// MaxFetchedDatapoints limits the total number of datapoints actually fetched by all queries at any given time.
	MaxFetchedDatapoints int64 `yaml:"maxFetchedDatapoints"`
}

// AsLimitManagerOptions converts this configuration to cost.LimitManagerOptions for MaxFetchedDatapoints.
func (l *GlobalLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions {
	return toLimitManagerOptions(l.MaxFetchedDatapoints)
}

// PerQueryLimitsConfiguration represents limits on resource usage within a single query. Zero or negative values imply no limit.
type PerQueryLimitsConfiguration struct {
	// PrivateMaxComputedDatapoints limits the number of datapoints that can be
	// returned by a query. It's determined purely
	// from the size of the time range and the step size (end - start / step).
	//
	// N.B.: the hacky "Private" prefix is to indicate that callers should use
	// LimitsConfiguration.MaxComputedDatapoints() instead of accessing
	// this field directly.
	PrivateMaxComputedDatapoints int64 `yaml:"maxComputedDatapoints"`

	// MaxFetchedDatapoints limits the number of datapoints actually used by a given query.
	MaxFetchedDatapoints int64 `yaml:"maxFetchedDatapoints"`
}

// AsLimitManagerOptions converts this configuration to cost.LimitManagerOptions for MaxFetchedDatapoints.
func (l *PerQueryLimitsConfiguration) AsLimitManagerOptions() cost.LimitManagerOptions {
	return toLimitManagerOptions(l.MaxFetchedDatapoints)
}

func toLimitManagerOptions(limit int64) cost.LimitManagerOptions {
	return cost.NewLimitManagerOptions().SetDefaultLimit(cost.Limit{
		Threshold: cost.Cost(limit),
		Enabled:   limit > 0,
	})
}

// IngestConfiguration is the configuration for ingestion server.
type IngestConfiguration struct {
	// Ingester is the configuration for storage based ingester.
	Ingester ingestm3msg.Configuration `yaml:"ingester"`

	// M3Msg is the configuration for m3msg server.
	M3Msg m3msg.Configuration `yaml:"m3msg"`
}

// CarbonConfiguration is the configuration for the carbon server.
type CarbonConfiguration struct {
	Ingester *CarbonIngesterConfiguration `yaml:"ingester"`
}

// CarbonIngesterConfiguration is the configuration struct for carbon ingestion.
type CarbonIngesterConfiguration struct {
	Debug          bool                              `yaml:"debug"`
	ListenAddress  string                            `yaml:"listenAddress"`
	MaxConcurrency int                               `yaml:"maxConcurrency"`
	Rules          []CarbonIngesterRuleConfiguration `yaml:"rules"`
}

// LookbackDurationOrDefault validates the LookbackDuration
func (c Configuration) LookbackDurationOrDefault() (time.Duration, error) {
	if c.LookbackDuration == nil {
		return defaultLookbackDuration, nil
	}

	v := *c.LookbackDuration
	if v < 0 {
		return 0, errors.New("lookbackDuration must be > 0")
	}

	return v, nil
}

// ListenAddressOrDefault returns the specified carbon ingester listen address if provided, or the
// default value if not.
func (c *CarbonIngesterConfiguration) ListenAddressOrDefault() string {
	if c.ListenAddress != "" {
		return c.ListenAddress
	}

	return defaultCarbonIngesterListenAddress
}

// RulesOrDefault returns the specified carbon ingester rules if provided, or generates reasonable
// defaults using the provided aggregated namespaces if not.
func (c *CarbonIngesterConfiguration) RulesOrDefault(namespaces m3.ClusterNamespaces) []CarbonIngesterRuleConfiguration {
	if len(c.Rules) > 0 {
		return c.Rules
	}

	if namespaces.NumAggregatedClusterNamespaces() == 0 {
		return nil
	}

	// Default to fanning out writes for all metrics to all aggregated namespaces if any exists.
	policies := []CarbonIngesterStoragePolicyConfiguration{}
	for _, ns := range namespaces {
		if ns.Options().Attributes().MetricsType == storage.AggregatedMetricsType {
			policies = append(policies, CarbonIngesterStoragePolicyConfiguration{
				Resolution: ns.Options().Attributes().Resolution,
				Retention:  ns.Options().Attributes().Retention,
			})
		}
	}

	if len(policies) == 0 {
		return nil
	}

	// Create a single catch-all rule with a policy for each of the aggregated namespaces we
	// enumerated above.
	aggregationEnabled := true
	return []CarbonIngesterRuleConfiguration{
		{
			Pattern: graphite.MatchAllPattern,
			Aggregation: CarbonIngesterAggregationConfiguration{
				Enabled: &aggregationEnabled,
				Type:    &defaultCarbonIngesterAggregationType,
			},
			Policies: policies,
		},
	}
}

// CarbonIngesterRuleConfiguration is the configuration struct for a carbon
// ingestion rule.
type CarbonIngesterRuleConfiguration struct {
	Pattern     string                                     `yaml:"pattern"`
	Aggregation CarbonIngesterAggregationConfiguration     `yaml:"aggregation"`
	Policies    []CarbonIngesterStoragePolicyConfiguration `yaml:"policies"`
}

// CarbonIngesterAggregationConfiguration is the configuration struct
// for the aggregation for a carbon ingest rule's storage policy.
type CarbonIngesterAggregationConfiguration struct {
	Enabled *bool             `yaml:"enabled"`
	Type    *aggregation.Type `yaml:"type"`
}

// EnabledOrDefault returns whether aggregation should be enabled based on the provided configuration,
// or a default value otherwise.
func (c *CarbonIngesterAggregationConfiguration) EnabledOrDefault() bool {
	if c.Enabled != nil {
		return *c.Enabled
	}

	return true
}

// TypeOrDefault returns the aggregation type that should be used based on the provided configuration,
// or a default value otherwise.
func (c *CarbonIngesterAggregationConfiguration) TypeOrDefault() aggregation.Type {
	if c.Type != nil {
		return *c.Type
	}

	return defaultCarbonIngesterAggregationType
}

// CarbonIngesterStoragePolicyConfiguration is the configuration struct for
// a carbon rule's storage policies.
type CarbonIngesterStoragePolicyConfiguration struct {
	Resolution time.Duration `yaml:"resolution" validate:"nonzero"`
	Retention  time.Duration `yaml:"retention" validate:"nonzero"`
}

// LocalConfiguration is the local embedded configuration if running
// coordinator embedded in the DB.
type LocalConfiguration struct {
	// Namespaces is the list of namespaces that the local embedded DB has.
	Namespaces []m3.ClusterStaticNamespaceConfiguration `yaml:"namespaces"`
}

// ClusterManagementConfiguration is configuration for the placemement,
// namespaces and database management endpoints (optional).
type ClusterManagementConfiguration struct {
	// Etcd is the client configuration for etcd.
	Etcd etcdclient.Configuration `yaml:"etcd"`
}

// RPCConfiguration is the RPC configuration for the coordinator for
// the GRPC server used for remote coordinator to coordinator calls.
type RPCConfiguration struct {
	// Enabled determines if coordinator RPC is enabled for remote calls.
	Enabled bool `yaml:"enabled"`

	// ListenAddress is the RPC server listen address.
	ListenAddress string `yaml:"listenAddress"`

	// RemoteListenAddresses is the remote listen addresses to call for remote
	// coordinator calls.
	RemoteListenAddresses []string `yaml:"remoteListenAddresses"`
}

// TagOptionsConfiguration is the configuration for shared tag options
// Currently only name, but can expand to cover deduplication settings, or other
// relevant options.
type TagOptionsConfiguration struct {
	// MetricName specifies the tag name that corresponds to the metric's name tag
	// If not provided, defaults to `__name__`.
	MetricName string `yaml:"metricName"`

	// BucketName specifies the tag name that corresponds to the metric's bucket.
	// If not provided, defaults to `le`.
	BucketName string `yaml:"bucketName"`

	// Scheme determines the default ID generation scheme. Defaults to TypeLegacy.
	Scheme models.IDSchemeType `yaml:"idScheme"`
}

// TagOptionsFromConfig translates tag option configuration into tag options.
func TagOptionsFromConfig(cfg TagOptionsConfiguration) (models.TagOptions, error) {
	opts := models.NewTagOptions()
	name := cfg.MetricName
	if name != "" {
		opts = opts.SetMetricName([]byte(name))
	}

	bucket := cfg.BucketName
	if bucket != "" {
		opts = opts.SetBucketName([]byte(bucket))
	}

	if cfg.Scheme == models.TypeDefault {
		// If no config has been set, error.
		docLink := xdocs.Path("how_to/query#migration")
		return nil, fmt.Errorf(errNoIDGenerationScheme, docLink)
	}

	opts = opts.SetIDSchemeType(cfg.Scheme)
	if err := opts.Validate(); err != nil {
		return nil, err
	}

	return opts, nil
}
<|endoftext|>