<commit_before>package collectors // import "bosun.org/cmd/scollector/collectors"

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/util"
)

var collectors []Collector

type Collector interface {
	Run(chan<- *opentsdb.DataPoint, <-chan struct{})
	Name() string
	Init()
}

const (
	osCPU              = "os.cpu"
	osCPUClock         = "os.cpu.clock"
	osDiskFree         = "os.disk.fs.space_free"
	osDiskPctFree      = "os.disk.fs.percent_free"
	osDiskTotal        = "os.disk.fs.space_total"
	osDiskUsed         = "os.disk.fs.space_used"
	osMemFree          = "os.mem.free"
	osMemPctFree       = "os.mem.percent_free"
	osMemTotal         = "os.mem.total"
	osMemUsed          = "os.mem.used"
	osNetBondBroadcast = "os.net.bond.packets_broadcast"
	osNetBondBytes     = "os.net.bond.bytes"
	osNetBondDropped   = "os.net.bond.dropped"
	osNetBondErrors    = "os.net.bond.errs"
	osNetBondMulticast = "os.net.bond.packets_multicast"
	osNetBondPackets   = "os.net.bond.packets"
	osNetBondUnicast   = "os.net.bond.packets_unicast"
	osNetBondIfSpeed   = "os.net.bond.ifspeed"
	osNetIfSpeed       = "os.net.ifspeed"
	osNetBroadcast     = "os.net.packets_broadcast"
	osNetBytes         = "os.net.bytes"
	osNetDropped       = "os.net.dropped"
	osNetErrors        = "os.net.errs"
	osNetMulticast     = "os.net.packets_multicast"
	osNetPackets       = "os.net.packets"
	osNetPauseFrames   = "os.net.pause_frames"
	osNetUnicast       = "os.net.packets_unicast"
	osSystemUptime     = "os.system.uptime"
	osNetMTU           = "os.net.mtu"
	osNetAdminStatus   = "os.net.admin_status"
	osNetOperStatus    = "os.net.oper_status"
)

const (
	osCPUClockDesc       = "The current speed of the processor in MHz."
	osDiskFreeDesc       = "The space_free property indicates in bytes how much free space is available on the disk."
	osDiskPctFreeDesc    = "The percent_free property indicates what percentage of the disk is available."
	osDiskTotalDesc      = "The space_total property indicates in bytes how much total space is on the disk."
	osDiskUsedDesc       = "The space_used property indicates in bytes how much space is used on the disk."
	osMemFreeDesc        = "Number, in bytes, of physical memory currently unused and available."
	osMemPctFreeDesc     = "The percent of free memory. In Linux free memory includes memory used by buffers and cache."
	osMemTotalDesc       = "Total amount, in bytes, of physical memory available to the operating system."
	osMemUsedDesc        = "The amount of used memory. In Linux this excludes memory used by buffers and cache."
	osNetBroadcastDesc   = "The rate at which broadcast packets are sent or received on the network interface."
	osNetBytesDesc       = "The rate at which bytes are sent or received over the network interface."
	osNetDroppedDesc     = "The number of packets that were chosen to be discarded even though no errors had been detected to prevent transmission."
	osNetErrorsDesc      = "The number of packets that could not be transmitted because of errors."
	osNetMulticastDesc   = "The rate at which multicast packets are sent or received on the network interface."
	osNetPacketsDesc     = "The rate at which packets are sent or received on the network interface."
	osNetUnicastDesc     = "The rate at which unicast packets are sent or received on the network interface."
	osNetIfSpeedDesc     = "The total link speed of the network interface in Megabits per second."
	osNetPauseFrameDesc  = "The rate of pause frames sent or recieved on the network interface. An overwhelmed network element can send a pause frame, which halts the transmission of the sender for a specified period of time."
	osSystemUptimeDesc   = "Seconds since last reboot."
	osNetMTUDesc         = "The maximum transmission unit for the ethernet frame."
	osNetAdminStatusDesc = "The desired state of the interface. The testing(3) state indicates that no operational packets can be passed. When a managed system initializes, all interfaces start with ifAdminStatus in the down(2) state. As a result of either explicit management action or per configuration information retained by the managed system, ifAdminStatus is then changed to either the up(1) or testing(3) states (or remains in the down(2) state)."
	osNetOperStatusDesc  = "The current operational state of the interface. The testing(3) state indicates that no operational packets can be passed. If ifAdminStatus is down(2) then ifOperStatus should be down(2). If ifAdminStatus is changed to up(1) then ifOperStatus should change to up(1) if the interface is ready to transmit and receive network traffic; it should change to dormant(5) if the interface is waiting for external actions (such as a serial line waiting for an incoming connection); it should remain in the down(2) state if and only if there is a fault that prevents it from going to the up(1) state; it should remain in the notPresent(6) state if the interface has missing (typically, hardware) components."
)

var (
	// DefaultFreq is the duration between collection intervals if none is
	// specified.
	DefaultFreq = time.Second * 15

	timestamp = time.Now().Unix()
	tlock     sync.Mutex
	AddTags   opentsdb.TagSet

	AddProcessDotNetConfig = func(params conf.ProcessDotNet) error {
		return fmt.Errorf("process_dotnet watching not implemented on this platform")
	}
	WatchProcessesDotNet = func() {}

	KeepalivedCommunity = ""
)

func init() {
	go func() {
		for t := range time.Tick(time.Second) {
			tlock.Lock()
			timestamp = t.Unix()
			tlock.Unlock()
		}
	}()
}

func now() (t int64) {
	tlock.Lock()
	t = timestamp
	tlock.Unlock()
	return
}

// Search returns all collectors matching the pattern s.
func Search(s []string) []Collector {
	if len(s) == 0 {
		return collectors
	}
	var r []Collector
	for _, c := range collectors {
		for _, p := range s {
			if strings.Contains(c.Name(), p) {
				r = append(r, c)
				break
			}
		}
	}
	return r
}

// Run runs specified collectors. Use nil for all collectors.
func Run(cs []Collector) (chan *opentsdb.DataPoint, chan struct{}) {
	if cs == nil {
		cs = collectors
	}
	ch := make(chan *opentsdb.DataPoint)
	quit := make(chan struct{})
	for _, c := range cs {
		go c.Run(ch, quit)
	}
	return ch, quit
}

type initFunc func(*conf.Conf)

var inits = []initFunc{}

func registerInit(i initFunc) {
	inits = append(inits, i)
}

func Init(c *conf.Conf) {
	for _, f := range inits {
		f(c)
	}
}

type MetricMeta struct {
	Metric   string
	TagSet   opentsdb.TagSet
	RateType metadata.RateType
	Unit     metadata.Unit
	Desc     string
}

// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	if b, ok := value.(bool); ok {
		if b {
			value = 1
		} else {
			value = 0
		}
	}
	tags := t.Copy()
	if host, present := tags["host"]; !present {
		tags["host"] = util.Hostname
	} else if host == "" {
		delete(tags, "host")
	}
	if rate != metadata.Unknown {
		metadata.AddMeta(name, nil, "rate", rate, false)
	}
	if unit != metadata.None {
		metadata.AddMeta(name, nil, "unit", unit, false)
	}
	if desc != "" {
		metadata.AddMeta(name, tags, "desc", desc, false)
	}

	tags = AddTags.Copy().Merge(tags)
	d := opentsdb.DataPoint{
		Metric:    name,
		Timestamp: ts,
		Value:     value,
		Tags:      tags,
	}
	*md = append(*md, &d)
}

// Add appends a new data point with given metric name, value, and tags. Tags
// may be nil. If tags is nil or does not contain a host key, it will be
// automatically added. If the value of the host key is the empty string, it
// will be removed (use this to prevent the normal auto-adding of the host tag).
func Add(md *opentsdb.MultiDataPoint, name string, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	AddTS(md, name, now(), value, t, rate, unit, desc)
}

func readLine(fname string, line func(string) error) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		if err := line(scanner.Text()); err != nil {
			return err
		}
	}
	return scanner.Err()
}

// IsDigit returns true if s consists of decimal digits.
func IsDigit(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) {
			return false
		}
	}
	return true
}

// IsAlNum returns true if s is alphanumeric.
func IsAlNum(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) && !unicode.IsLetter(ch) {
			return false
		}
	}
	return true
}

func TSys100NStoEpoch(nsec uint64) int64 {
	nsec -= 116444736000000000
	seconds := nsec / 1e7
	return int64(seconds)
}

func metaIfaces(f func(iface net.Interface, tags opentsdb.TagSet)) {
	ifaces, _ := net.Interfaces()
	for _, iface := range ifaces {
		if strings.HasPrefix(iface.Name, "lo") {
			continue
		}
		tags := opentsdb.TagSet{"iface": fmt.Sprint("Interface", iface.Index)}
		metadata.AddMeta("", tags, "name", iface.Name, true)
		if mac := strings.ToUpper(strings.Replace(iface.HardwareAddr.String(), ":", "", -1)); mac != "" {
			metadata.AddMeta("", tags, "mac", mac, true)
		}
		ads, _ := iface.Addrs()
		for i, ad := range ads {
			addr := strings.Split(ad.String(), "/")[0]
			metadata.AddMeta("", opentsdb.TagSet{"addr": fmt.Sprint("Addr", i)}.Merge(tags), "addr", addr, true)
		}
		if f != nil {
			f(iface, tags)
		}
	}
}
<commit_msg>cmd/scollector: Linux interface metadata changes<commit_after>package collectors // import "bosun.org/cmd/scollector/collectors"

import (
	"bufio"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"bosun.org/cmd/scollector/conf"
	"bosun.org/metadata"
	"bosun.org/opentsdb"
	"bosun.org/util"
)

var collectors []Collector

type Collector interface {
	Run(chan<- *opentsdb.DataPoint, <-chan struct{})
	Name() string
	Init()
}

const (
	osCPU              = "os.cpu"
	osCPUClock         = "os.cpu.clock"
	osDiskFree         = "os.disk.fs.space_free"
	osDiskPctFree      = "os.disk.fs.percent_free"
	osDiskTotal        = "os.disk.fs.space_total"
	osDiskUsed         = "os.disk.fs.space_used"
	osMemFree          = "os.mem.free"
	osMemPctFree       = "os.mem.percent_free"
	osMemTotal         = "os.mem.total"
	osMemUsed          = "os.mem.used"
	osNetBondBroadcast = "os.net.bond.packets_broadcast"
	osNetBondBytes     = "os.net.bond.bytes"
	osNetBondDropped   = "os.net.bond.dropped"
	osNetBondErrors    = "os.net.bond.errs"
	osNetBondMulticast = "os.net.bond.packets_multicast"
	osNetBondPackets   = "os.net.bond.packets"
	osNetBondUnicast   = "os.net.bond.packets_unicast"
	osNetBondIfSpeed   = "os.net.bond.ifspeed"
	osNetIfSpeed       = "os.net.ifspeed"
	osNetBroadcast     = "os.net.packets_broadcast"
	osNetBytes         = "os.net.bytes"
	osNetDropped       = "os.net.dropped"
	osNetErrors        = "os.net.errs"
	osNetMulticast     = "os.net.packets_multicast"
	osNetPackets       = "os.net.packets"
	osNetPauseFrames   = "os.net.pause_frames"
	osNetUnicast       = "os.net.packets_unicast"
	osSystemUptime     = "os.system.uptime"
	osNetMTU           = "os.net.mtu"
	osNetAdminStatus   = "os.net.admin_status"
	osNetOperStatus    = "os.net.oper_status"
)

const (
	osCPUClockDesc       = "The current speed of the processor in MHz."
	osDiskFreeDesc       = "The space_free property indicates in bytes how much free space is available on the disk."
	osDiskPctFreeDesc    = "The percent_free property indicates what percentage of the disk is available."
	osDiskTotalDesc      = "The space_total property indicates in bytes how much total space is on the disk."
	osDiskUsedDesc       = "The space_used property indicates in bytes how much space is used on the disk."
	osMemFreeDesc        = "Number, in bytes, of physical memory currently unused and available."
	osMemPctFreeDesc     = "The percent of free memory. In Linux free memory includes memory used by buffers and cache."
	osMemTotalDesc       = "Total amount, in bytes, of physical memory available to the operating system."
	osMemUsedDesc        = "The amount of used memory. In Linux this excludes memory used by buffers and cache."
	osNetBroadcastDesc   = "The rate at which broadcast packets are sent or received on the network interface."
	osNetBytesDesc       = "The rate at which bytes are sent or received over the network interface."
	osNetDroppedDesc     = "The number of packets that were chosen to be discarded even though no errors had been detected to prevent transmission."
	osNetErrorsDesc      = "The number of packets that could not be transmitted because of errors."
	osNetMulticastDesc   = "The rate at which multicast packets are sent or received on the network interface."
	osNetPacketsDesc     = "The rate at which packets are sent or received on the network interface."
	osNetUnicastDesc     = "The rate at which unicast packets are sent or received on the network interface."
	osNetIfSpeedDesc     = "The total link speed of the network interface in Megabits per second."
	osNetPauseFrameDesc  = "The rate of pause frames sent or recieved on the network interface. An overwhelmed network element can send a pause frame, which halts the transmission of the sender for a specified period of time."
	osSystemUptimeDesc   = "Seconds since last reboot."
	osNetMTUDesc         = "The maximum transmission unit for the ethernet frame."
	osNetAdminStatusDesc = "The desired state of the interface. The testing(3) state indicates that no operational packets can be passed. When a managed system initializes, all interfaces start with ifAdminStatus in the down(2) state. As a result of either explicit management action or per configuration information retained by the managed system, ifAdminStatus is then changed to either the up(1) or testing(3) states (or remains in the down(2) state)."
	osNetOperStatusDesc  = "The current operational state of the interface. The testing(3) state indicates that no operational packets can be passed. If ifAdminStatus is down(2) then ifOperStatus should be down(2). If ifAdminStatus is changed to up(1) then ifOperStatus should change to up(1) if the interface is ready to transmit and receive network traffic; it should change to dormant(5) if the interface is waiting for external actions (such as a serial line waiting for an incoming connection); it should remain in the down(2) state if and only if there is a fault that prevents it from going to the up(1) state; it should remain in the notPresent(6) state if the interface has missing (typically, hardware) components."
)

var (
	// DefaultFreq is the duration between collection intervals if none is
	// specified.
	DefaultFreq = time.Second * 15

	timestamp = time.Now().Unix()
	tlock     sync.Mutex
	AddTags   opentsdb.TagSet

	AddProcessDotNetConfig = func(params conf.ProcessDotNet) error {
		return fmt.Errorf("process_dotnet watching not implemented on this platform")
	}
	WatchProcessesDotNet = func() {}

	KeepalivedCommunity = ""
)

func init() {
	go func() {
		for t := range time.Tick(time.Second) {
			tlock.Lock()
			timestamp = t.Unix()
			tlock.Unlock()
		}
	}()
}

func now() (t int64) {
	tlock.Lock()
	t = timestamp
	tlock.Unlock()
	return
}

// Search returns all collectors matching the pattern s.
func Search(s []string) []Collector {
	if len(s) == 0 {
		return collectors
	}
	var r []Collector
	for _, c := range collectors {
		for _, p := range s {
			if strings.Contains(c.Name(), p) {
				r = append(r, c)
				break
			}
		}
	}
	return r
}

// Run runs specified collectors. Use nil for all collectors.
func Run(cs []Collector) (chan *opentsdb.DataPoint, chan struct{}) {
	if cs == nil {
		cs = collectors
	}
	ch := make(chan *opentsdb.DataPoint)
	quit := make(chan struct{})
	for _, c := range cs {
		go c.Run(ch, quit)
	}
	return ch, quit
}

type initFunc func(*conf.Conf)

var inits = []initFunc{}

func registerInit(i initFunc) {
	inits = append(inits, i)
}

func Init(c *conf.Conf) {
	for _, f := range inits {
		f(c)
	}
}

type MetricMeta struct {
	Metric   string
	TagSet   opentsdb.TagSet
	RateType metadata.RateType
	Unit     metadata.Unit
	Desc     string
}

// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	if b, ok := value.(bool); ok {
		if b {
			value = 1
		} else {
			value = 0
		}
	}
	tags := t.Copy()
	if host, present := tags["host"]; !present {
		tags["host"] = util.Hostname
	} else if host == "" {
		delete(tags, "host")
	}
	if rate != metadata.Unknown {
		metadata.AddMeta(name, nil, "rate", rate, false)
	}
	if unit != metadata.None {
		metadata.AddMeta(name, nil, "unit", unit, false)
	}
	if desc != "" {
		metadata.AddMeta(name, tags, "desc", desc, false)
	}

	tags = AddTags.Copy().Merge(tags)
	d := opentsdb.DataPoint{
		Metric:    name,
		Timestamp: ts,
		Value:     value,
		Tags:      tags,
	}
	*md = append(*md, &d)
}

// Add appends a new data point with given metric name, value, and tags. Tags
// may be nil. If tags is nil or does not contain a host key, it will be
// automatically added. If the value of the host key is the empty string, it
// will be removed (use this to prevent the normal auto-adding of the host tag).
func Add(md *opentsdb.MultiDataPoint, name string, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	AddTS(md, name, now(), value, t, rate, unit, desc)
}

func readLine(fname string, line func(string) error) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		if err := line(scanner.Text()); err != nil {
			return err
		}
	}
	return scanner.Err()
}

// IsDigit returns true if s consists of decimal digits.
func IsDigit(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) {
			return false
		}
	}
	return true
}

// IsAlNum returns true if s is alphanumeric.
func IsAlNum(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) && !unicode.IsLetter(ch) {
			return false
		}
	}
	return true
}

func TSys100NStoEpoch(nsec uint64) int64 {
	nsec -= 116444736000000000
	seconds := nsec / 1e7
	return int64(seconds)
}

func metaIfaces(f func(iface net.Interface, tags opentsdb.TagSet)) {
	ifaces, _ := net.Interfaces()
	for _, iface := range ifaces {
		if strings.HasPrefix(iface.Name, "lo") {
			continue
		}
		tags := opentsdb.TagSet{"iface": iface.Name}
		metadata.AddMeta("", tags, "name", iface.Name, true)
		if mac := strings.ToUpper(strings.Replace(iface.HardwareAddr.String(), ":", "", -1)); mac != "" {
			metadata.AddMeta("", tags, "mac", mac, true)
		}
		rawAds, _ := iface.Addrs()
		addrs := make([]string, len(rawAds))
		for i, rAd := range rawAds {
			addr := strings.Split(rAd.String(), "/")[0]
			addrs[i] = addr
		}
		j, _ := json.Marshal(addrs)
		metadata.AddMeta("", tags, "addresses", string(j), true)
		if f != nil {
			f(iface, tags)
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>indexdb: rename db field to ldb<commit_after><|endoftext|>


<commit_before>package memory

import (
	"sort"
	"strings"

	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	log "github.com/sirupsen/logrus"
)

type expressionFilter struct {
	// expr is the expression based on which this filter has been generated
	expr tagquery.Expression

	// test is a filter function which takes certain descriptive properties of a
	// MetricDefinition and returns a tagquery.FilterDecision type indicating
	// whether the MD satisfies this expression or not
	// this decision is made based on the metric tag index, the meta tag index
	// does not get take into account
	testByMetricTags tagquery.MetricDefinitionFilter

	// testByMetaTags is a filter function which has been generated from the meta
	// records that match this filter's expression.
	// it looks at certain descriptive properties of a MetricDefinition and decides
	// whether this metric definition satisfies the given expression based on the
	// meta tag index, it does not take the metric tag index into account
	testByMetaTags tagquery.MetricDefinitionFilter

	// the default decision which should be applied if none of test & testByMetaTags
	// have come to a conclusive decision
	defaultDecision tagquery.FilterDecision
}

// idFilter contains one or many filter functions which are used to filter metrics
// by the expressions from which its filters have been generated.
// once initialized it can be used concurrently by many worker routines
type idFilter struct {
	ctx     *TagQueryContext
	filters []expressionFilter
}

// newIdFilter takes a set of expressions and a tag query context, then it generates
// various filter functions from the expressions which are going to be used to decide
// whether a given metric matches the provided expressions
func newIdFilter(expressions tagquery.Expressions, ctx *TagQueryContext) *idFilter {
	res := idFilter{
		ctx:     ctx,
		filters: make([]expressionFilter, len(expressions)),
	}

	for i, expr := range expressions {
		res.filters[i] = expressionFilter{
			expr:             expr,
			testByMetricTags: expr.GetMetricDefinitionFilter(ctx.index.idHasTag),
			defaultDecision:  expr.GetDefaultDecision(),
		}

		if !ctx.useMetaTagIndex() {
			continue
		}

		// this is a performacnce optimization:
		// some expressions indicate that they'll likely result in a smaller result set
		// if they get inverted.
		// f.e. a!=b becomes a=b)
		// in this case we want to get the inverted set of meta records that matches
		// "expr", because when generating a off of this smaller result set less meta
		// record expressions will need to be checked, which will improve the filter
		// speed. if the meta record filter set has been generated from an inverted
		// expression, then its result will need to be inverted again to get the
		// correct result.
		// f.e. if a!=b previously has been inverted to a=b, and for a given MD a=b
		// results in true, then a!=b would result in false
		invertSetOfMetaRecords := expr.ResultIsSmallerWhenInverted()

		// if no meta records match this expression, then we don't need to generate
		// a meta record filter for it
		metaRecordIds := ctx.metaTagIndex.getMetaRecordIdsByExpression(expr, invertSetOfMetaRecords)
		if len(metaRecordIds) == 0 {
			continue
		}

		// if we don't use an inverted set of meta records, then we want to check if
		// all meta records involved in a meta tag filter use the "=" operator.
		// if this is the case then it is cheaper to build a set of acceptable tags
		// based on the meta record expressions and just check whether they are present
		// in a metric that gets filtered, compared to doing a full tag index lookup
		// to check whether a metric has one of the necessary meta tags associated
		// with it.
		var metaRecordFilters []tagquery.MetricDefinitionFilter
		records := make([]tagquery.MetaTagRecord, 0, len(metaRecordIds))
		for _, id := range metaRecordIds {
			record, ok := ctx.metaTagRecords.getMetaRecordById(id)
			if !ok {
				corruptIndex.Inc()
				log.Errorf("TagQueryContext: Tried to lookup a meta tag record id that does not exist, index is corrupted")
				continue
			}

			records = append(records, record)
		}

		onlyEqualOperators, singleExprPerRecord := viableOptimizations(invertSetOfMetaRecords, records)

		if onlyEqualOperators {
			// there are two different ways how we optimize for the case where all expressions
			// of all involved meta records are using the "=" operator. the first and fastest
			// way can only be used if each involved meta record only has one single expression,
			// otherwise we use the second way which is a bit more expensive but it also works
			// if some of the involved meta records have multiple expressions.
			if singleExprPerRecord {
				res.filters[i].testByMetaTags = metaRecordFilterBySetOfValidValues(records)
			} else {
				res.filters[i].testByMetaTags = metaRecordFilterBySetOfValidValueSets(records)
			}
		} else {
			for recordIdx := range records {
				metaRecordFilters = append(metaRecordFilters, records[recordIdx].GetMetricDefinitionFilter(ctx.index.idHasTag))
			}

			if invertSetOfMetaRecords {
				res.filters[i].testByMetaTags = metaRecordFilterInverted(metaRecordFilters, res.filters[i].defaultDecision)
			} else {
				res.filters[i].testByMetaTags = metaRecordFilterNormal(metaRecordFilters, res.filters[i].defaultDecision)
			}
		}
	}

	return &res
}

// viableOptimizations looks at a set of meta tag records and decides whether two possible
// optimizations can be applied when filtering by these records. it returns two bools to
// indicate which optimizations are or are not viable.
// if invertSetOfMetaRecords is true then none of these optimizations can be used.
//
// * the first bool refers to the optimization for sets of records which all have only one
//   expression and this expression is using the equal operator.
// * the second bool refers to the optimization for sets of records which all only have
//   expressions using the equal operator, but there may be more than one per record.
func viableOptimizations(invertSetOfMetaRecords bool, records []tagquery.MetaTagRecord) (bool, bool) {
	if invertSetOfMetaRecords {
		return false, false
	}
	singleExprPerRecord := true
	for recordIdx := range records {
		for exprIdx := range records[recordIdx].Expressions {
			if records[recordIdx].Expressions[exprIdx].GetOperator() != tagquery.EQUAL {
				return false, false
			}
		}
		if len(records[recordIdx].Expressions) > 1 {
			singleExprPerRecord = false
		}
	}

	return true, singleExprPerRecord
}

// metaRecordFilterBySetOfValidValues creates a filter function to filter by a meta tag
// which only involves meta records of which each only has exactly one expression and that
// expression is using the "=" operator. this is quite a narrow scenario, but since it is
// a very common use case it makes sense to optimize for it.
func metaRecordFilterBySetOfValidValues(records []tagquery.MetaTagRecord) tagquery.MetricDefinitionFilter {
	// we first build a set of valid tags and names.
	// since we know that each of the involved meta records uses exactly one expression
	// which is using the "=" operator we know that if a given metric's name matches a
	// value in validNames or if one of its tags matches a value in validValues then this
	// is sufficient to let it pass the filter.
	validValues := make(map[string]struct{})
	validNames := make(map[string]struct{})
	var builder strings.Builder
	for _, record := range records {
		if len(record.Expressions) < 1 {
			corruptIndex.Inc()
			log.Errorf("meta-record-filter-by-set-of-valid-values: received a meta record without any associated expressions")
			continue
		}

		if record.Expressions[0].GetKey() == "name" {
			validNames[record.Expressions[0].GetValue()] = struct{}{}
		} else {
			record.Expressions[0].StringIntoWriter(&builder)
			validValues[builder.String()] = struct{}{}
			builder.Reset()
		}
	}

	return func(_ schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for i := range tags {
			if _, ok := validValues[tags[i]]; ok {
				return tagquery.Pass
			}
		}
		if _, ok := validNames[name]; ok {
			return tagquery.Pass
		}
		return tagquery.None
	}
}

// metaRecordFilterBySetOfValidValueSets creates a filter function to filter by a meta tag
// which only involves meta records of which all expressions are only using the "=" operator,
// it is ok if one meta record uses multiple such expressions.
func metaRecordFilterBySetOfValidValueSets(records []tagquery.MetaTagRecord) tagquery.MetricDefinitionFilter {
	// we first build a set of tag and name value combinations of which each is sufficient
	// to pass the generated filter when a metric contains all values of one of these
	// combinations
	validValueSets := make([]struct {
		name string
		tags []string
	}, len(records))
	var builder strings.Builder
	for i := range records {
		validValueSets[i].tags = make([]string, 0, len(records[i].Expressions))
		for j := range records[i].Expressions {
			if records[i].Expressions[j].GetKey() == "name" {
				validValueSets[i].name = records[i].Expressions[j].GetValue()
			} else {
				records[i].Expressions[j].StringIntoWriter(&builder)
				validValueSets[i].tags = append(validValueSets[i].tags, builder.String())
				builder.Reset()
			}
		}

		// the tags must be sorted because that's a requirement for sliceContainsElements()
		sort.Strings(validValueSets[i].tags)
	}

	return func(_ schema.MKey, name string, tags []string) tagquery.FilterDecision {
		// iterate over the acceptable value combinations and check if one matches this metric
		for _, validValueSet := range validValueSets {
			if len(validValueSet.name) > 0 {
				if name != validValueSet.name {
					continue
				}
			}

			if sliceContainsElements(validValueSet.tags, tags) {
				return tagquery.Pass
			}
		}

		return tagquery.None
	}
}

// sliceContainsElements returns true if the elements in the slice "find"
// are all present int the slice "in". It requires both slices to be sorted
func sliceContainsElements(find, in []string) bool {
	var findIdx, inIdx int
	for {
		if findIdx == len(find) {
			return true
		}

		if inIdx == len(in) {
			return false
		}

		if find[findIdx] == in[inIdx] {
			findIdx++
			inIdx++
			continue
		}

		if find[findIdx] < in[inIdx] {
			return false
		}

		inIdx++
	}
}

func metaRecordFilterInverted(metaRecordFilters []tagquery.MetricDefinitionFilter, defaultDecision tagquery.FilterDecision) tagquery.MetricDefinitionFilter {
	return func(id schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for _, metaRecordFilter := range metaRecordFilters {
			decision := metaRecordFilter(id, name, tags)
			if decision == tagquery.None {
				decision = defaultDecision
			}

			if decision == tagquery.Pass {
				return tagquery.Fail
			}
		}

		return tagquery.Pass
	}
}

func metaRecordFilterNormal(metaRecordFilters []tagquery.MetricDefinitionFilter, defaultDecision tagquery.FilterDecision) tagquery.MetricDefinitionFilter {
	return func(id schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for _, metaRecordFilter := range metaRecordFilters {
			decision := metaRecordFilter(id, name, tags)
			if decision == tagquery.None {
				decision = defaultDecision
			}

			if decision == tagquery.Pass {
				return tagquery.Pass
			}
		}

		return tagquery.Fail
	}
}

// matches takes descriptive properties of a metric definition and runs them through all
// filters required to come to a decision whether this metric definition should be part
// of the result or not.
// it uses the filter functions that have previously been generated when this instance
// of idFilter was instantiated
func (f *idFilter) matches(id schema.MKey, name string, tags []string) bool {
	for i := range f.filters {
		decision := f.filters[i].testByMetricTags(id, name, tags)

		if decision == tagquery.Pass {
			continue
		}

		if decision == tagquery.Fail {
			return false
		}

		if f.filters[i].testByMetaTags != nil {
			decision = f.filters[i].testByMetaTags(id, name, tags)
		}

		if decision == tagquery.None {
			decision = f.filters[i].defaultDecision
		}

		if decision == tagquery.Pass {
			continue
		}

		return false
	}

	return true
}
<commit_msg>build strings by concatenating instead of using the string builder<commit_after>package memory

import (
	"sort"

	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	log "github.com/sirupsen/logrus"
)

type expressionFilter struct {
	// expr is the expression based on which this filter has been generated
	expr tagquery.Expression

	// test is a filter function which takes certain descriptive properties of a
	// MetricDefinition and returns a tagquery.FilterDecision type indicating
	// whether the MD satisfies this expression or not
	// this decision is made based on the metric tag index, the meta tag index
	// does not get take into account
	testByMetricTags tagquery.MetricDefinitionFilter

	// testByMetaTags is a filter function which has been generated from the meta
	// records that match this filter's expression.
	// it looks at certain descriptive properties of a MetricDefinition and decides
	// whether this metric definition satisfies the given expression based on the
	// meta tag index, it does not take the metric tag index into account
	testByMetaTags tagquery.MetricDefinitionFilter

	// the default decision which should be applied if none of test & testByMetaTags
	// have come to a conclusive decision
	defaultDecision tagquery.FilterDecision
}

// idFilter contains one or many filter functions which are used to filter metrics
// by the expressions from which its filters have been generated.
// once initialized it can be used concurrently by many worker routines
type idFilter struct {
	ctx     *TagQueryContext
	filters []expressionFilter
}

// newIdFilter takes a set of expressions and a tag query context, then it generates
// various filter functions from the expressions which are going to be used to decide
// whether a given metric matches the provided expressions
func newIdFilter(expressions tagquery.Expressions, ctx *TagQueryContext) *idFilter {
	res := idFilter{
		ctx:     ctx,
		filters: make([]expressionFilter, len(expressions)),
	}

	for i, expr := range expressions {
		res.filters[i] = expressionFilter{
			expr:             expr,
			testByMetricTags: expr.GetMetricDefinitionFilter(ctx.index.idHasTag),
			defaultDecision:  expr.GetDefaultDecision(),
		}

		if !ctx.useMetaTagIndex() {
			continue
		}

		// this is a performacnce optimization:
		// some expressions indicate that they'll likely result in a smaller result set
		// if they get inverted.
		// f.e. a!=b becomes a=b)
		// in this case we want to get the inverted set of meta records that matches
		// "expr", because when generating a off of this smaller result set less meta
		// record expressions will need to be checked, which will improve the filter
		// speed. if the meta record filter set has been generated from an inverted
		// expression, then its result will need to be inverted again to get the
		// correct result.
		// f.e. if a!=b previously has been inverted to a=b, and for a given MD a=b
		// results in true, then a!=b would result in false
		invertSetOfMetaRecords := expr.ResultIsSmallerWhenInverted()

		// if no meta records match this expression, then we don't need to generate
		// a meta record filter for it
		metaRecordIds := ctx.metaTagIndex.getMetaRecordIdsByExpression(expr, invertSetOfMetaRecords)
		if len(metaRecordIds) == 0 {
			continue
		}

		// if we don't use an inverted set of meta records, then we want to check if
		// all meta records involved in a meta tag filter use the "=" operator.
		// if this is the case then it is cheaper to build a set of acceptable tags
		// based on the meta record expressions and just check whether they are present
		// in a metric that gets filtered, compared to doing a full tag index lookup
		// to check whether a metric has one of the necessary meta tags associated
		// with it.
		var metaRecordFilters []tagquery.MetricDefinitionFilter
		records := make([]tagquery.MetaTagRecord, 0, len(metaRecordIds))
		for _, id := range metaRecordIds {
			record, ok := ctx.metaTagRecords.getMetaRecordById(id)
			if !ok {
				corruptIndex.Inc()
				log.Errorf("TagQueryContext: Tried to lookup a meta tag record id that does not exist, index is corrupted")
				continue
			}

			records = append(records, record)
		}

		onlyEqualOperators, singleExprPerRecord := viableOptimizations(invertSetOfMetaRecords, records)

		if onlyEqualOperators {
			// there are two different ways how we optimize for the case where all expressions
			// of all involved meta records are using the "=" operator. the first and fastest
			// way can only be used if each involved meta record only has one single expression,
			// otherwise we use the second way which is a bit more expensive but it also works
			// if some of the involved meta records have multiple expressions.
			if singleExprPerRecord {
				res.filters[i].testByMetaTags = metaRecordFilterBySetOfValidValues(records)
			} else {
				res.filters[i].testByMetaTags = metaRecordFilterBySetOfValidValueSets(records)
			}
		} else {
			for recordIdx := range records {
				metaRecordFilters = append(metaRecordFilters, records[recordIdx].GetMetricDefinitionFilter(ctx.index.idHasTag))
			}

			if invertSetOfMetaRecords {
				res.filters[i].testByMetaTags = metaRecordFilterInverted(metaRecordFilters, res.filters[i].defaultDecision)
			} else {
				res.filters[i].testByMetaTags = metaRecordFilterNormal(metaRecordFilters, res.filters[i].defaultDecision)
			}
		}
	}

	return &res
}

// viableOptimizations looks at a set of meta tag records and decides whether two possible
// optimizations can be applied when filtering by these records. it returns two bools to
// indicate which optimizations are or are not viable.
// if invertSetOfMetaRecords is true then none of these optimizations can be used.
//
// * the first bool refers to the optimization for sets of records which all have only one
//   expression and this expression is using the equal operator.
// * the second bool refers to the optimization for sets of records which all only have
//   expressions using the equal operator, but there may be more than one per record.
func viableOptimizations(invertSetOfMetaRecords bool, records []tagquery.MetaTagRecord) (bool, bool) {
	if invertSetOfMetaRecords {
		return false, false
	}
	singleExprPerRecord := true
	for recordIdx := range records {
		for exprIdx := range records[recordIdx].Expressions {
			if records[recordIdx].Expressions[exprIdx].GetOperator() != tagquery.EQUAL {
				return false, false
			}
		}
		if len(records[recordIdx].Expressions) > 1 {
			singleExprPerRecord = false
		}
	}

	return true, singleExprPerRecord
}

// metaRecordFilterBySetOfValidValues creates a filter function to filter by a meta tag
// which only involves meta records of which each only has exactly one expression and that
// expression is using the "=" operator. this is quite a narrow scenario, but since it is
// a very common use case it makes sense to optimize for it.
func metaRecordFilterBySetOfValidValues(records []tagquery.MetaTagRecord) tagquery.MetricDefinitionFilter {
	// we first build a set of valid tags and names.
	// since we know that each of the involved meta records uses exactly one expression
	// which is using the "=" operator we know that if a given metric's name matches a
	// value in validNames or if one of its tags matches a value in validValues then this
	// is sufficient to let it pass the filter.
	validValues := make(map[string]struct{})
	validNames := make(map[string]struct{})
	for _, record := range records {
		if len(record.Expressions) < 1 {
			corruptIndex.Inc()
			log.Errorf("meta-record-filter-by-set-of-valid-values: received a meta record without any associated expressions")
			continue
		}

		if record.Expressions[0].GetKey() == "name" {
			validNames[record.Expressions[0].GetValue()] = struct{}{}
		} else {
			validValues[record.Expressions[0].GetKey()+"="+record.Expressions[0].GetValue()] = struct{}{}
		}
	}

	return func(_ schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for i := range tags {
			if _, ok := validValues[tags[i]]; ok {
				return tagquery.Pass
			}
		}
		if _, ok := validNames[name]; ok {
			return tagquery.Pass
		}
		return tagquery.None
	}
}

// metaRecordFilterBySetOfValidValueSets creates a filter function to filter by a meta tag
// which only involves meta records of which all expressions are only using the "=" operator,
// it is ok if one meta record uses multiple such expressions.
func metaRecordFilterBySetOfValidValueSets(records []tagquery.MetaTagRecord) tagquery.MetricDefinitionFilter {
	// we first build a set of tag and name value combinations of which each is sufficient
	// to pass the generated filter when a metric contains all values of one of these
	// combinations
	validValueSets := make([]struct {
		name string
		tags []string
	}, len(records))
	for i := range records {
		validValueSets[i].tags = make([]string, 0, len(records[i].Expressions))
		for j := range records[i].Expressions {
			if records[i].Expressions[j].GetKey() == "name" {
				validValueSets[i].name = records[i].Expressions[j].GetValue()
			} else {
				validValueSets[i].tags = append(validValueSets[i].tags, records[i].Expressions[j].GetKey()+"="+records[i].Expressions[j].GetValue())
			}
		}

		// the tags must be sorted because that's a requirement for sliceContainsElements()
		sort.Strings(validValueSets[i].tags)
	}

	return func(_ schema.MKey, name string, tags []string) tagquery.FilterDecision {
		// iterate over the acceptable value combinations and check if one matches this metric
		for _, validValueSet := range validValueSets {
			if len(validValueSet.name) > 0 {
				if name != validValueSet.name {
					continue
				}
			}

			if sliceContainsElements(validValueSet.tags, tags) {
				return tagquery.Pass
			}
		}

		return tagquery.None
	}
}

// sliceContainsElements returns true if the elements in the slice "find"
// are all present int the slice "in". It requires both slices to be sorted
func sliceContainsElements(find, in []string) bool {
	var findIdx, inIdx int
	for {
		if findIdx == len(find) {
			return true
		}

		if inIdx == len(in) {
			return false
		}

		if find[findIdx] == in[inIdx] {
			findIdx++
			inIdx++
			continue
		}

		if find[findIdx] < in[inIdx] {
			return false
		}

		inIdx++
	}
}

func metaRecordFilterInverted(metaRecordFilters []tagquery.MetricDefinitionFilter, defaultDecision tagquery.FilterDecision) tagquery.MetricDefinitionFilter {
	return func(id schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for _, metaRecordFilter := range metaRecordFilters {
			decision := metaRecordFilter(id, name, tags)
			if decision == tagquery.None {
				decision = defaultDecision
			}

			if decision == tagquery.Pass {
				return tagquery.Fail
			}
		}

		return tagquery.Pass
	}
}

func metaRecordFilterNormal(metaRecordFilters []tagquery.MetricDefinitionFilter, defaultDecision tagquery.FilterDecision) tagquery.MetricDefinitionFilter {
	return func(id schema.MKey, name string, tags []string) tagquery.FilterDecision {
		for _, metaRecordFilter := range metaRecordFilters {
			decision := metaRecordFilter(id, name, tags)
			if decision == tagquery.None {
				decision = defaultDecision
			}

			if decision == tagquery.Pass {
				return tagquery.Pass
			}
		}

		return tagquery.Fail
	}
}

// matches takes descriptive properties of a metric definition and runs them through all
// filters required to come to a decision whether this metric definition should be part
// of the result or not.
// it uses the filter functions that have previously been generated when this instance
// of idFilter was instantiated
func (f *idFilter) matches(id schema.MKey, name string, tags []string) bool {
	for i := range f.filters {
		decision := f.filters[i].testByMetricTags(id, name, tags)

		if decision == tagquery.Pass {
			continue
		}

		if decision == tagquery.Fail {
			return false
		}

		if f.filters[i].testByMetaTags != nil {
			decision = f.filters[i].testByMetaTags(id, name, tags)
		}

		if decision == tagquery.None {
			decision = f.filters[i].defaultDecision
		}

		if decision == tagquery.Pass {
			continue
		}

		return false
	}

	return true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/jvikstedt/alarm-bot/configuration"
	"github.com/jvikstedt/alarm-bot/tracker"
)

var conf = configuration.NewConfiguration("./config.json")

func main() {
	for _, c := range conf.TestObjects {
		trackResult, err := tracker.Perform(c.URL, c.MatchString, c.Status)
		if err != nil {
			fmt.Print(err)
		} else {
			fmt.Print(trackResult)
		}
	}
}
<commit_msg>Allow setting config file path as env variable<commit_after>package main

import (
	"fmt"

	"os"

	"github.com/jvikstedt/alarm-bot/configuration"
	"github.com/jvikstedt/alarm-bot/tracker"
)

var conf *configuration.Configuration

func main() {
	setupConf()
	for _, c := range conf.TestObjects {
		trackResult, err := tracker.Perform(c.URL, c.MatchString, c.Status)
		if err != nil {
			fmt.Print(err)
		} else {
			fmt.Print(trackResult)
		}
	}
}

func setupConf() {
	confName := os.Getenv("ALARM_BOT_CONFIG")
	if confName == "" {
		confName = "./config.json"
	}
	conf = configuration.NewConfiguration(confName)
}
<|endoftext|>


<commit_before>/*
A basic Go interface to Oracle Berkeley DB XML.

http://www.oracle.com/us/products/database/berkeley-db/xml/index.html
*/
package dbxml

//. Imports

/*
#cgo LDFLAGS: -ldbxml
#include <stdlib.h>
#include "c_dbxml.h"
*/
import "C"

import (
	"errors"
	"runtime"
	"strings"
	"sync"
	"unsafe"
)

//. Types

// A database connection.
type Db struct {
	opened  bool
	db      C.c_dbxml
	lock    sync.Mutex
	queries map[uint64]*Query
	counter uint64
}

// An iterator over xml documents in the database.
type Docs struct {
	started bool
	opened  bool
	docs    C.c_dbxml_docs
	lock    sync.Mutex
	err     error
}

// A prepared query that can be run multiple times and interrupted while running.
type Query struct {
	db     *Db
	id     uint64
	opened bool
	query  C.c_dbxml_query
	lock   sync.Mutex
}

//. Variables

var (
	errclosed      = errors.New("Database is closed")
	errqueryclosed = errors.New("Query is closed")
	errempty       = errors.New("Query is empty")
	errbrackets    = errors.New("Query starts with '('")
)

//. Open & Close

// Open a database.
//
// Call db.Close() to ensure all write operations to the database are finished, before terminating the program.
func Open(filename string) (*Db, error) {
	db := &Db{
		queries: make(map[uint64]*Query),
	}
	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.db = C.c_dbxml_open(cs)
	if C.c_dbxml_error(db.db) != 0 {
		err := errors.New(C.GoString(C.c_dbxml_errstring(db.db)))
		C.c_dbxml_free(db.db)
		return db, err
	}
	db.opened = true
	runtime.SetFinalizer(db, (*Db).Close)
	return db, nil
}

// Close the database.
//
// This flushes all write operations to the database.
//
// This is called automaticly on garbage collection.
// Note that teminating the program does not call the garbage collector.
func (db *Db) Close() {
	db.lock.Lock()
	defer db.lock.Unlock()
	if db.opened {
		// Collect all the keys before starting to close, because closing will change the hash
		keys := make([]uint64, 0, len(db.queries))
		for key := range db.queries {
			keys = append(keys, key)
		}
		for _, key := range keys {
			db.queries[key].Close()
		}
		C.c_dbxml_free(db.db)
		db.opened = false
	}
}

//. Write

// Put an xml file from disc into the database.
func (db *Db) PutFile(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_file(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Put an xml document from memory into the database.
func (db *Db) PutXml(name string, data string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	csname := C.CString(name)
	defer C.free(unsafe.Pointer(csname))
	csdata := C.CString(data)
	defer C.free(unsafe.Pointer(csdata))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_xml(db.db, csname, csdata, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Merge a database from disc into this database.
func (db *Db) Merge(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_merge(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Remove an xml document from the database.
func (db *Db) Remove(name string) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_remove(db.db, cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

//. Read

// Get an xml document by name from the database.
func (db *Db) Get(name string) (string, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return "", errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	r := C.c_dbxml_get(db.db, cs)
	defer C.c_dbxml_result_free(r)
	s := C.GoString(C.c_dbxml_result_string(r))
	if C.c_dbxml_result_error(r) != 0 {
		return "", errors.New(s)
	}
	return s, nil
}

// Get the number of xml documents in the database.
func (db *Db) Size() (uint64, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return 0, errclosed
	}
	return uint64(C.c_dbxml_size(db.db)), nil
}

// Get all xml documents from the database.
//
// Example:
//
//      docs, err := db.All()
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) All() (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	docs.docs = C.c_dbxml_get_all(db.db)
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Get all xml documents that match the XPATH query from the database.
//
// Example:
//
//      docs, err := db.Query(xpath_query)
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) Query(query string) (*Docs, error) {
	q, err := db.Prepare(query)
	if err != nil {
		return &Docs{}, err
	}
	return q.Run()
}

// Prepare an XPATH query.
//
// The query can be run multiple times, and a running query can be cancelled by query.Cancel()
func (db *Db) Prepare(query string) (*Query, error) {
	q := &Query{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return q, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	q.query = C.c_dbxml_prepare_query(db.db, cs)
	if C.c_dbxml_get_prepared_error(q.query) != 0 {
		defer C.c_dbxml_query_free(q.query)
		return q, errors.New(C.GoString(C.c_dbxml_get_prepared_errstring(q.query)))
	}
	// No finalizer: query will be closed when database gets closed
	q.opened = true
	q.db = db
	q.id = db.counter
	db.counter++
	db.queries[q.id] = q
	return q, nil
}

// Run a prepared query.
func (query *Query) Run() (*Docs, error) {
	docs := &Docs{}
	query.lock.Lock()
	defer query.lock.Unlock()

	if !query.opened {
		return docs, errqueryclosed
	}
	docs.docs = C.c_dbxml_run_query(query.query)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		defer C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Cancel a running query.
func (query *Query) Cancel() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_cancel_query(query.query)
	}
}

// Iterate to the next xml document in the list, that was returned by db.All(), db.Query(query), or query.Run().
func (docs *Docs) Next() bool {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !docs.opened {
		return false
	}
	docs.err = nil
	if C.c_dbxml_docs_next(docs.docs) == 0 {
		if C.c_dbxml_get_query_error(docs.docs) != 0 {
			docs.err = errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
		}
		docs.close()
		docs.started = false
		return false
	}
	docs.started = true
	return true
}

// Get name of current xml document after call to docs.Next().
func (docs *Docs) Name() string {
	return docs.getNameContent(1)
}

// Get content of current xml document after call to docs.Next().
func (docs *Docs) Content() string {
	return docs.getNameContent(2)
}

// Get matched subtree from current xml document after call to docs.Next().
func (docs *Docs) Match() string {
	return docs.getNameContent(3)
}

func (docs *Docs) getNameContent(what int) string {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !(docs.opened && docs.started) {
		return ""
	}
	switch what {
	case 1:
		return C.GoString(C.c_dbxml_docs_name(docs.docs))
	case 2:
		return C.GoString(C.c_dbxml_docs_content(docs.docs))
	case 3:
		return C.GoString(C.c_dbxml_docs_match(docs.docs))
	}
	return ""
}

// Close iterator over xml documents in the database, that was returned by db.All(), db.Query(query), or query.Run().
//
// This is called automaticly if docs.Next() reaches false, but you can also call it inside a loop to exit it prematurely:
//
//      docs, _ := db.All()
//      for docs.Next() {
//          fmt.Println(docs.Name(), docs.Content())
//          if some_condition {
//              docs.Close()
//          }
//      }
func (docs *Docs) Close() {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	docs.close()
}

func (docs *Docs) close() {
	if docs.opened {
		C.c_dbxml_docs_free(docs.docs)
		docs.opened = false
	}
}

// Get the error, if any, after docs.Next() returned false.
func (docs *Docs) Error() error {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	return docs.err
}

// Close a query that was created with Prepare()
//
// This is called automaticly when the database is closed.
func (query *Query) Close() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_query_free(query.query)
		delete(query.db.queries, query.id)
		query.db = nil
		query.opened = false
	}
}

//. Check

// Check if query is valid without opening a database.
func Check(query string) error {
	query = strings.TrimSpace(query)
	if query == "" {
		return errempty
	}
	if query[0] == '(' {
		return errbrackets
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_check(cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Get DbXml version
func Version() (major, minor, patch int) {
	var majorp, minorp, patchp C.int
	C.c_dbxml_version(&majorp, &minorp, &patchp)
	return int(majorp), int(minorp), int(patchp)
}
<commit_msg>use lock while opening database<commit_after>/*
A basic Go interface to Oracle Berkeley DB XML.

http://www.oracle.com/us/products/database/berkeley-db/xml/index.html
*/
package dbxml

//. Imports

/*
#cgo LDFLAGS: -ldbxml
#include <stdlib.h>
#include "c_dbxml.h"
*/
import "C"

import (
	"errors"
	"runtime"
	"strings"
	"sync"
	"unsafe"
)

//. Types

// A database connection.
type Db struct {
	opened  bool
	db      C.c_dbxml
	lock    sync.Mutex
	queries map[uint64]*Query
	counter uint64
}

// An iterator over xml documents in the database.
type Docs struct {
	started bool
	opened  bool
	docs    C.c_dbxml_docs
	lock    sync.Mutex
	err     error
}

// A prepared query that can be run multiple times and interrupted while running.
type Query struct {
	db     *Db
	id     uint64
	opened bool
	query  C.c_dbxml_query
	lock   sync.Mutex
}

//. Variables

var (
	errclosed      = errors.New("Database is closed")
	errqueryclosed = errors.New("Query is closed")
	errempty       = errors.New("Query is empty")
	errbrackets    = errors.New("Query starts with '('")
	lock           sync.Mutex
)

//. Open & Close

// Open a database.
//
// Call db.Close() to ensure all write operations to the database are finished, before terminating the program.
func Open(filename string) (*Db, error) {
	lock.Lock()
	defer lock.Unlock()
	db := &Db{
		queries: make(map[uint64]*Query),
	}
	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.db = C.c_dbxml_open(cs)
	if C.c_dbxml_error(db.db) != 0 {
		err := errors.New(C.GoString(C.c_dbxml_errstring(db.db)))
		C.c_dbxml_free(db.db)
		return db, err
	}
	db.opened = true
	runtime.SetFinalizer(db, (*Db).Close)
	return db, nil
}

// Close the database.
//
// This flushes all write operations to the database.
//
// This is called automaticly on garbage collection.
// Note that teminating the program does not call the garbage collector.
func (db *Db) Close() {
	db.lock.Lock()
	defer db.lock.Unlock()
	if db.opened {
		// Collect all the keys before starting to close, because closing will change the hash
		keys := make([]uint64, 0, len(db.queries))
		for key := range db.queries {
			keys = append(keys, key)
		}
		for _, key := range keys {
			db.queries[key].Close()
		}
		C.c_dbxml_free(db.db)
		db.opened = false
	}
}

//. Write

// Put an xml file from disc into the database.
func (db *Db) PutFile(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_file(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Put an xml document from memory into the database.
func (db *Db) PutXml(name string, data string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	csname := C.CString(name)
	defer C.free(unsafe.Pointer(csname))
	csdata := C.CString(data)
	defer C.free(unsafe.Pointer(csdata))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_xml(db.db, csname, csdata, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Merge a database from disc into this database.
func (db *Db) Merge(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_merge(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Remove an xml document from the database.
func (db *Db) Remove(name string) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_remove(db.db, cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

//. Read

// Get an xml document by name from the database.
func (db *Db) Get(name string) (string, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return "", errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	r := C.c_dbxml_get(db.db, cs)
	defer C.c_dbxml_result_free(r)
	s := C.GoString(C.c_dbxml_result_string(r))
	if C.c_dbxml_result_error(r) != 0 {
		return "", errors.New(s)
	}
	return s, nil
}

// Get the number of xml documents in the database.
func (db *Db) Size() (uint64, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return 0, errclosed
	}
	return uint64(C.c_dbxml_size(db.db)), nil
}

// Get all xml documents from the database.
//
// Example:
//
//      docs, err := db.All()
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) All() (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	docs.docs = C.c_dbxml_get_all(db.db)
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Get all xml documents that match the XPATH query from the database.
//
// Example:
//
//      docs, err := db.Query(xpath_query)
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) Query(query string) (*Docs, error) {
	q, err := db.Prepare(query)
	if err != nil {
		return &Docs{}, err
	}
	return q.Run()
}

// Prepare an XPATH query.
//
// The query can be run multiple times, and a running query can be cancelled by query.Cancel()
func (db *Db) Prepare(query string) (*Query, error) {
	q := &Query{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return q, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	q.query = C.c_dbxml_prepare_query(db.db, cs)
	if C.c_dbxml_get_prepared_error(q.query) != 0 {
		defer C.c_dbxml_query_free(q.query)
		return q, errors.New(C.GoString(C.c_dbxml_get_prepared_errstring(q.query)))
	}
	// No finalizer: query will be closed when database gets closed
	q.opened = true
	q.db = db
	q.id = db.counter
	db.counter++
	db.queries[q.id] = q
	return q, nil
}

// Run a prepared query.
func (query *Query) Run() (*Docs, error) {
	docs := &Docs{}
	query.lock.Lock()
	defer query.lock.Unlock()

	if !query.opened {
		return docs, errqueryclosed
	}
	docs.docs = C.c_dbxml_run_query(query.query)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		defer C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Cancel a running query.
func (query *Query) Cancel() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_cancel_query(query.query)
	}
}

// Iterate to the next xml document in the list, that was returned by db.All(), db.Query(query), or query.Run().
func (docs *Docs) Next() bool {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !docs.opened {
		return false
	}
	docs.err = nil
	if C.c_dbxml_docs_next(docs.docs) == 0 {
		if C.c_dbxml_get_query_error(docs.docs) != 0 {
			docs.err = errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
		}
		docs.close()
		docs.started = false
		return false
	}
	docs.started = true
	return true
}

// Get name of current xml document after call to docs.Next().
func (docs *Docs) Name() string {
	return docs.getNameContent(1)
}

// Get content of current xml document after call to docs.Next().
func (docs *Docs) Content() string {
	return docs.getNameContent(2)
}

// Get matched subtree from current xml document after call to docs.Next().
func (docs *Docs) Match() string {
	return docs.getNameContent(3)
}

func (docs *Docs) getNameContent(what int) string {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !(docs.opened && docs.started) {
		return ""
	}
	switch what {
	case 1:
		return C.GoString(C.c_dbxml_docs_name(docs.docs))
	case 2:
		return C.GoString(C.c_dbxml_docs_content(docs.docs))
	case 3:
		return C.GoString(C.c_dbxml_docs_match(docs.docs))
	}
	return ""
}

// Close iterator over xml documents in the database, that was returned by db.All(), db.Query(query), or query.Run().
//
// This is called automaticly if docs.Next() reaches false, but you can also call it inside a loop to exit it prematurely:
//
//      docs, _ := db.All()
//      for docs.Next() {
//          fmt.Println(docs.Name(), docs.Content())
//          if some_condition {
//              docs.Close()
//          }
//      }
func (docs *Docs) Close() {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	docs.close()
}

func (docs *Docs) close() {
	if docs.opened {
		C.c_dbxml_docs_free(docs.docs)
		docs.opened = false
	}
}

// Get the error, if any, after docs.Next() returned false.
func (docs *Docs) Error() error {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	return docs.err
}

// Close a query that was created with Prepare()
//
// This is called automaticly when the database is closed.
func (query *Query) Close() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_query_free(query.query)
		delete(query.db.queries, query.id)
		query.db = nil
		query.opened = false
	}
}

//. Check

// Check if query is valid without opening a database.
func Check(query string) error {
	query = strings.TrimSpace(query)
	if query == "" {
		return errempty
	}
	if query[0] == '(' {
		return errbrackets
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_check(cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Get DbXml version
func Version() (major, minor, patch int) {
	var majorp, minorp, patchp C.int
	C.c_dbxml_version(&majorp, &minorp, &patchp)
	return int(majorp), int(minorp), int(patchp)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"index/suffixarray"
	"regexp/syntax"
	"runtime"
	"sort"
	"sync"
)

type line struct {
	off   int    // Offset of line
	value []byte // One line from source slice
}

type delta struct {
	off   int // Offset of existing value
	index int // Index of replacement value
}

// TODO Are there any errors that need handling?
func producer(input <-chan line, index *replaceIndex) []delta {

	var wg sync.WaitGroup
	runtime.GOMAXPROCS(runtime.NumCPU()) // Maximum CPU utilisation please!
	maxGophers := runtime.GOMAXPROCS(0)
	wg.Add(maxGophers)

	deltaChan := make(chan []delta)
	done := make(chan bool)
	defer close(done)

	fmt.Printf("Launching %v gophers ...\n", maxGophers)

	for i := 0; i < maxGophers; i++ {
		go func() {
			for t := range input {
				select {
				case deltaChan <- makeDeltas(t, index, i):
				case <-done:
					return
				}
			}
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(deltaChan)
	}()

	fmt.Printf("Generating deltas... \n")
	deltas := make([]delta, 0)
	for d := range deltaChan {
		if len(d) > 0 {
			deltas = append(deltas, d...)
		}
	}

	fmt.Printf("Got %v deltas.\n", len(deltas))

	sort.Sort(ByLine(deltas))
	return deltas
}

// Make deltas
func makeDeltas(t line, index *replaceIndex, id int) []delta {

	s := make([]delta, 0)

	lineIndex := suffixarray.New(t.value)

	// Look for each replacement word in the line
	for i := 0; i < index.len(); i++ {
		results := lineIndex.Lookup(index.readItem(i).find, -1)
		if len(results) > 0 {
			for _, p := range results {

				// We get this far then a match has been found

				// It's not a match if it's a partial word
				//x := p + len(index.readItem(i).find) - 1 // If it's not the end of the slice ...

				// TODO Is there a guarantee of left-to-right eval?
				// TODO Test this works and hasn't broken everything!!!

				// It's a full word match if it's end of line or word terminates in non-alpha
				if (p == len(t.value)) || (p < len(t.value) && !syntax.IsWordChar(rune(t.value[p+len(index.readItem(i).find)])+1)) {
					d := delta{off: t.off + p, index: i}
					s = append(s, d)
				}
				//if x < len(t.value) { // ... and the search term is short enough

				//	// If last word is followed by non-alpha, then it's a full match
				//	if syntax.IsWordChar(rune(t.value[x])) && !syntax.IsWordChar(rune(t.value[x+1])) {
				//		d := delta{off: t.off + p, index: i}
				//		s = append(s, d)
				//	}
				//} else {
				//	d := delta{off: t.off + p, index: i}
				//	s = append(s, d)
				//}

			}
		}
	}

	// TODO Optional memory optimiser. Use less system RAM at the expense of CPU.
	// debug.FreeOSMemory()

	return s
}

type ByLine []delta

func (d ByLine) Len() int {
	return len(d)
}

func (d ByLine) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByLine) Less(i, j int) bool {
	return d[i].off < d[j].off
}
<commit_msg>makeDelta() check for eol second.<commit_after>package main

import (
	"fmt"
	"index/suffixarray"
	"regexp/syntax"
	"runtime"
	"sort"
	"sync"
)

type line struct {
	off   int    // Offset of line
	value []byte // One line from source slice
}

type delta struct {
	off   int // Offset of existing value
	index int // Index of replacement value
}

// TODO Are there any errors that need handling?
func producer(input <-chan line, index *replaceIndex) []delta {

	var wg sync.WaitGroup
	runtime.GOMAXPROCS(runtime.NumCPU()) // Maximum CPU utilisation please!
	maxGophers := runtime.GOMAXPROCS(0)
	wg.Add(maxGophers)

	deltaChan := make(chan []delta)
	done := make(chan bool)
	defer close(done)

	fmt.Printf("Launching %v gophers ...\n", maxGophers)

	for i := 0; i < maxGophers; i++ {
		go func() {
			for t := range input {
				select {
				case deltaChan <- makeDeltas(t, index, i):
				case <-done:
					return
				}
			}
			wg.Done()
		}()
	}

	go func() {
		wg.Wait()
		close(deltaChan)
	}()

	fmt.Printf("Generating deltas... \n")
	deltas := make([]delta, 0)
	for d := range deltaChan {
		if len(d) > 0 {
			deltas = append(deltas, d...)
		}
	}

	fmt.Printf("Got %v deltas.\n", len(deltas))

	sort.Sort(ByLine(deltas))
	return deltas
}

// Make deltas
func makeDeltas(t line, index *replaceIndex, id int) []delta {

	s := make([]delta, 0)

	lineIndex := suffixarray.New(t.value)

	// Look for each replacement word in the line
	for i := 0; i < index.len(); i++ {
		results := lineIndex.Lookup(index.readItem(i).find, -1)
		if len(results) > 0 {
			for _, p := range results {

				// It's not a match if it's a partial word

				// TODO Is there a guarantee of left-to-right eval?
				// TODO Test this works and hasn't broken everything!!!

				// It's a full word match if it's end of line or word terminates in non-alpha
				if (p < len(t.value) && !syntax.IsWordChar(rune(t.value[p+len(index.readItem(i).find)])+1)) || (p == len(t.value)) {
					d := delta{off: t.off + p, index: i}
					s = append(s, d)
				}
			}
		}
	}

	// TODO Optional memory optimiser. Use less system RAM at the expense of CPU.
	// debug.FreeOSMemory()

	return s
}

type ByLine []delta

func (d ByLine) Len() int {
	return len(d)
}

func (d ByLine) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

func (d ByLine) Less(i, j int) bool {
	return d[i].off < d[j].off
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"fmt"
	"time"
)

// Partition is the configuration of a single admin partition. Admin Partitions are a Consul Enterprise feature.
type Partition struct {
	// Name is the name of the Partition.
	Name string `json:"Name"`

	// Description is where the user puts any information they want
	// about the partition. It is not used internally.
	Description string `json:"Description,omitempty"`

	// DeletedAt is the time when the Partition was marked for deletion
	// This is nullable so that we can omit if empty when encoding in JSON
	DeletedAt *time.Time `json:"DeletedAt,omitempty" alias:"deleted_at"`

	// CreateIndex is the Raft index at which the Partition was created
	CreateIndex uint64 `json:"CreateIndex,omitempty"`

	// ModifyIndex is the latest Raft index at which the Partition was modified.
	ModifyIndex uint64 `json:"ModifyIndex,omitempty"`
}

// PartitionDefaultName is the default partition value.
const PartitionDefaultName = "default"

// Partitions can be used to manage Partitions in Consul Enterprise..
type Partitions struct {
	c *Client
}

// Operator returns a handle to the operator endpoints.
func (c *Client) Partitions() *Partitions {
	return &Partitions{c}
}

func (p *Partitions) Create(ctx context.Context, partition *Partition, q *WriteOptions) (*Partition, *WriteMeta, error) {
	if partition.Name == "" {
		return nil, nil, fmt.Errorf("Must specify a Name for Partition creation")
	}

	r := p.c.newRequest("PUT", "/v1/partition")
	r.setWriteOptions(q)
	r.ctx = ctx
	r.obj = partition
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	var out Partition
	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}

	return &out, wm, nil
}

func (p *Partitions) Update(ctx context.Context, partition *Partition, q *WriteOptions) (*Partition, *WriteMeta, error) {
	if partition.Name == "" {
		return nil, nil, fmt.Errorf("Must specify a Name for Partition updating")
	}

	r := p.c.newRequest("PUT", "/v1/partition/"+partition.Name)
	r.setWriteOptions(q)
	r.ctx = ctx
	r.obj = partition
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	var out Partition
	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}

	return &out, wm, nil
}

func (p *Partitions) Read(ctx context.Context, name string, q *QueryOptions) (*Partition, *QueryMeta, error) {
	var out Partition
	r := p.c.newRequest("GET", "/v1/partition/"+name)
	r.setQueryOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	found, resp, err := requireNotFoundOrOK(resp)
	if err != nil {
		return nil, nil, err
	}

	qm := &QueryMeta{}
	parseQueryMeta(resp, qm)
	qm.RequestTime = rtt

	if !found {
		return nil, qm, nil
	}

	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}
	return &out, qm, nil
}

func (p *Partitions) Delete(ctx context.Context, name string, q *WriteOptions) (*WriteMeta, error) {
	r := p.c.newRequest("DELETE", "/v1/partition/"+name)
	r.setWriteOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	return wm, nil
}

func (p *Partitions) List(ctx context.Context, q *QueryOptions) ([]*Partition, *QueryMeta, error) {
	var out []*Partition
	r := p.c.newRequest("GET", "/v1/partitions")
	r.setQueryOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	qm := &QueryMeta{}
	parseQueryMeta(resp, qm)
	qm.RequestTime = rtt

	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}
	return out, qm, nil
}
<commit_msg>Clarify feature name in partition docstring<commit_after>package api

import (
	"context"
	"fmt"
	"time"
)

// Partition is the configuration of a single admin partition. Admin Partitions are a Consul Enterprise feature.
type Partition struct {
	// Name is the name of the Partition.
	Name string `json:"Name"`

	// Description is where the user puts any information they want
	// about the admin partition. It is not used internally.
	Description string `json:"Description,omitempty"`

	// DeletedAt is the time when the Partition was marked for deletion
	// This is nullable so that we can omit if empty when encoding in JSON
	DeletedAt *time.Time `json:"DeletedAt,omitempty" alias:"deleted_at"`

	// CreateIndex is the Raft index at which the Partition was created
	CreateIndex uint64 `json:"CreateIndex,omitempty"`

	// ModifyIndex is the latest Raft index at which the Partition was modified.
	ModifyIndex uint64 `json:"ModifyIndex,omitempty"`
}

// PartitionDefaultName is the default partition value.
const PartitionDefaultName = "default"

// Partitions can be used to manage Partitions in Consul Enterprise.
type Partitions struct {
	c *Client
}

// Operator returns a handle to the operator endpoints.
func (c *Client) Partitions() *Partitions {
	return &Partitions{c}
}

func (p *Partitions) Create(ctx context.Context, partition *Partition, q *WriteOptions) (*Partition, *WriteMeta, error) {
	if partition.Name == "" {
		return nil, nil, fmt.Errorf("Must specify a Name for Partition creation")
	}

	r := p.c.newRequest("PUT", "/v1/partition")
	r.setWriteOptions(q)
	r.ctx = ctx
	r.obj = partition
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	var out Partition
	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}

	return &out, wm, nil
}

func (p *Partitions) Update(ctx context.Context, partition *Partition, q *WriteOptions) (*Partition, *WriteMeta, error) {
	if partition.Name == "" {
		return nil, nil, fmt.Errorf("Must specify a Name for Partition updating")
	}

	r := p.c.newRequest("PUT", "/v1/partition/"+partition.Name)
	r.setWriteOptions(q)
	r.ctx = ctx
	r.obj = partition
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	var out Partition
	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}

	return &out, wm, nil
}

func (p *Partitions) Read(ctx context.Context, name string, q *QueryOptions) (*Partition, *QueryMeta, error) {
	var out Partition
	r := p.c.newRequest("GET", "/v1/partition/"+name)
	r.setQueryOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	found, resp, err := requireNotFoundOrOK(resp)
	if err != nil {
		return nil, nil, err
	}

	qm := &QueryMeta{}
	parseQueryMeta(resp, qm)
	qm.RequestTime = rtt

	if !found {
		return nil, qm, nil
	}

	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}
	return &out, qm, nil
}

func (p *Partitions) Delete(ctx context.Context, name string, q *WriteOptions) (*WriteMeta, error) {
	r := p.c.newRequest("DELETE", "/v1/partition/"+name)
	r.setWriteOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, err
	}

	wm := &WriteMeta{RequestTime: rtt}
	return wm, nil
}

func (p *Partitions) List(ctx context.Context, q *QueryOptions) ([]*Partition, *QueryMeta, error) {
	var out []*Partition
	r := p.c.newRequest("GET", "/v1/partitions")
	r.setQueryOptions(q)
	r.ctx = ctx
	rtt, resp, err := p.c.doRequest(r)
	if err != nil {
		return nil, nil, err
	}
	defer closeResponseBody(resp)
	if err := requireOK(resp); err != nil {
		return nil, nil, err
	}

	qm := &QueryMeta{}
	parseQueryMeta(resp, qm)
	qm.RequestTime = rtt

	if err := decodeBody(resp, &out); err != nil {
		return nil, nil, err
	}
	return out, qm, nil
}
<|endoftext|>


<commit_before>// REST API for Client side Hydra
package rest

import (
	"github.com/golang/glog"
	"io"
	"net/http"
	"strings"
	"time"
)

type Rest struct{}

type RestRequest struct {
	Req     string
	ReplyTo http.ResponseWriter
}

var waiting chan RestRequest
var outstanding chan RestRequest

func versionServer(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "hydra 0.1\n")
}

// hello world, the web server
func requestServer(w http.ResponseWriter, req *http.Request) {
	glog.Info("Incoming GET request to", req.URL.String())
	reqs := strings.Split(req.URL.String(), "/")
	reqNew := strings.Join(reqs[2:], " ")
	glog.Info("API request is:", reqNew)
	waiting <- RestRequest{reqNew + "\n", w}
	time.Sleep(time.Second)
}

func Create() *Rest {
	port := ":12345"
	glog.Info("Setting up HTTP server on ", port)

	//setup HTTP server
	http.HandleFunc("/request/", requestServer)
	http.HandleFunc("/version", versionServer)
	go func() {
		err := http.ListenAndServe(port, nil)
		if err != nil {
			glog.Fatal("ListenAndServe: ", err)
		}
	}()

	// setup channels
	waiting = make(chan RestRequest, 10)
	outstanding = make(chan RestRequest, 1)

	return &(Rest{})

}

func (r *Rest) Next() (string, bool) {
	glog.Info("Waiting for next request")
	restreq := <-waiting
	outstanding <- restreq
	glog.Info("Next request received: ", restreq.Req)
	return restreq.Req, true
}

func (r *Rest) Return(str string) {
	glog.Info("Response received: ", str)
	restreq := <-outstanding
	io.WriteString(restreq.ReplyTo, str)
	glog.Info("Response sent")

}
<commit_msg>main request handler<commit_after>// REST API for Client side Hydra
package rest

import (
	"github.com/golang/glog"
	"io"
	"net/http"
	"strings"
	"time"
)

type Rest struct{}

type RestRequest struct {
	Req     string
	ReplyTo http.ResponseWriter
}

var waiting chan RestRequest
var outstanding chan RestRequest

func versionServer(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "hydra 0.1\n")
}

func closeServer(w http.ResponseWriter, req *http.Request) {
	close(waiting)
	io.WriteString(w, "Will do\n")
}

// main request handler
func requestServer(w http.ResponseWriter, req *http.Request) {
	// NB: ResponseWriter needs to be used before this function exits
	glog.Info("Incoming GET request to", req.URL.String())
	reqs := strings.Split(req.URL.String(), "/")
	reqNew := strings.Join(reqs[2:], " ")
	glog.Info("API request is:", reqNew)
	waiting <- RestRequest{reqNew + "\n", w}

	//wait for response, else give up
	time.Sleep(time.Second)
}

func Create() *Rest {
	port := ":12345"
	glog.Info("Setting up HTTP server on ", port)

	//setup HTTP server
	http.HandleFunc("/request/", requestServer)
	http.HandleFunc("/close", closeServer)
	http.HandleFunc("/version", versionServer)
	go func() {
		err := http.ListenAndServe(port, nil)
		if err != nil {
			glog.Fatal("ListenAndServe: ", err)
		}
	}()

	// setup channels
	waiting = make(chan RestRequest, 10)
	outstanding = make(chan RestRequest, 1)

	return &(Rest{})

}

func (r *Rest) Next() (string, bool) {
	glog.Info("Waiting for next request")
	restreq, ok := <-waiting
	if !ok {
		return "", false
	}
	outstanding <- restreq
	glog.Info("Next request received: ", restreq.Req)
	return restreq.Req, true
}

func (r *Rest) Return(str string) {
	glog.Info("Response received: ", str)
	restreq := <-outstanding
	io.WriteString(restreq.ReplyTo, str)
	glog.Info("Response sent")

}
<|endoftext|>


<commit_before>package siri

import (
	"bytes"
	"text/template"
	"time"

	"github.com/jbowtie/gokogiri"
	"github.com/jbowtie/gokogiri/xml"
)

type XMLNotifyGeneralMessage struct {
	ResponseXMLStructure

	deliveries []*XMLGeneralMessageDelivery
}

type XMLGeneralMessageDelivery struct {
	ResponseXMLStructure

	subscriptionRef string
	subscriberRef   string

	xmlGeneralMessages              []*XMLGeneralMessage
	xmlGeneralMessagesCancellations []*XMLGeneralMessageCancellation
}

type SIRINotifyGeneralMessage struct {
	Address                   string
	ProducerRef               string
	RequestMessageRef         string
	ResponseMessageIdentifier string
	SubscriberRef             string
	SubscriptionIdentifier    string

	ResponseTimestamp time.Time

	Status      bool
	ErrorType   string
	ErrorNumber int
	ErrorText   string

	GeneralMessages []*SIRIGeneralMessage
}

const generalMessageNotifyTemplate = `<sw:NotifyGeneralMessage xmlns:sw="http://wsdl.siri.org.uk" xmlns:siri="http://www.siri.org.uk/siri">
	<ServiceDeliveryInfo>
		<siri:ResponseTimestamp>{{ .ResponseTimestamp.Format "2006-01-02T15:04:05.000Z07:00" }}</siri:ResponseTimestamp>
		<siri:ProducerRef>{{ .ProducerRef }}</siri:ProducerRef>{{ if .Address }}
		<siri:Address>{{ .Address }}</siri:Address>{{ end }}
		<siri:ResponseMessageIdentifier>{{ .ResponseMessageIdentifier }}</siri:ResponseMessageIdentifier>
		<siri:RequestMessageRef>{{ .RequestMessageRef }}</siri:RequestMessageRef>
	</ServiceDeliveryInfo>
	<Notification xmlns:ns2="http://www.ifopt.org.uk/acsb" xmlns:siri="http://www.ifopt.org.uk/ifopt" xmlns:ns4="http://datex2.eu/schema/2_0RC1/2_0" xmlns:siri="http://www.siri.org.uk/siri" xmlns:ns6="http://wsdl.siri.org.uk/siri">
		<siri:GeneralMessageDelivery version="2.0:FR-IDF-2.4">
			<siri:ResponseTimestamp>{{ .ResponseTimestamp.Format "2006-01-02T15:04:05.000Z07:00" }}</siri:ResponseTimestamp>
			<siri:RequestMessageRef>{{ .RequestMessageRef }}</siri:RequestMessageRef>
			<siri:SubscriberRef>{{ .SubscriberRef }}</siri:SubscriberRef>
			<siri:SubscriptionRef>{{ .SubscriptionIdentifier }}</siri:SubscriptionRef>
			<siri:Status>{{ .Status }}</siri:Status>{{ if not .Status }}
			<siri:ErrorCondition>{{ if eq .ErrorType "OtherError" }}
				<siri:OtherError number="{{ .ErrorNumber }}">{{ else }}
				<siri:{{ .ErrorType }}>{{ end }}
					<siri:ErrorText>{{ .ErrorText }}</siri:ErrorText>
				</siri:{{ .ErrorType }}>
			</siri:ErrorCondition>{{ else }}{{ range .GeneralMessages }}
			{{ .BuildGeneralMessageXML }}{{ end }}{{ end }}
		 </siri:GeneralMessageDelivery>
	</Notification>
	<NotificationExtension />
</sw:NotifyGeneralMessage>`

func NewXMLNotifyGeneralMessage(node xml.Node) *XMLNotifyGeneralMessage {
	xmlGeneralMessageResponse := &XMLNotifyGeneralMessage{}
	xmlGeneralMessageResponse.node = NewXMLNode(node)
	return xmlGeneralMessageResponse
}

func NewXMLNotifyGeneralMessageFromContent(content []byte) (*XMLNotifyGeneralMessage, error) {
	doc, err := gokogiri.ParseXml(content)
	if err != nil {
		return nil, err
	}
	response := NewXMLNotifyGeneralMessage(doc.Root().XmlNode)
	return response, nil
}

func NewXMLGeneralMessageDelivery(node XMLNode) *XMLGeneralMessageDelivery {
	delivery := &XMLGeneralMessageDelivery{}
	delivery.node = node
	return delivery
}

func (notify *XMLNotifyGeneralMessage) GeneralMessagesDeliveries() []*XMLGeneralMessageDelivery {
	if notify.deliveries == nil {
		deliveries := []*XMLGeneralMessageDelivery{}
		nodes := notify.findNodes("GeneralMessageDelivery")
		if nodes != nil {
			for _, node := range nodes {
				deliveries = append(deliveries, NewXMLGeneralMessageDelivery(node))
			}
		}
		notify.deliveries = deliveries
	}
	return notify.deliveries
}

func (delivery *XMLGeneralMessageDelivery) SubscriptionRef() string {
	if delivery.subscriptionRef == "" {
		delivery.subscriptionRef = delivery.findStringChildContent("SubscriptionRef")
	}
	return delivery.subscriptionRef
}

func (delivery *XMLGeneralMessageDelivery) SubscriberRef() string {
	if delivery.subscriberRef == "" {
		delivery.subscriberRef = delivery.findStringChildContent("SubscriberRef")
	}
	return delivery.subscriberRef
}

func (delivery *XMLGeneralMessageDelivery) XMLGeneralMessages() []*XMLGeneralMessage {
	if delivery.xmlGeneralMessages == nil {
		nodes := delivery.findNodes("GeneralMessage")
		if nodes != nil {
			for _, node := range nodes {
				delivery.xmlGeneralMessages = append(delivery.xmlGeneralMessages, NewXMLGeneralMessage(node))
			}
		}
	}
	return delivery.xmlGeneralMessages
}

func (delivery *XMLGeneralMessageDelivery) XMLGeneralMessagesCancellations() []*XMLGeneralMessageCancellation {
	if delivery.xmlGeneralMessagesCancellations == nil {
		cancellations := []*XMLGeneralMessageCancellation{}
		nodes := delivery.findNodes("GeneralMessageCancellation")
		if nodes != nil {
			for _, node := range nodes {
				cancellations = append(cancellations, NewXMLCancelledGeneralMessage(node))
			}
		}
		delivery.xmlGeneralMessagesCancellations = cancellations
	}
	return delivery.xmlGeneralMessagesCancellations
}

func (notify *SIRINotifyGeneralMessage) BuildXML() (string, error) {
	var buffer bytes.Buffer
	var notifyDelivery = template.Must(template.New("generalMessageNotifyTemplate").Parse(generalMessageNotifyTemplate))
	if err := notifyDelivery.Execute(&buffer, notify); err != nil {
		return "", err
	}
	return buffer.String(), nil
}
<commit_msg>Fix NotifyGeneralMessage template<commit_after>package siri

import (
	"bytes"
	"text/template"
	"time"

	"github.com/jbowtie/gokogiri"
	"github.com/jbowtie/gokogiri/xml"
)

type XMLNotifyGeneralMessage struct {
	ResponseXMLStructure

	deliveries []*XMLGeneralMessageDelivery
}

type XMLGeneralMessageDelivery struct {
	ResponseXMLStructure

	subscriptionRef string
	subscriberRef   string

	xmlGeneralMessages              []*XMLGeneralMessage
	xmlGeneralMessagesCancellations []*XMLGeneralMessageCancellation
}

type SIRINotifyGeneralMessage struct {
	Address                   string
	ProducerRef               string
	RequestMessageRef         string
	ResponseMessageIdentifier string
	SubscriberRef             string
	SubscriptionIdentifier    string

	ResponseTimestamp time.Time

	Status      bool
	ErrorType   string
	ErrorNumber int
	ErrorText   string

	GeneralMessages []*SIRIGeneralMessage
}

const generalMessageNotifyTemplate = `<sw:NotifyGeneralMessage xmlns:sw="http://wsdl.siri.org.uk" xmlns:siri="http://www.siri.org.uk/siri">
	<ServiceDeliveryInfo>
		<siri:ResponseTimestamp>{{ .ResponseTimestamp.Format "2006-01-02T15:04:05.000Z07:00" }}</siri:ResponseTimestamp>
		<siri:ProducerRef>{{ .ProducerRef }}</siri:ProducerRef>{{ if .Address }}
		<siri:Address>{{ .Address }}</siri:Address>{{ end }}
		<siri:ResponseMessageIdentifier>{{ .ResponseMessageIdentifier }}</siri:ResponseMessageIdentifier>
		<siri:RequestMessageRef>{{ .RequestMessageRef }}</siri:RequestMessageRef>
	</ServiceDeliveryInfo>
	<Notification xmlns:ns2="http://www.ifopt.org.uk/acsb" xmlns:siri="http://www.ifopt.org.uk/ifopt" xmlns:ns4="http://datex2.eu/schema/2_0RC1/2_0" xmlns:siri="http://www.siri.org.uk/siri" xmlns:ns6="http://wsdl.siri.org.uk/siri">
		<siri:GeneralMessageDelivery version="2.0:FR-IDF-2.4" xmlns:stif="http://wsdl.siri.org.uk/siri">
			<siri:ResponseTimestamp>{{ .ResponseTimestamp.Format "2006-01-02T15:04:05.000Z07:00" }}</siri:ResponseTimestamp>
			<siri:RequestMessageRef>{{ .RequestMessageRef }}</siri:RequestMessageRef>
			<siri:SubscriberRef>{{ .SubscriberRef }}</siri:SubscriberRef>
			<siri:SubscriptionRef>{{ .SubscriptionIdentifier }}</siri:SubscriptionRef>
			<siri:Status>{{ .Status }}</siri:Status>{{ if not .Status }}
			<siri:ErrorCondition>{{ if eq .ErrorType "OtherError" }}
				<siri:OtherError number="{{ .ErrorNumber }}">{{ else }}
				<siri:{{ .ErrorType }}>{{ end }}
					<siri:ErrorText>{{ .ErrorText }}</siri:ErrorText>
				</siri:{{ .ErrorType }}>
			</siri:ErrorCondition>{{ else }}{{ range .GeneralMessages }}
			{{ .BuildGeneralMessageXML }}{{ end }}{{ end }}
		 </siri:GeneralMessageDelivery>
	</Notification>
	<NotificationExtension />
</sw:NotifyGeneralMessage>`

func NewXMLNotifyGeneralMessage(node xml.Node) *XMLNotifyGeneralMessage {
	xmlGeneralMessageResponse := &XMLNotifyGeneralMessage{}
	xmlGeneralMessageResponse.node = NewXMLNode(node)
	return xmlGeneralMessageResponse
}

func NewXMLNotifyGeneralMessageFromContent(content []byte) (*XMLNotifyGeneralMessage, error) {
	doc, err := gokogiri.ParseXml(content)
	if err != nil {
		return nil, err
	}
	response := NewXMLNotifyGeneralMessage(doc.Root().XmlNode)
	return response, nil
}

func NewXMLGeneralMessageDelivery(node XMLNode) *XMLGeneralMessageDelivery {
	delivery := &XMLGeneralMessageDelivery{}
	delivery.node = node
	return delivery
}

func (notify *XMLNotifyGeneralMessage) GeneralMessagesDeliveries() []*XMLGeneralMessageDelivery {
	if notify.deliveries == nil {
		deliveries := []*XMLGeneralMessageDelivery{}
		nodes := notify.findNodes("GeneralMessageDelivery")
		if nodes != nil {
			for _, node := range nodes {
				deliveries = append(deliveries, NewXMLGeneralMessageDelivery(node))
			}
		}
		notify.deliveries = deliveries
	}
	return notify.deliveries
}

func (delivery *XMLGeneralMessageDelivery) SubscriptionRef() string {
	if delivery.subscriptionRef == "" {
		delivery.subscriptionRef = delivery.findStringChildContent("SubscriptionRef")
	}
	return delivery.subscriptionRef
}

func (delivery *XMLGeneralMessageDelivery) SubscriberRef() string {
	if delivery.subscriberRef == "" {
		delivery.subscriberRef = delivery.findStringChildContent("SubscriberRef")
	}
	return delivery.subscriberRef
}

func (delivery *XMLGeneralMessageDelivery) XMLGeneralMessages() []*XMLGeneralMessage {
	if delivery.xmlGeneralMessages == nil {
		nodes := delivery.findNodes("GeneralMessage")
		if nodes != nil {
			for _, node := range nodes {
				delivery.xmlGeneralMessages = append(delivery.xmlGeneralMessages, NewXMLGeneralMessage(node))
			}
		}
	}
	return delivery.xmlGeneralMessages
}

func (delivery *XMLGeneralMessageDelivery) XMLGeneralMessagesCancellations() []*XMLGeneralMessageCancellation {
	if delivery.xmlGeneralMessagesCancellations == nil {
		cancellations := []*XMLGeneralMessageCancellation{}
		nodes := delivery.findNodes("GeneralMessageCancellation")
		if nodes != nil {
			for _, node := range nodes {
				cancellations = append(cancellations, NewXMLCancelledGeneralMessage(node))
			}
		}
		delivery.xmlGeneralMessagesCancellations = cancellations
	}
	return delivery.xmlGeneralMessagesCancellations
}

func (notify *SIRINotifyGeneralMessage) BuildXML() (string, error) {
	var buffer bytes.Buffer
	var notifyDelivery = template.Must(template.New("generalMessageNotifyTemplate").Parse(generalMessageNotifyTemplate))
	if err := notifyDelivery.Execute(&buffer, notify); err != nil {
		return "", err
	}
	return buffer.String(), nil
}
<|endoftext|>