<commit_before>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/xml"
	"flag"
	"log"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"

	libvirt "github.com/libvirt/libvirt-go"

	"./libvirt_schema"
)

var (
	libvirtUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "", "up"),
		"Whether scraping libvirt's metrics was successful.",
		nil,
		nil)

	libvirtBlockRdBytesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "read_bytes_total"),
		"Number of bytes read from a block device, in bytes.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockRdReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "read_requests_total"),
		"Number of read requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockRdTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "read_seconds_total"),
		"Amount of time spent reading from a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockWrBytesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "write_bytes_total"),
		"Number of bytes written from a block device, in bytes.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockWrReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "write_requests_total"),
		"Number of write requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockWrTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "write_seconds_total"),
		"Amount of time spent writing from a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockFlushReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "flush_requests_total"),
		"Number of flush requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtBlockFlushTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "block_stats", "flush_seconds_total"),
		"Amount of time spent flushing of a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
)

// CollectDomain extracts Prometheus metrics from a libvirt domain.
func CollectDomain(ch chan<- prometheus.Metric, domain *libvirt.Domain) error {
	domainName, err := domain.GetName()
	if err != nil {
		return err
	}

	xmlDesc, err := domain.GetXMLDesc(0)
	if err != nil {
		return err
	}

	var desc libvirt_schema.Domain
	err = xml.Unmarshal([]byte(xmlDesc), &desc)
	if err != nil {
		return err
	}

	// Report block device statistics.
	for _, disk := range desc.Devices.Disks {
		blockStats, err := domain.BlockStats(disk.Target.Device)
		if err != nil {
			return err
		}
		if blockStats.RdBytesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockRdBytesDesc,
				prometheus.CounterValue,
				float64(blockStats.RdBytes),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.RdReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockRdReqDesc,
				prometheus.CounterValue,
				float64(blockStats.RdReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.RdTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockRdTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.RdTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrBytesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockWrBytesDesc,
				prometheus.CounterValue,
				float64(blockStats.WrBytes),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockWrReqDesc,
				prometheus.CounterValue,
				float64(blockStats.WrReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockWrTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.WrTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.FlushReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockFlushReqDesc,
				prometheus.CounterValue,
				float64(blockStats.FlushReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.FlushTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtBlockFlushTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.FlushTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		// Skip "Errs", as the documentation does not clearly
		// explain what this means.
	}

	return nil
}

// CollectFromLibvirt obtains Prometheus metrics from all domains in a
// libvirt setup.
func CollectFromLibvirt(ch chan<- prometheus.Metric, uri string) error {
	conn, err := libvirt.NewConnect(uri)
	if err != nil {
		return err
	}
	defer conn.Close()

	// First attempt to get a list of active domains using
	// ListAllDomains(). If this fails, the remote side is using a version
	// of libvirt older than 0.9.13. In that case, fall back to using
	// ListDomains() in combination with LookupDomainById().
	domains, err := conn.ListAllDomains(libvirt.CONNECT_LIST_DOMAINS_ACTIVE)
	if err == nil {
		for _, domain := range domains {
			defer domain.Free()
		}
		for _, domain := range domains {
			err = CollectDomain(ch, &domain)
			if err != nil {
				return err
			}
		}
	} else {
		domainIds, err := conn.ListDomains()
		if err != nil {
			return err
		}
		for _, id := range domainIds {
			domain, err := conn.LookupDomainById(id)
			if err == nil {
				err = CollectDomain(ch, domain)
				domain.Free()
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// LibvirtExporter implements a Prometheus exporter for libvirt state.
type LibvirtExporter struct {
	uri string
}

// NewLibvirtExporter creates a new Prometheus exporter for libvirt.
func NewLibvirtExporter(uri string) (*LibvirtExporter, error) {
	return &LibvirtExporter{
		uri: uri,
	}, nil
}

// Describe returns metadata for all Prometheus metrics that may be exported.
func (e *LibvirtExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- libvirtUpDesc

	ch <- libvirtBlockRdBytesDesc
	ch <- libvirtBlockRdReqDesc
	ch <- libvirtBlockRdTotalTimesDesc
	ch <- libvirtBlockWrBytesDesc
	ch <- libvirtBlockWrReqDesc
	ch <- libvirtBlockWrTotalTimesDesc
	ch <- libvirtBlockFlushReqDesc
	ch <- libvirtBlockFlushTotalTimesDesc
}

// Collect scrapes Prometheus metrics from libvirt.
func (e *LibvirtExporter) Collect(ch chan<- prometheus.Metric) {
	err := CollectFromLibvirt(ch, e.uri)
	if err == nil {
		ch <- prometheus.MustNewConstMetric(
			libvirtUpDesc,
			prometheus.GaugeValue,
			1.0)
	} else {
		log.Printf("Failed to scrape metrics: %s", err)
		ch <- prometheus.MustNewConstMetric(
			libvirtUpDesc,
			prometheus.GaugeValue,
			0.0)
	}
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9177", "Address to listen on for web interface and telemetry.")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		libvirtURI    = flag.String("libvirt.uri", "qemu:///system", "Libvirt URI from which to extract metrics.")
	)
	flag.Parse()

	exporter, err := NewLibvirtExporter(*libvirtURI)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>Libvirt Exporter</title>

Libvirt Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Export CPU and memory usage using GetInfo().<commit_after>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/xml"
	"flag"
	"log"
	"net/http"

	"github.com/prometheus/client_golang/prometheus"

	libvirt "github.com/libvirt/libvirt-go"

	"./libvirt_schema"
)

var (
	libvirtUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "", "up"),
		"Whether scraping libvirt's metrics was successful.",
		nil,
		nil)

	libvirtDomainInfoMaxMemDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_info", "maximum_memory_bytes"),
		"Maximum allowed memory of the domain, in bytes.",
		[]string{"domain"},
		nil)
	libvirtDomainInfoMemoryDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_info", "memory_usage_bytes"),
		"Memory usage of the domain, in bytes.",
		[]string{"domain"},
		nil)
	libvirtDomainInfoNrVirtCpuDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_info", "virtual_cpus"),
		"Number of virtual CPUs for the domain.",
		[]string{"domain"},
		nil)
	libvirtDomainInfoCpuTimeDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_info", "cpu_time_seconds_total"),
		"Amount of CPU time used by the domain, in seconds.",
		[]string{"domain"},
		nil)

	libvirtDomainBlockRdBytesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "read_bytes_total"),
		"Number of bytes read from a block device, in bytes.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockRdReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "read_requests_total"),
		"Number of read requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockRdTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "read_seconds_total"),
		"Amount of time spent reading from a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockWrBytesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "write_bytes_total"),
		"Number of bytes written from a block device, in bytes.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockWrReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "write_requests_total"),
		"Number of write requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockWrTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "write_seconds_total"),
		"Amount of time spent writing from a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockFlushReqDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "flush_requests_total"),
		"Number of flush requests from a block device.",
		[]string{"domain", "source_file", "target_device"},
		nil)
	libvirtDomainBlockFlushTotalTimesDesc = prometheus.NewDesc(
		prometheus.BuildFQName("libvirt", "domain_block_stats", "flush_seconds_total"),
		"Amount of time spent flushing of a block device, in seconds.",
		[]string{"domain", "source_file", "target_device"},
		nil)
)

// CollectDomain extracts Prometheus metrics from a libvirt domain.
func CollectDomain(ch chan<- prometheus.Metric, domain *libvirt.Domain) error {
	domainName, err := domain.GetName()
	if err != nil {
		return err
	}

	// Decode XML description of domain to get block device names, etc.
	xmlDesc, err := domain.GetXMLDesc(0)
	if err != nil {
		return err
	}
	var desc libvirt_schema.Domain
	err = xml.Unmarshal([]byte(xmlDesc), &desc)
	if err != nil {
		return err
	}

	// Report domain info.
	info, err := domain.GetInfo()
	if err != nil {
		return err
	}
	ch <- prometheus.MustNewConstMetric(
		libvirtDomainInfoMaxMemDesc,
		prometheus.GaugeValue,
		float64(info.MaxMem)*1024,
		domainName)
	ch <- prometheus.MustNewConstMetric(
		libvirtDomainInfoMemoryDesc,
		prometheus.GaugeValue,
		float64(info.Memory)*1024,
		domainName)
	ch <- prometheus.MustNewConstMetric(
		libvirtDomainInfoNrVirtCpuDesc,
		prometheus.GaugeValue,
		float64(info.NrVirtCpu),
		domainName)
	ch <- prometheus.MustNewConstMetric(
		libvirtDomainInfoCpuTimeDesc,
		prometheus.CounterValue,
		float64(info.CpuTime)/1e9,
		domainName)

	// Report block device statistics.
	for _, disk := range desc.Devices.Disks {
		blockStats, err := domain.BlockStats(disk.Target.Device)
		if err != nil {
			return err
		}
		if blockStats.RdBytesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockRdBytesDesc,
				prometheus.CounterValue,
				float64(blockStats.RdBytes),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.RdReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockRdReqDesc,
				prometheus.CounterValue,
				float64(blockStats.RdReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.RdTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockRdTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.RdTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrBytesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockWrBytesDesc,
				prometheus.CounterValue,
				float64(blockStats.WrBytes),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockWrReqDesc,
				prometheus.CounterValue,
				float64(blockStats.WrReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.WrTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockWrTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.WrTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.FlushReqSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockFlushReqDesc,
				prometheus.CounterValue,
				float64(blockStats.FlushReq),
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		if blockStats.FlushTotalTimesSet {
			ch <- prometheus.MustNewConstMetric(
				libvirtDomainBlockFlushTotalTimesDesc,
				prometheus.CounterValue,
				float64(blockStats.FlushTotalTimes)/1e9,
				domainName,
				disk.Source.File,
				disk.Target.Device)
		}
		// Skip "Errs", as the documentation does not clearly
		// explain what this means.
	}

	return nil
}

// CollectFromLibvirt obtains Prometheus metrics from all domains in a
// libvirt setup.
func CollectFromLibvirt(ch chan<- prometheus.Metric, uri string) error {
	conn, err := libvirt.NewConnect(uri)
	if err != nil {
		return err
	}
	defer conn.Close()

	// First attempt to get a list of active domains using
	// ListAllDomains(). If this fails, the remote side is using a version
	// of libvirt older than 0.9.13. In that case, fall back to using
	// ListDomains() in combination with LookupDomainById().
	domains, err := conn.ListAllDomains(libvirt.CONNECT_LIST_DOMAINS_ACTIVE)
	if err == nil {
		for _, domain := range domains {
			defer domain.Free()
		}
		for _, domain := range domains {
			err = CollectDomain(ch, &domain)
			if err != nil {
				return err
			}
		}
	} else {
		domainIds, err := conn.ListDomains()
		if err != nil {
			return err
		}
		for _, id := range domainIds {
			domain, err := conn.LookupDomainById(id)
			if err == nil {
				err = CollectDomain(ch, domain)
				domain.Free()
				if err != nil {
					return err
				}
			}
		}
	}

	return nil
}

// LibvirtExporter implements a Prometheus exporter for libvirt state.
type LibvirtExporter struct {
	uri string
}

// NewLibvirtExporter creates a new Prometheus exporter for libvirt.
func NewLibvirtExporter(uri string) (*LibvirtExporter, error) {
	return &LibvirtExporter{
		uri: uri,
	}, nil
}

// Describe returns metadata for all Prometheus metrics that may be exported.
func (e *LibvirtExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- libvirtUpDesc

	ch <- libvirtDomainInfoMaxMemDesc
	ch <- libvirtDomainInfoMemoryDesc
	ch <- libvirtDomainInfoNrVirtCpuDesc
	ch <- libvirtDomainInfoCpuTimeDesc

	ch <- libvirtDomainBlockRdBytesDesc
	ch <- libvirtDomainBlockRdReqDesc
	ch <- libvirtDomainBlockRdTotalTimesDesc
	ch <- libvirtDomainBlockWrBytesDesc
	ch <- libvirtDomainBlockWrReqDesc
	ch <- libvirtDomainBlockWrTotalTimesDesc
	ch <- libvirtDomainBlockFlushReqDesc
	ch <- libvirtDomainBlockFlushTotalTimesDesc
}

// Collect scrapes Prometheus metrics from libvirt.
func (e *LibvirtExporter) Collect(ch chan<- prometheus.Metric) {
	err := CollectFromLibvirt(ch, e.uri)
	if err == nil {
		ch <- prometheus.MustNewConstMetric(
			libvirtUpDesc,
			prometheus.GaugeValue,
			1.0)
	} else {
		log.Printf("Failed to scrape metrics: %s", err)
		ch <- prometheus.MustNewConstMetric(
			libvirtUpDesc,
			prometheus.GaugeValue,
			0.0)
	}
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9177", "Address to listen on for web interface and telemetry.")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		libvirtURI    = flag.String("libvirt.uri", "qemu:///system", "Libvirt URI from which to extract metrics.")
	)
	flag.Parse()

	exporter, err := NewLibvirtExporter(*libvirtURI)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>Libvirt Exporter</title>

Libvirt Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/definition-mapper/libmapper/providers/aws/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/aws/definition"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapRoute53Zones : Maps the zones from a given input payload.
func MapRoute53Zones(d *definition.Definition) []*components.Route53Zone {
	var zones []*components.Route53Zone

	for _, zone := range d.Route53Zones {
		z := &components.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
			Vpc:     zone.Vpc,
			Tags:    mapTagsServiceOnly(d.Name),
		}

		for _, record := range zone.Records {
			r := components.Record{
				Entry:         record.Entry,
				Type:          record.Type,
				Instances:     record.Instances,
				Loadbalancers: record.Loadbalancers,
				RDSClusters:   record.RDSClusters,
				RDSInstances:  record.RDSInstances,
				Values:        record.Values,
				TTL:           record.TTL,
			}

			z.Records = append(z.Records, r)
		}

		z.SetDefaultVariables()

		zones = append(zones, z)
	}

	return zones
}

// MapDefinitionRoute53Zones : Maps zones from the internal format to the input definition format
func MapDefinitionRoute53Zones(g *graph.Graph) []definition.Route53Zone {
	var zones []definition.Route53Zone

	for _, gzone := range g.GetComponents().ByType("route53") {
		zone := gzone.(*components.Route53Zone)

		z := definition.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
		}

		for _, record := range zone.Records {
			r := definition.Record{
				Entry:         record.Entry,
				Type:          record.Type,
				TTL:           record.TTL,
				Instances:     record.Instances,
				Loadbalancers: record.Loadbalancers,
				RDSClusters:   record.RDSClusters,
				RDSInstances:  record.RDSInstances,
				Values:        record.Values,
			}

			for i := len(r.Values) - 1; i >= 0; i++ {
				if len(r.Values[i]) < 2 {
					continue
				}
				if r.Values[i][0:2] == "$(" {
					r.Values = append(r.Values[:i], r.Values[i+1:]...)
				}
			}

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}
<commit_msg>fixed route53 mapping incrementer<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/definition-mapper/libmapper/providers/aws/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/aws/definition"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapRoute53Zones : Maps the zones from a given input payload.
func MapRoute53Zones(d *definition.Definition) []*components.Route53Zone {
	var zones []*components.Route53Zone

	for _, zone := range d.Route53Zones {
		z := &components.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
			Vpc:     zone.Vpc,
			Tags:    mapTagsServiceOnly(d.Name),
		}

		for _, record := range zone.Records {
			r := components.Record{
				Entry:         record.Entry,
				Type:          record.Type,
				Instances:     record.Instances,
				Loadbalancers: record.Loadbalancers,
				RDSClusters:   record.RDSClusters,
				RDSInstances:  record.RDSInstances,
				Values:        record.Values,
				TTL:           record.TTL,
			}

			z.Records = append(z.Records, r)
		}

		z.SetDefaultVariables()

		zones = append(zones, z)
	}

	return zones
}

// MapDefinitionRoute53Zones : Maps zones from the internal format to the input definition format
func MapDefinitionRoute53Zones(g *graph.Graph) []definition.Route53Zone {
	var zones []definition.Route53Zone

	for _, gzone := range g.GetComponents().ByType("route53") {
		zone := gzone.(*components.Route53Zone)

		z := definition.Route53Zone{
			Name:    zone.Name,
			Private: zone.Private,
		}

		for _, record := range zone.Records {
			r := definition.Record{
				Entry:         record.Entry,
				Type:          record.Type,
				TTL:           record.TTL,
				Instances:     record.Instances,
				Loadbalancers: record.Loadbalancers,
				RDSClusters:   record.RDSClusters,
				RDSInstances:  record.RDSInstances,
				Values:        record.Values,
			}

			for i := len(r.Values) - 1; i >= 0; i-- {
				if len(r.Values[i]) < 2 {
					continue
				}
				if r.Values[i][0:2] == "$(" {
					r.Values = append(r.Values[:i], r.Values[i+1:]...)
				}
			}

			z.Records = append(z.Records, r)
		}

		zones = append(zones, z)
	}

	return zones
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src storage.Object

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	o *storage.Object) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket: bucket,
		src:    *o,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket, depending on whether the backing object has
// been deleted.
func (op *ObjectProxy) Name() string {
	return op.src.Name
}

// Return the generation of the object from which the current contents of this
// proxy were branched. If Sync has been successfully called, this is the
// generation most recently returned by Sync. Otherwise it is the generation
// from which the proxy was created.
func (op *ObjectProxy) SourceGeneration() int64 {
	return op.src.Generation
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size int64, clobbered bool, err error) {
	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		size = fi.Size()
	} else {
		size = op.src.Size
	}

	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: op.Name()}
	o, err := op.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		clobbered = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (o.Generation != op.src.Generation)

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been modified, save its current contents
// to GCS and return a generation number for a generation with exactly those
// contents. Do so with a precondition such that the creation will fail if the
// source generation is not current. In that case, return an error of type
// *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen int64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.src.Generation
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = op.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.src.Name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &op.src.Generation,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Update our state.
	op.src = *o
	op.dirty = false

	// Return the generation number.
	gen = op.src.Generation

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Open the object for reading.
	req := &gcs.ReadObjectRequest{
		Name:       name,
		Generation: generation,
	}

	var rc io.ReadCloser
	if rc, err = bucket.NewReader(ctx, req); err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Copy to the file.
	if _, err = io.Copy(f, rc); err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Close.
	if err = rc.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.Name(), op.src.Generation)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Added StatResult.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src storage.Object

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

type StatResult struct {
	// The current size in bytes of the content, including any local
	// modifications that have not been Sync'd.
	Size int64

	// Has the object changed out from under us in GCS? If so, Sync will fail.
	Clobbered bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	o *storage.Object) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket: bucket,
		src:    *o,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket, depending on whether the backing object has
// been deleted.
func (op *ObjectProxy) Name() string {
	return op.src.Name
}

// Return the generation of the object from which the current contents of this
// proxy were branched. If Sync has been successfully called, this is the
// generation most recently returned by Sync. Otherwise it is the generation
// from which the proxy was created.
func (op *ObjectProxy) SourceGeneration() int64 {
	return op.src.Generation
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size int64, clobbered bool, err error) {
	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		size = fi.Size()
	} else {
		size = op.src.Size
	}

	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: op.Name()}
	o, err := op.bucket.StatObject(ctx, req)

	// Special case: "not found" means we have been clobbered.
	if _, ok := err.(*gcs.NotFoundError); ok {
		err = nil
		clobbered = true
		return
	}

	// Propagate other errors.
	if err != nil {
		err = fmt.Errorf("StatObject: %v", err)
		return
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (o.Generation != op.src.Generation)

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been modified, save its current contents
// to GCS and return a generation number for a generation with exactly those
// contents. Do so with a precondition such that the creation will fail if the
// source generation is not current. In that case, return an error of type
// *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen int64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.src.Generation
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = op.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.src.Name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &op.src.Generation,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Update our state.
	op.src = *o
	op.dirty = false

	// Return the generation number.
	gen = op.src.Generation

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Open the object for reading.
	req := &gcs.ReadObjectRequest{
		Name:       name,
		Generation: generation,
	}

	var rc io.ReadCloser
	if rc, err = bucket.NewReader(ctx, req); err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Copy to the file.
	if _, err = io.Copy(f, rc); err != nil {
		err = fmt.Errorf("Copy: %v", err)
		return
	}

	// Close.
	if err = rc.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.Name(), op.src.Generation)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return an error of type *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.srcGeneration
		return
	}

	// TODO(jacobsa): Add a test that ensures we don't screw up the seek position
	// within the file when reading below. Sync then dirty then Sync.

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	signedSrcGeneration := int64(op.srcGeneration)
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &signedSrcGeneration,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Make sure the server didn't return a silly generation number.
	//
	// TODO(jacobsa): Push unsigned generation numbers and a guarantee on zero
	// into package gcs, including checking results from the server, and remove
	// this.
	if o.Generation <= 0 {
		err = fmt.Errorf(
			"CreateObject returned invalid generation number: %v",
			o.Generation)

		return
	}

	gen = uint64(o.Generation)

	// Update our state.
	op.srcGeneration = gen
	op.dirty = false

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Fixed a bug involving the file's seek position.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket:        bucket,
		name:          name,
		srcGeneration: srcGeneration,
	}

	// For "doesn't exist" source generations, we must establish an empty local
	// file and mark the proxy dirty.
	if srcGeneration == 0 {
		if err = op.ensureLocalFile(ctx); err != nil {
			return
		}

		op.dirty = true
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	// TODO(jacobsa): Make sure the dirty flag modification below is tested by
	// removing it and looking for a failure.
	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return an error of type *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.srcGeneration
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = op.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	signedSrcGeneration := int64(op.srcGeneration)
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &signedSrcGeneration,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Make sure the server didn't return a silly generation number.
	//
	// TODO(jacobsa): Push unsigned generation numbers and a guarantee on zero
	// into package gcs, including checking results from the server, and remove
	// this.
	if o.Generation <= 0 {
		err = fmt.Errorf(
			"CreateObject returned invalid generation number: %v",
			o.Generation)

		return
	}

	gen = uint64(o.Generation)

	// Update our state.
	op.srcGeneration = gen
	op.dirty = false

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation uint64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		f.Close()
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		panic("TODO")
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/argusdusty/Ferret"
	"io/ioutil"
	"net/http"
	"strconv"
)

var Correction = func(b []byte) [][]byte { return ferret.ErrorCorrect(b, ferret.LowercaseLetters) }
var LengthSorter = func(s string, v interface{}, l int, i int) float64 { return -float64(l + i) }
var FreqSorter = func(s string, v interface{}, l int, i int) float64 { return float64(v.(uint64)) }
var Converter = ferret.UnicodeToLowerASCII
var ValueIds = map[string]SearchItem{}
var SearchEngine *ferret.InvertedSuffix

func main() {
	fmt.Println("Hi, I am Rustic Search Server!")
	fmt.Println("Run rusticsearch -h for help with parameters")
	http.HandleFunc("/", handler)
	port := flag.Int("port", 8080, "a serving TCP port")
	flag.Parse()
	Data, err := ioutil.ReadFile("search_index.csv")
	if err != nil {
		fmt.Println("search_index.csv not found :(")
		panic(err)
	}

	fmt.Println("Parsing search_index.csv...")
	Words := make([]string, 0)
	Values := make([]interface{}, 0)
	for i, Vals := range bytes.Split(Data, []byte("\n")) {
		WordFreq := bytes.Split(Vals, []byte("----------> "))
		if len(WordFreq) != 2 {
			fmt.Printf("Bollocks! search_index.csv line: %v breaks everything: \n \"%v\" \n I quit! \n", i, string(Vals))
			panic(Vals)
		}

		Words = append(Words, string(WordFreq[0]))
		// to add some priority mechanism in here in the future
		Values = append(Values, 10)
		ValueIds[string(WordFreq[0])] = SearchItem{string(WordFreq[0]), string(WordFreq[1])}
	}

	fmt.Println("Created index...")
	SearchEngine = ferret.New(Words, Words, Values, Converter)

	fmt.Println("~~~~~~~~~~~~~~~~~~~~~~~")
	fmt.Printf("   Starting server at port %v... \n", ":"+strconv.Itoa(*port))
	fmt.Println("~~~~~~~~~~~~~~~~~~~~~~~")
	err = http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if err != nil {
		fmt.Printf("Dang it! Error at ListenAndServe: %v \n", err)
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	h := w.Header()
	h.Set("Access-Control-Allow-Origin", "*")
	h.Set("Access-Control-Allow-Methods", "POST, GET, PUT, PATCH, DELETE, OPTIONS")
	h.Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, API-Date, Host, Authorization, Key, User-Token")
	h.Set("Access-Control-Max-Age", "1728000")

	fmt.Println("ANOTHER REQUEST: %s", r.URL.Path)
	results, _ := SearchEngine.Query(r.URL.Path[1:], 5)
	output := make([]SearchItem, 0)
	for _, word := range results {
		fmt.Printf("~~~~~~~~~~~~: %v \n", string(word))
		output = append(output, ValueIds[string(word)])
		fmt.Println("Here is the value added: ", ValueIds[string(word)])
	}
	searchResults, err := json.Marshal(SearchResult{output})
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(w, string(searchResults))
}
<commit_msg>minor stuff<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/argusdusty/Ferret"
	"io/ioutil"
	"net/http"
	"strconv"
)

var port = flag.Int("port", 8080, "a serving TCP port")
var Correction = func(b []byte) [][]byte { return ferret.ErrorCorrect(b, ferret.LowercaseLetters) }
var LengthSorter = func(s string, v interface{}, l int, i int) float64 { return -float64(l + i) }
var FreqSorter = func(s string, v interface{}, l int, i int) float64 { return float64(v.(uint64)) }
var Converter = ferret.UnicodeToLowerASCII
var ValueIds = map[string]SearchItem{}
var SearchEngine *ferret.InvertedSuffix

func main() {
	fmt.Println("Hi, I am Rustic Search Server!")
	fmt.Println("Call rusticsearch -h for help with parameters")
	flag.Parse()

	http.HandleFunc("/", handler)
	Data, err := ioutil.ReadFile("search_index.csv")
	if err != nil {
		fmt.Println("search_index.csv not found :(")
		panic(err)
	}

	fmt.Println("Parsing search_index.csv...")
	Words := make([]string, 0)
	Values := make([]interface{}, 0)
	for i, Vals := range bytes.Split(Data, []byte("\n")) {
		WordFreq := bytes.Split(Vals, []byte("----------> "))
		if len(WordFreq) != 2 {
			fmt.Printf("Bollocks! search_index.csv line: %v breaks everything: \n \"%v\" \n I quit! \n", i, string(Vals))
			panic(Vals)
		}

		Words = append(Words, string(WordFreq[0]))
		// to add some priority mechanism in here in the future
		Values = append(Values, 10)
		ValueIds[string(WordFreq[0])] = SearchItem{string(WordFreq[0]), string(WordFreq[1])}
	}

	fmt.Println("Created index...")
	SearchEngine = ferret.New(Words, Words, Values, Converter)

	fmt.Println("~~~~~~~~~~~~~~~~~~~~~~~")
	fmt.Printf("   Starting server at port %v... \n", ":"+strconv.Itoa(*port))
	fmt.Println("~~~~~~~~~~~~~~~~~~~~~~~")
	err = http.ListenAndServe(":"+strconv.Itoa(*port), nil)
	if err != nil {
		fmt.Printf("Dang it! Error at ListenAndServe: %v \n", err)
		panic(err)
	}
}

func handler(w http.ResponseWriter, r *http.Request) {
	h := w.Header()
	h.Set("Access-Control-Allow-Origin", "*")
	h.Set("Access-Control-Allow-Methods", "POST, GET, PUT, PATCH, DELETE, OPTIONS")
	h.Set("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, API-Date, Host, Authorization, Key, User-Token")
	h.Set("Access-Control-Max-Age", "1728000")

	fmt.Println("ANOTHER REQUEST: %s", r.URL.Path)
	results, _ := SearchEngine.Query(r.URL.Path[1:], 5)
	output := make([]SearchItem, 0)
	for _, word := range results {
		fmt.Printf("~~~~~~~~~~~~: %v \n", string(word))
		output = append(output, ValueIds[string(word)])
		fmt.Println("Here is the value added: ", ValueIds[string(word)])
	}
	searchResults, err := json.Marshal(SearchResult{output})
	if err != nil {
		panic(err)
	}
	fmt.Fprintf(w, string(searchResults))
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"net/http"
)

// TODO: add the missing methods
const (
	// HTTP GET request
	MethodGet = "GET"
	// HTTP Post request
	MethodPost = "POST"
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// When creating the multiplexer, the user needs to specify a list of allowed
// domains. The server will only serve requests target to those domains and
// otherwise will reply with HTTP 404 Not Found.
// TODO(@mihalimara22, @mattiasgrenfeldt): add a link to docs/ explaining
// why this is done.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only. This host name must be in the list of allowed domains passed
// when creating the ServeMux. Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux     *http.ServeMux
	domains map[string]bool
	disp    Dispatcher

	// Maps patterns to handlers supporting multiple HTTP methods.
	handlers map[string]methodHandler

	// Maps interceptor key to interceptor.
	interceps map[string]Interceptor
}

// NewServeMux allocates and returns a new ServeMux
func NewServeMux(d Dispatcher, domains ...string) *ServeMux {
	// TODO(@mattiasgrenfeldt, @mihalimara22): make domains a variadic of string **literals**.
	dm := map[string]bool{}
	for _, host := range domains {
		dm[host] = true
	}
	return &ServeMux{
		mux:       http.NewServeMux(),
		domains:   dm,
		disp:      d,
		handlers:  map[string]methodHandler{},
		interceps: map[string]Interceptor{},
	}
}

// Handle registers a handler for the given pattern and method. If another
// handler is already registered for the same pattern and method, Handle panics.
func (m *ServeMux) Handle(pattern string, method string, h Handler) {
	mh, ok := m.handlers[pattern]
	if !ok {
		mh := methodHandler{
			handlers:     map[string]Handler{method: h},
			domains:      m.domains,
			disp:         m.disp,
			muxInterceps: m.interceps,
		}

		m.handlers[pattern] = mh
		m.mux.Handle(pattern, mh)
		return
	}

	if _, ok := mh.handlers[method]; ok {
		panic("method already registered")
	}
	mh.handlers[method] = h
}

// Install installs interceps which will be applied to all requests coming to
// the ServeMux. The key is used to access the interceptor and should be unique.
//
// TODO(@empijei, @grenfeldt, @kele, @mihalimara22): Right now you could install
// the same interceptor twice with different keys, we need to figure out how
// exactly we want to avoid that and how we define key uniqueness.
func (m *ServeMux) Install(key string, p Interceptor) {
	m.interceps[key] = p
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

// methodHandler is collection of handlers based on the request method.
type methodHandler struct {
	// Maps an HTTP method to its handler
	handlers     map[string]Handler
	domains      map[string]bool
	disp         Dispatcher
	muxInterceps map[string]Interceptor
}

// ServeHTTP dispatches the request to the handler associated with
// the incoming request's method after calling the Before function of all
// ServeMux interceptors the handler is registered on.
func (m methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	rw := NewResponseWriter(m.disp, w, m.muxInterceps)
	ir := NewIncomingRequest(r)

	if !m.domains[r.Host] {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	h, ok := m.handlers[r.Method]
	if !ok {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}

	for _, intercep := range m.muxInterceps {
		if res := intercep.Before(rw, ir); res.written {
			return
		}
	}

	h.ServeHTTP(rw, ir)
}
<commit_msg>ServeMux.Install now panics when the same key is used twice when installing interceptors. Also fixed nit in godoc comment.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package safehttp

import (
	"net/http"
)

// TODO: add the missing methods
const (
	// HTTP GET request
	MethodGet = "GET"
	// HTTP Post request
	MethodPost = "POST"
)

// ServeMux is an HTTP request multiplexer. It matches the URL of each incoming
// request against a list of registered patterns and calls the handler for
// the pattern that most closely matches the URL.
//
// When creating the multiplexer, the user needs to specify a list of allowed
// domains. The server will only serve requests target to those domains and
// otherwise will reply with HTTP 404 Not Found.
// TODO(@mihalimara22, @mattiasgrenfeldt): add a link to docs/ explaining
// why this is done.
//
// Patterns names are fixed, rooted paths, like "/favicon.ico", or rooted
// subtrees like "/images/" (note the trailing slash). Longer patterns take
// precedence over shorter ones, so that if there are handlers registered for
// both "/images/" and "/images/thumbnails/", the latter handler will be called
// for paths beginning "/images/thumbnails/" and the former will receive
// requests for any other paths in the "/images/" subtree.
//
// Note that since a pattern ending in a slash names a rooted subtree, the
// pattern "/" matches all paths not matched by other registered patterns,
// not just the URL with Path == "/".
//
// If a subtree has been registered and a request is received naming the subtree
// root without its trailing slash, ServeMux redirects that request to
// the subtree root (adding the trailing slash). This behavior can be overridden
// with a separate registration for the path without the trailing slash. For
// example, registering "/images/" causes ServeMux to redirect a request for
// "/images" to "/images/", unless "/images" has been registered separately.
//
// Patterns may optionally begin with a host name, restricting matches to URLs
// on that host only. This host name must be in the list of allowed domains passed
// when creating the ServeMux. Host-specific patterns take precedence over general
// patterns, so that a handler might register for the two patterns "/codesearch"
// and "codesearch.google.com/" without also taking over requests for
// "http://www.google.com/".
//
// ServeMux also takes care of sanitizing the URL request path and the Host
// header, stripping the port number and redirecting any request containing . or
// .. elements or repeated slashes to an equivalent, cleaner URL.
//
// Multiple handlers can be registered for a single pattern, as long as they
// handle different HTTP methods.
type ServeMux struct {
	mux     *http.ServeMux
	domains map[string]bool
	disp    Dispatcher

	// Maps patterns to handlers supporting multiple HTTP methods.
	handlers map[string]methodHandler

	// Maps interceptor key to interceptor.
	interceps map[string]Interceptor
}

// NewServeMux allocates and returns a new ServeMux
func NewServeMux(d Dispatcher, domains ...string) *ServeMux {
	// TODO(@mattiasgrenfeldt, @mihalimara22): make domains a variadic of string **literals**.
	dm := map[string]bool{}
	for _, host := range domains {
		dm[host] = true
	}
	return &ServeMux{
		mux:       http.NewServeMux(),
		domains:   dm,
		disp:      d,
		handlers:  map[string]methodHandler{},
		interceps: map[string]Interceptor{},
	}
}

// Handle registers a handler for the given pattern and method. If another
// handler is already registered for the same pattern and method, Handle panics.
func (m *ServeMux) Handle(pattern string, method string, h Handler) {
	mh, ok := m.handlers[pattern]
	if !ok {
		mh := methodHandler{
			handlers:     map[string]Handler{method: h},
			domains:      m.domains,
			disp:         m.disp,
			muxInterceps: m.interceps,
		}

		m.handlers[pattern] = mh
		m.mux.Handle(pattern, mh)
		return
	}

	if _, ok := mh.handlers[method]; ok {
		panic("method already registered")
	}
	mh.handlers[method] = h
}

// Install installs an Interceptor. Interceptor keys need to be unique. If an
// Interceptor with the same key has already been installed, Install panics.
//
// TODO(@empijei, @grenfeldt, @kele, @mihalimara22): Right now you could install
// the same interceptor twice with different keys, we need to figure out how
// exactly we want to avoid that and how we define key uniqueness.
func (m *ServeMux) Install(key string, i Interceptor) {
	if _, exists := m.interceps[key]; exists {
		panic("interceptor with same key already installed")
	}
	m.interceps[key] = i
}

// ServeHTTP dispatches the request to the handler whose method matches the
// incoming request and whose pattern most closely matches the request URL.
func (m *ServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	m.mux.ServeHTTP(w, r)
}

// methodHandler is a collection of handlers based on the request method.
type methodHandler struct {
	// Maps an HTTP method to its handler
	handlers     map[string]Handler
	domains      map[string]bool
	disp         Dispatcher
	muxInterceps map[string]Interceptor
}

// ServeHTTP dispatches the request to the handler associated with
// the incoming request's method after calling the Before function of all
// ServeMux interceptors the handler is registered on.
func (m methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	rw := NewResponseWriter(m.disp, w, m.muxInterceps)
	ir := NewIncomingRequest(r)

	if !m.domains[r.Host] {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	h, ok := m.handlers[r.Method]
	if !ok {
		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
		return
	}

	for _, intercep := range m.muxInterceps {
		if res := intercep.Before(rw, ir); res.written {
			return
		}
	}

	h.ServeHTTP(rw, ir)
}
<|endoftext|>