<commit_before>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

const NotRecentlyCrawled int = 0
const AlreadyCrawled int = -1

func ReadFromQueue(
	inboundChannel <-chan amqp.Delivery,
	rootURLs []*url.URL,
	ttlHashSet *ttl_hash_set.TTLHashSet,
	blacklistPaths []string,
	crawlerThreads int,
) chan *CrawlerMessageItem {
	outboundChannel := make(chan *CrawlerMessageItem, crawlerThreads)

	readLoop := func(
		inbound <-chan amqp.Delivery,
		outbound chan<- *CrawlerMessageItem,
		ttlHashSet *ttl_hash_set.TTLHashSet,
		blacklistPaths []string,
	) {
		for item := range inbound {
			start := time.Now()
			message := NewCrawlerMessageItem(item, rootURLs, blacklistPaths)

			if message.IsBlacklisted() {
				item.Ack(false)
				log.Debugln("URL is blacklisted (acknowledging):", message.URL())
				continue
			}

			crawlCount, err := ttlHashSet.Get(message.URL())
			if err != nil {
				item.Reject(true)
				log.Errorln("Couldn't check existence of (rejecting):", message.URL(), err)
				continue
			}

			if crawlCount == AlreadyCrawled {
				log.Debugln("URL read from queue already crawled:", message.URL())
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (ReadFromQueue): ", message.URL())
				}
				continue
			}

			outbound <- message

			util.StatsDTiming("read_from_queue", start, time.Now())
		}
	}

	go readLoop(inboundChannel, outboundChannel, ttlHashSet, blacklistPaths)

	return outboundChannel
}

func CrawlURL(
	ttlHashSet *ttl_hash_set.TTLHashSet,
	crawlChannel <-chan *CrawlerMessageItem,
	crawler *http_crawler.Crawler,
	crawlerThreads int,
	maxCrawlRetries int,
) <-chan *CrawlerMessageItem {
	if crawlerThreads < 1 {
		panic("cannot start a negative or zero number of crawler threads")
	}

	extractChannel := make(chan *CrawlerMessageItem, 2)

	crawlLoop := func(
		ttlHashSet *ttl_hash_set.TTLHashSet,
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
		crawler *http_crawler.Crawler,
		maxCrawlRetries int,
	) {
		for item := range crawl {
			start := time.Now()
			u, err := url.Parse(item.URL())
			if err != nil {
				item.Reject(false)
				log.Warningln("Couldn't crawl, invalid URL (rejecting):", item.URL(), err)
				continue
			}

			crawlCount, err := ttlHashSet.Get(u.String())
			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't confirm existence of URL (rejecting):", u.String(), err)
				continue
			}

			if crawlCount == maxCrawlRetries {
				item.Reject(false)
				log.Errorf("Aborting crawl of URL which has been retried %d times (rejecting): %s", maxCrawlRetries, u.String())
				continue
			}

			log.Debugln("Starting crawl of URL:", u)
			response, err := crawler.Crawl(u)
			if err != nil {
				switch err {
				case http_crawler.ErrRetryRequest5XX, http_crawler.ErrRetryRequest429:
					switch err {
					case http_crawler.ErrRetryRequest5XX:
						ttlHashSet.Incr(u.String())
					case http_crawler.ErrRetryRequest429:
						sleepTime := 5 * time.Second

						// Back off from crawling for a few seconds.
						log.Warningf("Sleeping for: %v. Received 429 HTTP status", sleepTime)
						time.Sleep(sleepTime)
					}

					item.Reject(true)
					log.Warningln("Couldn't crawl (requeueing):", u.String(), err)
				case http_crawler.ErrRedirect:
					setErr := ttlHashSet.Set(u.String(), AlreadyCrawled)
					if setErr != nil {
						log.Errorln("Couldn't mark item as already crawled:", u.String(), setErr)
					}

					item.Reject(false)
					// log at INFO because redirect URLs are not a concern
					log.Debugln("Couldn't crawl (rejecting):", u.String(), err)
				default:
					item.Reject(false)
					log.Warningln("Couldn't crawl (rejecting):", u.String(), err)
				}

				continue
			}

			item.Response = response

			if item.Response.AcceptedContentType() {
				extract <- item
			} else {
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (CrawlURL): ", item.URL())
				}

				err = ttlHashSet.Set(item.URL(), AlreadyCrawled)
				if err != nil {
					log.Errorln("Couldn't mark item as already crawled:", item.URL(), err)
				}
			}

			util.StatsDTiming("crawl_url", start, time.Now())
		}
	}

	for i := 1; i <= crawlerThreads; i++ {
		go crawlLoop(ttlHashSet, crawlChannel, extractChannel, crawler, maxCrawlRetries)
	}

	return extractChannel
}

func WriteItemToDisk(basePath string, crawlChannel <-chan *CrawlerMessageItem) <-chan *CrawlerMessageItem {
	extractChannel := make(chan *CrawlerMessageItem, 2)

	writeLoop := func(
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
	) {
		for item := range crawl {
			start := time.Now()
			relativeFilePath, err := item.RelativeFilePath()

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't retrieve relative file path for item (rejecting):", item.URL(), err)
				continue
			}

			filePath := filepath.Join(basePath, relativeFilePath)
			dirPath := filepath.Dir(filePath)
			err = os.MkdirAll(dirPath, 0755)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't created directories for item (rejecting):", filePath, err)
				continue
			}

			err = ioutil.WriteFile(filePath, item.Response.Body, 0644)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			log.Debugln("Wrote URL body to disk for:", item.URL())

			contentType, err := item.Response.ParseContentType()
			if err != nil {
				log.Errorln("Couldn't determine Content-Type for item (rejecting):", item, err)
				item.Reject(false)
				continue
			}

			// Only send HTML pages for URL extraction. All other
			// pages should be written directly to disk and acknowledged.
			if contentType == http_crawler.HTML {
				extract <- item
			} else {
				item.Ack(false)
			}

			util.StatsDTiming("write_to_disk", start, time.Now())
		}
	}

	go writeLoop(crawlChannel, extractChannel)

	return extractChannel
}

func ExtractURLs(extractChannel <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	extractLoop := func(
		extract <-chan *CrawlerMessageItem,
		publish chan<- string,
		acknowledge chan<- *CrawlerMessageItem,
	) {
		for item := range extract {
			start := time.Now()
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Errorln("ExtractURLs (rejecting):", string(item.Body), err)

				continue
			}

			log.Debugln("Extracted URLs:", len(urls))

			for _, u := range urls {
				publish <- u.String()
			}

			acknowledge <- item

			util.StatsDTiming("extract_urls", start, time.Now())
		}
	}

	go extractLoop(extractChannel, publishChannel, acknowledgeChannel)

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.Manager, publish <-chan string) {
	for url := range publish {
		start := time.Now()
		crawlCount, err := ttlHashSet.Get(url)

		if err != nil {
			log.Errorln("Couldn't check existence of URL:", url, err)
			continue
		}

		if crawlCount == AlreadyCrawled {
			log.Debugln("URL extracted from page already crawled:", url)
		} else if crawlCount == NotRecentlyCrawled {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}

		util.StatsDGauge("publish_urls", int64(len(publish)))
		util.StatsDTiming("publish_urls", start, time.Now())
	}
}

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		start := time.Now()
		url := item.URL()

		err := ttlHashSet.Set(url, AlreadyCrawled)
		if err != nil {
			item.Reject(false)
			log.Errorln("Acknowledge failed (rejecting):", url, err)
			continue
		}

		if err = item.Ack(false); err != nil {
			log.Errorln("Ack failed (AcknowledgeItem): ", item.URL())
		}
		log.Debugln("Acknowledged:", url)

		util.StatsDTiming("acknowledge_item", start, time.Now())
	}
}
<commit_msg>Fix grammar in error message<commit_after>package main

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/alphagov/govuk_crawler_worker/http_crawler"
	"github.com/alphagov/govuk_crawler_worker/queue"
	"github.com/alphagov/govuk_crawler_worker/ttl_hash_set"
	"github.com/alphagov/govuk_crawler_worker/util"
	"github.com/streadway/amqp"
)

const NotRecentlyCrawled int = 0
const AlreadyCrawled int = -1

func ReadFromQueue(
	inboundChannel <-chan amqp.Delivery,
	rootURLs []*url.URL,
	ttlHashSet *ttl_hash_set.TTLHashSet,
	blacklistPaths []string,
	crawlerThreads int,
) chan *CrawlerMessageItem {
	outboundChannel := make(chan *CrawlerMessageItem, crawlerThreads)

	readLoop := func(
		inbound <-chan amqp.Delivery,
		outbound chan<- *CrawlerMessageItem,
		ttlHashSet *ttl_hash_set.TTLHashSet,
		blacklistPaths []string,
	) {
		for item := range inbound {
			start := time.Now()
			message := NewCrawlerMessageItem(item, rootURLs, blacklistPaths)

			if message.IsBlacklisted() {
				item.Ack(false)
				log.Debugln("URL is blacklisted (acknowledging):", message.URL())
				continue
			}

			crawlCount, err := ttlHashSet.Get(message.URL())
			if err != nil {
				item.Reject(true)
				log.Errorln("Couldn't check existence of (rejecting):", message.URL(), err)
				continue
			}

			if crawlCount == AlreadyCrawled {
				log.Debugln("URL read from queue already crawled:", message.URL())
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (ReadFromQueue): ", message.URL())
				}
				continue
			}

			outbound <- message

			util.StatsDTiming("read_from_queue", start, time.Now())
		}
	}

	go readLoop(inboundChannel, outboundChannel, ttlHashSet, blacklistPaths)

	return outboundChannel
}

func CrawlURL(
	ttlHashSet *ttl_hash_set.TTLHashSet,
	crawlChannel <-chan *CrawlerMessageItem,
	crawler *http_crawler.Crawler,
	crawlerThreads int,
	maxCrawlRetries int,
) <-chan *CrawlerMessageItem {
	if crawlerThreads < 1 {
		panic("cannot start a negative or zero number of crawler threads")
	}

	extractChannel := make(chan *CrawlerMessageItem, 2)

	crawlLoop := func(
		ttlHashSet *ttl_hash_set.TTLHashSet,
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
		crawler *http_crawler.Crawler,
		maxCrawlRetries int,
	) {
		for item := range crawl {
			start := time.Now()
			u, err := url.Parse(item.URL())
			if err != nil {
				item.Reject(false)
				log.Warningln("Couldn't crawl, invalid URL (rejecting):", item.URL(), err)
				continue
			}

			crawlCount, err := ttlHashSet.Get(u.String())
			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't confirm existence of URL (rejecting):", u.String(), err)
				continue
			}

			if crawlCount == maxCrawlRetries {
				item.Reject(false)
				log.Errorf("Aborting crawl of URL which has been retried %d times (rejecting): %s", maxCrawlRetries, u.String())
				continue
			}

			log.Debugln("Starting crawl of URL:", u)
			response, err := crawler.Crawl(u)
			if err != nil {
				switch err {
				case http_crawler.ErrRetryRequest5XX, http_crawler.ErrRetryRequest429:
					switch err {
					case http_crawler.ErrRetryRequest5XX:
						ttlHashSet.Incr(u.String())
					case http_crawler.ErrRetryRequest429:
						sleepTime := 5 * time.Second

						// Back off from crawling for a few seconds.
						log.Warningf("Sleeping for: %v. Received 429 HTTP status", sleepTime)
						time.Sleep(sleepTime)
					}

					item.Reject(true)
					log.Warningln("Couldn't crawl (requeueing):", u.String(), err)
				case http_crawler.ErrRedirect:
					setErr := ttlHashSet.Set(u.String(), AlreadyCrawled)
					if setErr != nil {
						log.Errorln("Couldn't mark item as already crawled:", u.String(), setErr)
					}

					item.Reject(false)
					// log at INFO because redirect URLs are not a concern
					log.Debugln("Couldn't crawl (rejecting):", u.String(), err)
				default:
					item.Reject(false)
					log.Warningln("Couldn't crawl (rejecting):", u.String(), err)
				}

				continue
			}

			item.Response = response

			if item.Response.AcceptedContentType() {
				extract <- item
			} else {
				if err = item.Ack(false); err != nil {
					log.Errorln("Ack failed (CrawlURL): ", item.URL())
				}

				err = ttlHashSet.Set(item.URL(), AlreadyCrawled)
				if err != nil {
					log.Errorln("Couldn't mark item as already crawled:", item.URL(), err)
				}
			}

			util.StatsDTiming("crawl_url", start, time.Now())
		}
	}

	for i := 1; i <= crawlerThreads; i++ {
		go crawlLoop(ttlHashSet, crawlChannel, extractChannel, crawler, maxCrawlRetries)
	}

	return extractChannel
}

func WriteItemToDisk(basePath string, crawlChannel <-chan *CrawlerMessageItem) <-chan *CrawlerMessageItem {
	extractChannel := make(chan *CrawlerMessageItem, 2)

	writeLoop := func(
		crawl <-chan *CrawlerMessageItem,
		extract chan<- *CrawlerMessageItem,
	) {
		for item := range crawl {
			start := time.Now()
			relativeFilePath, err := item.RelativeFilePath()

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't retrieve relative file path for item (rejecting):", item.URL(), err)
				continue
			}

			filePath := filepath.Join(basePath, relativeFilePath)
			dirPath := filepath.Dir(filePath)
			err = os.MkdirAll(dirPath, 0755)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't create directories for item (rejecting):", filePath, err)
				continue
			}

			err = ioutil.WriteFile(filePath, item.Response.Body, 0644)

			if err != nil {
				item.Reject(false)
				log.Errorln("Couldn't write to disk (rejecting):", filePath, err)
				continue
			}

			log.Debugln("Wrote URL body to disk for:", item.URL())

			contentType, err := item.Response.ParseContentType()
			if err != nil {
				log.Errorln("Couldn't determine Content-Type for item (rejecting):", item, err)
				item.Reject(false)
				continue
			}

			// Only send HTML pages for URL extraction. All other
			// pages should be written directly to disk and acknowledged.
			if contentType == http_crawler.HTML {
				extract <- item
			} else {
				item.Ack(false)
			}

			util.StatsDTiming("write_to_disk", start, time.Now())
		}
	}

	go writeLoop(crawlChannel, extractChannel)

	return extractChannel
}

func ExtractURLs(extractChannel <-chan *CrawlerMessageItem) (<-chan string, <-chan *CrawlerMessageItem) {
	publishChannel := make(chan string, 100)
	acknowledgeChannel := make(chan *CrawlerMessageItem, 1)

	extractLoop := func(
		extract <-chan *CrawlerMessageItem,
		publish chan<- string,
		acknowledge chan<- *CrawlerMessageItem,
	) {
		for item := range extract {
			start := time.Now()
			urls, err := item.ExtractURLs()
			if err != nil {
				item.Reject(false)
				log.Errorln("ExtractURLs (rejecting):", string(item.Body), err)

				continue
			}

			log.Debugln("Extracted URLs:", len(urls))

			for _, u := range urls {
				publish <- u.String()
			}

			acknowledge <- item

			util.StatsDTiming("extract_urls", start, time.Now())
		}
	}

	go extractLoop(extractChannel, publishChannel, acknowledgeChannel)

	return publishChannel, acknowledgeChannel
}

func PublishURLs(ttlHashSet *ttl_hash_set.TTLHashSet, queueManager *queue.Manager, publish <-chan string) {
	for url := range publish {
		start := time.Now()
		crawlCount, err := ttlHashSet.Get(url)

		if err != nil {
			log.Errorln("Couldn't check existence of URL:", url, err)
			continue
		}

		if crawlCount == AlreadyCrawled {
			log.Debugln("URL extracted from page already crawled:", url)
		} else if crawlCount == NotRecentlyCrawled {
			err = queueManager.Publish("#", "text/plain", url)
			if err != nil {
				log.Fatalln("Delivery failed:", url, err)
			}
		}

		util.StatsDGauge("publish_urls", int64(len(publish)))
		util.StatsDTiming("publish_urls", start, time.Now())
	}
}

func AcknowledgeItem(inbound <-chan *CrawlerMessageItem, ttlHashSet *ttl_hash_set.TTLHashSet) {
	for item := range inbound {
		start := time.Now()
		url := item.URL()

		err := ttlHashSet.Set(url, AlreadyCrawled)
		if err != nil {
			item.Reject(false)
			log.Errorln("Acknowledge failed (rejecting):", url, err)
			continue
		}

		if err = item.Ack(false); err != nil {
			log.Errorln("Ack failed (AcknowledgeItem): ", item.URL())
		}
		log.Debugln("Acknowledged:", url)

		util.StatsDTiming("acknowledge_item", start, time.Now())
	}
}
<|endoftext|>


<commit_before>package xmetrics

import (
	"fmt"

	"github.com/Comcast/webpa-common/logging"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	gokitprometheus "github.com/go-kit/kit/metrics/prometheus"
	"github.com/go-kit/kit/metrics/provider"
	"github.com/prometheus/client_golang/prometheus"
)

// PrometheusProvider is a Prometheus-specific version of go-kit's metrics.Provider.  Use this interface
// when interacting directly with Prometheus.
type PrometheusProvider interface {
	NewCounterVec(name string) *prometheus.CounterVec
	NewCounterVecEx(namespace, subsystem, name string) *prometheus.CounterVec

	NewGaugeVec(name string) *prometheus.GaugeVec
	NewGaugeVecEx(namespace, subsystem, name string) *prometheus.GaugeVec

	NewHistogramVec(name string) *prometheus.HistogramVec
	NewHistogramVecEx(namespace, subsystem, name string) *prometheus.HistogramVec

	NewSummaryVec(name string) *prometheus.SummaryVec
	NewSummaryVecEx(namespace, subsystem, name string) *prometheus.SummaryVec
}

// Registry is the core abstraction for this package.  It is a Prometheus gatherer and a go-kit metrics.Provider all in one.
//
// The Provider implementation works slightly differently than the go-kit implementation.  For any metric that is already defined
// the provider returns a new go-kit wrapper for that metric.  Additionally, new metrics (including ad hoc metrics) are cached
// and returned by subsequent calles to the Provider methods.
type Registry interface {
	PrometheusProvider
	provider.Provider
	prometheus.Gatherer
}

// registry is the internal Registry implementation
type registry struct {
	prometheus.Gatherer
	prometheus.Registerer

	namespace     string
	subsystem     string
	preregistered map[string]prometheus.Collector
}

func (r *registry) NewCounterVec(name string) *prometheus.CounterVec {
	return r.NewCounterVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewCounterVecEx(namespace, subsystem, name string) *prometheus.CounterVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if counterVec, ok := existing.(*prometheus.CounterVec); ok {
			return counterVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a counter", key))
	}

	counterVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(counterVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.CounterVec)
		} else {
			panic(err)
		}
	}

	return counterVec
}

func (r *registry) NewCounter(name string) metrics.Counter {
	return gokitprometheus.NewCounter(r.NewCounterVec(name))
}

func (r *registry) NewGaugeVec(name string) *prometheus.GaugeVec {
	return r.NewGaugeVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewGaugeVecEx(namespace, subsystem, name string) *prometheus.GaugeVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if gaugeVec, ok := existing.(*prometheus.GaugeVec); ok {
			return gaugeVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a gauge", key))
	}

	gaugeVec := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(gaugeVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.GaugeVec)
		} else {
			panic(err)
		}
	}

	return gaugeVec
}

func (r *registry) NewGauge(name string) metrics.Gauge {
	return gokitprometheus.NewGauge(r.NewGaugeVec(name))
}

func (r *registry) NewHistogramVec(name string) *prometheus.HistogramVec {
	return r.NewHistogramVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewHistogramVecEx(namespace, subsystem, name string) *prometheus.HistogramVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if histogramVec, ok := existing.(*prometheus.HistogramVec); ok {
			return histogramVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a histogram", key))
	}

	histogramVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(histogramVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			panic(err)
		}
	}

	return histogramVec
}

// NewHistogram has some special logic over and above the go-kit implementations.  This method allows a summary or
// a histogram as the underlying metric for the go-kit metrics.Histogram.
func (r *registry) NewHistogram(name string, _ int) metrics.Histogram {
	key := prometheus.BuildFQName(r.namespace, r.subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		switch e := existing.(type) {
		case *prometheus.HistogramVec:
			return gokitprometheus.NewHistogram(e)
		case *prometheus.SummaryVec:
			return gokitprometheus.NewSummary(e)
		default:
			panic(fmt.Errorf("The preregistered metric %s is not a histogram or a summary", key))
		}
	}

	return gokitprometheus.NewHistogram(r.NewHistogramVec(name))
}

func (r *registry) NewSummaryVec(name string) *prometheus.SummaryVec {
	return r.NewSummaryVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewSummaryVecEx(namespace, subsystem, name string) *prometheus.SummaryVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if summaryVec, ok := existing.(*prometheus.SummaryVec); ok {
			return summaryVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a histogram", key))
	}

	summaryVec := prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(summaryVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.SummaryVec)
		} else {
			panic(err)
		}
	}

	return summaryVec
}

// Stop is just here to implement metrics.Provider.  This method is a noop.
func (r *registry) Stop() {
}

// NewRegistry creates an xmetrics.Registry from an externally supplied set of Options and a set
// of modules, which are functions that just return Metrics to register.  The module functions are
// expected to come from application or library code, and are to define any built-in metrics.  Metrics
// present in the options will override any corresponding metric from modules.
func NewRegistry(o *Options, modules ...Module) (Registry, error) {
	logger := o.logger()

	// merge all the metrics, allowing options to override modules
	merger := NewMerger().
		Logger(logger).
		DefaultNamespace(o.namespace()).
		DefaultSubsystem(o.subsystem()).
		AddModules(false, modules...).
		AddModules(true, o.Module)

	if merger.Err() != nil {
		return nil, merger.Err()
	}

	var (
		pr = o.registry()
		r  = &registry{
			Registerer:    pr,
			Gatherer:      pr,
			namespace:     o.namespace(),
			subsystem:     o.subsystem(),
			preregistered: make(map[string]prometheus.Collector),
		}
	)

	for name, metric := range merger.Merged() {
		// merged metrics will have namespace and subsystem set appropriately
		metricLogger := log.With(
			logger,
			"name", metric.Name,
			"namespace", metric.Namespace,
			"subsystem", metric.Subsystem,
			"fqn", name,
		)

		metricLogger.Log(
			level.Key(), level.DebugValue(),
			logging.MessageKey(), "registering merged metric",
		)

		c, err := NewCollector(metric)
		if err != nil {
			metricLogger.Log(
				level.Key(), level.ErrorValue(),
				logging.MessageKey(), "unable to create collector for metric",
				logging.ErrorKey(), err,
			)

			return nil, err
		}

		if err := pr.Register(c); err != nil {
			metricLogger.Log(
				level.Key(), level.ErrorValue(),
				logging.MessageKey(), "unable to register collector for metric",
				logging.ErrorKey(), err,
			)

			return nil, err
		}

		r.preregistered[name] = c
	}

	return r, nil
}

// MustNewRegistry is like NewRegistry, except that it panics when NewRegistry would return an error.
func MustNewRegistry(o *Options, modules ...Module) Registry {
	r, err := NewRegistry(o, modules...)
	if err != nil {
		panic(err)
	}

	return r
}
<commit_msg>Forgot type<commit_after>package xmetrics

import (
	"fmt"

	"github.com/Comcast/webpa-common/logging"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/go-kit/kit/metrics"
	gokitprometheus "github.com/go-kit/kit/metrics/prometheus"
	"github.com/go-kit/kit/metrics/provider"
	"github.com/prometheus/client_golang/prometheus"
)

// PrometheusProvider is a Prometheus-specific version of go-kit's metrics.Provider.  Use this interface
// when interacting directly with Prometheus.
type PrometheusProvider interface {
	NewCounterVec(name string) *prometheus.CounterVec
	NewCounterVecEx(namespace, subsystem, name string) *prometheus.CounterVec

	NewGaugeVec(name string) *prometheus.GaugeVec
	NewGaugeVecEx(namespace, subsystem, name string) *prometheus.GaugeVec

	NewHistogramVec(name string) *prometheus.HistogramVec
	NewHistogramVecEx(namespace, subsystem, name string) *prometheus.HistogramVec

	NewSummaryVec(name string) *prometheus.SummaryVec
	NewSummaryVecEx(namespace, subsystem, name string) *prometheus.SummaryVec
}

// Registry is the core abstraction for this package.  It is a Prometheus gatherer and a go-kit metrics.Provider all in one.
//
// The Provider implementation works slightly differently than the go-kit implementation.  For any metric that is already defined
// the provider returns a new go-kit wrapper for that metric.  Additionally, new metrics (including ad hoc metrics) are cached
// and returned by subsequent calles to the Provider methods.
type Registry interface {
	PrometheusProvider
	provider.Provider
	prometheus.Gatherer
}

// registry is the internal Registry implementation
type registry struct {
	prometheus.Gatherer
	prometheus.Registerer

	namespace     string
	subsystem     string
	preregistered map[string]prometheus.Collector
}

func (r *registry) NewCounterVec(name string) *prometheus.CounterVec {
	return r.NewCounterVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewCounterVecEx(namespace, subsystem, name string) *prometheus.CounterVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if counterVec, ok := existing.(*prometheus.CounterVec); ok {
			return counterVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a counter", key))
	}

	counterVec := prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(counterVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.CounterVec)
		} else {
			panic(err)
		}
	}

	return counterVec
}

func (r *registry) NewCounter(name string) metrics.Counter {
	return gokitprometheus.NewCounter(r.NewCounterVec(name))
}

func (r *registry) NewGaugeVec(name string) *prometheus.GaugeVec {
	return r.NewGaugeVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewGaugeVecEx(namespace, subsystem, name string) *prometheus.GaugeVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if gaugeVec, ok := existing.(*prometheus.GaugeVec); ok {
			return gaugeVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a gauge", key))
	}

	gaugeVec := prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(gaugeVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.GaugeVec)
		} else {
			panic(err)
		}
	}

	return gaugeVec
}

func (r *registry) NewGauge(name string) metrics.Gauge {
	return gokitprometheus.NewGauge(r.NewGaugeVec(name))
}

func (r *registry) NewHistogramVec(name string) *prometheus.HistogramVec {
	return r.NewHistogramVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewHistogramVecEx(namespace, subsystem, name string) *prometheus.HistogramVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if histogramVec, ok := existing.(*prometheus.HistogramVec); ok {
			return histogramVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a histogram", key))
	}

	histogramVec := prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(histogramVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.HistogramVec)
		} else {
			panic(err)
		}
	}

	return histogramVec
}

// NewHistogram has some special logic over and above the go-kit implementations.  This method allows a summary or
// a histogram as the underlying metric for the go-kit metrics.Histogram.
func (r *registry) NewHistogram(name string, _ int) metrics.Histogram {
	key := prometheus.BuildFQName(r.namespace, r.subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		switch e := existing.(type) {
		case *prometheus.HistogramVec:
			return gokitprometheus.NewHistogram(e)
		case *prometheus.SummaryVec:
			return gokitprometheus.NewSummary(e)
		default:
			panic(fmt.Errorf("The preregistered metric %s is not a histogram or a summary", key))
		}
	}

	return gokitprometheus.NewHistogram(r.NewHistogramVec(name))
}

func (r *registry) NewSummaryVec(name string) *prometheus.SummaryVec {
	return r.NewSummaryVecEx(r.namespace, r.subsystem, name)
}

func (r *registry) NewSummaryVecEx(namespace, subsystem, name string) *prometheus.SummaryVec {
	key := prometheus.BuildFQName(namespace, subsystem, name)
	if existing, ok := r.preregistered[key]; ok {
		if summaryVec, ok := existing.(*prometheus.SummaryVec); ok {
			return summaryVec
		}

		panic(fmt.Errorf("The preregistered metric %s is not a histogram", key))
	}

	summaryVec := prometheus.NewSummaryVec(
		prometheus.SummaryOpts{
			Namespace: namespace,
			Subsystem: subsystem,
			Name:      name,
			Help:      name,
		},
		[]string{},
	)

	if err := r.Register(summaryVec); err != nil {
		if already, ok := err.(prometheus.AlreadyRegisteredError); ok {
			return already.ExistingCollector.(*prometheus.SummaryVec)
		} else {
			panic(err)
		}
	}

	return summaryVec
}

// Stop is just here to implement metrics.Provider.  This method is a noop.
func (r *registry) Stop() {
}

// NewRegistry creates an xmetrics.Registry from an externally supplied set of Options and a set
// of modules, which are functions that just return Metrics to register.  The module functions are
// expected to come from application or library code, and are to define any built-in metrics.  Metrics
// present in the options will override any corresponding metric from modules.
func NewRegistry(o *Options, modules ...Module) (Registry, error) {
	logger := o.logger()

	// merge all the metrics, allowing options to override modules
	merger := NewMerger().
		Logger(logger).
		DefaultNamespace(o.namespace()).
		DefaultSubsystem(o.subsystem()).
		AddModules(false, modules...).
		AddModules(true, o.Module)

	if merger.Err() != nil {
		return nil, merger.Err()
	}

	var (
		pr = o.registry()
		r  = &registry{
			Registerer:    pr,
			Gatherer:      pr,
			namespace:     o.namespace(),
			subsystem:     o.subsystem(),
			preregistered: make(map[string]prometheus.Collector),
		}
	)

	for name, metric := range merger.Merged() {
		// merged metrics will have namespace and subsystem set appropriately
		metricLogger := log.With(
			logger,
			"name", metric.Name,
			"namespace", metric.Namespace,
			"subsystem", metric.Subsystem,
			"type", metric.Type,
			"fqn", name,
		)

		metricLogger.Log(
			level.Key(), level.DebugValue(),
			logging.MessageKey(), "registering merged metric",
		)

		c, err := NewCollector(metric)
		if err != nil {
			metricLogger.Log(
				level.Key(), level.ErrorValue(),
				logging.MessageKey(), "unable to create collector for metric",
				logging.ErrorKey(), err,
			)

			return nil, err
		}

		if err := pr.Register(c); err != nil {
			metricLogger.Log(
				level.Key(), level.ErrorValue(),
				logging.MessageKey(), "unable to register collector for metric",
				logging.ErrorKey(), err,
			)

			return nil, err
		}

		r.preregistered[name] = c
	}

	return r, nil
}

// MustNewRegistry is like NewRegistry, except that it panics when NewRegistry would return an error.
func MustNewRegistry(o *Options, modules ...Module) Registry {
	r, err := NewRegistry(o, modules...)
	if err != nil {
		panic(err)
	}

	return r
}
<|endoftext|>


<commit_before>package xlsx

import (
	"bytes"
	"encoding/xml"
	. "gopkg.in/check.v1"
)

type WorksheetSuite struct{}

var _ = Suite(&WorksheetSuite{})

// Test we can succesfully unmarshal the sheetN.xml files within and
// XLSX file into an xlsxWorksheet struct (and it's related children).
func (w *WorksheetSuite) TestUnmarshallWorksheet(c *C) {
	var sheetxml = bytes.NewBufferString(
		`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
        <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
                   xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
          <sheetPr filterMode="false">
            <pageSetUpPr fitToPage="false"/>
          </sheetPr>
          <dimension ref="A1:B2"/>
          <sheetViews>
            <sheetView colorId="64"
                       defaultGridColor="true"
                       rightToLeft="false"
                       showFormulas="false"
                       showGridLines="true"
                       showOutlineSymbols="true"
                       showRowColHeaders="true"
                       showZeros="true"
                       tabSelected="true"
                       topLeftCell="A1"
                       view="normal"
                       windowProtection="false"
                       workbookViewId="0"
                       zoomScale="100"
                       zoomScaleNormal="100"
                       zoomScalePageLayoutView="100">
              <selection activeCell="B2"
                         activeCellId="0"
                         pane="topLeft"
                         sqref="B2"/>
            </sheetView>
          </sheetViews>
          <sheetFormatPr defaultRowHeight="15" defaultColWidth="8">
          </sheetFormatPr>
          <cols>
            <col collapsed="false"
                 hidden="false"
                 max="1025"
                 min="1"
                 style="0"
                 width="10.5748987854251"/>
          </cols>
          <sheetData>
            <row collapsed="false"
                 customFormat="false"
                 customHeight="false"
                 hidden="false"
                 ht="14.9"
                 outlineLevel="0"
                 r="1">
              <c r="A1"
                 s="1"
                 t="s">
                <v>0</v>
              </c>
              <c r="B1"
                 s="0"
                 t="s">
                <v>1</v>
              </c>
            </row>
            <row collapsed="false"
                 customFormat="false"
                 customHeight="false"
                 hidden="false"
                 ht="14.9"
                 outlineLevel="0"
                 r="2">
              <c r="A2"
                 s="0"
                 t="s">
                <v>2</v>
              </c>
              <c r="B2"
                 s="2"
                 t="s">
                <v>3</v>
              </c>
            </row>
          </sheetData>
          <printOptions headings="false"
                        gridLines="false"
                        gridLinesSet="true"
                        horizontalCentered="false"
                        verticalCentered="false"/>
          <pageMargins left="0.7"
                       right="0.7"
                       top="0.7875"
                       bottom="0.7875"
                       header="0.511805555555555"
                       footer="0.511805555555555"/>
          <pageSetup blackAndWhite="false"
                     cellComments="none"
                     copies="1"
                     draft="false"
                     firstPageNumber="0"
                     fitToHeight="1"
                     fitToWidth="1"
                     horizontalDpi="300"
                     orientation="portrait"
                     pageOrder="downThenOver"
                     paperSize="9"
                     scale="100"
                     useFirstPageNumber="false"
                     usePrinterDefaults="false"
                     verticalDpi="300"/>
          <headerFooter differentFirst="false"
                        differentOddEven="false">
            <oddHeader>
            </oddHeader>
            <oddFooter>
            </oddFooter>
          </headerFooter>
        </worksheet>`)
	worksheet := new(xlsxWorksheet)
	err := xml.NewDecoder(sheetxml).Decode(worksheet)
	c.Assert(err, IsNil)
	c.Assert(worksheet.Dimension.Ref, Equals, "A1:B2")
	c.Assert(worksheet.SheetData.Row, HasLen, 2)
	c.Assert(worksheet.SheetFormatPr.DefaultRowHeight, Equals, 15.0)
	c.Assert(worksheet.SheetFormatPr.DefaultColWidth, Equals, 8.0)
	row := worksheet.SheetData.Row[0]
	c.Assert(row.R, Equals, 1)
	c.Assert(row.C, HasLen, 2)
	cell := row.C[0]
	c.Assert(cell.R, Equals, "A1")
	c.Assert(cell.T, Equals, "s")
	c.Assert(cell.V, Equals, "0")
}
<commit_msg>Assert that MergeCells is correctly read from sheet.xml<commit_after>package xlsx

import (
	"bytes"
	"encoding/xml"

	. "gopkg.in/check.v1"
)

type WorksheetSuite struct{}

var _ = Suite(&WorksheetSuite{})

// Test we can succesfully unmarshal the sheetN.xml files within and
// XLSX file into an xlsxWorksheet struct (and it's related children).
func (w *WorksheetSuite) TestUnmarshallWorksheet(c *C) {
	var sheetxml = bytes.NewBufferString(
		`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
        <worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
                   xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
          <sheetPr filterMode="false">
            <pageSetUpPr fitToPage="false"/>
          </sheetPr>
          <dimension ref="A1:B2"/>
          <sheetViews>
            <sheetView colorId="64"
                       defaultGridColor="true"
                       rightToLeft="false"
                       showFormulas="false"
                       showGridLines="true"
                       showOutlineSymbols="true"
                       showRowColHeaders="true"
                       showZeros="true"
                       tabSelected="true"
                       topLeftCell="A1"
                       view="normal"
                       windowProtection="false"
                       workbookViewId="0"
                       zoomScale="100"
                       zoomScaleNormal="100"
                       zoomScalePageLayoutView="100">
              <selection activeCell="B2"
                         activeCellId="0"
                         pane="topLeft"
                         sqref="B2"/>
            </sheetView>
          </sheetViews>
          <sheetFormatPr defaultRowHeight="15" defaultColWidth="8">
          </sheetFormatPr>
          <cols>
            <col collapsed="false"
                 hidden="false"
                 max="1025"
                 min="1"
                 style="0"
                 width="10.5748987854251"/>
          </cols>
          <sheetData>
            <row collapsed="false"
                 customFormat="false"
                 customHeight="false"
                 hidden="false"
                 ht="14.9"
                 outlineLevel="0"
                 r="1">
              <c r="A1"
                 s="1"
                 t="s">
                <v>0</v>
              </c>
              <c r="B1"
                 s="0"
                 t="s">
                <v>1</v>
              </c>
            </row>
            <row collapsed="false"
                 customFormat="false"
                 customHeight="false"
                 hidden="false"
                 ht="14.9"
                 outlineLevel="0"
                 r="2">
              <c r="A2"
                 s="0"
                 t="s">
                <v>2</v>
              </c>
              <c r="B2"
                 s="2"
                 t="s">
                <v>3</v>
              </c>
            </row>
          </sheetData>
          <printOptions headings="false"
                        gridLines="false"
                        gridLinesSet="true"
                        horizontalCentered="false"
                        verticalCentered="false"/>
          <pageMargins left="0.7"
                       right="0.7"
                       top="0.7875"
                       bottom="0.7875"
                       header="0.511805555555555"
                       footer="0.511805555555555"/>
          <pageSetup blackAndWhite="false"
                     cellComments="none"
                     copies="1"
                     draft="false"
                     firstPageNumber="0"
                     fitToHeight="1"
                     fitToWidth="1"
                     horizontalDpi="300"
                     orientation="portrait"
                     pageOrder="downThenOver"
                     paperSize="9"
                     scale="100"
                     useFirstPageNumber="false"
                     usePrinterDefaults="false"
                     verticalDpi="300"/>
          <headerFooter differentFirst="false"
                        differentOddEven="false">
            <oddHeader>
            </oddHeader>
            <oddFooter>
            </oddFooter>
          </headerFooter>
        </worksheet>`)
	worksheet := new(xlsxWorksheet)
	err := xml.NewDecoder(sheetxml).Decode(worksheet)
	c.Assert(err, IsNil)
	c.Assert(worksheet.Dimension.Ref, Equals, "A1:B2")
	c.Assert(worksheet.SheetData.Row, HasLen, 2)
	c.Assert(worksheet.SheetFormatPr.DefaultRowHeight, Equals, 15.0)
	c.Assert(worksheet.SheetFormatPr.DefaultColWidth, Equals, 8.0)
	row := worksheet.SheetData.Row[0]
	c.Assert(row.R, Equals, 1)
	c.Assert(row.C, HasLen, 2)
	cell := row.C[0]
	c.Assert(cell.R, Equals, "A1")
	c.Assert(cell.T, Equals, "s")
	c.Assert(cell.V, Equals, "0")
}

// MergeCells information is correctly read from the worksheet.
func (w *WorksheetSuite) TestUnmarshallWorksheetWithMergeCells(c *C) {
	var sheetxml = bytes.NewBufferString(
		`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:mx="http://schemas.microsoft.com/office/mac/excel/2008/main" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:mv="urn:schemas-microsoft-com:mac:vml" xmlns:x14="http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" xmlns:x14ac="http://schemas.microsoft.com/office/spreadsheetml/2009/9/ac" xmlns:xm="http://schemas.microsoft.com/office/excel/2006/main">
  <sheetViews>
    <sheetView workbookViewId="0"/>
  </sheetViews>
  <sheetFormatPr customHeight="1" defaultColWidth="17.29" defaultRowHeight="15.0"/>
  <cols>
    <col customWidth="1" min="1" max="6" width="14.43"/>
  </cols>
  <sheetData>
    <row r="1" ht="15.75" customHeight="1">
      <c r="A1" s="1" t="s">
        <v>0</v>
      </c>
    </row>
    <row r="2" ht="15.75" customHeight="1">
      <c r="A2" s="1" t="s">
        <v>1</v>
      </c>
      <c r="B2" s="1" t="s">
        <v>2</v>
      </c>
    </row>
  </sheetData>
  <mergeCells count="1">
    <mergeCell ref="A1:B1"/>
  </mergeCells>
  <drawing r:id="rId1"/>
</worksheet>
`)
	worksheet := new(xlsxWorksheet)
	err := xml.NewDecoder(sheetxml).Decode(worksheet)
	c.Assert(err, IsNil)
	c.Assert(worksheet.MergeCells, NotNil)
	c.Assert(worksheet.MergeCells.Count, Equals, 1)
	mergeCell := worksheet.MergeCells.Cells[0]
	c.Assert(mergeCell.Ref, Equals, "A1:B1")
}
<|endoftext|>


<commit_before>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unquoteCStyle(ss), ""
	}
	return unquoteCStyle(ss[:tabi]), ss[tabi+1:]
}

// C-style name unquoting.
// it is from https://github.com/git/git/blob/77556354bb7ac50450e3b28999e3576969869068/quote.c#L345-L413
func unquoteCStyle(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unquote
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, line)
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := (lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew)
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<commit_msg>Redundant parentheses.<commit_after>package diff

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const (
	tokenDiffGit        = "diff --git" // diff --git a/sample.old.txt b/sample.new.txt
	tokenOldFile        = "---"        // --- sample.old.txt	2016-10-13 05:09:35.820791185 +0900
	tokenNewFile        = "+++"        // +++ sample.new.txt	2016-10-13 05:15:26.839245048 +0900
	tokenStartHunk      = "@@"         // @@ -1,3 +1,4 @@
	tokenUnchangedLine  = " "          //  unchanged, contextual line
	tokenAddedLine      = "+"          // +added line
	tokenDeletedLine    = "-"          // -deleted line
	tokenNoNewlineAtEOF = `\`          // \ No newline at end of file
)

var (
	// ErrNoNewFile represents error which there are no expected new file line.
	ErrNoNewFile = errors.New("no expected new file line") // +++ newfile
	// ErrNoHunks represents error which there are no expected hunks.
	ErrNoHunks = errors.New("no expected hunks") // @@ -1,3 +1,4 @@
)

// ErrInvalidHunkRange represents invalid line of hunk range. @@ -1,3 +1,4 @@
type ErrInvalidHunkRange struct {
	invalid string
}

func (e *ErrInvalidHunkRange) Error() string {
	return fmt.Sprintf("invalid hunk range: %v", e.invalid)
}

// ParseMultiFile parses a multi-file unified diff.
func ParseMultiFile(r io.Reader) ([]*FileDiff, error) {
	return (&multiFileParser{r: bufio.NewReader(r)}).Parse()
}

type multiFileParser struct {
	r *bufio.Reader
}

func (p *multiFileParser) Parse() ([]*FileDiff, error) {
	var fds []*FileDiff
	fp := &fileParser{r: p.r}
	for {
		fd, err := fp.Parse()
		if err != nil || fd == nil {
			break
		}
		fds = append(fds, fd)
	}
	return fds, nil
}

// ParseFile parses a file unified diff.
func ParseFile(r io.Reader) (*FileDiff, error) {
	return (&fileParser{r: bufio.NewReader(r)}).Parse()
}

type fileParser struct {
	r *bufio.Reader
}

func (p *fileParser) Parse() (*FileDiff, error) {
	fd := &FileDiff{}
	fd.Extended = parseExtendedHeader(p.r)
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		if err == io.EOF && len(fd.Extended) > 0 {
			return fd, nil
		}
		return nil, nil
	}
	if bytes.HasPrefix(b, []byte(tokenOldFile)) {
		// parse `--- sample.old.txt	2016-10-13 05:09:35.820791185 +0900`
		oldline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathOld, fd.TimeOld = parseFileHeader(oldline)
		// parse `+++ sample.new.txt	2016-10-13 05:09:35.820791185 +0900`
		if b, err := p.r.Peek(len(tokenNewFile)); err != nil || !bytes.HasPrefix(b, []byte(tokenNewFile)) {
			return nil, ErrNoNewFile
		}
		newline, _ := readline(p.r) // ignore err because we know it can read something
		fd.PathNew, fd.TimeNew = parseFileHeader(newline)
	}
	// parse hunks
	fd.Hunks, err = p.parseHunks()
	if err != nil {
		return nil, err
	}
	return fd, nil
}

func (p *fileParser) parseHunks() ([]*Hunk, error) {
	b, err := p.r.Peek(len(tokenOldFile))
	if err != nil {
		return nil, ErrNoHunks
	}
	if !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		b, err := p.r.Peek(len(tokenDiffGit))
		if err != nil {
			return nil, ErrNoHunks
		}
		if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
			// git diff may contain a file diff with empty hunks.
			// e.g. delete an empty file.
			return []*Hunk{}, nil
		}
		return nil, ErrNoHunks
	}
	var hunks []*Hunk
	hp := &hunkParser{r: p.r}
	for {
		h, err := hp.Parse()
		if err != nil {
			return nil, err
		}
		if h == nil {
			break
		}
		hunks = append(hunks, h)
	}
	return hunks, nil
}

// parseFileHeader parses file header line and returns filename and timestamp.
// timestamp may be empty.
func parseFileHeader(line string) (filename, timestamp string) {
	// strip `+++ ` or `--- `
	ss := line[len(tokenOldFile)+1:]
	tabi := strings.LastIndex(ss, "\t")
	if tabi == -1 {
		return unquoteCStyle(ss), ""
	}
	return unquoteCStyle(ss[:tabi]), ss[tabi+1:]
}

// C-style name unquoting.
// it is from https://github.com/git/git/blob/77556354bb7ac50450e3b28999e3576969869068/quote.c#L345-L413
func unquoteCStyle(str string) string {
	if !strings.HasPrefix(str, `"`) {
		// no need to unquote
		return str
	}
	str = strings.TrimPrefix(strings.TrimSuffix(str, `"`), `"`)

	res := make([]byte, 0, len(str))
	r := strings.NewReader(str)
LOOP:
	for {
		ch, err := r.ReadByte()
		if err != nil {
			break
		}
		if ch != '\\' {
			res = append(res, ch)
			continue
		}

		ch, err = r.ReadByte()
		if err != nil {
			break
		}
		switch ch {
		case 'a':
			res = append(res, '\a')
		case 'b':
			res = append(res, '\b')
		case 't':
			res = append(res, '\t')
		case 'n':
			res = append(res, '\n')
		case 'v':
			res = append(res, '\v')
		case 'f':
			res = append(res, '\f')
		case 'r':
			res = append(res, '\r')
		case '"':
			res = append(res, '"')
		case '\\':
			res = append(res, '\\')
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			if err := r.UnreadByte(); err != nil {
				break LOOP
			}
			var oct [3]byte
			if n, _ := r.Read(oct[:]); n < 3 {
				res = append(res, oct[:n]...)
				break LOOP
			}
			ch, err := strconv.ParseUint(string(oct[:]), 8, 8)
			if err != nil {
				res = append(res, oct[:]...)
				break
			}
			res = append(res, byte(ch))
		default:
			res = append(res, ch)
		}
	}

	return string(res)
}

func parseExtendedHeader(r *bufio.Reader) []string {
	var es []string
	b, err := r.Peek(len(tokenDiffGit))
	if err != nil {
		return nil
	}
	// if starts with 'diff --git', parse extended header
	if bytes.HasPrefix(b, []byte(tokenDiffGit)) {
		diffgitline, _ := readline(r) // ignore err because we know it can read something
		es = append(es, diffgitline)
		for {
			b, err := r.Peek(len(tokenDiffGit))
			if err != nil || bytes.HasPrefix(b, []byte(tokenOldFile)) || bytes.HasPrefix(b, []byte(tokenDiffGit)) {
				break
			}
			line, _ := readline(r)
			es = append(es, line)
		}
	}
	return es
}

type hunkParser struct {
	r        *bufio.Reader
	lnumdiff int
}

func (p *hunkParser) Parse() (*Hunk, error) {
	if b, err := p.r.Peek(len(tokenStartHunk)); err != nil || !bytes.HasPrefix(b, []byte(tokenStartHunk)) {
		return nil, nil
	}
	rangeline, _ := readline(p.r)
	hr, err := parseHunkRange(rangeline)
	if err != nil {
		return nil, err
	}
	hunk := &Hunk{
		StartLineOld:  hr.lold,
		LineLengthOld: hr.sold,
		StartLineNew:  hr.lnew,
		LineLengthNew: hr.snew,
		Section:       hr.section,
	}
	lold := hr.lold
	lnew := hr.lnew
endhunk:
	for !p.done(lold, lnew, hr) {
		b, err := p.r.Peek(1)
		if err != nil {
			break
		}
		token := string(b)
		switch token {
		case tokenUnchangedLine, tokenAddedLine, tokenDeletedLine:
			p.lnumdiff++
			l, _ := readline(p.r)
			line := &Line{Content: l[len(token):]} // trim first token
			switch token {
			case tokenUnchangedLine:
				line.Type = LineUnchanged
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				line.LnumNew = lnew
				lold++
				lnew++
			case tokenAddedLine:
				line.Type = LineAdded
				line.LnumDiff = p.lnumdiff
				line.LnumNew = lnew
				lnew++
			case tokenDeletedLine:
				line.Type = LineDeleted
				line.LnumDiff = p.lnumdiff
				line.LnumOld = lold
				lold++
			}
			hunk.Lines = append(hunk.Lines, line)
		case tokenNoNewlineAtEOF:
			// skip \ No newline at end of file. just consume line
			readline(p.r)
		default:
			break endhunk
		}
	}
	p.lnumdiff++ // count up by an additional hunk
	return hunk, nil
}

func (p *hunkParser) done(lold, lnew int, hr *hunkrange) bool {
	end := lold >= hr.lold+hr.sold && lnew >= hr.lnew+hr.snew
	if b, err := p.r.Peek(1); err != nil || (string(b) != tokenNoNewlineAtEOF && end) {
		return true
	}
	return false
}

// @@ -l,s +l,s @@ optional section heading
type hunkrange struct {
	lold, sold, lnew, snew int
	section                string
}

// @@ -lold[,sold] +lnew[,snew] @@[ section]
// 0  1              2            3   4
func parseHunkRange(rangeline string) (*hunkrange, error) {
	ps := strings.SplitN(rangeline, " ", 5)
	invalidErr := &ErrInvalidHunkRange{invalid: rangeline}
	hunkrange := &hunkrange{}
	if len(ps) < 4 || ps[0] != "@@" || ps[3] != "@@" {
		return nil, invalidErr
	}
	old := ps[1] // -lold[,sold]
	if !strings.HasPrefix(old, "-") {
		return nil, invalidErr
	}
	lold, sold, err := parseLS(old[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lold = lold
	hunkrange.sold = sold
	new := ps[2] // +lnew[,snew]
	if !strings.HasPrefix(new, "+") {
		return nil, invalidErr
	}
	lnew, snew, err := parseLS(new[1:])
	if err != nil {
		return nil, invalidErr
	}
	hunkrange.lnew = lnew
	hunkrange.snew = snew
	if len(ps) == 5 {
		hunkrange.section = ps[4]
	}
	return hunkrange, nil
}

// l[,s]
func parseLS(ls string) (l, s int, err error) {
	ss := strings.SplitN(ls, ",", 2)
	l, err = strconv.Atoi(ss[0])
	if err != nil {
		return 0, 0, err
	}
	if len(ss) == 2 {
		s, err = strconv.Atoi(ss[1])
		if err != nil {
			return 0, 0, err
		}
	} else {
		s = 1
	}
	return l, s, nil
}

// readline reads lines from bufio.Reader with size limit. It consumes
// remaining content even if the line size reaches size limit.
func readline(r *bufio.Reader) (string, error) {
	line, isPrefix, err := r.ReadLine()
	if err != nil {
		return "", err
	}
	// consume all remaining line content
	for isPrefix {
		_, isPrefix, _ = r.ReadLine()
	}
	return string(line), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routing

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/matrix-org/dendrite/clientapi/httputil"
	"github.com/matrix-org/dendrite/clientapi/jsonerror"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/internal/eventutil"
	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/syncapi/types"
	"github.com/matrix-org/dendrite/userapi/api"

	"github.com/matrix-org/util"
)

// GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
func GetAccountData(
	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
	userID string, roomID string, dataType string,
) util.JSONResponse {
	if userID != device.UserID {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden("userID does not match the current user"),
		}
	}

	dataReq := api.QueryAccountDataRequest{
		UserID:   userID,
		DataType: dataType,
		RoomID:   roomID,
	}
	dataRes := api.QueryAccountDataResponse{}
	if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
		return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err))
	}

	var data json.RawMessage
	var ok bool
	if roomID != "" {
		data, ok = dataRes.RoomAccountData[roomID][dataType]
	} else {
		data, ok = dataRes.GlobalAccountData[dataType]
	}
	if ok {
		return util.JSONResponse{
			Code: http.StatusOK,
			JSON: data,
		}
	}

	return util.JSONResponse{
		Code: http.StatusNotFound,
		JSON: jsonerror.NotFound("data not found"),
	}
}

// SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
func SaveAccountData(
	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
) util.JSONResponse {
	if userID != device.UserID {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden("userID does not match the current user"),
		}
	}

	defer req.Body.Close() // nolint: errcheck

	if req.Body == http.NoBody {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.NotJSON("Content not JSON"),
		}
	}

	if dataType == "m.fully_read" {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden("Unable to set read marker"),
		}
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
		return jsonerror.InternalServerError()
	}

	if !json.Valid(body) {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("Bad JSON content"),
		}
	}

	dataReq := api.InputAccountDataRequest{
		UserID:      userID,
		DataType:    dataType,
		RoomID:      roomID,
		AccountData: json.RawMessage(body),
	}
	dataRes := api.InputAccountDataResponse{}
	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
		return util.ErrorResponse(err)
	}

	var ignoredUsers *types.IgnoredUsers
	if dataType == "m.ignored_user_list" {
		ignoredUsers = &types.IgnoredUsers{}
		_ = json.Unmarshal(body, ignoredUsers)
	}

	// TODO: user API should do this since it's account data
	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
		return jsonerror.InternalServerError()
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: struct{}{},
	}
}

type fullyReadEvent struct {
	EventID string `json:"event_id"`
}

// SaveReadMarker implements POST /rooms/{roomId}/read_markers
func SaveReadMarker(
	req *http.Request,
	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
) util.JSONResponse {
	// Verify that the user is a member of this room
	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
	if resErr != nil {
		return *resErr
	}

	var r eventutil.ReadMarkerJSON
	resErr = httputil.UnmarshalJSONRequest(req, &r)
	if resErr != nil {
		return *resErr
	}

	if r.FullyRead == "" {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"),
		}
	}

	data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
	if err != nil {
		return jsonerror.InternalServerError()
	}

	dataReq := api.InputAccountDataRequest{
		UserID:      device.UserID,
		DataType:    "m.fully_read",
		RoomID:      roomID,
		AccountData: data,
	}
	dataRes := api.InputAccountDataResponse{}
	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
		return util.ErrorResponse(err)
	}

	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
		return jsonerror.InternalServerError()
	}

	// Handle the read receipt that may be included in the read marker
	if r.Read != "" {
		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: struct{}{},
	}
}
<commit_msg>Prevent setting push rules via account data endpoint (#2333)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routing

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/matrix-org/dendrite/clientapi/httputil"
	"github.com/matrix-org/dendrite/clientapi/jsonerror"
	"github.com/matrix-org/dendrite/clientapi/producers"
	"github.com/matrix-org/dendrite/internal/eventutil"
	roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/syncapi/types"
	"github.com/matrix-org/dendrite/userapi/api"

	"github.com/matrix-org/util"
)

// GetAccountData implements GET /user/{userId}/[rooms/{roomid}/]account_data/{type}
func GetAccountData(
	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
	userID string, roomID string, dataType string,
) util.JSONResponse {
	if userID != device.UserID {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden("userID does not match the current user"),
		}
	}

	dataReq := api.QueryAccountDataRequest{
		UserID:   userID,
		DataType: dataType,
		RoomID:   roomID,
	}
	dataRes := api.QueryAccountDataResponse{}
	if err := userAPI.QueryAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.QueryAccountData failed")
		return util.ErrorResponse(fmt.Errorf("userAPI.QueryAccountData: %w", err))
	}

	var data json.RawMessage
	var ok bool
	if roomID != "" {
		data, ok = dataRes.RoomAccountData[roomID][dataType]
	} else {
		data, ok = dataRes.GlobalAccountData[dataType]
	}
	if ok {
		return util.JSONResponse{
			Code: http.StatusOK,
			JSON: data,
		}
	}

	return util.JSONResponse{
		Code: http.StatusNotFound,
		JSON: jsonerror.NotFound("data not found"),
	}
}

// SaveAccountData implements PUT /user/{userId}/[rooms/{roomId}/]account_data/{type}
func SaveAccountData(
	req *http.Request, userAPI api.UserInternalAPI, device *api.Device,
	userID string, roomID string, dataType string, syncProducer *producers.SyncAPIProducer,
) util.JSONResponse {
	if userID != device.UserID {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden("userID does not match the current user"),
		}
	}

	defer req.Body.Close() // nolint: errcheck

	if req.Body == http.NoBody {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.NotJSON("Content not JSON"),
		}
	}

	if dataType == "m.fully_read" || dataType == "m.push_rules" {
		return util.JSONResponse{
			Code: http.StatusForbidden,
			JSON: jsonerror.Forbidden(fmt.Sprintf("Unable to modify %q using this API", dataType)),
		}
	}

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("ioutil.ReadAll failed")
		return jsonerror.InternalServerError()
	}

	if !json.Valid(body) {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("Bad JSON content"),
		}
	}

	dataReq := api.InputAccountDataRequest{
		UserID:      userID,
		DataType:    dataType,
		RoomID:      roomID,
		AccountData: json.RawMessage(body),
	}
	dataRes := api.InputAccountDataResponse{}
	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
		return util.ErrorResponse(err)
	}

	var ignoredUsers *types.IgnoredUsers
	if dataType == "m.ignored_user_list" {
		ignoredUsers = &types.IgnoredUsers{}
		_ = json.Unmarshal(body, ignoredUsers)
	}

	// TODO: user API should do this since it's account data
	if err := syncProducer.SendData(userID, roomID, dataType, nil, ignoredUsers); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
		return jsonerror.InternalServerError()
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: struct{}{},
	}
}

type fullyReadEvent struct {
	EventID string `json:"event_id"`
}

// SaveReadMarker implements POST /rooms/{roomId}/read_markers
func SaveReadMarker(
	req *http.Request,
	userAPI api.UserInternalAPI, rsAPI roomserverAPI.RoomserverInternalAPI,
	syncProducer *producers.SyncAPIProducer, device *api.Device, roomID string,
) util.JSONResponse {
	// Verify that the user is a member of this room
	resErr := checkMemberInRoom(req.Context(), rsAPI, device.UserID, roomID)
	if resErr != nil {
		return *resErr
	}

	var r eventutil.ReadMarkerJSON
	resErr = httputil.UnmarshalJSONRequest(req, &r)
	if resErr != nil {
		return *resErr
	}

	if r.FullyRead == "" {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("Missing m.fully_read mandatory field"),
		}
	}

	data, err := json.Marshal(fullyReadEvent{EventID: r.FullyRead})
	if err != nil {
		return jsonerror.InternalServerError()
	}

	dataReq := api.InputAccountDataRequest{
		UserID:      device.UserID,
		DataType:    "m.fully_read",
		RoomID:      roomID,
		AccountData: data,
	}
	dataRes := api.InputAccountDataResponse{}
	if err := userAPI.InputAccountData(req.Context(), &dataReq, &dataRes); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("userAPI.InputAccountData failed")
		return util.ErrorResponse(err)
	}

	if err := syncProducer.SendData(device.UserID, roomID, "m.fully_read", &r, nil); err != nil {
		util.GetLogger(req.Context()).WithError(err).Error("syncProducer.SendData failed")
		return jsonerror.InternalServerError()
	}

	// Handle the read receipt that may be included in the read marker
	if r.Read != "" {
		return SetReceipt(req, syncProducer, device, roomID, "m.read", r.Read)
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: struct{}{},
	}
}
<|endoftext|>


<commit_before>// Package valida implements JMBG, MBS, Municipal ID and OIB numbers validations.
package valida

import "strconv"

// isValid validate input string.
func isValid(in string) bool {
	i, err := strconv.ParseInt(in, 10, 64)
	if i > 0 && err == nil {
		return true
	}
	return false
}

// JMBG validate JMBG number.
func JMBG(in string) bool {
	if len(in) != 13 || in == "0000000000000" {
		return false
	}
	// get first 12 chars
	digits := in[:len(in)-1]
	coef := 7
	zzz := 0
	for _, r := range digits {
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
		if coef == 2 {
			coef = 8
		}
		coef--
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}

	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 0 && last == 0
	ok2 := ost > 1 && ost < 11 && last == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// MBS validate MBS number.
func MBS(in string) bool {
	if (len(in) != 8 && len(in) != 12) ||
		in == "00000000" || in == "000000000000" {
		return false
	}
	// get first 7 chars
	digits := in[:7]
	zzz := 0
	for i, r := range digits {
		coef := 8 - i
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
	}
	// exit if control char not digit
	control, err := strconv.Atoi(string(in[7:8]))
	if err != nil {
		return false
	}
	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 1 && control == 0
	ok2 := ost > 1 && ost < 11 && control == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// MID validate Municipal ID number.
func MID(in string) bool {
	if len(in) != 4 || in == "0000" {
		return false
	}
	// get first 3 chars
	digits := in[:len(in)-1]
	zzz := 0
	for i, r := range digits {
		coef := 4 - i
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}
	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 1 && last == 0
	ok2 := ost > 1 && ost < 11 && last == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// OIB validate OIB number.
func OIB(in string) bool {
	if len(in) != 11 {
		return false
	}
	// get first 10 chars
	digits := in[:len(in)-1]
	//
	o := 10
	for _, r := range digits {
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		o += d
		o = o % 10
		if o == 0 {
			o = 10
		}
		o *= 2
		o = o % 11
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}
	// calc control char
	ctrl := 11 - o
	if ctrl == 10 {
		ctrl = 0
	}
	return ctrl == last
}
<commit_msg>Add isValid into JMBG,MBS,MID<commit_after>// Package valida implements JMBG, MBS, Municipal ID and OIB numbers validations.
package valida

import "strconv"

// isValid validate input string.
func isValid(in string) bool {
	i, err := strconv.ParseInt(in, 10, 64)
	if i > 0 && err == nil {
		return true
	}
	return false
}

// JMBG validate JMBG number.
func JMBG(in string) bool {
	if len(in) != 13 || !isValid(in) {
		return false
	}
	// get first 12 chars
	digits := in[:len(in)-1]
	coef := 7
	zzz := 0
	for _, r := range digits {
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
		if coef == 2 {
			coef = 8
		}
		coef--
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}

	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 0 && last == 0
	ok2 := ost > 1 && ost < 11 && last == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// MBS validate MBS number.
func MBS(in string) bool {
	if (len(in) != 8 && len(in) != 12) || !isValid(in) {
		return false
	}
	// get first 7 chars
	digits := in[:7]
	zzz := 0
	for i, r := range digits {
		coef := 8 - i
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
	}
	// exit if control char not digit
	control, err := strconv.Atoi(string(in[7:8]))
	if err != nil {
		return false
	}
	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 1 && control == 0
	ok2 := ost > 1 && ost < 11 && control == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// MID validate Municipal ID number.
func MID(in string) bool {
	if len(in) != 4 || !isValid(in) {
		return false
	}
	// get first 3 chars
	digits := in[:len(in)-1]
	zzz := 0
	for i, r := range digits {
		coef := 4 - i
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		zzz += d * coef
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}
	ost := zzz % 11
	raz := 11 - ost
	ok1 := ost == 1 && last == 0
	ok2 := ost > 1 && ost < 11 && last == raz
	if ok1 || ok2 {
		return true
	}
	return false
}

// OIB validate OIB number.
func OIB(in string) bool {
	if len(in) != 11 {
		return false
	}
	// get first 10 chars
	digits := in[:len(in)-1]
	//
	o := 10
	for _, r := range digits {
		// exit if char not digit
		d, err := strconv.Atoi(string(r))
		if err != nil {
			return false
		}
		o += d
		o = o % 10
		if o == 0 {
			o = 10
		}
		o *= 2
		o = o % 11
	}
	// exit if last char not digit
	last, err := strconv.Atoi(string(in[len(in)-1:]))
	if err != nil {
		return false
	}
	// calc control char
	ctrl := 11 - o
	if ctrl == 10 {
		ctrl = 0
	}
	return ctrl == last
}
<|endoftext|>


<commit_before>package victor

import (
	"fmt"
	"github.com/brettbuddin/victor/pkg/chat"
	_ "github.com/brettbuddin/victor/pkg/chat/campfire"
	_ "github.com/brettbuddin/victor/pkg/chat/shell"
	_ "github.com/brettbuddin/victor/pkg/chat/slack"
	"github.com/brettbuddin/victor/pkg/httpserver"
	"github.com/brettbuddin/victor/pkg/store"
	"github.com/gorilla/mux"
	"log"
	"os"
	"strings"
)

type Runner interface {
    Run() error
    Stop()
}

type Chatter interface {
    Name() string
	HandleFunc(string, HandlerFunc)
	Handle(string, Handler)
	Direct(string) string
	Receive(chat.Message)
	Chat() chat.Adapter
}

type Persister interface {
	Store() store.Store
}

type Robot interface {
    Runner
    Chatter
    Persister

    HTTP() *mux.Router
}

type robot struct {
	*dispatch
	name     string
	http     *httpserver.Server
	httpAddr string
	router   *mux.Router
	store    store.Store
	adapter  chat.Adapter
	incoming chan chat.Message
	stop     chan struct{}
}

// New returns a robot
func New(adapterName, robotName, httpAddr string) *robot {
	initFunc, err := chat.Load(adapterName)

	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	bot := &robot{
		name:     robotName,
		http:     httpserver.New(),
		store:    store.NewMemoryStore(),
		incoming: make(chan chat.Message),
		httpAddr: httpAddr,
		stop:     make(chan struct{}),
	}

	bot.dispatch = newDispatch(bot)
	bot.adapter = initFunc(bot)
	bot.router = handlers(bot)

	defaults(bot)
	return bot
}

// Direct wraps a regexp pattern in the necessary pattern
// for a direct command to the bot.
func (r *robot) Direct(exp string) string {
	return strings.Join([]string{
		"(?i)", // flags
		"\\A",  // begin
		"(?:(?:@)?" + r.name + "[:,]?\\s*|/)", // bot name
		"(?:" + exp + ")",                     // expression
		"\\z",                                 // end
	}, "")
}

func (r *robot) Receive(m chat.Message) {
	r.incoming <- m
}

// Run starts the robot.
func (r *robot) Run() error {
	go r.adapter.Run()
	go func() {
		for {
			select {
			case <-r.stop:
				return
			case m := <-r.incoming:
				if strings.ToLower(m.UserName()) != r.name {
					go r.ProcessMessage(m)
				}
			}
		}
	}()

	r.http.Handle("/", r.router)
	return r.http.ListenAndServe(r.httpAddr)
}

func (r *robot) Stop() {
	r.adapter.Stop()
	r.stop <- struct{}{}
	close(r.incoming)
	r.http.Stop()
}

func (r *robot) Name() string {
	return r.name
}

func (r *robot) Store() store.Store {
	return r.store
}

func (r *robot) HTTP() *mux.Router {
	return r.router
}

func (r *robot) Chat() chat.Adapter {
	return r.adapter
}

func OnlyAllow(userNames []string, action func(s State)) func(State) {
	return func(s State) {
		actual := s.Message().UserName()
		for _, name := range userNames {
			if name == actual {
				action(s)
				return
			}
		}

		s.Chat().Send(s.Message().ChannelID(), fmt.Sprintf("Sorry, %s. I can't let you do that.", actual))
	}
}
<commit_msg>Vim doesn't seem to be auto-fmting my code anymore.<commit_after>package victor

import (
	"fmt"
	"github.com/brettbuddin/victor/pkg/chat"
	_ "github.com/brettbuddin/victor/pkg/chat/campfire"
	_ "github.com/brettbuddin/victor/pkg/chat/shell"
	_ "github.com/brettbuddin/victor/pkg/chat/slack"
	"github.com/brettbuddin/victor/pkg/httpserver"
	"github.com/brettbuddin/victor/pkg/store"
	"github.com/gorilla/mux"
	"log"
	"os"
	"strings"
)

type Runner interface {
	Run() error
	Stop()
}

type Chatter interface {
	Name() string
	HandleFunc(string, HandlerFunc)
	Handle(string, Handler)
	Direct(string) string
	Receive(chat.Message)
	Chat() chat.Adapter
}

type Persister interface {
	Store() store.Store
}

type Robot interface {
	Runner
	Chatter
	Persister

	HTTP() *mux.Router
}

type robot struct {
	*dispatch
	name     string
	http     *httpserver.Server
	httpAddr string
	router   *mux.Router
	store    store.Store
	adapter  chat.Adapter
	incoming chan chat.Message
	stop     chan struct{}
}

// New returns a robot
func New(adapterName, robotName, httpAddr string) *robot {
	initFunc, err := chat.Load(adapterName)

	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	bot := &robot{
		name:     robotName,
		http:     httpserver.New(),
		store:    store.NewMemoryStore(),
		incoming: make(chan chat.Message),
		httpAddr: httpAddr,
		stop:     make(chan struct{}),
	}

	bot.dispatch = newDispatch(bot)
	bot.adapter = initFunc(bot)
	bot.router = handlers(bot)

	defaults(bot)
	return bot
}

// Direct wraps a regexp pattern in the necessary pattern
// for a direct command to the bot.
func (r *robot) Direct(exp string) string {
	return strings.Join([]string{
		"(?i)", // flags
		"\\A",  // begin
		"(?:(?:@)?" + r.name + "[:,]?\\s*|/)", // bot name
		"(?:" + exp + ")",                     // expression
		"\\z",                                 // end
	}, "")
}

func (r *robot) Receive(m chat.Message) {
	r.incoming <- m
}

// Run starts the robot.
func (r *robot) Run() error {
	go r.adapter.Run()
	go func() {
		for {
			select {
			case <-r.stop:
				return
			case m := <-r.incoming:
				if strings.ToLower(m.UserName()) != r.name {
					go r.ProcessMessage(m)
				}
			}
		}
	}()

	r.http.Handle("/", r.router)
	return r.http.ListenAndServe(r.httpAddr)
}

func (r *robot) Stop() {
	r.adapter.Stop()
	r.stop <- struct{}{}
	close(r.incoming)
	r.http.Stop()
}

func (r *robot) Name() string {
	return r.name
}

func (r *robot) Store() store.Store {
	return r.store
}

func (r *robot) HTTP() *mux.Router {
	return r.router
}

func (r *robot) Chat() chat.Adapter {
	return r.adapter
}

func OnlyAllow(userNames []string, action func(s State)) func(State) {
	return func(s State) {
		actual := s.Message().UserName()
		for _, name := range userNames {
			if name == actual {
				action(s)
				return
			}
		}

		s.Chat().Send(s.Message().ChannelID(), fmt.Sprintf("Sorry, %s. I can't let you do that.", actual))
	}
}
<|endoftext|>