<commit_before>package gotana

import (
	"encoding/csv"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
	"io"
	"net"
	"net/http"
	URL "net/url"
	"os"
	"strings"
	"sync"
	"time"
)

type SaveableItem interface {
	Validate() bool
	RecordData() []string
}

type recordWriter interface {
	Write(record []string) error
	Flush()
}

type ScrapingHandlerFunc func(ScrapedItem, chan<- SaveableItem)

const (
	REQUEST_LIMIT_MILLISECOND = 100
	TIMEOUT_DIALER            = time.Duration(time.Second * 30)
	TIMEOUT_REQUEST           = time.Duration(time.Second * 30)
	TIMEOUT_TLS               = time.Duration(time.Second * 10)
)

func GetHref(t html.Token) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	return
}

type Extension interface {
	ScraperStarted(scraper *Scraper)
	ScraperStopped(scraper *Scraper)
	ItemScraped(scraper *Scraper, item SaveableItem)
}

type Extractable interface {
	Extract(io.ReadCloser, func(string))
}

type LinkExtractor struct {
	Extractable
}

func (extractor *LinkExtractor) Extract(r io.ReadCloser, callback func(string)) {
	z := html.NewTokenizer(r)
	defer r.Close()

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			return
		case tt == html.StartTagToken:
			t := z.Token()

			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			ok, url := GetHref(t)
			if !ok {
				continue
			}
			callback(url)
		}
	}
}

type ScrapedItem struct {
	Url      string
	Response http.Response
	scraper  Scraper
}

func (proxy ScrapedItem) String() (result string) {
	result = fmt.Sprintf("Result of scraping: %s", proxy.Url)
	return
}

func (proxy ScrapedItem) CheckIfRedirected() bool {
	return proxy.Url != proxy.Response.Request.URL.String()
}

func (proxy ScrapedItem) finalResponseBody() (io.ReadCloser, error) {
	if proxy.CheckIfRedirected() {
		client := NewHTTPClient()
		response, err := client.Get(proxy.Response.Request.URL.String())
		if err != nil {
			return nil, err
		}
		return response.Body, nil
	}
	return proxy.Response.Body, nil
}

func (proxy ScrapedItem) HTMLDocument() (document *goquery.Document, err error) {
	responseBody, err := proxy.finalResponseBody()

	if err == nil {
		document, err = goquery.NewDocumentFromReader(responseBody)
	}

	return
}

type Runnable interface {
	Run() (err error)
}

type Engine struct {
	limitCrawl        int
	limitFail         int
	handler           ScrapingHandlerFunc
	finished          int
	scrapers          []*Scraper
	requestMiddleware []RequestMiddlewareFunc
	extensions        []Extension
	chDone            chan *Scraper
	chScraped         chan ScrapedItem
	chItems           chan SaveableItem
	TcpAddress        string
	OutFileName       string
	Meta              *EngineMeta
}

func (engine *Engine) SetHandler(handler ScrapingHandlerFunc) *Engine {
	engine.handler = handler
	return engine
}

func (engine *Engine) IncrFinishedCounter() {
	engine.finished += 1
}

func (engine Engine) Done() bool {
	return len(engine.scrapers) == engine.finished
}

func (engine *Engine) scrapingLoop() {
	Logger().Info("Starting scraping loop")

	f, writer := GetWriter(engine)

	if f != nil {
		defer f.Close()
	}

	for {
		select {
		case proxy, ok := <-engine.chScraped:
			if !ok {
				break
			}
			if engine.handler != nil {
				engine.handler(proxy, engine.chItems)
			}
			if proxy.scraper.handler != nil {
				proxy.scraper.handler(proxy, engine.chItems)
			}

		case scraper, ok := <-engine.chDone:
			Logger().Infof("Stopped %s", scraper)
			engine.IncrFinishedCounter()
			if !ok {
				break
			}
		case item, ok := <-engine.chItems:
			if !ok {
				break
			}
			SaveItem(item, writer)
		}
		if engine.Done() {
			break
		}
	}
}

func (engine *Engine) startTCPServer() {
	if engine.TcpAddress != "" {
		server := NewTCPServer(engine.TcpAddress, engine)
		server.Start()
	}
}

func (engine *Engine) Run() {
	defer engine.Cleanup()

	for _, scraper := range engine.scrapers {
		go scraper.Start()
	}

	go engine.startTCPServer()
	engine.scrapingLoop()
}

func (engine *Engine) StopScrapers() {
	for _, scraper := range engine.scrapers {
		go scraper.Stop()
	}
}

func (engine *Engine) Cleanup() {
	close(engine.chDone)
	close(engine.chScraped)
	close(engine.chItems)
}

func (engine *Engine) PushScraper(scrapers ...*Scraper) *Engine {
	for _, scraper := range scrapers {
		engine.Meta.ScraperStats[scraper.name] = NewScraperMeta()
		scraper.engine = engine
		Logger().Debugf("Attached new scraper %s", scraper)
	}
	engine.scrapers = append(engine.scrapers, scrapers...)
	return engine
}

func (engine *Engine) UseMiddleware(middleware ...RequestMiddlewareFunc) *Engine {
	engine.requestMiddleware = append(engine.requestMiddleware, middleware...)
	return engine
}

func (engine *Engine) UseExtension(extensions ...Extension) *Engine {
	engine.extensions = append(engine.extensions, extensions...)
	return engine
}

func (engine *Engine) FromConfig(config *SpiderConfig) *Engine {
	engine.TcpAddress = config.TcpAddress
	engine.OutFileName = config.OutFileName

	for _, data := range config.Spiders {
		extractor := defaultExtractor()
		switch data.Extractor {
		case "link":
			extractor = &LinkExtractor{}
		default:
			break
		}
		scraper := NewScraper(data.Name, data.Url, extractor)
		engine.PushScraper(scraper)
	}

	return engine
}

type Scraper struct {
	crawled      int
	successful   int
	failed       int
	handler      ScrapingHandlerFunc
	fetchMutex   *sync.Mutex
	crawledMutex *sync.Mutex
	name         string
	domain       string
	baseUrl      string
	CurrentUrl   string
	fetchedUrls  map[string]bool
	engine       *Engine
	extractor    Extractable
	chDone       chan struct{}
	chRequestUrl chan string
}

func (scraper *Scraper) MarkAsFetched(url string) {
	scraper.fetchMutex.Lock()
	defer scraper.fetchMutex.Unlock()

	scraper.CurrentUrl = url
	scraper.fetchedUrls[url] = true
}

func (scraper *Scraper) CheckIfShouldStop() (ok bool) {
	scraper.crawledMutex.Lock()
	defer scraper.crawledMutex.Unlock()
	stats := scraper.engine.Meta.ScraperStats[scraper.name]

	if stats.crawled == scraper.engine.limitCrawl {
		Logger().Warningf("Crawl limit exceeded: %s", scraper)
		ok = true
	} else if stats.failed == scraper.engine.limitFail {
		Logger().Warningf("Fail limit exceeeded: %s", scraper)
		ok = true
	} else if stats.failed == 1 && scraper.crawled == 1 {
		Logger().Warningf("Base URL is corrupted: %s", scraper)
		ok = true
	}
	return
}

func (scraper *Scraper) CheckIfFetched(url string) (ok bool) {
	scraper.fetchMutex.Lock()
	defer scraper.fetchMutex.Unlock()

	_, ok = scraper.fetchedUrls[url]
	return
}

func (scraper *Scraper) CheckUrl(sourceUrl string) (ok bool, url string) {
	if strings.Contains(sourceUrl, scraper.domain) && strings.Index(sourceUrl, "http") == 0 {
		url = sourceUrl
		ok = true
	} else if strings.Index(sourceUrl, "/") == 0 {
		url = scraper.baseUrl + sourceUrl
		ok = true
	}
	return
}

func (scraper *Scraper) RunExtractor(resp http.Response) {
	defer SilentRecover("EXTRACTOR")

	scraper.extractor.Extract(resp.Body, func(url string) {
		ok, url := scraper.CheckUrl(url)

		if ok {
			scraper.chRequestUrl <- url
		}
	})
}

func (scraper *Scraper) Stop() {
	Logger().Infof("Stopping %s", scraper)
	scraper.chDone <- struct{}{}
	scraper.engine.chDone <- scraper
}

func (scraper *Scraper) Start() {
	Logger().Infof("Starting: %s", scraper)
	scraper.chRequestUrl <- scraper.baseUrl

	limiter := time.Tick(time.Millisecond * REQUEST_LIMIT_MILLISECOND)

	for {
		select {
		case url := <-scraper.chRequestUrl:
			<-limiter
			go scraper.Fetch(url)
		case <-scraper.chDone:
			return
		}
	}
	return
}

func (scraper *Scraper) Notify(url string, resp *http.Response) {
	scraper.engine.chScraped <- NewResultProxy(url, *scraper, *resp)
}

func (engine *Engine) PrepareRequest(request *http.Request) *http.Request {
	for _, middleware := range engine.requestMiddleware {
		request = middleware(request)
	}
	return request
}

func (scraper *Scraper) Fetch(url string) (resp *http.Response, err error) {
	if ok := scraper.CheckIfFetched(url); ok {
		return
	}
	scraper.MarkAsFetched(url)

	Logger().Infof("Fetching: %s", url)
	tic := time.Now()

	req, _ := http.NewRequest("GET", url, nil)
	req = scraper.engine.PrepareRequest(req)

	resp, err = NewHTTPClient().Do(req)

	statusCode := 0
	if err == nil {
		statusCode = resp.StatusCode
	}

	Logger().Debugf("[%d]Request to %s took: %s", statusCode, url, time.Since(tic))

	isSuccessful := (err == nil)

	scraper.engine.Meta.UpdateStats(scraper, isSuccessful, req, resp)

	if err == nil {
		scraper.Notify(url, resp)
		scraper.RunExtractor(*resp)
	} else {
		Logger().Warningf("Failed to crawl %s", url)
		Logger().Warning(err)
	}

	if scraper.CheckIfShouldStop() {
		scraper.Stop()
	}
	return
}

func (scraper *Scraper) SetHandler(handler ScrapingHandlerFunc) *Scraper {
	scraper.handler = handler
	return scraper
}

func (scraper *Scraper) String() (result string) {
	stats := scraper.engine.Meta.ScraperStats[scraper.name]
	result = fmt.Sprintf("<Scraper: %s>. Crawled: %d, successful: %d, failed: %d.",
		scraper.domain, stats.crawled, stats.successful, stats.failed)
	return
}

func NewEngine() (r *Engine) {
	r = &Engine{
		Meta:       NewEngineMeta(),
		limitCrawl: 10000,
		limitFail:  500,
		finished:   0,
		chDone:     make(chan *Scraper),
		chScraped:  make(chan ScrapedItem),
		chItems:    make(chan SaveableItem, 10),
	}
	return
}

func NewScraper(name string, sourceUrl string, extractor Extractable) (s *Scraper) {
	parsed, err := URL.Parse(sourceUrl)
	if err != nil {
		Logger().Infof("Inappropriate URL: %s", sourceUrl)
		return
	}

	if extractor == nil {
		Logger().Warning("Switching to default extractor")
		extractor = defaultExtractor()
	}

	s = &Scraper{
		name:         name,
		domain:       parsed.Host,
		baseUrl:      sourceUrl,
		fetchedUrls:  make(map[string]bool),
		crawledMutex: &sync.Mutex{},
		fetchMutex:   &sync.Mutex{},
		extractor:    extractor,
		chDone:       make(chan struct{}),
		chRequestUrl: make(chan string, 5),
	}
	return
}

func NewResultProxy(url string, scraper Scraper, resp http.Response) ScrapedItem {
	return ScrapedItem{
		Response: resp,
		Url:      url,
		scraper:  scraper,
	}
}

func NewHTTPClient() (client *http.Client) {
	client = &http.Client{
		Timeout: TIMEOUT_REQUEST,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: TIMEOUT_DIALER,
			}).Dial,
			TLSHandshakeTimeout: TIMEOUT_TLS,
		},
	}
	return
}

func defaultExtractor() Extractable {
	return &LinkExtractor{}
}

func GetWriter(engine *Engine) (*os.File, recordWriter) {
	if f, err := os.OpenFile(engine.OutFileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666); err == nil && f != nil {
		switch {
		case strings.HasSuffix(engine.OutFileName, ".csv"):
			Logger().Infof("Using CSV writer.")
			return f, csv.NewWriter(f)
		default:
			Logger().Warningf("Cannot write to: %s. Unsupported extension.", engine.OutFileName)
			return nil, nil
		}

	}
	return nil, nil
}

func SaveItem(item SaveableItem, writer recordWriter) {
	if writer == nil {
		return
	}

	if !item.Validate() {
		Logger().Warning("Item is not valid. Skipping...")
		return
	}

	defer writer.Flush()
	writer.Write(item.RecordData())
}

type SpiderConfig struct {
	Project     string `required:"true"`
	TcpAddress  string
	OutFileName string
	Spiders     []struct {
		Extractor string
		Name      string `required:"true"`
		Url       string `required:"true"`
	}
}

func NewSpiderConfig(file string) (config *SpiderConfig) {
	config = &SpiderConfig{}
	ProcessFile(config, file)
	return
}
<commit_msg>Added notifyExtensions<commit_after>package gotana

import (
	"encoding/csv"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
	"io"
	"net"
	"net/http"
	URL "net/url"
	"os"
	"strings"
	"sync"
	"time"
)

type SaveableItem interface {
	Validate() bool
	RecordData() []string
}

type recordWriter interface {
	Write(record []string) error
	Flush()
}

type ScrapingHandlerFunc func(ScrapedItem, chan<- SaveableItem)

const (
	EVENT_SCRAPER_OPENED      = "SCRAPER_OPENED"
	EVENT_SCRAPER_CLOSED      = "SCRAPER_CLOSED"
	EVENT_SAVEABLE_EXTRACTED  = "SAVEABLE_EXTRACTED"
	REQUEST_LIMIT_MILLISECOND = 100
	TIMEOUT_DIALER            = time.Duration(time.Second * 30)
	TIMEOUT_REQUEST           = time.Duration(time.Second * 30)
	TIMEOUT_TLS               = time.Duration(time.Second * 10)
)

func GetHref(t html.Token) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == "href" {
			href = a.Val
			ok = true
		}
	}

	return
}

type Extension interface {
	ScraperStarted(scraper *Scraper)
	ScraperStopped(scraper *Scraper)
	ItemScraped(scraper *Scraper, item SaveableItem)
}

type Extractable interface {
	Extract(io.ReadCloser, func(string))
}

type LinkExtractor struct {
	Extractable
}

func (extractor *LinkExtractor) Extract(r io.ReadCloser, callback func(string)) {
	z := html.NewTokenizer(r)
	defer r.Close()

	for {
		tt := z.Next()

		switch {
		case tt == html.ErrorToken:
			return
		case tt == html.StartTagToken:
			t := z.Token()

			isAnchor := t.Data == "a"
			if !isAnchor {
				continue
			}

			ok, url := GetHref(t)
			if !ok {
				continue
			}
			callback(url)
		}
	}
}

type ScrapedItem struct {
	Url      string
	Response http.Response
	scraper  Scraper
}

func (proxy ScrapedItem) String() (result string) {
	result = fmt.Sprintf("Result of scraping: %s", proxy.Url)
	return
}

func (proxy ScrapedItem) CheckIfRedirected() bool {
	return proxy.Url != proxy.Response.Request.URL.String()
}

func (proxy ScrapedItem) finalResponseBody() (io.ReadCloser, error) {
	if proxy.CheckIfRedirected() {
		client := NewHTTPClient()
		response, err := client.Get(proxy.Response.Request.URL.String())
		if err != nil {
			return nil, err
		}
		return response.Body, nil
	}
	return proxy.Response.Body, nil
}

func (proxy ScrapedItem) HTMLDocument() (document *goquery.Document, err error) {
	responseBody, err := proxy.finalResponseBody()

	if err == nil {
		document, err = goquery.NewDocumentFromReader(responseBody)
	}

	return
}

type Runnable interface {
	Run() (err error)
}

type Engine struct {
	limitCrawl        int
	limitFail         int
	handler           ScrapingHandlerFunc
	finished          int
	scrapers          []*Scraper
	requestMiddleware []RequestMiddlewareFunc
	extensions        []Extension
	chDone            chan *Scraper
	chScraped         chan ScrapedItem
	chItems           chan SaveableItem
	TcpAddress        string
	OutFileName       string
	Meta              *EngineMeta
}

func (engine *Engine) notifyExtensions(event string) {
	Logger().Warning(event)
}

func (engine *Engine) SetHandler(handler ScrapingHandlerFunc) *Engine {
	engine.handler = handler
	return engine
}

func (engine *Engine) IncrFinishedCounter() {
	engine.finished += 1
}

func (engine Engine) Done() bool {
	return len(engine.scrapers) == engine.finished
}

func (engine *Engine) scrapingLoop() {
	Logger().Info("Starting scraping loop")

	f, writer := GetWriter(engine)

	if f != nil {
		defer f.Close()
	}

	for {
		select {
		case proxy, ok := <-engine.chScraped:
			if !ok {
				break
			}
			if engine.handler != nil {
				engine.handler(proxy, engine.chItems)
			}
			if proxy.scraper.handler != nil {
				proxy.scraper.handler(proxy, engine.chItems)
			}

		case scraper, ok := <-engine.chDone:
			Logger().Infof("Stopped %s", scraper)
			engine.IncrFinishedCounter()
			if !ok {
				break
			}
		case item, ok := <-engine.chItems:
			if !ok {
				break
			}
			engine.notifyExtensions(EVENT_SAVEABLE_EXTRACTED)
			SaveItem(item, writer)
		}
		if engine.Done() {
			break
		}
	}
}

func (engine *Engine) startTCPServer() {
	if engine.TcpAddress != "" {
		server := NewTCPServer(engine.TcpAddress, engine)
		server.Start()
	}
}

func (engine *Engine) Run() {
	defer engine.Cleanup()

	for _, scraper := range engine.scrapers {
		go scraper.Start()
	}

	go engine.startTCPServer()
	engine.scrapingLoop()
}

func (engine *Engine) StopScrapers() {
	for _, scraper := range engine.scrapers {
		go scraper.Stop()
	}
}

func (engine *Engine) Cleanup() {
	close(engine.chDone)
	close(engine.chScraped)
	close(engine.chItems)
}

func (engine *Engine) PushScraper(scrapers ...*Scraper) *Engine {
	for _, scraper := range scrapers {
		engine.Meta.ScraperStats[scraper.name] = NewScraperMeta()
		scraper.engine = engine
		Logger().Debugf("Attached new scraper %s", scraper)
	}
	engine.scrapers = append(engine.scrapers, scrapers...)
	return engine
}

func (engine *Engine) UseMiddleware(middleware ...RequestMiddlewareFunc) *Engine {
	engine.requestMiddleware = append(engine.requestMiddleware, middleware...)
	return engine
}

func (engine *Engine) UseExtension(extensions ...Extension) *Engine {
	engine.extensions = append(engine.extensions, extensions...)
	return engine
}

func (engine *Engine) FromConfig(config *SpiderConfig) *Engine {
	engine.TcpAddress = config.TcpAddress
	engine.OutFileName = config.OutFileName

	for _, data := range config.Spiders {
		extractor := defaultExtractor()
		switch data.Extractor {
		case "link":
			extractor = &LinkExtractor{}
		default:
			break
		}
		scraper := NewScraper(data.Name, data.Url, extractor)
		engine.PushScraper(scraper)
	}

	return engine
}

type Scraper struct {
	crawled      int
	successful   int
	failed       int
	handler      ScrapingHandlerFunc
	fetchMutex   *sync.Mutex
	crawledMutex *sync.Mutex
	name         string
	domain       string
	baseUrl      string
	CurrentUrl   string
	fetchedUrls  map[string]bool
	engine       *Engine
	extractor    Extractable
	chDone       chan struct{}
	chRequestUrl chan string
}

func (scraper *Scraper) MarkAsFetched(url string) {
	scraper.fetchMutex.Lock()
	defer scraper.fetchMutex.Unlock()

	scraper.CurrentUrl = url
	scraper.fetchedUrls[url] = true
}

func (scraper *Scraper) CheckIfShouldStop() (ok bool) {
	scraper.crawledMutex.Lock()
	defer scraper.crawledMutex.Unlock()
	stats := scraper.engine.Meta.ScraperStats[scraper.name]

	if stats.crawled == scraper.engine.limitCrawl {
		Logger().Warningf("Crawl limit exceeded: %s", scraper)
		ok = true
	} else if stats.failed == scraper.engine.limitFail {
		Logger().Warningf("Fail limit exceeeded: %s", scraper)
		ok = true
	} else if stats.failed == 1 && scraper.crawled == 1 {
		Logger().Warningf("Base URL is corrupted: %s", scraper)
		ok = true
	}
	return
}

func (scraper *Scraper) CheckIfFetched(url string) (ok bool) {
	scraper.fetchMutex.Lock()
	defer scraper.fetchMutex.Unlock()

	_, ok = scraper.fetchedUrls[url]
	return
}

func (scraper *Scraper) CheckUrl(sourceUrl string) (ok bool, url string) {
	if strings.Contains(sourceUrl, scraper.domain) && strings.Index(sourceUrl, "http") == 0 {
		url = sourceUrl
		ok = true
	} else if strings.Index(sourceUrl, "/") == 0 {
		url = scraper.baseUrl + sourceUrl
		ok = true
	}
	return
}

func (scraper *Scraper) RunExtractor(resp http.Response) {
	defer SilentRecover("EXTRACTOR")

	scraper.extractor.Extract(resp.Body, func(url string) {
		ok, url := scraper.CheckUrl(url)

		if ok {
			scraper.chRequestUrl <- url
		}
	})
}

func (scraper *Scraper) Stop() {
	Logger().Infof("Stopping %s", scraper)
	scraper.engine.notifyExtensions(EVENT_SCRAPER_CLOSED)

	scraper.chDone <- struct{}{}
	scraper.engine.chDone <- scraper
}

func (scraper *Scraper) Start() {
	Logger().Infof("Starting: %s", scraper)
	scraper.engine.notifyExtensions(EVENT_SCRAPER_OPENED)

	scraper.chRequestUrl <- scraper.baseUrl

	limiter := time.Tick(time.Millisecond * REQUEST_LIMIT_MILLISECOND)

	for {
		select {
		case url := <-scraper.chRequestUrl:
			<-limiter
			go scraper.Fetch(url)
		case <-scraper.chDone:
			return
		}
	}
	return
}

func (scraper *Scraper) Notify(url string, resp *http.Response) {
	scraper.engine.chScraped <- NewResultProxy(url, *scraper, *resp)
}

func (engine *Engine) PrepareRequest(request *http.Request) *http.Request {
	for _, middleware := range engine.requestMiddleware {
		request = middleware(request)
	}
	return request
}

func (scraper *Scraper) Fetch(url string) (resp *http.Response, err error) {
	if ok := scraper.CheckIfFetched(url); ok {
		return
	}
	scraper.MarkAsFetched(url)

	Logger().Infof("Fetching: %s", url)
	tic := time.Now()

	req, _ := http.NewRequest("GET", url, nil)
	req = scraper.engine.PrepareRequest(req)

	resp, err = NewHTTPClient().Do(req)

	statusCode := 0
	if err == nil {
		statusCode = resp.StatusCode
	}

	Logger().Debugf("[%d]Request to %s took: %s", statusCode, url, time.Since(tic))

	isSuccessful := (err == nil)

	scraper.engine.Meta.UpdateStats(scraper, isSuccessful, req, resp)

	if err == nil {
		scraper.Notify(url, resp)
		scraper.RunExtractor(*resp)
	} else {
		Logger().Warningf("Failed to crawl %s", url)
		Logger().Warning(err)
	}

	if scraper.CheckIfShouldStop() {
		scraper.Stop()
	}
	return
}

func (scraper *Scraper) SetHandler(handler ScrapingHandlerFunc) *Scraper {
	scraper.handler = handler
	return scraper
}

func (scraper *Scraper) String() (result string) {
	stats := scraper.engine.Meta.ScraperStats[scraper.name]
	result = fmt.Sprintf("<Scraper: %s>. Crawled: %d, successful: %d, failed: %d.",
		scraper.domain, stats.crawled, stats.successful, stats.failed)
	return
}

func NewEngine() (r *Engine) {
	r = &Engine{
		Meta:       NewEngineMeta(),
		limitCrawl: 10000,
		limitFail:  500,
		finished:   0,
		chDone:     make(chan *Scraper),
		chScraped:  make(chan ScrapedItem),
		chItems:    make(chan SaveableItem, 10),
	}
	return
}

func NewScraper(name string, sourceUrl string, extractor Extractable) (s *Scraper) {
	parsed, err := URL.Parse(sourceUrl)
	if err != nil {
		Logger().Infof("Inappropriate URL: %s", sourceUrl)
		return
	}

	if extractor == nil {
		Logger().Warning("Switching to default extractor")
		extractor = defaultExtractor()
	}

	s = &Scraper{
		name:         name,
		domain:       parsed.Host,
		baseUrl:      sourceUrl,
		fetchedUrls:  make(map[string]bool),
		crawledMutex: &sync.Mutex{},
		fetchMutex:   &sync.Mutex{},
		extractor:    extractor,
		chDone:       make(chan struct{}),
		chRequestUrl: make(chan string, 5),
	}
	return
}

func NewResultProxy(url string, scraper Scraper, resp http.Response) ScrapedItem {
	return ScrapedItem{
		Response: resp,
		Url:      url,
		scraper:  scraper,
	}
}

func NewHTTPClient() (client *http.Client) {
	client = &http.Client{
		Timeout: TIMEOUT_REQUEST,
		Transport: &http.Transport{
			Dial: (&net.Dialer{
				Timeout: TIMEOUT_DIALER,
			}).Dial,
			TLSHandshakeTimeout: TIMEOUT_TLS,
		},
	}
	return
}

func defaultExtractor() Extractable {
	return &LinkExtractor{}
}

func GetWriter(engine *Engine) (*os.File, recordWriter) {
	if f, err := os.OpenFile(engine.OutFileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666); err == nil && f != nil {
		switch {
		case strings.HasSuffix(engine.OutFileName, ".csv"):
			Logger().Infof("Using CSV writer.")
			return f, csv.NewWriter(f)
		default:
			Logger().Warningf("Cannot write to: %s. Unsupported extension.", engine.OutFileName)
			return nil, nil
		}

	}
	return nil, nil
}

func SaveItem(item SaveableItem, writer recordWriter) {
	if writer == nil {
		return
	}

	if !item.Validate() {
		Logger().Warning("Item is not valid. Skipping...")
		return
	}

	defer writer.Flush()
	writer.Write(item.RecordData())
}

type SpiderConfig struct {
	Project     string `required:"true"`
	TcpAddress  string
	OutFileName string
	Spiders     []struct {
		Extractor string
		Name      string `required:"true"`
		Url       string `required:"true"`
	}
}

func NewSpiderConfig(file string) (config *SpiderConfig) {
	config = &SpiderConfig{}
	ProcessFile(config, file)
	return
}
<|endoftext|>


<commit_before>package mailjet

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
)

// DebugLevel defines the verbosity of the debug.
var DebugLevel int

// These are the different level of debug.
const (
	LevelNone      = iota // No debug.
	LevelDebug            // Debug without body.
	LevelDebugFull        // Debug with body.
)

// User-Agent is formated as "UserAgentBase/UserAgentVersion;runtime.Version()".
const (
	UserAgentBase    = "mailjet-api-v3-go"
	UserAgentVersion = "2.0.1"
)

const (
	apiBase  = "https://api.mailjet.com/v3"
	apiPath  = "REST"
	dataPath = "DATA"
)

// createRequest is the main core function.
func createRequest(method string, url string,
	payload interface{}, onlyFields []string,
	options ...RequestOptions) (req *http.Request, err error) {

	body, err := convertPayload(payload, onlyFields)
	if err != nil {
		return req, fmt.Errorf("creating request: %s\n", err)
	}
	req, err = http.NewRequest(method, url, bytes.NewBuffer(body))
	if err != nil {
		return req, fmt.Errorf("creating request: %s\n", err)
	}
	for _, option := range options {
		option(req)
	}
	userAgent(req)
	req.Header.Add("Accept", "application/json")
	return req, err
}

// converPayload returns payload casted in []byte.
// If the payload is a structure, it's encoded to JSON.
func convertPayload(payload interface{}, onlyFields []string) (body []byte, err error) {
	if payload != nil {
		switch t := payload.(type) {
		case string:
			body = []byte(t)
		case []byte:
			body = t
		default:
			v := reflect.Indirect(reflect.ValueOf(payload))
			if v.Kind() == reflect.Ptr {
				return convertPayload(v.Interface(), onlyFields)
			} else if v.Kind() == reflect.Struct {
				body, err = json.Marshal(buildMap(v, onlyFields))
				if err != nil {
					return body, err
				}
			}
		}
		if DebugLevel == LevelDebugFull {
			log.Println("Body:", string(body))
		}
	}
	return body, err
}

// buildMap returns a map with fields specified in onlyFields (all fields if nil)
// and without the read_only fields.
func buildMap(v reflect.Value, onlyFields []string) map[string]interface{} {
	res := make(map[string]interface{})
	if onlyFields != nil {
		for _, onlyField := range onlyFields {
			fieldType, exist := v.Type().FieldByName(onlyField)
			if exist {
				addFieldToMap(true, fieldType, v.FieldByName(onlyField), res)
			}
		}
	} else {
		for i := 0; i < v.NumField(); i++ {
			addFieldToMap(false, v.Type().Field(i), v.Field(i), res)
		}
	}
	return res
}

func addFieldToMap(onlyField bool, fieldType reflect.StructField,
	fieldValue reflect.Value, res map[string]interface{}) {
	if fieldType.Tag.Get("mailjet") != "read_only" {
		name, second := parseTag(fieldType.Tag.Get("json"))
		if name == "" {
			name = fieldType.Name
		}
		if !onlyField && second == "omitempty" &&
			isEmptyValue(fieldValue) {
			return
		}
		res[name] = fieldValue.Interface()
	}
}

func parseTag(tag string) (string, string) {
	if idx := strings.Index(tag, ","); idx != -1 {
		return tag[:idx], tag[idx+1:]
	}
	return tag, ""
}

func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

// userAgent add the User-Agent value to the request header.
func userAgent(req *http.Request) {
	ua := fmt.Sprintf("%s/%s;%s",
		UserAgentBase,
		UserAgentVersion,
		runtime.Version(),
	)
	req.Header.Add("User-Agent", ua)
}

func buildURL(info *Request) string {
	tokens := []string{apiBase, apiPath, info.Resource}
	if info.ID != 0 {
		id := strconv.FormatInt(info.ID, 10)
		tokens = append(tokens, id)
	} else if info.AltID != "" {
		tokens = append(tokens, string(info.AltID))
	}
	if info.Action != "" {
		tokens = append(tokens, info.Action)
	}
	if info.ActionID != 0 {
		actionID := strconv.FormatInt(info.ActionID, 10)
		tokens = append(tokens, actionID)
	}
	return strings.Join(tokens, "/")
}

func buildDataURL(info *DataRequest) string {
	tokens := []string{apiBase, dataPath, info.SourceType}
	if info.SourceTypeID != 0 {
		id := strconv.FormatInt(info.SourceTypeID, 10)
		tokens = append(tokens, id)
	}
	if info.DataType != "" {
		tokens = append(tokens, info.DataType)
		if info.MimeType != "" {
			tokens = append(tokens, info.MimeType)
		}
	}
	if info.DataTypeID != 0 {
		DataTypeID := strconv.FormatInt(info.DataTypeID, 10)
		tokens = append(tokens, DataTypeID)
	} else if info.LastID == true {
		tokens = append(tokens, "LAST")
	}
	return strings.Join(tokens, "/")
}

// readJsonResult decodes the API response, returns Count and Total values
// and stores the Data in the value pointed to by data.
func readJSONResult(r io.Reader, data interface{}) (int, int, error) {
	if DebugLevel == LevelDebugFull {
		r = io.TeeReader(r, debugOut)
		log.Print("Body: ")
		defer fmt.Fprintln(debugOut)
	}

	var res RequestResult
	res.Data = &data
	err := json.NewDecoder(r).Decode(&res)
	if err != nil {
		return 0, 0, fmt.Errorf("Error decoding API response: %s", err)
	}
	return res.Count, res.Total, nil
}

// NbAttempt defines the number of attempt
// for a request as long as StatusCode == 500.
var NbAttempt = 5

// doRequest is called to execute the request. Authentification is set
// with the public key and the secret key specified in MailjetClient.
func (c *httpClient) doRequest(req *http.Request) (resp *http.Response, err error) {
	debugRequest(req) //DEBUG
	req.SetBasicAuth(c.apiKeyPublic, c.apiKeyPrivate)
	for attempt := 0; attempt < NbAttempt; attempt++ {
		if resp != nil {
			resp.Body.Close()
		}
		resp, err = c.client.Do(req)
		if err != nil || (resp != nil && resp.StatusCode != 500) {
			break
		}
	}
	defer debugResponse(resp) //DEBUG
	if err != nil {
		if resp != nil {
			resp.Body.Close()
		}
		return nil, fmt.Errorf("Error getting %s: %s", req.URL, err)
	}
	err = checkResponseError(resp)
	return resp, err
}

// checkResponseError returns response error if the statuscode is < 200 or >= 400.
func checkResponseError(resp *http.Response) error {
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		var mailjetErr RequestError
		err := json.NewDecoder(resp.Body).Decode(&mailjetErr)
		resp.Body.Close()
		if err != nil {
			return fmt.Errorf("Unexpected server response code: %d: %s", resp.StatusCode, err)
		}
		return fmt.Errorf("Unexpected server response code: %d: %s (%s)",
			resp.StatusCode, mailjetErr.ErrorMessage, mailjetErr.ErrorInfo)
	}
	return nil
}

// debugRequest is a custom dump of the request.
// Method used, final URl called, and Header content are logged.
func debugRequest(req *http.Request) {
	if DebugLevel > LevelNone && req != nil {
		log.Printf("Method used is: %s\n", req.Method)
		log.Printf("Final URL is: %s\n", req.URL)
		log.Printf("Header is: %s\n", req.Header)
	}
}

// debugResponse is a custom dump of the response.
// Status and Header content are logged.
func debugResponse(resp *http.Response) {
	if DebugLevel > LevelNone && resp != nil {
		log.Printf("Status is: %s\n", resp.Status)
		log.Printf("Header is: %s\n", resp.Header)
	}
}<commit_msg>rewritten readJSONResult method<commit_after>package mailjet

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"runtime"
	"strconv"
	"strings"
)

// DebugLevel defines the verbosity of the debug.
var DebugLevel int

// These are the different level of debug.
const (
	LevelNone      = iota // No debug.
	LevelDebug            // Debug without body.
	LevelDebugFull        // Debug with body.
)

// User-Agent is formated as "UserAgentBase/UserAgentVersion;runtime.Version()".
const (
	UserAgentBase    = "mailjet-api-v3-go"
	UserAgentVersion = "2.0.1"
)

const (
	apiBase  = "https://api.mailjet.com/v3"
	apiPath  = "REST"
	dataPath = "DATA"
)

// createRequest is the main core function.
func createRequest(method string, url string,
	payload interface{}, onlyFields []string,
	options ...RequestOptions) (req *http.Request, err error) {

	body, err := convertPayload(payload, onlyFields)
	if err != nil {
		return req, fmt.Errorf("creating request: %s\n", err)
	}
	req, err = http.NewRequest(method, url, bytes.NewBuffer(body))
	if err != nil {
		return req, fmt.Errorf("creating request: %s\n", err)
	}
	for _, option := range options {
		option(req)
	}
	userAgent(req)
	req.Header.Add("Accept", "application/json")
	return req, err
}

// converPayload returns payload casted in []byte.
// If the payload is a structure, it's encoded to JSON.
func convertPayload(payload interface{}, onlyFields []string) (body []byte, err error) {
	if payload != nil {
		switch t := payload.(type) {
		case string:
			body = []byte(t)
		case []byte:
			body = t
		default:
			v := reflect.Indirect(reflect.ValueOf(payload))
			if v.Kind() == reflect.Ptr {
				return convertPayload(v.Interface(), onlyFields)
			} else if v.Kind() == reflect.Struct {
				body, err = json.Marshal(buildMap(v, onlyFields))
				if err != nil {
					return body, err
				}
			}
		}
		if DebugLevel == LevelDebugFull {
			log.Println("Body:", string(body))
		}
	}
	return body, err
}

// buildMap returns a map with fields specified in onlyFields (all fields if nil)
// and without the read_only fields.
func buildMap(v reflect.Value, onlyFields []string) map[string]interface{} {
	res := make(map[string]interface{})
	if onlyFields != nil {
		for _, onlyField := range onlyFields {
			fieldType, exist := v.Type().FieldByName(onlyField)
			if exist {
				addFieldToMap(true, fieldType, v.FieldByName(onlyField), res)
			}
		}
	} else {
		for i := 0; i < v.NumField(); i++ {
			addFieldToMap(false, v.Type().Field(i), v.Field(i), res)
		}
	}
	return res
}

func addFieldToMap(onlyField bool, fieldType reflect.StructField,
	fieldValue reflect.Value, res map[string]interface{}) {
	if fieldType.Tag.Get("mailjet") != "read_only" {
		name, second := parseTag(fieldType.Tag.Get("json"))
		if name == "" {
			name = fieldType.Name
		}
		if !onlyField && second == "omitempty" &&
			isEmptyValue(fieldValue) {
			return
		}
		res[name] = fieldValue.Interface()
	}
}

func parseTag(tag string) (string, string) {
	if idx := strings.Index(tag, ","); idx != -1 {
		return tag[:idx], tag[idx+1:]
	}
	return tag, ""
}

func isEmptyValue(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
		return v.Len() == 0
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Interface, reflect.Ptr:
		return v.IsNil()
	}
	return false
}

// userAgent add the User-Agent value to the request header.
func userAgent(req *http.Request) {
	ua := fmt.Sprintf("%s/%s;%s",
		UserAgentBase,
		UserAgentVersion,
		runtime.Version(),
	)
	req.Header.Add("User-Agent", ua)
}

func buildURL(info *Request) string {
	tokens := []string{apiBase, apiPath, info.Resource}
	if info.ID != 0 {
		id := strconv.FormatInt(info.ID, 10)
		tokens = append(tokens, id)
	} else if info.AltID != "" {
		tokens = append(tokens, string(info.AltID))
	}
	if info.Action != "" {
		tokens = append(tokens, info.Action)
	}
	if info.ActionID != 0 {
		actionID := strconv.FormatInt(info.ActionID, 10)
		tokens = append(tokens, actionID)
	}
	return strings.Join(tokens, "/")
}

func buildDataURL(info *DataRequest) string {
	tokens := []string{apiBase, dataPath, info.SourceType}
	if info.SourceTypeID != 0 {
		id := strconv.FormatInt(info.SourceTypeID, 10)
		tokens = append(tokens, id)
	}
	if info.DataType != "" {
		tokens = append(tokens, info.DataType)
		if info.MimeType != "" {
			tokens = append(tokens, info.MimeType)
		}
	}
	if info.DataTypeID != 0 {
		DataTypeID := strconv.FormatInt(info.DataTypeID, 10)
		tokens = append(tokens, DataTypeID)
	} else if info.LastID == true {
		tokens = append(tokens, "LAST")
	}
	return strings.Join(tokens, "/")
}

// readJsonResult decodes the API response, returns Count and Total values
// and stores the Data in the value pointed to by data.
func readJSONResult(r io.Reader, data interface{}) (int, int, error) {
	var res RequestResult
	res.Data = &data

	jsonBlob, err := ioutil.ReadAll(r) // ReadAll and store in jsonBlob (mandatory if we want to unmarshal two times)
	if err != nil {
		return 0, 0, fmt.Errorf("Error reading API response: %s", err)
	}
	if DebugLevel == LevelDebugFull {
		log.Println("Body: ", string(jsonBlob)) // DEBUG
	}

	err = json.Unmarshal(jsonBlob, &res) // First try with the RequestResult struct
	if err != nil {
		return 0, 0, fmt.Errorf("Error decoding API response: %s", err)
	} else if res.Total == 0  { // No result
		err = json.Unmarshal(jsonBlob, &data) // Trying directly with struct specified in parameter
		if err != nil {
			return 0, 0, fmt.Errorf("Error decoding API response: %s", err)
		}
		return 0, 0, nil // Count and Total are undetermined
	}
	return res.Count, res.Total, nil
}

// NbAttempt defines the number of attempt
// for a request as long as StatusCode == 500.
var NbAttempt = 5

// doRequest is called to execute the request. Authentification is set
// with the public key and the secret key specified in MailjetClient.
func (c *httpClient) doRequest(req *http.Request) (resp *http.Response, err error) {
	debugRequest(req) //DEBUG
	req.SetBasicAuth(c.apiKeyPublic, c.apiKeyPrivate)
	for attempt := 0; attempt < NbAttempt; attempt++ {
		if resp != nil {
			resp.Body.Close()
		}
		resp, err = c.client.Do(req)
		if err != nil || (resp != nil && resp.StatusCode != 500) {
			break
		}
	}
	defer debugResponse(resp) //DEBUG
	if err != nil {
		if resp != nil {
			resp.Body.Close()
		}
		return nil, fmt.Errorf("Error getting %s: %s", req.URL, err)
	}
	err = checkResponseError(resp)
	return resp, err
}

// checkResponseError returns response error if the statuscode is < 200 or >= 400.
func checkResponseError(resp *http.Response) error {
	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
		var mailjetErr RequestError
		err := json.NewDecoder(resp.Body).Decode(&mailjetErr)
		resp.Body.Close()
		if err != nil {
			return fmt.Errorf("Unexpected server response code: %d: %s", resp.StatusCode, err)
		}
		return fmt.Errorf("Unexpected server response code: %d: %s (%s)",
			resp.StatusCode, mailjetErr.ErrorMessage, mailjetErr.ErrorInfo)
	}
	return nil
}

// debugRequest is a custom dump of the request.
// Method used, final URl called, and Header content are logged.
func debugRequest(req *http.Request) {
	if DebugLevel > LevelNone && req != nil {
		log.Printf("Method used is: %s\n", req.Method)
		log.Printf("Final URL is: %s\n", req.URL)
		log.Printf("Header is: %s\n", req.Header)
	}
}

// debugResponse is a custom dump of the response.
// Status and Header content are logged.
func debugResponse(resp *http.Response) {
	if DebugLevel > LevelNone && resp != nil {
		log.Printf("Status is: %s\n", resp.Status)
		log.Printf("Header is: %s\n", resp.Header)
	}
}<|endoftext|>


<commit_before>/*
Package transl translates struct fields and store translations
in the same struct.
*/
package transl

import (
	"database/sql/driver"
	"encoding/json"
	"golang.org/x/net/context"
	"golang.org/x/text/language"
	"reflect"
	"sync"
)

var defaultLanguageString = "en"
var defaultLanguageTag = language.English

// SetDefaults redefines default language string and tag
func SetDefaults(str string, tag language.Tag) {
	defaultLanguageString = str
	defaultLanguageTag = tag
}

// StringTable is a type for struct field to hold translations
// e.g. Translations{"en": map[string]string{"name": "John"}}
type StringTable map[string]map[string]string

// Scan unmarshals translations from JSON
func (m *StringTable) Scan(value interface{}) error {
	return json.Unmarshal(value.([]byte), m)
}

// Value marshals translations to JSON
func (m StringTable) Value() (driver.Value, error) {
	b, err := json.Marshal(m)
	if err != nil {
		return nil, err
	}
	return string(b), nil
}

// Translate fills fields of `target` struct with translated values
//
func Translate(ctx context.Context, target interface{}) {
	meta := metas.getStructMeta(target)
	if !meta.valid {
		return
	}

	structValue := reflect.Indirect(reflect.ValueOf(target))

	translations, ok := structValue.FieldByName("Translations").Interface().(StringTable)
	if !ok || len(translations) == 0 {
		return
	}

	targetLanguages, ok := AcceptedLanguagesFromContext(ctx)
	if !ok || len(targetLanguages) == 0 {
		targetLanguages = []language.Tag{defaultLanguageTag}
	}

	for _, trF := range meta.fields {
		f := structValue.FieldByName(trF.name)
		if f.IsValid() && f.CanSet() && f.Kind() == reflect.String {
			translateField(f, trF.key, translations, targetLanguages)
		}
	}
}

func translateField(field reflect.Value, fieldName string, translations StringTable, targetLanguages []language.Tag) {
	matcher := getMatcher(fieldName, translations)
	effectiveLang, _, _ := matcher.Match(targetLanguages...)
	field.SetString(translations[effectiveLang.String()][fieldName])
}

var matchers = map[string]language.Matcher{}
var matchersMutex sync.RWMutex

func getMatcher(fieldName string, translations StringTable) language.Matcher {
	var langs []language.Tag
	var langsKey string

	defaultFound := false
	v, ok := translations[defaultLanguageString]
	if ok {
		_, ok = v[fieldName]
		if ok {
			defaultFound = true
			langs = []language.Tag{defaultLanguageTag}
			langsKey = defaultLanguageString
		}
	}
	if !defaultFound {
		langs = []language.Tag{}
	}

	for lang, tr := range translations {
		_, ok = tr[fieldName]
		if ok {
			// default language already in slice if needed
			if lang != defaultLanguageString {
				langs = append(langs, *getTagByString(lang))
				langsKey += lang
			}
		}
	}

	matchersMutex.RLock()
	matcher, ok := matchers[langsKey]
	matchersMutex.RUnlock()

	if ok {
		return matcher
	}

	matcher = language.NewMatcher(langs)

	matchersMutex.Lock()
	matchers[langsKey] = matcher
	matchersMutex.Unlock()

	return matcher
}

var tags = map[string]language.Tag{}
var tagsMutex sync.RWMutex

func getTagByString(s string) *language.Tag {
	tagsMutex.RLock()
	tag, ok := tags[s]
	tagsMutex.RUnlock()

	if ok {
		return &tag
	}

	tag = language.Make(s)

	tagsMutex.Lock()
	tags[s] = tag
	tagsMutex.Unlock()

	return &tag
}
<commit_msg>Optimize language list building for cache hit case<commit_after>/*
Package transl translates struct fields and store translations
in the same struct.
*/
package transl

import (
	"bytes"
	"database/sql/driver"
	"encoding/json"
	"golang.org/x/net/context"
	"golang.org/x/text/language"
	"reflect"
	"sync"
)

var defaultLanguageString = "en"
var defaultLanguageTag = language.English

// SetDefaults redefines default language string and tag
func SetDefaults(str string, tag language.Tag) {
	defaultLanguageString = str
	defaultLanguageTag = tag
}

// StringTable is a type for struct field to hold translations
// e.g. Translations{"en": map[string]string{"name": "John"}}
type StringTable map[string]map[string]string

// Scan unmarshals translations from JSON
func (m *StringTable) Scan(value interface{}) error {
	return json.Unmarshal(value.([]byte), m)
}

// Value marshals translations to JSON
func (m StringTable) Value() (driver.Value, error) {
	b, err := json.Marshal(m)
	if err != nil {
		return nil, err
	}
	return string(b), nil
}

// Translate fills fields of `target` struct with translated values
//
func Translate(ctx context.Context, target interface{}) {
	meta := metas.getStructMeta(target)
	if !meta.valid {
		return
	}

	structValue := reflect.Indirect(reflect.ValueOf(target))

	translations, ok := structValue.FieldByName("Translations").Interface().(StringTable)
	if !ok || len(translations) == 0 {
		return
	}

	targetLanguages, ok := AcceptedLanguagesFromContext(ctx)
	if !ok || len(targetLanguages) == 0 {
		targetLanguages = []language.Tag{defaultLanguageTag}
	}

	for _, trF := range meta.fields {
		f := structValue.FieldByName(trF.name)
		if f.IsValid() && f.CanSet() && f.Kind() == reflect.String {
			translateField(f, trF.key, translations, targetLanguages)
		}
	}
}

func translateField(field reflect.Value, fieldName string, translations StringTable, targetLanguages []language.Tag) {
	matcher := getMatcher(fieldName, translations)
	effectiveLang, _, _ := matcher.Match(targetLanguages...)
	field.SetString(translations[effectiveLang.String()][fieldName])
}

var matchers = map[string]language.Matcher{}
var matchersMutex sync.RWMutex

func getMatcher(fieldName string, translations StringTable) language.Matcher {
	var langsKeyBuffer bytes.Buffer

	// Build languages string key
	defaultFound := false
	v, ok := translations[defaultLanguageString]
	if ok {
		_, ok = v[fieldName]
		if ok {
			langsKeyBuffer.WriteString(defaultLanguageString)
		}
	}

	for lang, tr := range translations {
		_, ok := tr[fieldName]

		if ok {
			if lang == defaultLanguageString {
				defaultFound = true
			} else {
				langsKeyBuffer.WriteString(lang)
			}
		}
	}
	langsKey := langsKeyBuffer.String()

	// Return cached matcher for that string key if it's set
	matchersMutex.RLock()
	matcher, ok := matchers[langsKey]
	matchersMutex.RUnlock()

	if ok {
		return matcher
	}

	// Cache missed. Lets create matcher and add it to cache
	var langs []language.Tag

	if defaultFound {
		langs = []language.Tag{defaultLanguageTag}
	} else {
		langs = []language.Tag{}
	}

	for lang, tr := range translations {
		_, ok = tr[fieldName]
		if ok {
			// default language already in slice if needed
			if lang != defaultLanguageString {
				langs = append(langs, *getTagByString(lang))
			}
		}
	}

	matcher = language.NewMatcher(langs)

	matchersMutex.Lock()
	matchers[langsKey] = matcher
	matchersMutex.Unlock()

	return matcher
}

var tags = map[string]language.Tag{}
var tagsMutex sync.RWMutex

func getTagByString(s string) *language.Tag {
	tagsMutex.RLock()
	tag, ok := tags[s]
	tagsMutex.RUnlock()

	if ok {
		return &tag
	}

	tag = language.Make(s)

	tagsMutex.Lock()
	tags[s] = tag
	tagsMutex.Unlock()

	return &tag
}
<|endoftext|>


<commit_before>/*
Package cors is net/http handler to handle CORS related requests
as defined by http://www.w3.org/TR/cors/

You can configure it by passing an option struct to cors.New:

    c := cors.New(cors.Options{
        AllowedOrigins: []string{"foo.com"},
        AllowedMethods: []string{"GET", "POST", "DELETE"},
        AllowCredentials: true,
    })

Then insert the handler in the chain:

    handler = c.Handler(handler)

See Options documentation for more options.

The resulting handler is a standard net/http handler.
*/
package cors

import (
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// Options is a configuration container to setup the CORS middleware.
type Options struct {
	// AllowedOrigins is a list of origins a cross-domain request can be executed from.
	// If the special "*" value is present in the list, all origins will be allowed.
	// An origin may contain a wildcard (*) to replace 0 or more characters
	// (i.e.: http://*.domain.com). Usage of wildcards implies a small performance penality.
	// Only one wildcard can be used per origin.
	// Default value is ["*"]
	AllowedOrigins []string
	// AllowOriginFunc is a custom function to validate the origin. It take the origin
	// as argument and returns true if allowed or false otherwise. If this option is
	// set, the content of AllowedOrigins is ignored.
	AllowOriginFunc func(origin string) bool
	// AllowedMethods is a list of methods the client is allowed to use with
	// cross-domain requests. Default value is simple methods (GET and POST)
	AllowedMethods []string
	// AllowedHeaders is list of non simple headers the client is allowed to use with
	// cross-domain requests.
	// If the special "*" value is present in the list, all headers will be allowed.
	// Default value is [] but "Origin" is always appended to the list.
	AllowedHeaders []string
	// ExposedHeaders indicates which headers are safe to expose to the API of a CORS
	// API specification
	ExposedHeaders []string
	// AllowCredentials indicates whether the request can include user credentials like
	// cookies, HTTP authentication or client side SSL certificates.
	AllowCredentials bool
	// MaxAge indicates how long (in seconds) the results of a preflight request
	// can be cached
	MaxAge int
	// OptionsPassthrough instructs preflight to let other potential next handlers to
	// process the OPTIONS method. Turn this on if your application handles OPTIONS.
	OptionsPassthrough bool
	// Debugging flag adds additional output to debug server side CORS issues
	Debug bool
}

// Cors http handler
type Cors struct {
	// Debug logger
	Log *log.Logger
	// Set to true when allowed origins contains a "*"
	allowedOriginsAll bool
	// Normalized list of plain allowed origins
	allowedOrigins []string
	// List of allowed origins containing wildcards
	allowedWOrigins []wildcard
	// Optional origin validator function
	allowOriginFunc func(origin string) bool
	// Set to true when allowed headers contains a "*"
	allowedHeadersAll bool
	// Normalized list of allowed headers
	allowedHeaders []string
	// Normalized list of allowed methods
	allowedMethods []string
	// Normalized list of exposed headers
	exposedHeaders    []string
	allowCredentials  bool
	maxAge            int
	optionPassthrough bool
}

// New creates a new Cors handler with the provided options.
func New(options Options) *Cors {
	c := &Cors{
		exposedHeaders:    convert(options.ExposedHeaders, http.CanonicalHeaderKey),
		allowOriginFunc:   options.AllowOriginFunc,
		allowCredentials:  options.AllowCredentials,
		maxAge:            options.MaxAge,
		optionPassthrough: options.OptionsPassthrough,
	}
	if options.Debug {
		c.Log = log.New(os.Stdout, "[cors] ", log.LstdFlags)
	}

	// Normalize options
	// Note: for origins and methods matching, the spec requires a case-sensitive matching.
	// As it may error prone, we chose to ignore the spec here.

	// Allowed Origins
	if len(options.AllowedOrigins) == 0 {
		// Default is all origins
		c.allowedOriginsAll = true
	} else {
		c.allowedOrigins = []string{}
		c.allowedWOrigins = []wildcard{}
		for _, origin := range options.AllowedOrigins {
			// Normalize
			origin = strings.ToLower(origin)
			if origin == "*" {
				// If "*" is present in the list, turn the whole list into a match all
				c.allowedOriginsAll = true
				c.allowedOrigins = nil
				c.allowedWOrigins = nil
				break
			} else if i := strings.IndexByte(origin, '*'); i >= 0 {
				// Split the origin in two: start and end string without the *
				w := wildcard{origin[0:i], origin[i+1 : len(origin)]}
				c.allowedWOrigins = append(c.allowedWOrigins, w)
			} else {
				c.allowedOrigins = append(c.allowedOrigins, origin)
			}
		}
	}

	// Allowed Headers
	if len(options.AllowedHeaders) == 0 {
		// Use sensible defaults
		c.allowedHeaders = []string{"Origin", "Accept", "Content-Type"}
	} else {
		// Origin is always appended as some browsers will always request for this header at preflight
		c.allowedHeaders = convert(append(options.AllowedHeaders, "Origin"), http.CanonicalHeaderKey)
		for _, h := range options.AllowedHeaders {
			if h == "*" {
				c.allowedHeadersAll = true
				c.allowedHeaders = nil
				break
			}
		}
	}

	// Allowed Methods
	if len(options.AllowedMethods) == 0 {
		// Default is spec's "simple" methods
		c.allowedMethods = []string{"GET", "POST"}
	} else {
		c.allowedMethods = convert(options.AllowedMethods, strings.ToUpper)
	}

	return c
}

// Default creates a new Cors handler with default options
func Default() *Cors {
	return New(Options{})
}

// Handler apply the CORS specification on the request, and add relevant CORS headers
// as necessary.
func (c *Cors) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "OPTIONS" {
			c.logf("Handler: Preflight request")
			c.handlePreflight(w, r)
			// Preflight requests are standalone and should stop the chain as some other
			// middleware may not handle OPTIONS requests correctly. One typical example
			// is authentication middleware ; OPTIONS requests won't carry authentication
			// headers (see #1)
			if c.optionPassthrough {
				h.ServeHTTP(w, r)
			}
		} else {
			c.logf("Handler: Actual request")
			c.handleActualRequest(w, r)
			h.ServeHTTP(w, r)
		}
	})
}

// HandlerFunc provides Martini compatible handler
func (c *Cors) HandlerFunc(w http.ResponseWriter, r *http.Request) {
	if r.Method == "OPTIONS" {
		c.logf("HandlerFunc: Preflight request")
		c.handlePreflight(w, r)
	} else {
		c.logf("HandlerFunc: Actual request")
		c.handleActualRequest(w, r)
	}
}

// Negroni compatible interface
func (c *Cors) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	if r.Method == "OPTIONS" {
		c.logf("ServeHTTP: Preflight request")
		c.handlePreflight(w, r)
		// Preflight requests are standalone and should stop the chain as some other
		// middleware may not handle OPTIONS requests correctly. One typical example
		// is authentication middleware ; OPTIONS requests won't carry authentication
		// headers (see #1)
		if c.optionPassthrough {
			next(w, r)
		}
	} else {
		c.logf("ServeHTTP: Actual request")
		c.handleActualRequest(w, r)
		next(w, r)
	}
}

// handlePreflight handles pre-flight CORS requests
func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	origin := r.Header.Get("Origin")

	if r.Method != "OPTIONS" {
		c.logf("  Preflight aborted: %s!=OPTIONS", r.Method)
		return
	}
	// Always set Vary headers
	// see https://github.com/rs/cors/issues/10,
	//     https://github.com/rs/cors/commit/dbdca4d95feaa7511a46e6f1efb3b3aa505bc43f#commitcomment-12352001
	headers.Add("Vary", "Origin")
	headers.Add("Vary", "Access-Control-Request-Method")
	headers.Add("Vary", "Access-Control-Request-Headers")

	if origin == "" {
		c.logf("  Preflight aborted: empty origin")
		return
	}
	if !c.isOriginAllowed(origin) {
		c.logf("  Preflight aborted: origin '%s' not allowed", origin)
		return
	}

	reqMethod := r.Header.Get("Access-Control-Request-Method")
	if !c.isMethodAllowed(reqMethod) {
		c.logf("  Preflight aborted: method '%s' not allowed", reqMethod)
		return
	}
	reqHeaders := parseHeaderList(r.Header.Get("Access-Control-Request-Headers"))
	if !c.areHeadersAllowed(reqHeaders) {
		c.logf("  Preflight aborted: headers '%v' not allowed", reqHeaders)
		return
	}
	headers.Set("Access-Control-Allow-Origin", origin)
	// Spec says: Since the list of methods can be unbounded, simply returning the method indicated
	// by Access-Control-Request-Method (if supported) can be enough
	headers.Set("Access-Control-Allow-Methods", strings.ToUpper(reqMethod))
	if len(reqHeaders) > 0 {

		// Spec says: Since the list of headers can be unbounded, simply returning supported headers
		// from Access-Control-Request-Headers can be enough
		headers.Set("Access-Control-Allow-Headers", strings.Join(reqHeaders, ", "))
	}
	if c.allowCredentials {
		headers.Set("Access-Control-Allow-Credentials", "true")
	}
	if c.maxAge > 0 {
		headers.Set("Access-Control-Max-Age", strconv.Itoa(c.maxAge))
	}
	c.logf("  Preflight response headers: %v", headers)
}

// handleActualRequest handles simple cross-origin requests, actual request or redirects
func (c *Cors) handleActualRequest(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	origin := r.Header.Get("Origin")

	if r.Method == "OPTIONS" {
		c.logf("  Actual request no headers added: method == %s", r.Method)
		return
	}
	// Always set Vary, see https://github.com/rs/cors/issues/10
	headers.Add("Vary", "Origin")
	if origin == "" {
		c.logf("  Actual request no headers added: missing origin")
		return
	}
	if !c.isOriginAllowed(origin) {
		c.logf("  Actual request no headers added: origin '%s' not allowed", origin)
		return
	}

	// Note that spec does define a way to specifically disallow a simple method like GET or
	// POST. Access-Control-Allow-Methods is only used for pre-flight requests and the
	// spec doesn't instruct to check the allowed methods for simple cross-origin requests.
	// We think it's a nice feature to be able to have control on those methods though.
	if !c.isMethodAllowed(r.Method) {
		c.logf("  Actual request no headers added: method '%s' not allowed", r.Method)

		return
	}
	headers.Set("Access-Control-Allow-Origin", origin)
	if len(c.exposedHeaders) > 0 {
		headers.Set("Access-Control-Expose-Headers", strings.Join(c.exposedHeaders, ", "))
	}
	if c.allowCredentials {
		headers.Set("Access-Control-Allow-Credentials", "true")
	}
	c.logf("  Actual response added headers: %v", headers)
}

// convenience method. checks if debugging is turned on before printing
func (c *Cors) logf(format string, a ...interface{}) {
	if c.Log != nil {
		c.Log.Printf(format, a...)
	}
}

// isOriginAllowed checks if a given origin is allowed to perform cross-domain requests
// on the endpoint
func (c *Cors) isOriginAllowed(origin string) bool {
	if c.allowOriginFunc != nil {
		return c.allowOriginFunc(origin)
	}
	if c.allowedOriginsAll {
		return true
	}
	origin = strings.ToLower(origin)
	for _, o := range c.allowedOrigins {
		if o == origin {
			return true
		}
	}
	for _, w := range c.allowedWOrigins {
		if w.match(origin) {
			return true
		}
	}
	return false
}

// isMethodAllowed checks if a given method can be used as part of a cross-domain request
// on the endpoing
func (c *Cors) isMethodAllowed(method string) bool {
	if len(c.allowedMethods) == 0 {
		// If no method allowed, always return false, even for preflight request
		return false
	}
	method = strings.ToUpper(method)
	if method == "OPTIONS" {
		// Always allow preflight requests
		return true
	}
	for _, m := range c.allowedMethods {
		if m == method {
			return true
		}
	}
	return false
}

// areHeadersAllowed checks if a given list of headers are allowed to used within
// a cross-domain request.
func (c *Cors) areHeadersAllowed(requestedHeaders []string) bool {
	if c.allowedHeadersAll || len(requestedHeaders) == 0 {
		return true
	}
	for _, header := range requestedHeaders {
		header = http.CanonicalHeaderKey(header)
		found := false
		for _, h := range c.allowedHeaders {
			if h == header {
				found = true
			}
		}
		if !found {
			return false
		}
	}
	return true
}
<commit_msg>Add xhandler support<commit_after>/*
Package cors is net/http handler to handle CORS related requests
as defined by http://www.w3.org/TR/cors/

You can configure it by passing an option struct to cors.New:

    c := cors.New(cors.Options{
        AllowedOrigins: []string{"foo.com"},
        AllowedMethods: []string{"GET", "POST", "DELETE"},
        AllowCredentials: true,
    })

Then insert the handler in the chain:

    handler = c.Handler(handler)

See Options documentation for more options.

The resulting handler is a standard net/http handler.
*/
package cors

import (
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/rs/xhandler"
	"golang.org/x/net/context"
)

// Options is a configuration container to setup the CORS middleware.
type Options struct {
	// AllowedOrigins is a list of origins a cross-domain request can be executed from.
	// If the special "*" value is present in the list, all origins will be allowed.
	// An origin may contain a wildcard (*) to replace 0 or more characters
	// (i.e.: http://*.domain.com). Usage of wildcards implies a small performance penality.
	// Only one wildcard can be used per origin.
	// Default value is ["*"]
	AllowedOrigins []string
	// AllowOriginFunc is a custom function to validate the origin. It take the origin
	// as argument and returns true if allowed or false otherwise. If this option is
	// set, the content of AllowedOrigins is ignored.
	AllowOriginFunc func(origin string) bool
	// AllowedMethods is a list of methods the client is allowed to use with
	// cross-domain requests. Default value is simple methods (GET and POST)
	AllowedMethods []string
	// AllowedHeaders is list of non simple headers the client is allowed to use with
	// cross-domain requests.
	// If the special "*" value is present in the list, all headers will be allowed.
	// Default value is [] but "Origin" is always appended to the list.
	AllowedHeaders []string
	// ExposedHeaders indicates which headers are safe to expose to the API of a CORS
	// API specification
	ExposedHeaders []string
	// AllowCredentials indicates whether the request can include user credentials like
	// cookies, HTTP authentication or client side SSL certificates.
	AllowCredentials bool
	// MaxAge indicates how long (in seconds) the results of a preflight request
	// can be cached
	MaxAge int
	// OptionsPassthrough instructs preflight to let other potential next handlers to
	// process the OPTIONS method. Turn this on if your application handles OPTIONS.
	OptionsPassthrough bool
	// Debugging flag adds additional output to debug server side CORS issues
	Debug bool
}

// Cors http handler
type Cors struct {
	// Debug logger
	Log *log.Logger
	// Set to true when allowed origins contains a "*"
	allowedOriginsAll bool
	// Normalized list of plain allowed origins
	allowedOrigins []string
	// List of allowed origins containing wildcards
	allowedWOrigins []wildcard
	// Optional origin validator function
	allowOriginFunc func(origin string) bool
	// Set to true when allowed headers contains a "*"
	allowedHeadersAll bool
	// Normalized list of allowed headers
	allowedHeaders []string
	// Normalized list of allowed methods
	allowedMethods []string
	// Normalized list of exposed headers
	exposedHeaders    []string
	allowCredentials  bool
	maxAge            int
	optionPassthrough bool
}

// New creates a new Cors handler with the provided options.
func New(options Options) *Cors {
	c := &Cors{
		exposedHeaders:    convert(options.ExposedHeaders, http.CanonicalHeaderKey),
		allowOriginFunc:   options.AllowOriginFunc,
		allowCredentials:  options.AllowCredentials,
		maxAge:            options.MaxAge,
		optionPassthrough: options.OptionsPassthrough,
	}
	if options.Debug {
		c.Log = log.New(os.Stdout, "[cors] ", log.LstdFlags)
	}

	// Normalize options
	// Note: for origins and methods matching, the spec requires a case-sensitive matching.
	// As it may error prone, we chose to ignore the spec here.

	// Allowed Origins
	if len(options.AllowedOrigins) == 0 {
		// Default is all origins
		c.allowedOriginsAll = true
	} else {
		c.allowedOrigins = []string{}
		c.allowedWOrigins = []wildcard{}
		for _, origin := range options.AllowedOrigins {
			// Normalize
			origin = strings.ToLower(origin)
			if origin == "*" {
				// If "*" is present in the list, turn the whole list into a match all
				c.allowedOriginsAll = true
				c.allowedOrigins = nil
				c.allowedWOrigins = nil
				break
			} else if i := strings.IndexByte(origin, '*'); i >= 0 {
				// Split the origin in two: start and end string without the *
				w := wildcard{origin[0:i], origin[i+1 : len(origin)]}
				c.allowedWOrigins = append(c.allowedWOrigins, w)
			} else {
				c.allowedOrigins = append(c.allowedOrigins, origin)
			}
		}
	}

	// Allowed Headers
	if len(options.AllowedHeaders) == 0 {
		// Use sensible defaults
		c.allowedHeaders = []string{"Origin", "Accept", "Content-Type"}
	} else {
		// Origin is always appended as some browsers will always request for this header at preflight
		c.allowedHeaders = convert(append(options.AllowedHeaders, "Origin"), http.CanonicalHeaderKey)
		for _, h := range options.AllowedHeaders {
			if h == "*" {
				c.allowedHeadersAll = true
				c.allowedHeaders = nil
				break
			}
		}
	}

	// Allowed Methods
	if len(options.AllowedMethods) == 0 {
		// Default is spec's "simple" methods
		c.allowedMethods = []string{"GET", "POST"}
	} else {
		c.allowedMethods = convert(options.AllowedMethods, strings.ToUpper)
	}

	return c
}

// Default creates a new Cors handler with default options
func Default() *Cors {
	return New(Options{})
}

// Handler apply the CORS specification on the request, and add relevant CORS headers
// as necessary.
func (c *Cors) Handler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "OPTIONS" {
			c.logf("Handler: Preflight request")
			c.handlePreflight(w, r)
			// Preflight requests are standalone and should stop the chain as some other
			// middleware may not handle OPTIONS requests correctly. One typical example
			// is authentication middleware ; OPTIONS requests won't carry authentication
			// headers (see #1)
			if c.optionPassthrough {
				h.ServeHTTP(w, r)
			}
		} else {
			c.logf("Handler: Actual request")
			c.handleActualRequest(w, r)
			h.ServeHTTP(w, r)
		}
	})
}

// HandlerC is net/context aware handler
func (c *Cors) HandlerC(h xhandler.HandlerC) xhandler.HandlerC {
	return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		if r.Method == "OPTIONS" {
			c.logf("Handler: Preflight request")
			c.handlePreflight(w, r)
			// Preflight requests are standalone and should stop the chain as some other
			// middleware may not handle OPTIONS requests correctly. One typical example
			// is authentication middleware ; OPTIONS requests won't carry authentication
			// headers (see #1)
			if c.optionPassthrough {
				h.ServeHTTPC(ctx, w, r)
			}
		} else {
			c.logf("Handler: Actual request")
			c.handleActualRequest(w, r)
			h.ServeHTTPC(ctx, w, r)
		}
	})
}

// HandlerFunc provides Martini compatible handler
func (c *Cors) HandlerFunc(w http.ResponseWriter, r *http.Request) {
	if r.Method == "OPTIONS" {
		c.logf("HandlerFunc: Preflight request")
		c.handlePreflight(w, r)
	} else {
		c.logf("HandlerFunc: Actual request")
		c.handleActualRequest(w, r)
	}
}

// Negroni compatible interface
func (c *Cors) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	if r.Method == "OPTIONS" {
		c.logf("ServeHTTP: Preflight request")
		c.handlePreflight(w, r)
		// Preflight requests are standalone and should stop the chain as some other
		// middleware may not handle OPTIONS requests correctly. One typical example
		// is authentication middleware ; OPTIONS requests won't carry authentication
		// headers (see #1)
		if c.optionPassthrough {
			next(w, r)
		}
	} else {
		c.logf("ServeHTTP: Actual request")
		c.handleActualRequest(w, r)
		next(w, r)
	}
}

// handlePreflight handles pre-flight CORS requests
func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	origin := r.Header.Get("Origin")

	if r.Method != "OPTIONS" {
		c.logf("  Preflight aborted: %s!=OPTIONS", r.Method)
		return
	}
	// Always set Vary headers
	// see https://github.com/rs/cors/issues/10,
	//     https://github.com/rs/cors/commit/dbdca4d95feaa7511a46e6f1efb3b3aa505bc43f#commitcomment-12352001
	headers.Add("Vary", "Origin")
	headers.Add("Vary", "Access-Control-Request-Method")
	headers.Add("Vary", "Access-Control-Request-Headers")

	if origin == "" {
		c.logf("  Preflight aborted: empty origin")
		return
	}
	if !c.isOriginAllowed(origin) {
		c.logf("  Preflight aborted: origin '%s' not allowed", origin)
		return
	}

	reqMethod := r.Header.Get("Access-Control-Request-Method")
	if !c.isMethodAllowed(reqMethod) {
		c.logf("  Preflight aborted: method '%s' not allowed", reqMethod)
		return
	}
	reqHeaders := parseHeaderList(r.Header.Get("Access-Control-Request-Headers"))
	if !c.areHeadersAllowed(reqHeaders) {
		c.logf("  Preflight aborted: headers '%v' not allowed", reqHeaders)
		return
	}
	headers.Set("Access-Control-Allow-Origin", origin)
	// Spec says: Since the list of methods can be unbounded, simply returning the method indicated
	// by Access-Control-Request-Method (if supported) can be enough
	headers.Set("Access-Control-Allow-Methods", strings.ToUpper(reqMethod))
	if len(reqHeaders) > 0 {

		// Spec says: Since the list of headers can be unbounded, simply returning supported headers
		// from Access-Control-Request-Headers can be enough
		headers.Set("Access-Control-Allow-Headers", strings.Join(reqHeaders, ", "))
	}
	if c.allowCredentials {
		headers.Set("Access-Control-Allow-Credentials", "true")
	}
	if c.maxAge > 0 {
		headers.Set("Access-Control-Max-Age", strconv.Itoa(c.maxAge))
	}
	c.logf("  Preflight response headers: %v", headers)
}

// handleActualRequest handles simple cross-origin requests, actual request or redirects
func (c *Cors) handleActualRequest(w http.ResponseWriter, r *http.Request) {
	headers := w.Header()
	origin := r.Header.Get("Origin")

	if r.Method == "OPTIONS" {
		c.logf("  Actual request no headers added: method == %s", r.Method)
		return
	}
	// Always set Vary, see https://github.com/rs/cors/issues/10
	headers.Add("Vary", "Origin")
	if origin == "" {
		c.logf("  Actual request no headers added: missing origin")
		return
	}
	if !c.isOriginAllowed(origin) {
		c.logf("  Actual request no headers added: origin '%s' not allowed", origin)
		return
	}

	// Note that spec does define a way to specifically disallow a simple method like GET or
	// POST. Access-Control-Allow-Methods is only used for pre-flight requests and the
	// spec doesn't instruct to check the allowed methods for simple cross-origin requests.
	// We think it's a nice feature to be able to have control on those methods though.
	if !c.isMethodAllowed(r.Method) {
		c.logf("  Actual request no headers added: method '%s' not allowed", r.Method)

		return
	}
	headers.Set("Access-Control-Allow-Origin", origin)
	if len(c.exposedHeaders) > 0 {
		headers.Set("Access-Control-Expose-Headers", strings.Join(c.exposedHeaders, ", "))
	}
	if c.allowCredentials {
		headers.Set("Access-Control-Allow-Credentials", "true")
	}
	c.logf("  Actual response added headers: %v", headers)
}

// convenience method. checks if debugging is turned on before printing
func (c *Cors) logf(format string, a ...interface{}) {
	if c.Log != nil {
		c.Log.Printf(format, a...)
	}
}

// isOriginAllowed checks if a given origin is allowed to perform cross-domain requests
// on the endpoint
func (c *Cors) isOriginAllowed(origin string) bool {
	if c.allowOriginFunc != nil {
		return c.allowOriginFunc(origin)
	}
	if c.allowedOriginsAll {
		return true
	}
	origin = strings.ToLower(origin)
	for _, o := range c.allowedOrigins {
		if o == origin {
			return true
		}
	}
	for _, w := range c.allowedWOrigins {
		if w.match(origin) {
			return true
		}
	}
	return false
}

// isMethodAllowed checks if a given method can be used as part of a cross-domain request
// on the endpoing
func (c *Cors) isMethodAllowed(method string) bool {
	if len(c.allowedMethods) == 0 {
		// If no method allowed, always return false, even for preflight request
		return false
	}
	method = strings.ToUpper(method)
	if method == "OPTIONS" {
		// Always allow preflight requests
		return true
	}
	for _, m := range c.allowedMethods {
		if m == method {
			return true
		}
	}
	return false
}

// areHeadersAllowed checks if a given list of headers are allowed to used within
// a cross-domain request.
func (c *Cors) areHeadersAllowed(requestedHeaders []string) bool {
	if c.allowedHeadersAll || len(requestedHeaders) == 0 {
		return true
	}
	for _, header := range requestedHeaders {
		header = http.CanonicalHeaderKey(header)
		found := false
		for _, h := range c.allowedHeaders {
			if h == header {
				found = true
			}
		}
		if !found {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2016 Liam Stanley <me@liamstanley.io>. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package girc

import (
	"runtime"
	"strings"
	"sync"
	"time"
)

// ctcpDelim if the delimiter used for CTCP formatted events/messages.
const ctcpDelim byte = 0x01 // Prefix and suffix for CTCP messages.

// CTCPEvent is the necessary information from an IRC message.
type CTCPEvent struct {
	// Source is the author of the CTCP event.
	Source *Source
	// Command is the type of CTCP event. E.g. PING, TIME, VERSION.
	Command string
	// Text is the raw arguments following the command.
	Text string
	// Reply is true if the CTCP event is intended to be a reply to a
	// previous CTCP (e.g, if we sent one).
	Reply bool
}

// decodeCTCP codes an incoming CTCP event, if it is CTCP. nil is returned
// if the incoming event does not match a valid CTCP.
func decodeCTCP(e *Event) *CTCPEvent {
	// http://www.irchelp.org/protocol/ctcpspec.html

	// Must be targetting a user/channel, AND trailing must have
	// DELIM+TAG+DELIM minimum (at least 3 chars).
	if len(e.Params) != 1 || len(e.Trailing) < 3 {
		return nil
	}

	if (e.Command != "PRIVMSG" && e.Command != "NOTICE") || !IsValidNick(e.Params[0]) {
		return nil
	}

	if e.Trailing[0] != ctcpDelim || e.Trailing[len(e.Trailing)-1] != ctcpDelim {
		return nil
	}

	// Strip delimiters.
	text := e.Trailing[1 : len(e.Trailing)-1]

	s := strings.IndexByte(text, space)

	// Check to see if it only contains a tag.
	if s < 0 {
		for i := 0; i < len(text); i++ {
			// Check for A-Z, 0-9.
			if (text[i] < 0x41 || text[i] > 0x5A) && (text[i] < 0x30 || text[i] > 0x39) {
				return nil
			}
		}

		return &CTCPEvent{
			Source:  e.Source,
			Command: text,
			Reply:   e.Command == "NOTICE",
		}
	}

	// Loop through checking the tag first.
	for i := 0; i < s; i++ {
		// Check for A-Z, 0-9.
		if (text[i] < 0x41 || text[i] > 0x5A) && (text[i] < 0x30 || text[i] > 0x39) {
			return nil
		}
	}

	return &CTCPEvent{
		Source:  e.Source,
		Command: text[0:s],
		Text:    text[s+1 : len(text)],
		Reply:   e.Command == "NOTICE",
	}
}

// encodeCTCP encodes a CTCP event into a string, including delimiters.
func encodeCTCP(ctcp *CTCPEvent) (out string) {
	if ctcp == nil {
		return ""
	}

	return encodeCTCPRaw(ctcp.Command, ctcp.Text)
}

// encodeCTCPRaw is much like encodeCTCP, however accepts a raw command and
// string as input.
func encodeCTCPRaw(cmd, text string) (out string) {
	if len(cmd) <= 0 {
		return ""
	}

	out = string(ctcpDelim) + cmd

	if len(text) > 0 {
		out += string(space) + text
	}

	return out + string(ctcpDelim)
}

// CTCP handles the storage and execution of CTCP handlers against incoming
// CTCP events.
type CTCP struct {
	disableDefault bool
	// mu is the mutex that should be used when accessing callbacks.
	mu sync.RWMutex
	// handlers is a map of CTCP message -> functions.
	handlers map[string]CTCPHandler
}

// newCTCP returns a new clean CTCP handler.
func newCTCP() *CTCP {
	return &CTCP{handlers: map[string]CTCPHandler{}}
}

// call executes the necessary CTCP handler for the incoming event/CTCP
// command.
func (c *CTCP) call(event *CTCPEvent, client *Client) {
	// TODO: Wildcard CTCP catching?
	c.mu.RLock()
	if _, ok := c.handlers[event.Command]; !ok {
		c.mu.RUnlock()

		// Send a ERRMSG reply.
		client.SendCTCPReply(event.Source.Name, CTCP_ERRMSG, "that is an unknown CTCP query")
		return
	}

	c.handlers[event.Command](client, event)
	c.mu.RUnlock()
}

// parseCMD parses a CTCP command/tag, ensuring it's valid. If not, an empty
// string is returned.
func (c *CTCP) parseCMD(cmd string) string {
	cmd = strings.ToUpper(cmd)

	for i := 0; i < len(cmd); i++ {
		// Check for A-Z, 0-9.
		if (cmd[i] < 0x41 || cmd[i] > 0x5A) && (cmd[i] < 0x30 || cmd[i] > 0x39) {
			return ""
		}
	}

	return cmd
}

// Set saves handler for execution upon a matching incoming CTCP event.
// Use SetBg if the handler may take an extended period of time to execute.
func (c *CTCP) Set(cmd string, handler func(client *Client, ctcp *CTCPEvent)) {
	if cmd = c.parseCMD(cmd); cmd == "" {
		return
	}

	c.mu.Lock()
	c.handlers[cmd] = CTCPHandler(handler)
	c.mu.Unlock()
}

// SetBg is much like Set, however the handler is executed in the background,
// ensuring that event handling isn't hung during long running tasks.
func (c *CTCP) SetBg(cmd string, handler func(client *Client, ctcp *CTCPEvent)) {
	c.Set(cmd, func(client *Client, ctcp *CTCPEvent) {
		go handler(client, ctcp)
	})
}

// Clear removes currently setup handler for cmd, if one is set. This will
// also disable default handlers for a specific cmd.
func (c *CTCP) Clear(cmd string) {
	if cmd = c.parseCMD(cmd); cmd == "" {
		return
	}

	c.mu.Lock()
	delete(c.handlers, cmd)
	c.mu.Unlock()
}

// ClearAll removes all currently setup and re-sets the default handlers,
// unless configured not to. See Client.Config.DisableDefaultCTCP.
func (c *CTCP) ClearAll() {
	c.mu.Lock()
	c.handlers = map[string]CTCPHandler{}
	c.mu.Unlock()

	// Register necessary handlers.
	c.addDefaultHandlers()
}

// CTCPHandler is a type that represents the function necessary to
// implement a CTCP handler.
type CTCPHandler func(client *Client, ctcp *CTCPEvent)

// addDefaultHandlers adds some useful default CTCP response handlers, unless
// requested by the client not to.
func (c *CTCP) addDefaultHandlers() {
	if c.disableDefault {
		return
	}

	c.SetBg(CTCP_PING, handleCTCPPing)
	c.SetBg(CTCP_PONG, handleCTCPPong)
	c.SetBg(CTCP_VERSION, handleCTCPVersion)
	c.SetBg(CTCP_SOURCE, handleCTCPSource)
	c.SetBg(CTCP_TIME, handleCTCPTime)
}

// handleCTCPPing replies with a ping and whatever was originally requested.
func handleCTCPPing(client *Client, ctcp *CTCPEvent) {
	if ctcp.Reply {
		return
	}
	client.SendCTCPReply(ctcp.Source.Name, CTCP_PING, ctcp.Text)
}

// handleCTCPPong replies with a pong.
func handleCTCPPong(client *Client, ctcp *CTCPEvent) {
	if ctcp.Reply {
		return
	}
	client.SendCTCPReply(ctcp.Source.Name, CTCP_PONG, "")
}

// handleCTCPVersion replies with the name of the client, Go version, as well
// as the os type (darwin, linux, windows, etc) and architecture type (x86,
// arm, etc).
func handleCTCPVersion(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReplyf(
		ctcp.Source.Name, CTCP_VERSION,
		"girc (github.com/lrstanley/girc) using %s (%s, %s)",
		runtime.Version(), runtime.GOOS, runtime.GOARCH,
	)
}

// handleCTCPSource replies with the public git location of this library.
func handleCTCPSource(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReply(ctcp.Source.Name, CTCP_SOURCE, "https://github.com/lrstanley/girc")
}

// handleCTCPTime replies with a RFC 1123 (Z) formatted version of Go's
// local time.
func handleCTCPTime(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReply(ctcp.Source.Name, CTCP_TIME, ":"+time.Now().Format(time.RFC1123Z))
}
<commit_msg>add support for wildcard CTCP handlers<commit_after>// Copyright 2016 Liam Stanley <me@liamstanley.io>. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package girc

import (
	"runtime"
	"strings"
	"sync"
	"time"
)

// ctcpDelim if the delimiter used for CTCP formatted events/messages.
const ctcpDelim byte = 0x01 // Prefix and suffix for CTCP messages.

// CTCPEvent is the necessary information from an IRC message.
type CTCPEvent struct {
	// Source is the author of the CTCP event.
	Source *Source
	// Command is the type of CTCP event. E.g. PING, TIME, VERSION.
	Command string
	// Text is the raw arguments following the command.
	Text string
	// Reply is true if the CTCP event is intended to be a reply to a
	// previous CTCP (e.g, if we sent one).
	Reply bool
}

// decodeCTCP codes an incoming CTCP event, if it is CTCP. nil is returned
// if the incoming event does not match a valid CTCP.
func decodeCTCP(e *Event) *CTCPEvent {
	// http://www.irchelp.org/protocol/ctcpspec.html

	// Must be targetting a user/channel, AND trailing must have
	// DELIM+TAG+DELIM minimum (at least 3 chars).
	if len(e.Params) != 1 || len(e.Trailing) < 3 {
		return nil
	}

	if (e.Command != "PRIVMSG" && e.Command != "NOTICE") || !IsValidNick(e.Params[0]) {
		return nil
	}

	if e.Trailing[0] != ctcpDelim || e.Trailing[len(e.Trailing)-1] != ctcpDelim {
		return nil
	}

	// Strip delimiters.
	text := e.Trailing[1 : len(e.Trailing)-1]

	s := strings.IndexByte(text, space)

	// Check to see if it only contains a tag.
	if s < 0 {
		for i := 0; i < len(text); i++ {
			// Check for A-Z, 0-9.
			if (text[i] < 0x41 || text[i] > 0x5A) && (text[i] < 0x30 || text[i] > 0x39) {
				return nil
			}
		}

		return &CTCPEvent{
			Source:  e.Source,
			Command: text,
			Reply:   e.Command == "NOTICE",
		}
	}

	// Loop through checking the tag first.
	for i := 0; i < s; i++ {
		// Check for A-Z, 0-9.
		if (text[i] < 0x41 || text[i] > 0x5A) && (text[i] < 0x30 || text[i] > 0x39) {
			return nil
		}
	}

	return &CTCPEvent{
		Source:  e.Source,
		Command: text[0:s],
		Text:    text[s+1 : len(text)],
		Reply:   e.Command == "NOTICE",
	}
}

// encodeCTCP encodes a CTCP event into a string, including delimiters.
func encodeCTCP(ctcp *CTCPEvent) (out string) {
	if ctcp == nil {
		return ""
	}

	return encodeCTCPRaw(ctcp.Command, ctcp.Text)
}

// encodeCTCPRaw is much like encodeCTCP, however accepts a raw command and
// string as input.
func encodeCTCPRaw(cmd, text string) (out string) {
	if len(cmd) <= 0 {
		return ""
	}

	out = string(ctcpDelim) + cmd

	if len(text) > 0 {
		out += string(space) + text
	}

	return out + string(ctcpDelim)
}

// CTCP handles the storage and execution of CTCP handlers against incoming
// CTCP events.
type CTCP struct {
	disableDefault bool
	// mu is the mutex that should be used when accessing callbacks.
	mu sync.RWMutex
	// handlers is a map of CTCP message -> functions.
	handlers map[string]CTCPHandler
}

// newCTCP returns a new clean CTCP handler.
func newCTCP() *CTCP {
	return &CTCP{handlers: map[string]CTCPHandler{}}
}

// call executes the necessary CTCP handler for the incoming event/CTCP
// command.
func (c *CTCP) call(event *CTCPEvent, client *Client) {
	c.mu.RLock()
	defer c.mu.RUnlock()

	// Support wildcard CTCP event handling. Gets executed first before
	// regular event handlers.
	if _, ok := c.handlers["*"]; ok {
		c.handlers[event.Command](client, event)
	}

	if _, ok := c.handlers[event.Command]; !ok {
		// Send a ERRMSG reply.
		client.SendCTCPReply(event.Source.Name, CTCP_ERRMSG, "that is an unknown CTCP query")
		return
	}

	c.handlers[event.Command](client, event)
}

// parseCMD parses a CTCP command/tag, ensuring it's valid. If not, an empty
// string is returned.
func (c *CTCP) parseCMD(cmd string) string {
	cmd = strings.ToUpper(cmd)

	for i := 0; i < len(cmd); i++ {
		// Check for A-Z, 0-9.
		if (cmd[i] < 0x41 || cmd[i] > 0x5A) && (cmd[i] < 0x30 || cmd[i] > 0x39) {
			return ""
		}
	}

	return cmd
}

// Set saves handler for execution upon a matching incoming CTCP event.
// Use SetBg if the handler may take an extended period of time to execute.
// If you would like to have a handler which will catch ALL CTCP requests,
// simply use "*" in place of the command.
func (c *CTCP) Set(cmd string, handler func(client *Client, ctcp *CTCPEvent)) {
	if cmd = c.parseCMD(cmd); cmd == "" {
		return
	}

	c.mu.Lock()
	c.handlers[cmd] = CTCPHandler(handler)
	c.mu.Unlock()
}

// SetBg is much like Set, however the handler is executed in the background,
// ensuring that event handling isn't hung during long running tasks. See Set
// for more information.
func (c *CTCP) SetBg(cmd string, handler func(client *Client, ctcp *CTCPEvent)) {
	c.Set(cmd, func(client *Client, ctcp *CTCPEvent) {
		go handler(client, ctcp)
	})
}

// Clear removes currently setup handler for cmd, if one is set. This will
// also disable default handlers for a specific cmd.
func (c *CTCP) Clear(cmd string) {
	if cmd = c.parseCMD(cmd); cmd == "" {
		return
	}

	c.mu.Lock()
	delete(c.handlers, cmd)
	c.mu.Unlock()
}

// ClearAll removes all currently setup and re-sets the default handlers,
// unless configured not to. See Client.Config.DisableDefaultCTCP.
func (c *CTCP) ClearAll() {
	c.mu.Lock()
	c.handlers = map[string]CTCPHandler{}
	c.mu.Unlock()

	// Register necessary handlers.
	c.addDefaultHandlers()
}

// CTCPHandler is a type that represents the function necessary to
// implement a CTCP handler.
type CTCPHandler func(client *Client, ctcp *CTCPEvent)

// addDefaultHandlers adds some useful default CTCP response handlers, unless
// requested by the client not to.
func (c *CTCP) addDefaultHandlers() {
	if c.disableDefault {
		return
	}

	c.SetBg(CTCP_PING, handleCTCPPing)
	c.SetBg(CTCP_PONG, handleCTCPPong)
	c.SetBg(CTCP_VERSION, handleCTCPVersion)
	c.SetBg(CTCP_SOURCE, handleCTCPSource)
	c.SetBg(CTCP_TIME, handleCTCPTime)
}

// handleCTCPPing replies with a ping and whatever was originally requested.
func handleCTCPPing(client *Client, ctcp *CTCPEvent) {
	if ctcp.Reply {
		return
	}
	client.SendCTCPReply(ctcp.Source.Name, CTCP_PING, ctcp.Text)
}

// handleCTCPPong replies with a pong.
func handleCTCPPong(client *Client, ctcp *CTCPEvent) {
	if ctcp.Reply {
		return
	}
	client.SendCTCPReply(ctcp.Source.Name, CTCP_PONG, "")
}

// handleCTCPVersion replies with the name of the client, Go version, as well
// as the os type (darwin, linux, windows, etc) and architecture type (x86,
// arm, etc).
func handleCTCPVersion(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReplyf(
		ctcp.Source.Name, CTCP_VERSION,
		"girc (github.com/lrstanley/girc) using %s (%s, %s)",
		runtime.Version(), runtime.GOOS, runtime.GOARCH,
	)
}

// handleCTCPSource replies with the public git location of this library.
func handleCTCPSource(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReply(ctcp.Source.Name, CTCP_SOURCE, "https://github.com/lrstanley/girc")
}

// handleCTCPTime replies with a RFC 1123 (Z) formatted version of Go's
// local time.
func handleCTCPTime(client *Client, ctcp *CTCPEvent) {
	client.SendCTCPReply(ctcp.Source.Name, CTCP_TIME, ":"+time.Now().Format(time.RFC1123Z))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bwolf/metricsystem"
	"math"
	"strings"
)

type Measurement struct {
	Name  string
	Value float64
}

type Weather struct {
	stationId    int
	measurements []Measurement
}

func (weather *Weather) StationId() int {
	return weather.stationId
}

func (weather *Weather) Measurements() []Measurement {
	return weather.measurements
}

func calcDewPoint(humid, temp float64) float64 {
	m := 17.62
	tn := 243.12

	if temp <= 0 {
		m = 22.46
		tn = 272.62
	}
	k := (math.Log10(humid)-2)/0.4343 + (m*temp)/(tn+temp)
	return tn * k / (m - k)
}

func normalizeMeasurement(m *Measurement) *Measurement {
	index := strings.LastIndex(m.Name, "_")
	if index == -1 {
		// fmt.Println("Nothing to normalize for key", m.Name)
		newName := strings.Replace(m.Name, "-", "_", -1)
		return &Measurement{Name: newName, Value: m.Value}
	}
	metricPrefix := m.Name[index+1:]
	newName := strings.Replace(m.Name[0:index], "-", "_", -1)
	newValue := metricsystem.ScaleByMetricSystemPrefix(m.Value, metricPrefix)
	return &Measurement{Name: newName, Value: newValue}
}

func transformMeasurements(meas []Measurement) []Measurement {
	res := make([]Measurement, len(meas))
	for i, m := range meas {
		mm := normalizeMeasurement(&m)
		res[i] = *mm
	}
	return res
}

// Patch dew point into data set based on rh_true and temp
func LazyMonkeyPatchDewPoint(weather *Weather) {
	var gotRhTrue, gotTemp bool
	var rhTrue, temp Measurement
	for _, m := range weather.measurements {
		if m.Name == "rh_true" {
			gotRhTrue = true
			rhTrue = m
		} else if m.Name == "temp" {
			gotTemp = true
			temp = m
		}
		if gotRhTrue && gotTemp {
			break
		}
	}

	if gotRhTrue && gotTemp {
		dp := calcDewPoint(rhTrue.Value, temp.Value)
		// fmt.Printf("Dew point from h %f, t %f: %f will be patched in\n",
		// rhTrue.Value, temp.Value, dp)

		weather.measurements = append(weather.measurements,
			Measurement{Name: "dew_point", Value: dp})
	}
}

func ParseWeather(js interface{}) (error, *Weather) {
	m := js.(map[string]interface{})

	rawWeather, ok := m["weather"]
	if !ok {
		return fmt.Errorf("No weather in Json %v", js), nil
	}

	weather := Weather{}
	weatherMap := rawWeather.(map[string]interface{})
	for k, v := range weatherMap {
		switch vv := v.(type) {
		case string:
			return fmt.Errorf("%s is unsupported string %s", k, vv), nil
		case int:
			return fmt.Errorf("%s is unsupported int %d", k, vv), nil
		case float64:
			if k == "station-id" { // Decoded as float64, also look like int
				weather.stationId = int(vv)
			} else {
				meas := Measurement{Name: k, Value: vv}
				weather.measurements = append(weather.measurements, meas)
			}
		default:
			return fmt.Errorf("%s is of unsupported type %v", k, vv), nil
		}
	}

	// Ensure there is a station-id in weather
	if weather.stationId == 0 {
		return fmt.Errorf("Missing 'station-id' in weather dataset %v", rawWeather), nil
	}

	weather.measurements = transformMeasurements(weather.measurements)

	return nil, &weather
}

func ParseJson(rawInput []byte) (error, interface{}) {
	var js interface{}
	err := json.Unmarshal(rawInput, &js)
	if err != nil {
		return fmt.Errorf("Invalid JSON: %v\n", err), nil
	}
	return nil, js
}
<commit_msg>Comment dew point calculation.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/bwolf/metricsystem"
	"math"
	"strings"
)

type Measurement struct {
	Name  string
	Value float64
}

type Weather struct {
	stationId    int
	measurements []Measurement
}

func (weather *Weather) StationId() int {
	return weather.stationId
}

func (weather *Weather) Measurements() []Measurement {
	return weather.measurements
}

// Based on Sensiron data sheet for SHT7x (http://is.gd/i8rS20).
func calcDewPoint(humid, temp float64) float64 {
	m := 17.62
	tn := 243.12

	if temp <= 0 {
		m = 22.46
		tn = 272.62
	}
	k := (math.Log10(humid)-2)/0.4343 + (m*temp)/(tn+temp)
	return tn * k / (m - k)
}

func normalizeMeasurement(m *Measurement) *Measurement {
	index := strings.LastIndex(m.Name, "_")
	if index == -1 {
		// fmt.Println("Nothing to normalize for key", m.Name)
		newName := strings.Replace(m.Name, "-", "_", -1)
		return &Measurement{Name: newName, Value: m.Value}
	}
	metricPrefix := m.Name[index+1:]
	newName := strings.Replace(m.Name[0:index], "-", "_", -1)
	newValue := metricsystem.ScaleByMetricSystemPrefix(m.Value, metricPrefix)
	return &Measurement{Name: newName, Value: newValue}
}

func transformMeasurements(meas []Measurement) []Measurement {
	res := make([]Measurement, len(meas))
	for i, m := range meas {
		mm := normalizeMeasurement(&m)
		res[i] = *mm
	}
	return res
}

// Patch dew point into data set based on rh_true and temp
func LazyMonkeyPatchDewPoint(weather *Weather) {
	var gotRhTrue, gotTemp bool
	var rhTrue, temp Measurement
	for _, m := range weather.measurements {
		if m.Name == "rh_true" {
			gotRhTrue = true
			rhTrue = m
		} else if m.Name == "temp" {
			gotTemp = true
			temp = m
		}
		if gotRhTrue && gotTemp {
			break
		}
	}

	if gotRhTrue && gotTemp {
		dp := calcDewPoint(rhTrue.Value, temp.Value)
		// fmt.Printf("Dew point from h %f, t %f: %f will be patched in\n",
		// rhTrue.Value, temp.Value, dp)

		weather.measurements = append(weather.measurements,
			Measurement{Name: "dew_point", Value: dp})
	}
}

func ParseWeather(js interface{}) (error, *Weather) {
	m := js.(map[string]interface{})

	rawWeather, ok := m["weather"]
	if !ok {
		return fmt.Errorf("No weather in Json %v", js), nil
	}

	weather := Weather{}
	weatherMap := rawWeather.(map[string]interface{})
	for k, v := range weatherMap {
		switch vv := v.(type) {
		case string:
			return fmt.Errorf("%s is unsupported string %s", k, vv), nil
		case int:
			return fmt.Errorf("%s is unsupported int %d", k, vv), nil
		case float64:
			if k == "station-id" { // Decoded as float64, also look like int
				weather.stationId = int(vv)
			} else {
				meas := Measurement{Name: k, Value: vv}
				weather.measurements = append(weather.measurements, meas)
			}
		default:
			return fmt.Errorf("%s is of unsupported type %v", k, vv), nil
		}
	}

	// Ensure there is a station-id in weather
	if weather.stationId == 0 {
		return fmt.Errorf("Missing 'station-id' in weather dataset %v", rawWeather), nil
	}

	weather.measurements = transformMeasurements(weather.measurements)

	return nil, &weather
}

func ParseJson(rawInput []byte) (error, interface{}) {
	var js interface{}
	err := json.Unmarshal(rawInput, &js)
	if err != nil {
		return fmt.Errorf("Invalid JSON: %v\n", err), nil
	}
	return nil, js
}
<|endoftext|>