<commit_before>// Package api provides rest-like server
package api

import (
	"context"
	"crypto/subtle"
	"encoding/xml"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"
	"github.com/pkg/errors"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/youtube"
	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
)

//go:generate moq -out mocks/yt_service.go -pkg mocks -skip-ensure -fmt goimports . YoutubeSvc
//go:generate moq -out mocks/store.go -pkg mocks -skip-ensure -fmt goimports . Store

// Server provides HTTP API
type Server struct {
	Version       string
	Conf          config.Conf
	Store         Store
	YoutubeSvc    YoutubeSvc
	TemplLocation string
	AdminPasswd   string

	httpServer *http.Server
	cache      lcw.LoadingCache
	templates  *template.Template
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
	StoreRSS(chanID, rss string) error
	RemoveEntry(entry ytfeed.Entry) error
}

// Store provides access to feed data
type Store interface {
	Load(fmFeed string, max int, skipJunk bool) ([]feed.Item, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	log.Printf("[INFO] starting server on port %d", port)
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*3), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	if s.TemplLocation == "" {
		s.TemplLocation = "webapp/templates/*"
	}
	log.Printf("[DEBUG] loading templates from %s", s.TemplLocation)
	s.templates = template.Must(template.ParseGlob(s.TemplLocation))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()), middleware.GetHead)
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(logger.AnonymizeIP))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"), logger.IPfn(logger.AnonymizeIP))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Head("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {

		auth := rest.BasicAuth(func(user, passwd string) bool {
			return (subtle.ConstantTimeCompare([]byte(s.AdminPasswd), []byte(passwd)) +
				subtle.ConstantTimeCompare([]byte("admin"), []byte(user))) == 2
		})

		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"), logger.IPfn(logger.AnonymizeIP))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
		r.With(auth).Post("/rss/generate", s.regenerateRSSCtrl)
		r.With(auth).Delete("/entry/{channel}/{video}", s.removeEntryCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}

		if mkdirErr := os.MkdirAll(s.Conf.YouTube.FilesLocation, 0o750); mkdirErr != nil {
			log.Printf("[ERROR] failed to create directory %s, %v", s.Conf.YouTube.FilesLocation, mkdirErr)
		}

		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")

	data, err := s.cache.Get("feed::"+feedName, func() (interface{}, error) {
		items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
		if err != nil {
			return nil, err
		}

		for i, itm := range items {
			// add ts suffix to titles
			switch s.Conf.Feeds[feedName].ExtendDateTitle {
			case "yyyyddmm":
				items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
			case "yyyymmdd":
				items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
			}
		}

		rss := feed.Rss2{
			Version:        "2.0",
			ItemList:       items,
			Title:          s.Conf.Feeds[feedName].Title,
			Description:    s.Conf.Feeds[feedName].Description,
			Language:       s.Conf.Feeds[feedName].Language,
			Link:           s.Conf.Feeds[feedName].Link,
			PubDate:        items[0].PubDate,
			LastBuildDate:  time.Now().Format(time.RFC822Z),
			ItunesAuthor:   s.Conf.Feeds[feedName].Author,
			ItunesExplicit: "no",
			ItunesOwner: &feed.ItunesOwner{
				Name:  "Feed Master",
				Email: "umputun@feed-master.com",
			},
			NsItunes: "http://www.itunes.com/dtds/podcast-1.0.dtd",
			NsMedia:  "http://search.yahoo.com/mrss/",
		}

		// replace link to UI page
		if s.Conf.System.BaseURL != "" {
			baseURL := strings.TrimSuffix(s.Conf.System.BaseURL, "/")
			rss.Link = baseURL + "/feed/" + feedName
			imagesURL := baseURL + "/images/" + feedName
			rss.ItunesImage = &feed.ItunesImg{URL: imagesURL}
			rss.MediaThumbnail = &feed.MediaThumbnail{URL: imagesURL}
		}

		b, err := xml.MarshalIndent(&rss, "", "  ")
		if err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
			return nil, errors.Wrapf(err, "failed to marshal rss for %s", feedName)
		}

		res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)

		// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
		res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
		res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

		return res, nil
	})

	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	res, ok := data.(string)
	if !ok {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to convert feed")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

// POST /yt/rss/generate - generates rss for all (each) youtube channels
func (s *Server) regenerateRSSCtrl(w http.ResponseWriter, r *http.Request) {

	for _, f := range s.Conf.YouTube.Channels {
		res, err := s.YoutubeSvc.RSSFeed(youtube.FeedInfo{ID: f.ID})
		if err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt rss for "+f.ID)
			return
		}
		if err := s.YoutubeSvc.StoreRSS(f.ID, res); err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to store yt rss for "+f.ID)
			return
		}
	}
	rest.RenderJSON(w, rest.JSON{"status": "ok", "feeds": len(s.Conf.YouTube.Channels)})
}

// DELETE /yt/entry/{channel}/{video} - deletes entry from youtube channel and videID
func (s *Server) removeEntryCtrl(w http.ResponseWriter, r *http.Request) {
	err := s.YoutubeSvc.RemoveEntry(ytfeed.Entry{ChannelID: chi.URLParam(r, "channel"), VideoID: chi.URLParam(r, "video")})
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to remove entry")
		return
	}
	rest.RenderJSON(w, rest.JSON{"status": "ok", "removed": chi.URLParam(r, "video")})
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<commit_msg>add endpoint to image.png<commit_after>// Package api provides rest-like server
package api

import (
	"context"
	"crypto/subtle"
	"encoding/xml"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"
	"github.com/pkg/errors"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/youtube"
	ytfeed "github.com/umputun/feed-master/app/youtube/feed"
)

//go:generate moq -out mocks/yt_service.go -pkg mocks -skip-ensure -fmt goimports . YoutubeSvc
//go:generate moq -out mocks/store.go -pkg mocks -skip-ensure -fmt goimports . Store

// Server provides HTTP API
type Server struct {
	Version       string
	Conf          config.Conf
	Store         Store
	YoutubeSvc    YoutubeSvc
	TemplLocation string
	AdminPasswd   string

	httpServer *http.Server
	cache      lcw.LoadingCache
	templates  *template.Template
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
	StoreRSS(chanID, rss string) error
	RemoveEntry(entry ytfeed.Entry) error
}

// Store provides access to feed data
type Store interface {
	Load(fmFeed string, max int, skipJunk bool) ([]feed.Item, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	log.Printf("[INFO] starting server on port %d", port)
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*3), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	if s.TemplLocation == "" {
		s.TemplLocation = "webapp/templates/*"
	}
	log.Printf("[DEBUG] loading templates from %s", s.TemplLocation)
	s.templates = template.Must(template.ParseGlob(s.TemplLocation))

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()), middleware.GetHead)
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))
	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"), logger.IPfn(logger.AnonymizeIP))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}.png", s.getImageCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"), logger.IPfn(logger.AnonymizeIP))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Head("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {

		auth := rest.BasicAuth(func(user, passwd string) bool {
			return (subtle.ConstantTimeCompare([]byte(s.AdminPasswd), []byte(passwd)) +
				subtle.ConstantTimeCompare([]byte("admin"), []byte(user))) == 2
		})

		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"), logger.IPfn(logger.AnonymizeIP))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
		r.With(auth).Post("/rss/generate", s.regenerateRSSCtrl)
		r.With(auth).Delete("/entry/{channel}/{video}", s.removeEntryCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}

		if mkdirErr := os.MkdirAll(s.Conf.YouTube.FilesLocation, 0o750); mkdirErr != nil {
			log.Printf("[ERROR] failed to create directory %s, %v", s.Conf.YouTube.FilesLocation, mkdirErr)
		}

		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")

	data, err := s.cache.Get("feed::"+feedName, func() (interface{}, error) {
		items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
		if err != nil {
			return nil, err
		}

		for i, itm := range items {
			// add ts suffix to titles
			switch s.Conf.Feeds[feedName].ExtendDateTitle {
			case "yyyyddmm":
				items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
			case "yyyymmdd":
				items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
			}
		}

		rss := feed.Rss2{
			Version:        "2.0",
			ItemList:       items,
			Title:          s.Conf.Feeds[feedName].Title,
			Description:    s.Conf.Feeds[feedName].Description,
			Language:       s.Conf.Feeds[feedName].Language,
			Link:           s.Conf.Feeds[feedName].Link,
			PubDate:        items[0].PubDate,
			LastBuildDate:  time.Now().Format(time.RFC822Z),
			ItunesAuthor:   s.Conf.Feeds[feedName].Author,
			ItunesExplicit: "no",
			ItunesOwner: &feed.ItunesOwner{
				Name:  "Feed Master",
				Email: "umputun@feed-master.com",
			},
			NsItunes: "http://www.itunes.com/dtds/podcast-1.0.dtd",
			NsMedia:  "http://search.yahoo.com/mrss/",
		}

		// replace link to UI page
		if s.Conf.System.BaseURL != "" {
			baseURL := strings.TrimSuffix(s.Conf.System.BaseURL, "/")
			rss.Link = baseURL + "/feed/" + feedName
			imagesURL := baseURL + "/images/" + feedName
			rss.ItunesImage = &feed.ItunesImg{URL: imagesURL}
			rss.MediaThumbnail = &feed.MediaThumbnail{URL: imagesURL}
		}

		b, err := xml.MarshalIndent(&rss, "", "  ")
		if err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
			return nil, errors.Wrapf(err, "failed to marshal rss for %s", feedName)
		}

		res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)

		// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
		res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
		res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

		return res, nil
	})

	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	res, ok := data.(string)
	if !ok {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to convert feed")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

// POST /yt/rss/generate - generates rss for all (each) youtube channels
func (s *Server) regenerateRSSCtrl(w http.ResponseWriter, r *http.Request) {

	for _, f := range s.Conf.YouTube.Channels {
		res, err := s.YoutubeSvc.RSSFeed(youtube.FeedInfo{ID: f.ID})
		if err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt rss for "+f.ID)
			return
		}
		if err := s.YoutubeSvc.StoreRSS(f.ID, res); err != nil {
			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to store yt rss for "+f.ID)
			return
		}
	}
	rest.RenderJSON(w, rest.JSON{"status": "ok", "feeds": len(s.Conf.YouTube.Channels)})
}

// DELETE /yt/entry/{channel}/{video} - deletes entry from youtube channel and videID
func (s *Server) removeEntryCtrl(w http.ResponseWriter, r *http.Request) {
	err := s.YoutubeSvc.RemoveEntry(ytfeed.Entry{ChannelID: chi.URLParam(r, "channel"), VideoID: chi.URLParam(r, "video")})
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to remove entry")
		return
	}
	rest.RenderJSON(w, rest.JSON{"status": "ok", "removed": chi.URLParam(r, "video")})
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package apiserver

import (
	stderrors "errors"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/rpc"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/common"
	"sync"
)

func newStateServer(srv *Server, rpcConn *rpc.Conn) *initialRoot {
	r := &initialRoot{
		srv:     srv,
		rpcConn: rpcConn,
	}
	r.admin = &srvAdmin{
		root: r,
	}
	return r
}

// initialRoot implements the API that a client first sees
// when connecting to the API. We start serving a different
// API once the user has logged in.
type initialRoot struct {
	srv     *Server
	rpcConn *rpc.Conn

	admin *srvAdmin
}

// Admin returns an object that provides API access
// to methods that can be called even when not
// authenticated.
func (r *initialRoot) Admin(id string) (*srvAdmin, error) {
	if id != "" {
		// Safeguard id for possible future use.
		return nil, common.ErrBadId
	}
	return r.admin, nil
}

// srvAdmin is the only object that unlogged-in
// clients can access. It holds any methods
// that are needed to log in.
type srvAdmin struct {
	mu       sync.Mutex
	root     *initialRoot
	loggedIn bool
}

var errAlreadyLoggedIn = stderrors.New("already logged in")

// Login logs in with the provided credentials.
// All subsequent requests on the connection will
// act as the authenticated user.
func (a *srvAdmin) Login(c params.Creds) error {
	a.mu.Lock()
	defer a.mu.Unlock()
	if a.loggedIn {
		// This can only happen if Login is called concurrently.
		return errAlreadyLoggedIn
	}
	entity, err := a.root.srv.state.Authenticator(c.AuthTag)
	if err != nil && !errors.IsNotFoundError(err) {
		return err
	}
	// We return the same error when an entity
	// does not exist as for a bad password, so that
	// we don't allow unauthenticated users to find information
	// about existing entities.
	if err != nil || !entity.PasswordValid(c.Password) {
		return common.ErrBadCreds
	}
	// We have authenticated the user; now choose an appropriate API
	// to serve to them.
	newRoot, err := a.apiRootForEntity(entity, c)
	if err != nil {
		return err
	}

	// All good, start serving the new root.
	if err := a.root.rpcConn.Serve(newRoot, serverError); err != nil {
		return err
	}
	return nil
}

func (a *srvAdmin) apiRootForEntity(entity state.TaggedAuthenticator, c params.Creds) (interface{}, error) {
	// TODO(rog) choose appropriate object to serve.
	newRoot := newSrvRoot(a.root.srv, entity)

	// If this is a machine agent connecting, we need to check the
	// nonce matches, otherwise the wrong agent might be trying to
	// connect.
	machine, ok := entity.(*state.Machine)
	if ok && !machine.CheckProvisioned(c.MachineNonce) {
		return nil, common.ErrNotProvisioned
	} else if ok && machine.CheckProvisioned(c.MachineNonce) {
		// The machine agent has connected, so start a pinger to announce
		// it's now alive.
		pinger, err := machine.SetAgentAlive()
		if err != nil {
			return nil, err
		}
		newRoot.resources.Register(pinger)
	}
	return newRoot, nil
}
<commit_msg>Changes after review<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package apiserver

import (
	stderrors "errors"
	"launchpad.net/juju-core/errors"
	"launchpad.net/juju-core/rpc"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/common"
	"sync"
)

func newStateServer(srv *Server, rpcConn *rpc.Conn) *initialRoot {
	r := &initialRoot{
		srv:     srv,
		rpcConn: rpcConn,
	}
	r.admin = &srvAdmin{
		root: r,
	}
	return r
}

// initialRoot implements the API that a client first sees
// when connecting to the API. We start serving a different
// API once the user has logged in.
type initialRoot struct {
	srv     *Server
	rpcConn *rpc.Conn

	admin *srvAdmin
}

// Admin returns an object that provides API access
// to methods that can be called even when not
// authenticated.
func (r *initialRoot) Admin(id string) (*srvAdmin, error) {
	if id != "" {
		// Safeguard id for possible future use.
		return nil, common.ErrBadId
	}
	return r.admin, nil
}

// srvAdmin is the only object that unlogged-in
// clients can access. It holds any methods
// that are needed to log in.
type srvAdmin struct {
	mu       sync.Mutex
	root     *initialRoot
	loggedIn bool
}

var errAlreadyLoggedIn = stderrors.New("already logged in")

// Login logs in with the provided credentials.
// All subsequent requests on the connection will
// act as the authenticated user.
func (a *srvAdmin) Login(c params.Creds) error {
	a.mu.Lock()
	defer a.mu.Unlock()
	if a.loggedIn {
		// This can only happen if Login is called concurrently.
		return errAlreadyLoggedIn
	}
	entity, err := a.root.srv.state.Authenticator(c.AuthTag)
	if err != nil && !errors.IsNotFoundError(err) {
		return err
	}
	// We return the same error when an entity
	// does not exist as for a bad password, so that
	// we don't allow unauthenticated users to find information
	// about existing entities.
	if err != nil || !entity.PasswordValid(c.Password) {
		return common.ErrBadCreds
	}
	// We have authenticated the user; now choose an appropriate API
	// to serve to them.
	newRoot, err := a.apiRootForEntity(entity, c)
	if err != nil {
		return err
	}

	// All good, start serving the new root.
	if err := a.root.rpcConn.Serve(newRoot, serverError); err != nil {
		return err
	}
	return nil
}

func (a *srvAdmin) apiRootForEntity(entity state.TaggedAuthenticator, c params.Creds) (interface{}, error) {
	// TODO(rog) choose appropriate object to serve.
	newRoot := newSrvRoot(a.root.srv, entity)

	// If this is a machine agent connecting, we need to check the
	// nonce matches, otherwise the wrong agent might be trying to
	// connect.
	machine, ok := entity.(*state.Machine)
	if ok {
		if !machine.CheckProvisioned(c.MachineNonce) {
			return nil, common.ErrNotProvisioned
		}
		// The machine agent has connected, so start a pinger to announce
		// it's now alive.
		pinger, err := machine.SetAgentAlive()
		if err != nil {
			return nil, err
		}
		newRoot.resources.Register(pinger)
	}
	return newRoot, nil
}
<|endoftext|>


<commit_before>package knowledge

// https://dinosaurscode.xyz/go/2016/06/19/golang-mysql-authentication/
// go get github.com/go-sql-driver/mysql

import "nli-go/lib/mentalese"
import "database/sql"
import (
	_ "github.com/go-sql-driver/mysql"
	"nli-go/lib/common"
	"fmt"
	"strings"
	"log"
)

type MySqlFactBase struct{
	db *sql.DB
	tableDescriptions map[string] []string
	ds2db []mentalese.Rule
	matcher *mentalese.RelationMatcher
}

func NewMySqlFactBase(domain string, username string, password string, database string, ds2db []mentalese.Rule) *MySqlFactBase {

	db, err := sql.Open("mysql", username + ":" + password + "@/" + database)
	if err != nil {
		panic(err.Error())
	}

	return &MySqlFactBase{ db: db, tableDescriptions: map[string] []string{}, ds2db: ds2db, matcher: mentalese.NewRelationMatcher() }
}

func (factBase MySqlFactBase) AddTableDescription(tableName string, columns []string) {
	factBase.tableDescriptions[tableName] = columns
}

// todo: remove code duplication
func (factBase MySqlFactBase) Bind(goal mentalese.Relation) []mentalese.Binding {

common.LoggerActive=true

	common.LogTree("MySqlFactBase.Bind", goal)

	bindings := []mentalese.Binding{}

	for _, ds2db := range factBase.ds2db {

		// gender(14, G), gender(A, male) => externalBinding: G = male
		externalBinding, match := factBase.matcher.MatchTwoRelations(goal, ds2db.Goal, mentalese.Binding{})
		if match {

			// gender(14, G), gender(A, male) => internalBinding: A = 14
			internalBinding, _ := factBase.matcher.MatchTwoRelations(ds2db.Goal, goal, mentalese.Binding{})

			// create a version of the conditions with bound variables
			boundConditions := factBase.matcher.BindRelationSetSingleBinding(ds2db.Pattern, internalBinding)
			// match this bound version to the database
			internalBindings, match := factBase.MatchSequenceToDatabase(boundConditions)

			if match {
				for _, binding := range internalBindings {
					bindings = append(bindings, externalBinding.Intersection(binding))
				}
			}
		}
	}

	common.LogTree("MySqlFactBase.Bind", bindings)

	common.LoggerActive=false

	return bindings
}

// Matches a sequence of relations to the relations of the MySql database
// sequence: [ marriages(A, C) person(A, 'John', _, _) ]
func (factBase MySqlFactBase) MatchSequenceToDatabase(sequence mentalese.RelationSet) ([]mentalese.Binding, bool){

	common.LogTree("MatchSequenceToDatabase", sequence)

	// bindings using database level variables
	sequenceBindings := []mentalese.Binding{}
	match := true

	for _, relation := range sequence {

		relationBindings := []mentalese.Binding{}

		if (len(relationBindings) == 0) {

			resultBindings := factBase.matchRelationToDatabase(relation)
			relationBindings = append(relationBindings, resultBindings...)

		} else {

			for _, binding := range sequenceBindings {
				boundRelation := factBase.matcher.BindSingleRelationSingleBinding(relation, binding)
				resultBindings := factBase.matchRelationToDatabase(boundRelation)
				relationBindings = append(relationBindings, resultBindings...)
			}
		}

		if len(relationBindings) == 0 {
			match = false
			break
		}

		sequenceBindings = relationBindings
	}

	common.LogTree("MatchSequenceToDatabase", sequenceBindings, match)

	return sequenceBindings, match
}

func (factBase MySqlFactBase) matchRelationToDatabase(needleRelation mentalese.Relation) []mentalese.Binding {

	common.LogTree("matchRelationToDatabase", needleRelation)

	dbBindings := []mentalese.Binding{}

	table := needleRelation.Predicate
	columns := factBase.tableDescriptions[table]

	whereClause := ""
	var values = []interface{}{}

	for i, argument := range needleRelation.Arguments {
		column := columns[i]
		if argument.TermType != mentalese.Term_anonymousVariable && argument.TermType != mentalese.Term_variable {
			whereClause += " AND " + column + " = ?"
			values = append(values, argument.TermValue)
		}
	}

	columnClause := strings.Join(columns, ", ")

	query := "SELECT " + columnClause + " FROM " + table + " WHERE TRUE" + whereClause
	fmt.Printf("        %s, %v\n", query, values)

	rows, err := factBase.db.Query(query, values...)
	if err != nil {
		panic(err.Error())
	}

	defer rows.Close()
	for rows.Next() {

		binding := mentalese.Binding{}

		// prepare an array of result value references
		resultValues := []string{}

		for range columns {
			resultValues = append(resultValues, "")
		}
		resultValueRefs := []interface{}{}
		for i := range columns {
			resultValueRefs = append(resultValueRefs, &resultValues[i])
		}

		// query all rows
		if err := rows.Scan(resultValueRefs...); err != nil {
			log.Fatal(err)
		}

		for i, argument := range needleRelation.Arguments {
			if argument.IsVariable() {
				variable := argument.TermValue
				binding[variable] = mentalese.Term{ TermType: mentalese.Term_stringConstant, TermValue: resultValues[i] }
			}
		}

		dbBindings = append(dbBindings, binding)
	}

	common.LogTree("matchRelationToDatabase", dbBindings)

	return dbBindings
}<commit_msg>Fixing bindings of MySqlFactBase<commit_after>package knowledge

// https://dinosaurscode.xyz/go/2016/06/19/golang-mysql-authentication/
// go get github.com/go-sql-driver/mysql

import "nli-go/lib/mentalese"
import "database/sql"
import (
	_ "github.com/go-sql-driver/mysql"
	"nli-go/lib/common"
	"strings"
	"log"
)

type MySqlFactBase struct{
	db *sql.DB
	tableDescriptions map[string] []string
	ds2db []mentalese.Rule
	matcher *mentalese.RelationMatcher
}

func NewMySqlFactBase(domain string, username string, password string, database string, ds2db []mentalese.Rule) *MySqlFactBase {

	db, err := sql.Open("mysql", username + ":" + password + "@/" + database)
	if err != nil {
		panic(err.Error())
	}

	return &MySqlFactBase{ db: db, tableDescriptions: map[string] []string{}, ds2db: ds2db, matcher: mentalese.NewRelationMatcher() }
}

func (factBase MySqlFactBase) AddTableDescription(tableName string, columns []string) {
	factBase.tableDescriptions[tableName] = columns
}

// todo: remove code duplication
func (factBase MySqlFactBase) Bind(goal mentalese.Relation) []mentalese.Binding {

	common.LogTree("MySqlFactBase.Bind", goal)

	bindings := []mentalese.Binding{}

	for _, ds2db := range factBase.ds2db {

		// gender(14, G), gender(A, male) => externalBinding: G = male
		externalBinding, match := factBase.matcher.MatchTwoRelations(goal, ds2db.Goal, mentalese.Binding{})
		if match {

			// gender(14, G), gender(A, male) => internalBinding: A = 14
			internalBinding, _ := factBase.matcher.MatchTwoRelations(ds2db.Goal, goal, mentalese.Binding{})

			// create a version of the conditions with bound variables
			boundConditions := factBase.matcher.BindRelationSetSingleBinding(ds2db.Pattern, internalBinding)
			// match this bound version to the database
			internalBindings, match := factBase.MatchSequenceToDatabase(boundConditions)

			if match {
				for _, binding := range internalBindings {
					bindings = append(bindings, externalBinding.Intersection(binding))
				}
			}
		}
	}

	common.LogTree("MySqlFactBase.Bind", bindings)

	return bindings
}

// Matches a sequence of relations to the relations of the MySql database
// sequence: [ marriages(A, C) person(A, 'John', _, _) ]
// return: [ { C: 1, A: 5 } ]
func (factBase MySqlFactBase) MatchSequenceToDatabase(sequence mentalese.RelationSet) ([]mentalese.Binding, bool) {

	common.LogTree("MatchSequenceToDatabase", sequence)

	// bindings using database level variables
	sequenceBindings := []mentalese.Binding{}
	match := true

	for _, relation := range sequence {

		relationBindings := []mentalese.Binding{}

		if len(relationBindings) == 0 {

			resultBindings := factBase.matchRelationToDatabase(relation)
			relationBindings = resultBindings

		} else {

			// go through the bindings resulting from previous relation
			for _, binding := range sequenceBindings {

				boundRelation := factBase.matcher.BindSingleRelationSingleBinding(relation, binding)
				resultBindings := factBase.matchRelationToDatabase(boundRelation)

				// found bindings must be extended with the bindings already present
				for _, resultBinding := range resultBindings {
					newRelationBinding := binding.Merge(resultBinding)
					relationBindings = append(relationBindings, newRelationBinding)
				}
			}
		}

		sequenceBindings = relationBindings

		if len(sequenceBindings) == 0 {
			match = false
			break
		}
	}

	common.LogTree("MatchSequenceToDatabase", sequenceBindings, match)

	return sequenceBindings, match
}

// Matches needleRelation to all relations in the database
// Returns a set of bindings
func (factBase MySqlFactBase) matchRelationToDatabase(needleRelation mentalese.Relation) []mentalese.Binding {

	common.LogTree("matchRelationToDatabase", needleRelation)

	dbBindings := []mentalese.Binding{}

	table := needleRelation.Predicate
	columns := factBase.tableDescriptions[table]

	whereClause := ""
	var values = []interface{}{}

	for i, argument := range needleRelation.Arguments {
		column := columns[i]
		if argument.TermType != mentalese.Term_anonymousVariable && argument.TermType != mentalese.Term_variable {
			whereClause += " AND " + column + " = ?"
			values = append(values, argument.TermValue)
		}
	}

	columnClause := strings.Join(columns, ", ")

	query := "SELECT " + columnClause + " FROM " + table + " WHERE TRUE" + whereClause
//	fmt.Printf("        %s, %v\n", query, values)

	rows, err := factBase.db.Query(query, values...)
	if err != nil {
		panic(err.Error())
	}

	defer rows.Close()
	for rows.Next() {

		binding := mentalese.Binding{}

		// prepare an array of result value references
		resultValues := []string{}

		for range columns {
			resultValues = append(resultValues, "")
		}
		resultValueRefs := []interface{}{}
		for i := range columns {
			resultValueRefs = append(resultValueRefs, &resultValues[i])
		}

		// query all rows
		if err := rows.Scan(resultValueRefs...); err != nil {
			log.Fatal(err)
		}

		for i, argument := range needleRelation.Arguments {
			if argument.IsVariable() {
				variable := argument.TermValue
				binding[variable] = mentalese.Term{ TermType: mentalese.Term_stringConstant, TermValue: resultValues[i] }
			}
		}

		dbBindings = append(dbBindings, binding)
	}

	common.LogTree("matchRelationToDatabase", dbBindings)

	return dbBindings
}<|endoftext|>


<commit_before>package dynamo

import (
	"encoding/binary"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/mathutil"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/util"
)

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc         dynamodbiface.DynamoDBAPI
	tablePrefix string
}

type timeSlot struct {
	tableName string
	itemEpoch int64
}

const (
	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

// NewDynamoDB creates a new DynamoDB.
func NewDynamoDB() *DynamoDB {
	return &DynamoDB{
		svc: dynamodb.New(
			session.New(),
			aws.NewConfig().WithRegion(config.Config.DynamoDBRegion),
		),
		tablePrefix: config.Config.DynamoDBTablePrefix,
	}
}

func (d *DynamoDB) Ping() error {
	var params *dynamodb.DescribeLimitsInput
	_, err := d.svc.DescribeLimits(params)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (series.SeriesMap, error) {
	slots, step := selectTimeSlots(start, end, d.tablePrefix)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	c := make(chan interface{})
	for _, slot := range slots {
		for _, names := range nameGroups {
			d.concurrentBatchGet(slot, names, step, c)
		}
	}
	sm := make(series.SeriesMap, len(nameGroups))
	for i := 0; i < len(slots)*len(nameGroups); i++ {
		ret := <-c
		switch ret.(type) {
		case series.SeriesMap:
			sm.MergePointsToMap(ret.(series.SeriesMap))
		case error:
			return nil, errors.WithStack(ret.(error))
		}
	}
	return sm, nil
}

func batchGetResultToMap(resp *dynamodb.BatchGetItemOutput, step int) series.SeriesMap {
	sm := make(series.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["MetricName"].S)
			points := make(series.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				points = append(points, series.NewDataPoint(t, v))
			}
			sm[name] = series.NewSeriesPoint(name, points, step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(slot *timeSlot, names []string, step int) (series.SeriesMap, error) {
	var keys []map[string]*dynamodb.AttributeValue
	for _, name := range names {
		keys = append(keys, map[string]*dynamodb.AttributeValue{
			"MetricName": {S: aws.String(name)},
			"Timestamp":  {N: aws.String(fmt.Sprintf("%d", slot.itemEpoch))},
		})
	}
	items := make(map[string]*dynamodb.KeysAndAttributes)
	items[slot.tableName] = &dynamodb.KeysAndAttributes{Keys: keys}
	params := &dynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	resp, err := d.svc.BatchGetItem(params)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return series.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err, "Failed to BatchGetItem %s %d %s %d",
			slot.tableName, slot.itemEpoch, strings.Join(names, ","), step,
		)
	}
	return batchGetResultToMap(resp, step), nil
}

func (d *DynamoDB) concurrentBatchGet(slot *timeSlot, names []string, step int, c chan<- interface{}) {
	go func() {
		resp, err := d.batchGet(slot, names, step)
		if err != nil {
			c <- errors.WithStack(err)
		} else {
			c <- resp
		}
	}()
}

func selectTimeSlots(startTime, endTime time.Time, tablePrefix string) ([]*timeSlot, int) {
	var (
		tableName      string
		step           int
		tableEpochStep int
		itemEpochStep  int
	)
	diffTime := endTime.Sub(startTime)
	if oneYear <= diffTime {
		tableName = tablePrefix + "-1d360d"
		tableEpochStep = oneYearSeconds
		itemEpochStep = tableEpochStep
		step = 60 * 60 * 24
	} else if oneWeek <= diffTime {
		tableName = tablePrefix + "-1h7d"
		tableEpochStep = 60 * 60 * 24 * 7
		itemEpochStep = tableEpochStep
		step = 60 * 60
	} else if oneDay <= diffTime {
		tableName = tablePrefix + "-5m1d"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = tableEpochStep
		step = 5 * 60
	} else {
		tableName = tablePrefix + "-1m1h"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = 60 * 60
		step = 60
	}

	slots := make([]*timeSlot, 0, 5)
	startTableEpoch := startTime.Unix() - startTime.Unix()%int64(tableEpochStep)
	endTableEpoch := endTime.Unix()
	for tableEpoch := startTableEpoch; tableEpoch < endTableEpoch; tableEpoch += int64(tableEpochStep) {
		startItemEpoch := mathutil.MaxInt64(tableEpoch, startTime.Unix()-startTime.Unix()%int64(itemEpochStep))
		endItemEpoch := mathutil.MinInt64(tableEpoch+int64(tableEpochStep), endTime.Unix())
		for itemEpoch := startItemEpoch; itemEpoch < endItemEpoch; itemEpoch += int64(itemEpochStep) {
			slot := timeSlot{
				tableName: fmt.Sprintf("%s-%d", tableName, tableEpoch),
				itemEpoch: itemEpoch,
			}
			slots = append(slots, &slot)
		}
	}

	return slots, step
}
<commit_msg>Forget to add comment<commit_after>package dynamo

import (
	"encoding/binary"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
	"github.com/pkg/errors"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/mathutil"
	"github.com/yuuki/diamondb/lib/series"
	"github.com/yuuki/diamondb/lib/util"
)

// DynamoDB provides a dynamodb client.
type DynamoDB struct {
	svc         dynamodbiface.DynamoDBAPI
	tablePrefix string
}

type timeSlot struct {
	tableName string
	itemEpoch int64
}

const (
	oneYear time.Duration = time.Duration(24*360) * time.Hour
	oneWeek time.Duration = time.Duration(24*7) * time.Hour
	oneDay  time.Duration = time.Duration(24*1) * time.Hour
)

var (
	dynamodbBatchLimit = 100

	oneYearSeconds = int(oneYear.Seconds())
	oneWeekSeconds = int(oneWeek.Seconds())
	oneDaySeconds  = int(oneDay.Seconds())
)

// NewDynamoDB creates a new DynamoDB.
func NewDynamoDB() *DynamoDB {
	return &DynamoDB{
		svc: dynamodb.New(
			session.New(),
			aws.NewConfig().WithRegion(config.Config.DynamoDBRegion),
		),
		tablePrefix: config.Config.DynamoDBTablePrefix,
	}
}

// Ping pings DynamoDB endpoint.
func (d *DynamoDB) Ping() error {
	var params *dynamodb.DescribeLimitsInput
	_, err := d.svc.DescribeLimits(params)
	if err != nil {
		return errors.WithStack(err)
	}
	return nil
}

// Fetch fetches datapoints by name from start until end.
func (d *DynamoDB) Fetch(name string, start, end time.Time) (series.SeriesMap, error) {
	slots, step := selectTimeSlots(start, end, d.tablePrefix)
	nameGroups := util.GroupNames(util.SplitName(name), dynamodbBatchLimit)
	c := make(chan interface{})
	for _, slot := range slots {
		for _, names := range nameGroups {
			d.concurrentBatchGet(slot, names, step, c)
		}
	}
	sm := make(series.SeriesMap, len(nameGroups))
	for i := 0; i < len(slots)*len(nameGroups); i++ {
		ret := <-c
		switch ret.(type) {
		case series.SeriesMap:
			sm.MergePointsToMap(ret.(series.SeriesMap))
		case error:
			return nil, errors.WithStack(ret.(error))
		}
	}
	return sm, nil
}

func batchGetResultToMap(resp *dynamodb.BatchGetItemOutput, step int) series.SeriesMap {
	sm := make(series.SeriesMap, len(resp.Responses))
	for _, xs := range resp.Responses {
		for _, x := range xs {
			name := (*x["MetricName"].S)
			points := make(series.DataPoints, 0, len(x["Values"].BS))
			for _, y := range x["Values"].BS {
				t := int64(binary.BigEndian.Uint64(y[0:8]))
				v := math.Float64frombits(binary.BigEndian.Uint64(y[8:]))
				points = append(points, series.NewDataPoint(t, v))
			}
			sm[name] = series.NewSeriesPoint(name, points, step)
		}
	}
	return sm
}

func (d *DynamoDB) batchGet(slot *timeSlot, names []string, step int) (series.SeriesMap, error) {
	var keys []map[string]*dynamodb.AttributeValue
	for _, name := range names {
		keys = append(keys, map[string]*dynamodb.AttributeValue{
			"MetricName": {S: aws.String(name)},
			"Timestamp":  {N: aws.String(fmt.Sprintf("%d", slot.itemEpoch))},
		})
	}
	items := make(map[string]*dynamodb.KeysAndAttributes)
	items[slot.tableName] = &dynamodb.KeysAndAttributes{Keys: keys}
	params := &dynamodb.BatchGetItemInput{
		RequestItems:           items,
		ReturnConsumedCapacity: aws.String("NONE"),
	}
	resp, err := d.svc.BatchGetItem(params)
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				// Don't handle ResourceNotFoundException as error
				// bacause diamondb web return length 0 series as 200.
				return series.SeriesMap{}, nil
			}
		}
		return nil, errors.Wrapf(err, "Failed to BatchGetItem %s %d %s %d",
			slot.tableName, slot.itemEpoch, strings.Join(names, ","), step,
		)
	}
	return batchGetResultToMap(resp, step), nil
}

func (d *DynamoDB) concurrentBatchGet(slot *timeSlot, names []string, step int, c chan<- interface{}) {
	go func() {
		resp, err := d.batchGet(slot, names, step)
		if err != nil {
			c <- errors.WithStack(err)
		} else {
			c <- resp
		}
	}()
}

func selectTimeSlots(startTime, endTime time.Time, tablePrefix string) ([]*timeSlot, int) {
	var (
		tableName      string
		step           int
		tableEpochStep int
		itemEpochStep  int
	)
	diffTime := endTime.Sub(startTime)
	if oneYear <= diffTime {
		tableName = tablePrefix + "-1d360d"
		tableEpochStep = oneYearSeconds
		itemEpochStep = tableEpochStep
		step = 60 * 60 * 24
	} else if oneWeek <= diffTime {
		tableName = tablePrefix + "-1h7d"
		tableEpochStep = 60 * 60 * 24 * 7
		itemEpochStep = tableEpochStep
		step = 60 * 60
	} else if oneDay <= diffTime {
		tableName = tablePrefix + "-5m1d"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = tableEpochStep
		step = 5 * 60
	} else {
		tableName = tablePrefix + "-1m1h"
		tableEpochStep = 60 * 60 * 24
		itemEpochStep = 60 * 60
		step = 60
	}

	slots := make([]*timeSlot, 0, 5)
	startTableEpoch := startTime.Unix() - startTime.Unix()%int64(tableEpochStep)
	endTableEpoch := endTime.Unix()
	for tableEpoch := startTableEpoch; tableEpoch < endTableEpoch; tableEpoch += int64(tableEpochStep) {
		startItemEpoch := mathutil.MaxInt64(tableEpoch, startTime.Unix()-startTime.Unix()%int64(itemEpochStep))
		endItemEpoch := mathutil.MinInt64(tableEpoch+int64(tableEpochStep), endTime.Unix())
		for itemEpoch := startItemEpoch; itemEpoch < endItemEpoch; itemEpoch += int64(itemEpochStep) {
			slot := timeSlot{
				tableName: fmt.Sprintf("%s-%d", tableName, tableEpoch),
				itemEpoch: itemEpoch,
			}
			slots = append(slots, &slot)
		}
	}

	return slots, step
}
<|endoftext|>


<commit_before>// +build !confonly

package dns

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"

	dns_feature "v2ray.com/core/features/dns"

	"golang.org/x/net/dns/dnsmessage"
	"v2ray.com/core/common"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol/dns"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal/pubsub"
	"v2ray.com/core/common/task"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/transport/internet"
)

// DoHNameServer implemented DNS over HTTPS (RFC8484) Wire Format,
// which is compatible with traditional dns over udp(RFC1035),
// thus most of the DOH implementation is copied from udpns.go
type DoHNameServer struct {
	sync.RWMutex
	ips        map[string]record
	pub        *pubsub.Service
	cleanup    *task.Periodic
	reqID      uint32
	clientIP   net.IP
	httpClient *http.Client
	dohURL     string
	name       string
}

// NewDoHNameServer creates DOH client object for remote resolving
func NewDoHNameServer(url *url.URL, dispatcher routing.Dispatcher, clientIP net.IP) (*DoHNameServer, error) {

	newError("DNS: created Remote DOH client for ", url.String()).AtInfo().WriteToLog()
	s := baseDOHNameServer(url, "DOH", clientIP)

	// Dispatched connection will be closed (interrupted) after each request
	// This makes DOH inefficient without a keep-alived connection
	// See: core/app/proxyman/outbound/handler.go:113
	// Using mux (https request wrapped in a stream layer) improves the situation.
	// Recommend to use NewDoHLocalNameServer (DOHL:) if v2ray instance is running on
	//  a normal network eg. the server side of v2ray
	tr := &http.Transport{
		MaxIdleConns:        30,
		IdleConnTimeout:     90 * time.Second,
		TLSHandshakeTimeout: 30 * time.Second,
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			dest, err := net.ParseDestination(network + ":" + addr)
			if err != nil {
				return nil, err
			}

			link, err := dispatcher.Dispatch(ctx, dest)
			if err != nil {
				return nil, err
			}
			return net.NewConnection(
				net.ConnectionInputMulti(link.Writer),
				net.ConnectionOutputMulti(link.Reader),
			), nil
		},
	}

	dispatchedClient := &http.Client{
		Transport: tr,
		Timeout:   60 * time.Second,
	}

	s.httpClient = dispatchedClient
	return s, nil
}

// NewDoHLocalNameServer creates DOH client object for local resolving
func NewDoHLocalNameServer(url *url.URL, clientIP net.IP) *DoHNameServer {
	url.Scheme = "https"
	s := baseDOHNameServer(url, "DOHL", clientIP)
	tr := &http.Transport{
		IdleConnTimeout: 90 * time.Second,
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			dest, err := net.ParseDestination(network + ":" + addr)
			if err != nil {
				return nil, err
			}
			conn, err := internet.DialSystem(ctx, dest, nil)
			if err != nil {
				return nil, err
			}
			return conn, nil
		},
	}
	s.httpClient = &http.Client{
		Timeout:   time.Second * 180,
		Transport: tr,
	}
	newError("DNS: created Local DOH client for ", url.String()).AtInfo().WriteToLog()
	return s
}

func baseDOHNameServer(url *url.URL, prefix string, clientIP net.IP) *DoHNameServer {

	s := &DoHNameServer{
		ips:      make(map[string]record),
		clientIP: clientIP,
		pub:      pubsub.NewService(),
		name:     prefix + "//" + url.Host,
		dohURL:   url.String(),
	}
	s.cleanup = &task.Periodic{
		Interval: time.Minute,
		Execute:  s.Cleanup,
	}

	return s
}

// Name returns client name
func (s *DoHNameServer) Name() string {
	return s.name
}

// Cleanup clears expired items from cache
func (s *DoHNameServer) Cleanup() error {
	now := time.Now()
	s.Lock()
	defer s.Unlock()

	if len(s.ips) == 0 {
		return newError("nothing to do. stopping...")
	}

	for domain, record := range s.ips {
		if record.A != nil && record.A.Expire.Before(now) {
			record.A = nil
		}
		if record.AAAA != nil && record.AAAA.Expire.Before(now) {
			record.AAAA = nil
		}

		if record.A == nil && record.AAAA == nil {
			newError(s.name, " cleanup ", domain).AtDebug().WriteToLog()
			delete(s.ips, domain)
		} else {
			s.ips[domain] = record
		}
	}

	if len(s.ips) == 0 {
		s.ips = make(map[string]record)
	}

	return nil
}

func (s *DoHNameServer) updateIP(req *dnsRequest, ipRec *IPRecord) {
	elapsed := time.Since(req.start)

	s.Lock()
	rec := s.ips[req.domain]
	updated := false

	switch req.reqType {
	case dnsmessage.TypeA:
		if isNewer(rec.A, ipRec) {
			rec.A = ipRec
			updated = true
		}
	case dnsmessage.TypeAAAA:
		addr := make([]net.Address, 0)
		for _, ip := range ipRec.IP {
			if len(ip.IP()) == net.IPv6len {
				addr = append(addr, ip)
			}
		}
		ipRec.IP = addr
		if isNewer(rec.AAAA, ipRec) {
			rec.AAAA = ipRec
			updated = true
		}
	}
	newError(s.name, " got answer: ", req.domain, " ", req.reqType, " -> ", ipRec.IP, " ", elapsed).AtInfo().WriteToLog()

	if updated {
		s.ips[req.domain] = rec
	}
	switch req.reqType {
	case dnsmessage.TypeA:
		s.pub.Publish(req.domain+"4", nil)
	case dnsmessage.TypeAAAA:
		s.pub.Publish(req.domain+"6", nil)
	}
	s.Unlock()
	common.Must(s.cleanup.Start())
}

func (s *DoHNameServer) newReqID() uint16 {
	return uint16(atomic.AddUint32(&s.reqID, 1))
}

func (s *DoHNameServer) sendQuery(ctx context.Context, domain string, option IPOption) {
	newError(s.name, " querying: ", domain).AtInfo().WriteToLog(session.ExportIDToError(ctx))

	reqs := buildReqMsgs(domain, option, s.newReqID, genEDNS0Options(s.clientIP))

	var deadline time.Time
	if d, ok := ctx.Deadline(); ok {
		deadline = d
	} else {
		deadline = time.Now().Add(time.Second * 8)
	}

	for _, req := range reqs {

		go func(r *dnsRequest) {

			// generate new context for each req, using same context
			// may cause reqs all aborted if any one encounter an error
			dnsCtx := context.Background()

			// reserve internal dns server requested Inbound
			if inbound := session.InboundFromContext(ctx); inbound != nil {
				dnsCtx = session.ContextWithInbound(dnsCtx, inbound)
			}

			dnsCtx = session.ContextWithContent(dnsCtx, &session.Content{
				Protocol:      "https",
				SkipRoutePick: true,
			})

			// forced to use mux for DOH
			dnsCtx = session.ContextWithMuxPrefered(dnsCtx, true)

			dnsCtx, cancel := context.WithDeadline(dnsCtx, deadline)
			defer cancel()

			b, err := dns.PackMessage(r.msg)
			if err != nil {
				newError("failed to pack dns query").Base(err).AtError().WriteToLog()
				return
			}
			resp, err := s.dohHTTPSContext(dnsCtx, b.Bytes())
			if err != nil {
				newError("failed to retrieve response").Base(err).AtError().WriteToLog()
				return
			}
			rec, err := parseResponse(resp)
			if err != nil {
				newError("failed to handle DOH response").Base(err).AtError().WriteToLog()
				return
			}
			s.updateIP(r, rec)
		}(req)
	}
}

func (s *DoHNameServer) dohHTTPSContext(ctx context.Context, b []byte) ([]byte, error) {

	body := bytes.NewBuffer(b)
	req, err := http.NewRequest("POST", s.dohURL, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/dns-message")
	req.Header.Add("Content-Type", "application/dns-message")

	resp, err := s.httpClient.Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		io.Copy(ioutil.Discard, resp.Body) // flush resp.Body so that the conn is reusable
		return nil, fmt.Errorf("DOH server returned code %d", resp.StatusCode)
	}

	return ioutil.ReadAll(resp.Body)
}

func (s *DoHNameServer) findIPsForDomain(domain string, option IPOption) ([]net.IP, error) {
	s.RLock()
	record, found := s.ips[domain]
	s.RUnlock()

	if !found {
		return nil, errRecordNotFound
	}

	var ips []net.Address
	var lastErr error
	if option.IPv6Enable && record.AAAA != nil && record.AAAA.RCode == dnsmessage.RCodeSuccess {
		aaaa, err := record.AAAA.getIPs()
		if err != nil {
			lastErr = err
		}
		ips = append(ips, aaaa...)
	}

	if option.IPv4Enable && record.A != nil && record.A.RCode == dnsmessage.RCodeSuccess {
		a, err := record.A.getIPs()
		if err != nil {
			lastErr = err
		}
		ips = append(ips, a...)
	}

	if len(ips) > 0 {
		return toNetIP(ips), nil
	}

	if lastErr != nil {
		return nil, lastErr
	}

	if (option.IPv4Enable && record.A != nil) || (option.IPv6Enable && record.AAAA != nil) {
		return nil, dns_feature.ErrEmptyResponse
	}

	return nil, errRecordNotFound
}

// QueryIP is called from dns.Server->queryIPTimeout
func (s *DoHNameServer) QueryIP(ctx context.Context, domain string, option IPOption) ([]net.IP, error) {
	fqdn := Fqdn(domain)

	ips, err := s.findIPsForDomain(fqdn, option)
	if err != errRecordNotFound {
		newError(s.name, " cache HIT ", domain, " -> ", ips).Base(err).AtDebug().WriteToLog()
		return ips, err
	}

	// ipv4 and ipv6 belong to different subscription groups
	var sub4, sub6 *pubsub.Subscriber
	if option.IPv4Enable {
		sub4 = s.pub.Subscribe(fqdn + "4")
		defer sub4.Close()
	}
	if option.IPv6Enable {
		sub6 = s.pub.Subscribe(fqdn + "6")
		defer sub6.Close()
	}
	done := make(chan interface{})
	go func() {
		if sub4 != nil {
			select {
			case <-sub4.Wait():
			case <-ctx.Done():
			}
		}
		if sub6 != nil {
			select {
			case <-sub6.Wait():
			case <-ctx.Done():
			}
		}
		close(done)
	}()
	s.sendQuery(ctx, fqdn, option)

	for {
		ips, err := s.findIPsForDomain(fqdn, option)
		if err != errRecordNotFound {
			return ips, err
		}

		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-done:
		}
	}
}
<commit_msg>Enable ForceAttemptHTTP2 on DoH client<commit_after>// +build !confonly

package dns

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"sync"
	"sync/atomic"
	"time"

	dns_feature "v2ray.com/core/features/dns"

	"golang.org/x/net/dns/dnsmessage"
	"v2ray.com/core/common"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol/dns"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal/pubsub"
	"v2ray.com/core/common/task"
	"v2ray.com/core/features/routing"
	"v2ray.com/core/transport/internet"
)

// DoHNameServer implemented DNS over HTTPS (RFC8484) Wire Format,
// which is compatible with traditional dns over udp(RFC1035),
// thus most of the DOH implementation is copied from udpns.go
type DoHNameServer struct {
	sync.RWMutex
	ips        map[string]record
	pub        *pubsub.Service
	cleanup    *task.Periodic
	reqID      uint32
	clientIP   net.IP
	httpClient *http.Client
	dohURL     string
	name       string
}

// NewDoHNameServer creates DOH client object for remote resolving
func NewDoHNameServer(url *url.URL, dispatcher routing.Dispatcher, clientIP net.IP) (*DoHNameServer, error) {

	newError("DNS: created Remote DOH client for ", url.String()).AtInfo().WriteToLog()
	s := baseDOHNameServer(url, "DOH", clientIP)

	// Dispatched connection will be closed (interrupted) after each request
	// This makes DOH inefficient without a keep-alived connection
	// See: core/app/proxyman/outbound/handler.go:113
	// Using mux (https request wrapped in a stream layer) improves the situation.
	// Recommend to use NewDoHLocalNameServer (DOHL:) if v2ray instance is running on
	//  a normal network eg. the server side of v2ray
	tr := &http.Transport{
		MaxIdleConns:        30,
		IdleConnTimeout:     90 * time.Second,
		TLSHandshakeTimeout: 30 * time.Second,
		ForceAttemptHTTP2:   true,
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			dest, err := net.ParseDestination(network + ":" + addr)
			if err != nil {
				return nil, err
			}

			link, err := dispatcher.Dispatch(ctx, dest)
			if err != nil {
				return nil, err
			}
			return net.NewConnection(
				net.ConnectionInputMulti(link.Writer),
				net.ConnectionOutputMulti(link.Reader),
			), nil
		},
	}

	dispatchedClient := &http.Client{
		Transport: tr,
		Timeout:   60 * time.Second,
	}

	s.httpClient = dispatchedClient
	return s, nil
}

// NewDoHLocalNameServer creates DOH client object for local resolving
func NewDoHLocalNameServer(url *url.URL, clientIP net.IP) *DoHNameServer {
	url.Scheme = "https"
	s := baseDOHNameServer(url, "DOHL", clientIP)
	tr := &http.Transport{
		IdleConnTimeout:   90 * time.Second,
		ForceAttemptHTTP2: true,
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			dest, err := net.ParseDestination(network + ":" + addr)
			if err != nil {
				return nil, err
			}
			conn, err := internet.DialSystem(ctx, dest, nil)
			if err != nil {
				return nil, err
			}
			return conn, nil
		},
	}
	s.httpClient = &http.Client{
		Timeout:   time.Second * 180,
		Transport: tr,
	}
	newError("DNS: created Local DOH client for ", url.String()).AtInfo().WriteToLog()
	return s
}

func baseDOHNameServer(url *url.URL, prefix string, clientIP net.IP) *DoHNameServer {

	s := &DoHNameServer{
		ips:      make(map[string]record),
		clientIP: clientIP,
		pub:      pubsub.NewService(),
		name:     prefix + "//" + url.Host,
		dohURL:   url.String(),
	}
	s.cleanup = &task.Periodic{
		Interval: time.Minute,
		Execute:  s.Cleanup,
	}

	return s
}

// Name returns client name
func (s *DoHNameServer) Name() string {
	return s.name
}

// Cleanup clears expired items from cache
func (s *DoHNameServer) Cleanup() error {
	now := time.Now()
	s.Lock()
	defer s.Unlock()

	if len(s.ips) == 0 {
		return newError("nothing to do. stopping...")
	}

	for domain, record := range s.ips {
		if record.A != nil && record.A.Expire.Before(now) {
			record.A = nil
		}
		if record.AAAA != nil && record.AAAA.Expire.Before(now) {
			record.AAAA = nil
		}

		if record.A == nil && record.AAAA == nil {
			newError(s.name, " cleanup ", domain).AtDebug().WriteToLog()
			delete(s.ips, domain)
		} else {
			s.ips[domain] = record
		}
	}

	if len(s.ips) == 0 {
		s.ips = make(map[string]record)
	}

	return nil
}

func (s *DoHNameServer) updateIP(req *dnsRequest, ipRec *IPRecord) {
	elapsed := time.Since(req.start)

	s.Lock()
	rec := s.ips[req.domain]
	updated := false

	switch req.reqType {
	case dnsmessage.TypeA:
		if isNewer(rec.A, ipRec) {
			rec.A = ipRec
			updated = true
		}
	case dnsmessage.TypeAAAA:
		addr := make([]net.Address, 0)
		for _, ip := range ipRec.IP {
			if len(ip.IP()) == net.IPv6len {
				addr = append(addr, ip)
			}
		}
		ipRec.IP = addr
		if isNewer(rec.AAAA, ipRec) {
			rec.AAAA = ipRec
			updated = true
		}
	}
	newError(s.name, " got answer: ", req.domain, " ", req.reqType, " -> ", ipRec.IP, " ", elapsed).AtInfo().WriteToLog()

	if updated {
		s.ips[req.domain] = rec
	}
	switch req.reqType {
	case dnsmessage.TypeA:
		s.pub.Publish(req.domain+"4", nil)
	case dnsmessage.TypeAAAA:
		s.pub.Publish(req.domain+"6", nil)
	}
	s.Unlock()
	common.Must(s.cleanup.Start())
}

func (s *DoHNameServer) newReqID() uint16 {
	return uint16(atomic.AddUint32(&s.reqID, 1))
}

func (s *DoHNameServer) sendQuery(ctx context.Context, domain string, option IPOption) {
	newError(s.name, " querying: ", domain).AtInfo().WriteToLog(session.ExportIDToError(ctx))

	reqs := buildReqMsgs(domain, option, s.newReqID, genEDNS0Options(s.clientIP))

	var deadline time.Time
	if d, ok := ctx.Deadline(); ok {
		deadline = d
	} else {
		deadline = time.Now().Add(time.Second * 8)
	}

	for _, req := range reqs {

		go func(r *dnsRequest) {

			// generate new context for each req, using same context
			// may cause reqs all aborted if any one encounter an error
			dnsCtx := context.Background()

			// reserve internal dns server requested Inbound
			if inbound := session.InboundFromContext(ctx); inbound != nil {
				dnsCtx = session.ContextWithInbound(dnsCtx, inbound)
			}

			dnsCtx = session.ContextWithContent(dnsCtx, &session.Content{
				Protocol:      "https",
				SkipRoutePick: true,
			})

			// forced to use mux for DOH
			dnsCtx = session.ContextWithMuxPrefered(dnsCtx, true)

			dnsCtx, cancel := context.WithDeadline(dnsCtx, deadline)
			defer cancel()

			b, err := dns.PackMessage(r.msg)
			if err != nil {
				newError("failed to pack dns query").Base(err).AtError().WriteToLog()
				return
			}
			resp, err := s.dohHTTPSContext(dnsCtx, b.Bytes())
			if err != nil {
				newError("failed to retrieve response").Base(err).AtError().WriteToLog()
				return
			}
			rec, err := parseResponse(resp)
			if err != nil {
				newError("failed to handle DOH response").Base(err).AtError().WriteToLog()
				return
			}
			s.updateIP(r, rec)
		}(req)
	}
}

func (s *DoHNameServer) dohHTTPSContext(ctx context.Context, b []byte) ([]byte, error) {

	body := bytes.NewBuffer(b)
	req, err := http.NewRequest("POST", s.dohURL, body)
	if err != nil {
		return nil, err
	}

	req.Header.Add("Accept", "application/dns-message")
	req.Header.Add("Content-Type", "application/dns-message")

	resp, err := s.httpClient.Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		io.Copy(ioutil.Discard, resp.Body) // flush resp.Body so that the conn is reusable
		return nil, fmt.Errorf("DOH server returned code %d", resp.StatusCode)
	}

	return ioutil.ReadAll(resp.Body)
}

func (s *DoHNameServer) findIPsForDomain(domain string, option IPOption) ([]net.IP, error) {
	s.RLock()
	record, found := s.ips[domain]
	s.RUnlock()

	if !found {
		return nil, errRecordNotFound
	}

	var ips []net.Address
	var lastErr error
	if option.IPv6Enable && record.AAAA != nil && record.AAAA.RCode == dnsmessage.RCodeSuccess {
		aaaa, err := record.AAAA.getIPs()
		if err != nil {
			lastErr = err
		}
		ips = append(ips, aaaa...)
	}

	if option.IPv4Enable && record.A != nil && record.A.RCode == dnsmessage.RCodeSuccess {
		a, err := record.A.getIPs()
		if err != nil {
			lastErr = err
		}
		ips = append(ips, a...)
	}

	if len(ips) > 0 {
		return toNetIP(ips), nil
	}

	if lastErr != nil {
		return nil, lastErr
	}

	if (option.IPv4Enable && record.A != nil) || (option.IPv6Enable && record.AAAA != nil) {
		return nil, dns_feature.ErrEmptyResponse
	}

	return nil, errRecordNotFound
}

// QueryIP is called from dns.Server->queryIPTimeout
func (s *DoHNameServer) QueryIP(ctx context.Context, domain string, option IPOption) ([]net.IP, error) {
	fqdn := Fqdn(domain)

	ips, err := s.findIPsForDomain(fqdn, option)
	if err != errRecordNotFound {
		newError(s.name, " cache HIT ", domain, " -> ", ips).Base(err).AtDebug().WriteToLog()
		return ips, err
	}

	// ipv4 and ipv6 belong to different subscription groups
	var sub4, sub6 *pubsub.Subscriber
	if option.IPv4Enable {
		sub4 = s.pub.Subscribe(fqdn + "4")
		defer sub4.Close()
	}
	if option.IPv6Enable {
		sub6 = s.pub.Subscribe(fqdn + "6")
		defer sub6.Close()
	}
	done := make(chan interface{})
	go func() {
		if sub4 != nil {
			select {
			case <-sub4.Wait():
			case <-ctx.Done():
			}
		}
		if sub6 != nil {
			select {
			case <-sub6.Wait():
			case <-ctx.Done():
			}
		}
		close(done)
	}()
	s.sendQuery(ctx, fqdn, option)

	for {
		ips, err := s.findIPsForDomain(fqdn, option)
		if err != errRecordNotFound {
			return ips, err
		}

		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-done:
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1
	log.Printf("[Scanner.findBlock] cur %d, remaining %d\n", s.idx, remaining)

	if remaining <= 0 {
		log.Println("[Scanner.findBlock] last block")
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		log.Println("[Scanner.findBlock] next block is past key")
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		log.Printf("[Scanner.blockFor] curBlock after key %s (cur: %d, start: %s)\n",
			hex.EncodeToString(key),
			s.idx,
			hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
		)
		return nil, nil, false
	}

	idx := s.findBlock(key)
	log.Printf("[Scanner.blockFor] findBlock key: %s. Picked %d (starts: %s). Cur: %d (starts: %s)\n",
		hex.EncodeToString(key),
		idx,
		hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
		s.idx,
		hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
	)

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			log.Printf("[Scanner.blockFor] read err %s (key: %s, idx: %d, start: %s)\n",
				err,
				hex.EncodeToString(key),
				idx,
				hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
			)
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetFirst] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetAll] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		if bytes.Compare(key, keyBytes) == 0 {
			if first {
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
	}

	return nil, acc, len(acc) > 0
}
<commit_msg>more debug when walking blocks<commit_after>// Copyright (C) 2014 Daniel Harrison

package hfile

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"log"
	"sort"
)

type Scanner struct {
	reader  *Reader
	idx     int
	buf     *bytes.Reader
	lastKey *[]byte
}

func NewScanner(r *Reader) Scanner {
	return Scanner{r, 0, nil, nil}
}

func (s *Scanner) Reset() {
	s.idx = 0
	s.buf = nil
	s.lastKey = nil
}

func (s *Scanner) findBlock(key []byte) int {
	remaining := len(s.reader.index) - s.idx - 1
	log.Printf("[Scanner.findBlock] cur %d, remaining %d\n", s.idx, remaining)

	if remaining <= 0 {
		log.Println("[Scanner.findBlock] last block")
		return s.idx // s.cur is the last block, so it is only choice.
	}

	if s.reader.index[s.idx+1].IsAfter(key) {
		log.Println("[Scanner.findBlock] next block is past key")
		return s.idx
	}

	offset := sort.Search(remaining, func(i int) bool {
		return s.reader.index[s.idx+i+1].IsAfter(key)
	})

	return s.idx + offset
}

func (s *Scanner) CheckIfKeyOutOfOrder(key []byte) error {
	if s.lastKey != nil && bytes.Compare(*s.lastKey, key) > 0 {
		return fmt.Errorf("Keys our of order! %v > %v", *s.lastKey, key)
	}
	s.lastKey = &key
	return nil
}

func (s *Scanner) blockFor(key []byte) (*bytes.Reader, error, bool) {
	err := s.CheckIfKeyOutOfOrder(key)
	if err != nil {
		return nil, err, false
	}

	if s.reader.index[s.idx].IsAfter(key) {
		log.Printf("[Scanner.blockFor] curBlock after key %s (cur: %d, start: %s)\n",
			hex.EncodeToString(key),
			s.idx,
			hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
		)
		return nil, nil, false
	}

	idx := s.findBlock(key)
	log.Printf("[Scanner.blockFor] findBlock (key: %s) picked %d (starts: %s). Cur: %d (starts: %s)\n",
		hex.EncodeToString(key),
		idx,
		hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
		s.idx,
		hex.EncodeToString(s.reader.index[s.idx].firstKeyBytes),
	)

	if idx != s.idx || s.buf == nil { // need to load a new block
		data, err := s.reader.GetBlock(idx)
		if err != nil {
			log.Printf("[Scanner.blockFor] read err %s (key: %s, idx: %d, start: %s)\n",
				err,
				hex.EncodeToString(key),
				idx,
				hex.EncodeToString(s.reader.index[idx].firstKeyBytes),
			)
			return nil, err, false
		}
		s.idx = idx
		s.buf = data
	} else {
		log.Println("[Scanner.blockFor] Re-using current block")
	}

	return s.buf, nil, true
}

func (s *Scanner) GetFirst(key []byte) ([]byte, error, bool) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetFirst] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err, ok
	}

	value, _, found := getValuesFromBuffer(data, key, true)
	return value, nil, found
}

func (s *Scanner) GetAll(key []byte) ([][]byte, error) {
	data, err, ok := s.blockFor(key)

	if !ok {
		log.Printf("[Scanner.GetAll] No Block for key: %s (err: %s, found: %s)\n", hex.EncodeToString(key), err, ok)
		return nil, err
	}

	_, found, _ := getValuesFromBuffer(data, key, false)
	return found, err
}

func getValuesFromBuffer(buf *bytes.Reader, key []byte, first bool) ([]byte, [][]byte, bool) {
	var acc [][]byte

	log.Printf("[Scanner.getValuesFromBuffer] buf before %d\n", buf.Len())
	for buf.Len() > 0 {
		var keyLen, valLen uint32
		binary.Read(buf, binary.BigEndian, &keyLen)
		binary.Read(buf, binary.BigEndian, &valLen)
		keyBytes := make([]byte, keyLen)
		valBytes := make([]byte, valLen)
		buf.Read(keyBytes)
		buf.Read(valBytes)
		cmp := bytes.Compare(keyBytes, key)
		if cmp == 0 {
			if first {
				log.Printf("[Scanner.getValuesFromBuffer] buf after %d\n", buf.Len())
				return valBytes, nil, true
			} else {
				acc = append(acc, valBytes)
			}
		}
		if cmp > 0 {
			log.Printf("[Scanner.getValuesFromBuffer] past key %s vs %s. buf remaining %d\n",
				hex.EncodeToString(key),
				hex.EncodeToString(keyBytes),
				buf.Len(),
			)
			buf.Seek(-(int64(keyLen + valLen + 8)), 1)
			return nil, acc, len(acc) > 0
		}
	}
	log.Printf("[Scanner.getValuesFromBuffer] walked off block\n")
	return nil, acc, len(acc) > 0
}
<|endoftext|>


<commit_before>package webface

import (
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/tsukanov/steaminfo-go/storage"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

func makeRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", homeHandler)
	r.HandleFunc("/history/{appid:[0-9]+}.json", historyHandler)
	r.HandleFunc("/search", searchHandler)
	r.HandleFunc("/about/", aboutHandler)
	return r
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	t, err := template.ParseFiles(
		"webface/templates/base.html",
		"webface/templates/home.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, r.URL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func historyHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	appId, err := strconv.Atoi(vars["appid"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	name, err := storage.GetName(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	history, err := storage.AllUsageHistory(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	type jason struct {
		Name    string     `json:"name"`
		History [][2]int64 `json:"history"`
	}
	result := jason{
		Name:    name,
		History: history,
	}
	b, err := json.Marshal(result)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func searchHandler(w http.ResponseWriter, r *http.Request) {
	queries := r.URL.Query()
	query := queries["q"][0]
	results, err := storage.Search(query)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	b, err := json.Marshal(results)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	t, err := template.ParseFiles(
		"webface/templates/base.html",
		"webface/templates/about.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

// Start starts FastCGI server at 127.0.0.1:9000
func Start() {
	log.Println("Starting server...")
	l, err := net.Listen("tcp", "127.0.0.1:9000")
	if err != nil {
		log.Fatal("Failed to start server!", err)
	}
	fcgi.Serve(l, makeRouter())
}

// StartDev starts a development server at localhost:8080
func StartDev() {
	log.Println("Starting development server (localhost:8080)...")
	http.ListenAndServe(":8080", makeRouter())
}
<commit_msg>More path fixes!<commit_after>package webface

import (
	"bitbucket.org/kardianos/osext"
	"encoding/json"
	"github.com/gorilla/mux"
	"github.com/tsukanov/steaminfo-go/storage"
	"html/template"
	"log"
	"net"
	"net/http"
	"net/http/fcgi"
	"strconv"
)

func makeRouter() *mux.Router {
	r := mux.NewRouter()
	r.HandleFunc("/", homeHandler)
	r.HandleFunc("/history/{appid:[0-9]+}.json", historyHandler)
	r.HandleFunc("/search", searchHandler)
	r.HandleFunc("/about/", aboutHandler)
	return r
}

func homeHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/home.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, r.URL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

func historyHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	appId, err := strconv.Atoi(vars["appid"])
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	name, err := storage.GetName(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	history, err := storage.AllUsageHistory(appId)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	type jason struct {
		Name    string     `json:"name"`
		History [][2]int64 `json:"history"`
	}
	result := jason{
		Name:    name,
		History: history,
	}
	b, err := json.Marshal(result)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func searchHandler(w http.ResponseWriter, r *http.Request) {
	queries := r.URL.Query()
	query := queries["q"][0]
	results, err := storage.Search(query)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	b, err := json.Marshal(results)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(b)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	exeloc, err := osext.ExecutableFolder()
	t, err := template.ParseFiles(
		exeloc+"webface/templates/base.html",
		exeloc+"webface/templates/about.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
	err = t.Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		log.Println(err)
		return
	}
}

// Start starts FastCGI server at 127.0.0.1:9000
func Start() {
	log.Println("Starting server...")
	l, err := net.Listen("tcp", "127.0.0.1:9000")
	if err != nil {
		log.Fatal("Failed to start server!", err)
	}
	fcgi.Serve(l, makeRouter())
}

// StartDev starts a development server at localhost:8080
func StartDev() {
	log.Println("Starting development server (localhost:8080)...")
	http.ListenAndServe(":8080", makeRouter())
}
<|endoftext|>


<commit_before>package webhook

import (
	"sync/atomic"
	"time"
)

const (
	DEFAULT_EXPIRATION_DURATION time.Duration = time.Second * 300
)

// W is the structure that represents the Webhook listener
// data we share.
//
// (Note to Wes: this follows the golang naming conventions.  webhook.Webhook "stutters",
// and this type is really the central type of this package.  Calling it a single letter is the norm.
// This could also go in the server package, in which case I'd change the name to Webhook, since
// service.Webhook works better.  See https://blog.golang.org/package-names)
type W struct {
	// Configuration for message delivery
	Config struct {
		// The URL to deliver messages to.
		URL string `json:"url"`

		// The content-type to set the messages to (unless specified by WRP).
		ContentType string `json:"content_type"`

		// The secret to use for the SHA1 HMAC.
		// Optional, set to "" to disable behavior.
		Secret string `json:"secret,omitempty"`
	} `json:"config"`

	// The URL to notify when we cut off a client due to overflow.
	// Optional, set to "" to disable behavior
	FailureURL string `json:"failure_url"`

	// The list of regular expressions to match event type against.
	Events []string `json:"events"`

	// Matcher type contains values to match against the metadata.
	Matcher struct {
		// The list of regular expressions to match device id type against.
		DeviceId []string `json:"device_id"`
	} `json:"matcher,omitempty"`

	// The specified duration for this hook to live
	Duration time.Duration `json:"duration"`

	// The absolute time when this hook is to be disabled
	Until time.Time `json:"until"`

	// The address that performed the registration
	Address string `json:"registered_from_address"`
}

// ID creates the canonical string identifing a WebhookListener
func (w *W) ID() string {
	return w.Config.URL
}

// durationValidator performs a check on a W.Duration value.
// if found to be invalid it is set to the DEFAULT_EXPIRATION_DURATION
func (w *W) durationValidator() {
	if w.Duration <= 0 || w.Duration > DEFAULT_EXPIRATION_DURATION {
		w.Duration = DEFAULT_EXPIRATION_DURATION
	}
}

// List is a read-only random access interface to a set of W's
// We don't necessarily need an implementation of just this interface alone.
type List interface {
	Len() int
	Get(int) *W
}

// UpdatableList is mutable list that can be updated en masse
type UpdatableList interface {
	List

	// Update performs a bulk update of this webhooks known to this list
	Update([]W)

	// Filter atomically filters the elements of this list
	Filter(func([]W) []W)
}

type updatableList struct {
	value atomic.Value
}

func (ul *updatableList) set(list []W) {
	ul.value.Store(list)
}

func (ul *updatableList) Len() int {
	if list, ok := ul.value.Load().([]W); ok {
		return len(list)
	}

	return 0
}

func (ul *updatableList) Get(index int) *W {
	if list, ok := ul.value.Load().([]W); ok {
		return &list[index]
	}

	// TODO: design choice.  may want to panic here, to mimic
	// the behavior of the golang runtime for slices.  Alternatively,
	// could return a second parameter that is an error (consistentHash does that).
	return nil
}

func (ul *updatableList) Update(newItems []W) {
	for _, newItem := range newItems {
		found := false
		var items []*W
		for i := 0; i < ul.Len(); i++ {
			items = append(items, ul.Get(i))
		}

		// for new items.  we don't want to change a valid expiration time.
		if &newItem.Until == nil || newItem.Until.Equal(time.Time{}) {
			newItem.durationValidator()
			newItem.Until = time.Now().Add(newItem.Duration)
		}

		// we want to add items that will expire in the future
		if newItem.Until.After(time.Now()) {
			for i := 0; i < len(items) && !found; i++ {
				if items[i].ID() == newItem.ID() {
					found = true

					items[i].Matcher = newItem.Matcher
					items[i].Events = newItem.Events
					items[i].Config.ContentType = newItem.Config.ContentType
					items[i].Config.Secret = newItem.Config.Secret
				}
			}

			// add item
			if !found {
				items = append(items, &newItem)
			}

			var itemsCopy []W
			for _, i := range items {
				itemsCopy = append(itemsCopy, *i)
			}

			// store items
			ul.set(itemsCopy)
		}
	}
}

func (ul *updatableList) Filter(filter func([]W) []W) {
	if list, ok := ul.value.Load().([]W); ok {
		copyOf := make([]W, len(list))
		for i, w := range list {
			copyOf[i] = w
		}

		ul.set(filter(copyOf))
	}
}

// NewList just creates an UpdatableList.  Don't forget:
// NewList(nil) is valid!
func NewList(initial []W) UpdatableList {
	ul := &updatableList{}
	ul.Update(initial)
	return ul
}
<commit_msg>fixes inconsistent listener registration.<commit_after>package webhook

import (
	"sync/atomic"
	"time"
)

const (
	DEFAULT_EXPIRATION_DURATION time.Duration = time.Second * 300
)

// W is the structure that represents the Webhook listener
// data we share.
//
// (Note to Wes: this follows the golang naming conventions.  webhook.Webhook "stutters",
// and this type is really the central type of this package.  Calling it a single letter is the norm.
// This could also go in the server package, in which case I'd change the name to Webhook, since
// service.Webhook works better.  See https://blog.golang.org/package-names)
type W struct {
	// Configuration for message delivery
	Config struct {
		// The URL to deliver messages to.
		URL string `json:"url"`

		// The content-type to set the messages to (unless specified by WRP).
		ContentType string `json:"content_type"`

		// The secret to use for the SHA1 HMAC.
		// Optional, set to "" to disable behavior.
		Secret string `json:"secret,omitempty"`
	} `json:"config"`

	// The URL to notify when we cut off a client due to overflow.
	// Optional, set to "" to disable behavior
	FailureURL string `json:"failure_url"`

	// The list of regular expressions to match event type against.
	Events []string `json:"events"`

	// Matcher type contains values to match against the metadata.
	Matcher struct {
		// The list of regular expressions to match device id type against.
		DeviceId []string `json:"device_id"`
	} `json:"matcher,omitempty"`

	// The specified duration for this hook to live
	Duration time.Duration `json:"duration"`

	// The absolute time when this hook is to be disabled
	Until time.Time `json:"until"`

	// The address that performed the registration
	Address string `json:"registered_from_address"`
}

// ID creates the canonical string identifing a WebhookListener
func (w *W) ID() string {
	return w.Config.URL
}

// durationValidator performs a check on a W.Duration value.
// if found to be invalid it is set to the DEFAULT_EXPIRATION_DURATION
func (w *W) durationValidator() {
	if w.Duration <= 0 || w.Duration > DEFAULT_EXPIRATION_DURATION {
		w.Duration = DEFAULT_EXPIRATION_DURATION
	}
}

// List is a read-only random access interface to a set of W's
// We don't necessarily need an implementation of just this interface alone.
type List interface {
	Len() int
	Get(int) *W
}

// UpdatableList is mutable list that can be updated en masse
type UpdatableList interface {
	List

	// Update performs a bulk update of this webhooks known to this list
	Update([]W)

	// Filter atomically filters the elements of this list
	Filter(func([]W) []W)
}

type updatableList struct {
	value atomic.Value
}

func (ul *updatableList) set(list []W) {
	ul.value.Store(list)
}

func (ul *updatableList) Len() int {
	if list, ok := ul.value.Load().([]W); ok {
		return len(list)
	}

	return 0
}

func (ul *updatableList) Get(index int) *W {
	if list, ok := ul.value.Load().([]W); ok {
		return &list[index]
	}

	// TODO: design choice.  may want to panic here, to mimic
	// the behavior of the golang runtime for slices.  Alternatively,
	// could return a second parameter that is an error (consistentHash does that).
	return nil
}

func (ul *updatableList) Update(newItems []W) {
	for _, newItem := range newItems {
		found := false
		var items []*W
		for i := 0; i < ul.Len(); i++ {
			items = append(items, ul.Get(i))
		}

		// for new items.  we don't want to change a valid expiration time.
		if &newItem.Until == nil || newItem.Until.Equal(time.Time{}) {
			newItem.durationValidator()
			newItem.Until = time.Now().Add(newItem.Duration)
		}

		// we want to add items that will expire in the future
		if newItem.Until.After(time.Now()) {
			for i := 0; i < len(items) && !found; i++ {
				if items[i].ID() == newItem.ID() {
					found = true

					items[i].Matcher = newItem.Matcher
					items[i].Events = newItem.Events
					items[i].Config.ContentType = newItem.Config.ContentType
					items[i].Config.Secret = newItem.Config.Secret
					items[i].Until = newItem.Until
				}
			}

			// add item
			if !found {
				items = append(items, &newItem)
			}

			var itemsCopy []W
			for _, i := range items {
				itemsCopy = append(itemsCopy, *i)
			}

			// store items
			ul.set(itemsCopy)
		}
	}
}

func (ul *updatableList) Filter(filter func([]W) []W) {
	if list, ok := ul.value.Load().([]W); ok {
		copyOf := make([]W, len(list))
		for i, w := range list {
			copyOf[i] = w
		}

		ul.set(filter(copyOf))
	}
}

// NewList just creates an UpdatableList.  Don't forget:
// NewList(nil) is valid!
func NewList(initial []W) UpdatableList {
	ul := &updatableList{}
	ul.Update(initial)
	return ul
}
<|endoftext|>