<commit_before>// Package api provides rest-like server
package api

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
	"github.com/umputun/feed-master/app/youtube"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       config.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}
		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
		NsMedia:       "http://search.yahoo.com/mrss/",
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		baseURL := s.Conf.System.BaseURL
		if strings.HasSuffix(baseURL, "/") {
			baseURL = strings.TrimSuffix(baseURL, "/")
		}
		rss.Link = baseURL + "/feed/" + feedName
		imagesURL := baseURL + "/images/" + feedName
		rss.ItunesImage = feed.ItunesImg{URL: imagesURL}
		rss.MediaThumbnail = feed.MediaThumbnail{URL: imagesURL}
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)
	// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
	res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
	res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<commit_msg>lint: simplify suffix trim<commit_after>// Package api provides rest-like server
package api

import (
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/didip/tollbooth/v6"
	"github.com/didip/tollbooth_chi"
	"github.com/go-chi/chi/v5"
	"github.com/go-chi/chi/v5/middleware"
	"github.com/go-chi/render"
	"github.com/go-pkgz/lcw"
	log "github.com/go-pkgz/lgr"
	"github.com/go-pkgz/rest"
	"github.com/go-pkgz/rest/logger"

	"github.com/umputun/feed-master/app/config"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
	"github.com/umputun/feed-master/app/youtube"
)

// Server provides HTTP API
type Server struct {
	Version    string
	Conf       config.Conf
	Store      *proc.BoltDB
	YoutubeSvc YoutubeSvc
	httpServer *http.Server
	cache      lcw.LoadingCache
}

// YoutubeSvc provides access to youtube's audio rss
type YoutubeSvc interface {
	RSSFeed(cinfo youtube.FeedInfo) (string, error)
}

// Run starts http server for API with all routes
func (s *Server) Run(ctx context.Context, port int) {
	var err error
	if s.cache, err = lcw.NewExpirableCache(lcw.TTL(time.Minute*5), lcw.MaxCacheSize(10*1024*1024)); err != nil {
		log.Printf("[PANIC] failed to make loading cache, %v", err)
		return
	}

	go func() {
		<-ctx.Done()
		if s.httpServer != nil {
			if clsErr := s.httpServer.Close(); clsErr != nil {
				log.Printf("[ERROR] failed to close proxy http server, %v", clsErr)
			}
		}
	}()

	s.httpServer = &http.Server{
		Addr:              fmt.Sprintf(":%d", port),
		Handler:           s.router(),
		ReadHeaderTimeout: 5 * time.Second,
		WriteTimeout:      30 * time.Second,
		IdleTimeout:       30 * time.Second,
	}
	err = s.httpServer.ListenAndServe()
	log.Printf("[WARN] http server terminated, %s", err)
}

func (s *Server) router() *chi.Mux {
	router := chi.NewRouter()
	router.Use(middleware.RealIP, rest.Recoverer(log.Default()))
	router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second))
	router.Use(rest.AppInfo("feed-master", "umputun", s.Version), rest.Ping)
	router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(5, nil)))

	router.Group(func(rimg chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[DEBUG]"))
		rimg.Use(l.Handler)
		rimg.Get("/images/{name}", s.getImageCtrl)
		rimg.Get("/image/{name}", s.getImageCtrl)
		rimg.Head("/image/{name}", s.getImageHeadCtrl)
		rimg.Head("/images/{name}", s.getImageHeadCtrl)
	})

	router.Group(func(rrss chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		rrss.Use(l.Handler)
		rrss.Get("/rss/{name}", s.getFeedCtrl)
		rrss.Get("/list", s.getListCtrl)
		rrss.Get("/feed/{name}", s.getFeedPageCtrl)
		rrss.Get("/feed/{name}/sources", s.getSourcesPageCtrl)
		rrss.Get("/feeds", s.getFeedsPageCtrl)
	})

	router.Route("/yt", func(r chi.Router) {
		l := logger.New(logger.Log(log.Default()), logger.Prefix("[INFO]"))
		r.Use(l.Handler)
		r.Get("/rss/{channel}", s.getYoutubeFeedCtrl)
	})

	if s.Conf.YouTube.BaseURL != "" {
		baseYtURL, parseErr := url.Parse(s.Conf.YouTube.BaseURL)
		if parseErr != nil {
			log.Printf("[ERROR] failed to parse base url %s, %v", s.Conf.YouTube.BaseURL, parseErr)
		}
		ytfs, fsErr := rest.NewFileServer(baseYtURL.Path, s.Conf.YouTube.FilesLocation)
		if fsErr == nil {
			router.Mount(baseYtURL.Path, ytfs)
		} else {
			log.Printf("[WARN] can't start static file server for yt, %v", fsErr)
		}
	}

	fs, err := rest.NewFileServer("/static", filepath.Join("webapp", "static"))
	if err == nil {
		router.Mount("/static", fs)
	} else {
		log.Printf("[WARN] can't start static file server, %v", err)
	}
	return router
}

// GET /rss/{name} - returns rss for given feeds set
func (s *Server) getFeedCtrl(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "name")
	items, err := s.Store.Load(feedName, s.Conf.System.MaxTotal, true)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, err, "failed to get feed")
		return
	}

	for i, itm := range items {
		// add ts suffix to titles
		switch s.Conf.Feeds[feedName].ExtendDateTitle {
		case "yyyyddmm":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-02-01"))
		case "yyyymmdd":
			items[i].Title = fmt.Sprintf("%s (%s)", itm.Title, itm.DT.Format("2006-01-02"))
		}
	}

	rss := feed.Rss2{
		Version:       "2.0",
		ItemList:      items,
		Title:         s.Conf.Feeds[feedName].Title,
		Description:   s.Conf.Feeds[feedName].Description,
		Language:      s.Conf.Feeds[feedName].Language,
		Link:          s.Conf.Feeds[feedName].Link,
		PubDate:       items[0].PubDate,
		LastBuildDate: time.Now().Format(time.RFC822Z),
		NsItunes:      "http://www.itunes.com/dtds/podcast-1.0.dtd",
		NsMedia:       "http://search.yahoo.com/mrss/",
	}

	// replace link to UI page
	if s.Conf.System.BaseURL != "" {
		baseURL := strings.TrimSuffix(s.Conf.System.BaseURL, "/")
		rss.Link = baseURL + "/feed/" + feedName
		imagesURL := baseURL + "/images/" + feedName
		rss.ItunesImage = feed.ItunesImg{URL: imagesURL}
		rss.MediaThumbnail = feed.MediaThumbnail{URL: imagesURL}
	}

	b, err := xml.MarshalIndent(&rss, "", "  ")
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to marshal rss")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res := `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + string(b)
	// this hack to avoid having different items for marshal and unmarshal due to "itunes" namespace
	res = strings.Replace(res, "<duration>", "<itunes:duration>", -1)
	res = strings.Replace(res, "</duration>", "</itunes:duration>", -1)

	_, _ = fmt.Fprintf(w, "%s", res)
}

// GET /image/{name}
func (s *Server) getImageCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			fmt.Errorf("image %s not found", fm), "failed to load image")
		return
	}

	b, err := os.ReadFile(feedConf.Image)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest,
			errors.New("can't read  "+chi.URLParam(r, "name")), "failed to read image")
		return
	}
	w.Header().Set("Content-Type", "image/png")
	if _, err := w.Write(b); err != nil {
		log.Printf("[WARN] failed to send image, %s", err)
	}
}

// HEAD /image/{name}
func (s *Server) getImageHeadCtrl(w http.ResponseWriter, r *http.Request) {
	fm := chi.URLParam(r, "name")
	fm = strings.TrimSuffix(fm, ".png")
	feedConf, found := s.Conf.Feeds[fm]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	info, err := os.Stat(feedConf.Image)
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "image/png")
	w.Header().Set("Content-Length", strconv.Itoa(int(info.Size())))
	w.WriteHeader(http.StatusOK)
}

// GET /list - returns feed's image
func (s *Server) getListCtrl(w http.ResponseWriter, r *http.Request) {
	feeds := s.feeds()
	render.JSON(w, r, feeds)
}

// GET /yt/rss/{channel} - returns rss for given youtube channel
func (s *Server) getYoutubeFeedCtrl(w http.ResponseWriter, r *http.Request) {
	channel := chi.URLParam(r, "channel")

	fi := youtube.FeedInfo{ID: channel}
	for _, f := range s.Conf.YouTube.Channels {
		if f.ID == channel {
			fi = f
			break
		}
	}

	res, err := s.YoutubeSvc.RSSFeed(fi)
	if err != nil {
		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, "failed to read yt list")
		return
	}

	w.Header().Set("Content-Type", "application/xml; charset=UTF-8")
	res = `<?xml version="1.0" encoding="UTF-8"?>` + "\n" + res
	_, _ = fmt.Fprintf(w, "%s", res)
}

func (s *Server) feeds() []string {
	feeds := make([]string, 0, len(s.Conf.Feeds))
	for k := range s.Conf.Feeds {
		feeds = append(feeds, k)
	}
	return feeds
}
<|endoftext|>


<commit_before>package bettermail

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	log_ "log"
	"net/http"
	"strings"
	"time"

	"github.com/mailgun/mailgun-go"

	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

var templates map[string]*Template

type MailgunConfig struct {
	Domain    string
	APIKey    string
	PublicKey string
	Recipient string
}

var config MailgunConfig

func init() {
	initConfig()
	templates = loadTemplates()

	http.HandleFunc("/hook", hookHandler)
	http.HandleFunc("/hook-test-harness", hookTestHarnessHandler)
	http.HandleFunc("/test-mail-send", testMailSendHandler)
	http.HandleFunc("/_ah/bounce", bounceHandler)
	http.HandleFunc("/test-email-thread", testEmailThreadHandler)
}

func initConfig() {
	path := "config/mailgun"
	if appengine.IsDevAppServer() {
		path = path + "-dev"
	}
	path += ".json"
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		log_.Panicf("Could not read config from %s: %s", path, err.Error())
	}
	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		log_.Panicf("Could not parse config %s: %s", configBytes, err.Error())
	}
}

type EmailThread struct {
	CommitSHA string `datastore:",noindex"`
	Subject   string `datastore:",noindex"`
	MessageID string `datastore:",noindex"`
}

func createThread(sha string, subject string, messageId string, c context.Context) {
	thread := EmailThread{
		CommitSHA: sha,
		Subject:   subject,
		MessageID: messageId,
	}
	key := datastore.NewKey(c, "EmailThread", sha, 0, nil)
	_, err := datastore.Put(c, key, &thread)
	if err != nil {
		log.Errorf(c, "Error creating thread: %s", err)
	} else {
		log.Infof(c, "Created thread: %v", thread)
	}
}

func getEmailThreadForCommit(sha string, c context.Context) *EmailThread {
	thread := new(EmailThread)
	key := datastore.NewKey(c, "EmailThread", sha, 0, nil)
	err := datastore.Get(c, key, thread)
	if err != nil {
		log.Infof(c, "No thread found for SHA = %s", sha)
		return nil
	}
	return thread
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	eventType := r.Header.Get("X-Github-Event")
	email, commits, err := handlePayload(eventType, r.Body, c)
	if err != nil {
		log.Errorf(c, "Error %s handling %s payload", err, eventType)
		http.Error(w, "Error handling payload", http.StatusInternalServerError)
		return
	}
	if email == nil {
		fmt.Fprint(w, "Unhandled event type: %s", eventType)
		log.Warningf(c, "Unhandled event type: %s", eventType)
		return
	}
	msg, id, err := sendEmail(email, c)
	if commits != nil {
		for _, commit := range commits {
			createThread(commit.SHA, email.Subject, id, c)
		}
	}
	if err != nil {
		log.Errorf(c, "Could not send mail: %s %s", err, msg)
		http.Error(w, "Could not send mail", http.StatusInternalServerError)
		return
	}
	log.Infof(c, "Sent message id=%s", id)
	fmt.Fprint(w, "OK")
}

type Email struct {
	SenderName string
	SenderUserName string
	Subject string
	HTMLBody string
	Headers map [string]string
}

func sendEmail(email* Email, c context.Context) (msg string, id string, err error) {
	httpc := urlfetch.Client(c)
	mg := mailgun.NewMailgun(
		config.Domain,
		config.APIKey,
		config.PublicKey,
	)
	mg.SetClient(httpc)
	sender := fmt.Sprintf("%s <%s@%s>", email.SenderName, email.SenderUserName, config.Domain)
	message := mg.NewMessage(
		sender,
		email.Subject,
		email.HTMLBody,
		config.Recipient,
	)
	message.SetHtml(email.HTMLBody)
	for header, value := range email.Headers {
		message.AddHeader(header, value)
	}
	msg, id, err = mg.Send(message)
	if err != nil {
		log.Errorf(c, "Failed to send message: %v, ID %v, %+v", err, id, msg)
	} else {
		log.Infof(c, "Sent message: %s", id)
	}
	return msg, id, err
}

func handlePayload(eventType string, payloadReader io.Reader, c context.Context) (*Email, []DisplayCommit, error) {
	decoder := json.NewDecoder(payloadReader)
	if eventType == "push" {
		var payload PushPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, nil, err
		}
		return handlePushPayload(payload, c)
	} else if eventType == "commit_comment" {
		var payload CommitCommentPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, nil, err
		}
		email, err := handleCommitCommentPayload(payload, c)
		return email, nil, err
	}
	return nil, nil, nil
}

func handlePushPayload(payload PushPayload, c context.Context) (*Email, []DisplayCommit, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")

	displayCommits := make([]DisplayCommit, 0)
	for i := range payload.Commits {
		displayCommits = append(displayCommits, newDisplayCommit(&payload.Commits[i], payload.Sender, payload.Repo, location, c))
	}
	branchName := (*payload.Ref)[11:]
	branchUrl := fmt.Sprintf("https://github.com/%s/tree/%s", *payload.Repo.FullName, branchName)
	pushedDate := payload.Repo.PushedAt.In(location)
	// Last link is a link so that the GitHub Gmail extension
	// (https://github.com/muan/github-gmail) will open the diff view.
	extensionUrl := displayCommits[0].URL
	if len(displayCommits) > 1 {
		extensionUrl = *payload.Compare
	}
	var data = map[string]interface{}{
		"Payload":                  payload,
		"Commits":                  displayCommits,
		"BranchName":               branchName,
		"BranchURL":                branchUrl,
		"PushedDisplayDate":        safeFormattedDate(pushedDate.Format(DisplayDateFormat)),
		"PushedDisplayDateTooltip": pushedDate.Format(DisplayDateFullFormat),
		"ExtensionURL":             extensionUrl,
	}
	var mailHtml bytes.Buffer
	if err := templates["push"].Execute(&mailHtml, data); err != nil {
		return nil, nil, err
	}

	senderUserName := *payload.Pusher.Name
	senderName := senderUserName
	// We don't have the display name in the pusher, but usually it's one of the
	// commiters, so get it from there (without having to do any extra API
	// requests)
	for _, commit := range payload.Commits {
		if *commit.Author.Username == senderUserName {
			senderName = *commit.Author.Name
			break
		}
		if *commit.Committer.Username == senderUserName {
			senderName = *commit.Committer.Name
			break
		}
	}

	subjectCommit := displayCommits[0]
	subject := fmt.Sprintf("[%s] %s: %s", *payload.Repo.FullName, subjectCommit.ShortSHA, subjectCommit.Title)

	message := &Email{
		SenderName: senderName,
		SenderUserName: senderUserName,
		Subject:  subject,
		HTMLBody: mailHtml.String(),
	}
	return message, displayCommits, nil
}

func handleCommitCommentPayload(payload CommitCommentPayload, c context.Context) (*Email, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")
	updatedDate := payload.Comment.UpdatedAt.In(location)

	commitSHA := *payload.Comment.CommitID
	commitShortSHA := commitSHA[:7]
	commitURL := *payload.Repo.HTMLURL + "/commit/" + commitSHA

	body := *payload.Comment.Body
	if len(body) > 0 {
		body = renderMessageMarkdown(body, payload.Repo, c)
	}

	var data = map[string]interface{}{
		"Payload":            payload,
		"Comment":            payload.Comment,
		"Sender":             payload.Sender,
		"Repo":               payload.Repo,
		"ShortSHA":           commitShortSHA,
		"Body":               body,
		"CommitURL":          commitURL,
		"UpdatedDisplayDate": safeFormattedDate(updatedDate.Format(DisplayDateFormat)),
	}

	var mailHtml bytes.Buffer
	if err := templates["commit-comment"].Execute(&mailHtml, data); err != nil {
		return nil, err
	}

	senderUserName := *payload.Sender.Login
	senderName := senderUserName

	thread := getEmailThreadForCommit(commitSHA, c)
	subject := fmt.Sprintf("[%s] %s", *payload.Repo.FullName, commitShortSHA)
	messageId := ""
	if thread != nil {
		subject = thread.Subject
		messageId = thread.MessageID
	}
	// We don't control the message ID, but hopefully subject-basic threading
	// wil work.
	subject = "Re: " + subject

	message := &Email{
		SenderName: senderName,
		SenderUserName: senderUserName,
		Subject: subject,
		HTMLBody: mailHtml.String(),
		Headers: map [string]string {
			"In-Reply-To": messageId,
		},
	}
	return message, nil
}

func hookTestHarnessHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["hook-test-harness"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		eventType := r.FormValue("event_type")
		payload := r.FormValue("payload")
		c := appengine.NewContext(r)

		message, _, err := handlePayload(eventType, strings.NewReader(payload), c)
		var data = map[string]interface{}{
			"EventType":  eventType,
			"Payload":    payload,
			"Message":    message,
			"MessageErr": err,
		}
		templates["hook-test-harness"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}

func bounceHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if b, err := ioutil.ReadAll(r.Body); err == nil {
		log.Warningf(c, "Bounce: %s", string(b))
	} else {
		log.Warningf(c, "Bounce: <unreadable body>")
	}
}

func testEmailThreadHandler(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		http.Error(w, "", http.StatusMethodNotAllowed)
		return
	}
	values := r.URL.Query()
	sha, ok := values["sha"]
	if !ok || len(sha) < 1 {
		http.Error(w, "Need to specify sha param", http.StatusInternalServerError)
		return
	}
	c := appengine.NewContext(r)
	thread := getEmailThreadForCommit(sha[0], c)
	if thread == nil {
		http.Error(w, "No thread found", http.StatusInternalServerError)
	}
	fmt.Fprintf(w, "Subject: %s\n", thread.Subject)
	fmt.Fprintf(w, "MessageID: %s\n", thread.MessageID)
}

func testMailSendHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["test-mail-send"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		c := appengine.NewContext(r)
		email := &Email{
			SenderName:   r.FormValue("sender"),
			SenderUserName:   r.FormValue("sender"),
			Subject:  r.FormValue("subject"),
			HTMLBody: r.FormValue("html_body"),
		}
		_, id, err := sendEmail(email, c)
		var data = map[string]interface{}{
			"Message": email,
			"SendErr": err,
			"Id": id,
		}
		templates["test-mail-send"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}
<commit_msg>Don't set header when messageId is empty (#5)<commit_after>package bettermail

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	log_ "log"
	"net/http"
	"strings"
	"time"

	"github.com/mailgun/mailgun-go"

	"golang.org/x/net/context"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
)

var templates map[string]*Template

type MailgunConfig struct {
	Domain    string
	APIKey    string
	PublicKey string
	Recipient string
}

var config MailgunConfig

func init() {
	initConfig()
	templates = loadTemplates()

	http.HandleFunc("/hook", hookHandler)
	http.HandleFunc("/hook-test-harness", hookTestHarnessHandler)
	http.HandleFunc("/test-mail-send", testMailSendHandler)
	http.HandleFunc("/_ah/bounce", bounceHandler)
	http.HandleFunc("/test-email-thread", testEmailThreadHandler)
}

func initConfig() {
	path := "config/mailgun"
	if appengine.IsDevAppServer() {
		path = path + "-dev"
	}
	path += ".json"
	configBytes, err := ioutil.ReadFile(path)
	if err != nil {
		log_.Panicf("Could not read config from %s: %s", path, err.Error())
	}
	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		log_.Panicf("Could not parse config %s: %s", configBytes, err.Error())
	}
}

type EmailThread struct {
	CommitSHA string `datastore:",noindex"`
	Subject   string `datastore:",noindex"`
	MessageID string `datastore:",noindex"`
}

func createThread(sha string, subject string, messageId string, c context.Context) {
	thread := EmailThread{
		CommitSHA: sha,
		Subject:   subject,
		MessageID: messageId,
	}
	key := datastore.NewKey(c, "EmailThread", sha, 0, nil)
	_, err := datastore.Put(c, key, &thread)
	if err != nil {
		log.Errorf(c, "Error creating thread: %s", err)
	} else {
		log.Infof(c, "Created thread: %v", thread)
	}
}

func getEmailThreadForCommit(sha string, c context.Context) *EmailThread {
	thread := new(EmailThread)
	key := datastore.NewKey(c, "EmailThread", sha, 0, nil)
	err := datastore.Get(c, key, thread)
	if err != nil {
		log.Infof(c, "No thread found for SHA = %s", sha)
		return nil
	}
	return thread
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	eventType := r.Header.Get("X-Github-Event")
	email, commits, err := handlePayload(eventType, r.Body, c)
	if err != nil {
		log.Errorf(c, "Error %s handling %s payload", err, eventType)
		http.Error(w, "Error handling payload", http.StatusInternalServerError)
		return
	}
	if email == nil {
		fmt.Fprint(w, "Unhandled event type: %s", eventType)
		log.Warningf(c, "Unhandled event type: %s", eventType)
		return
	}
	msg, id, err := sendEmail(email, c)
	if commits != nil {
		for _, commit := range commits {
			createThread(commit.SHA, email.Subject, id, c)
		}
	}
	if err != nil {
		log.Errorf(c, "Could not send mail: %s %s", err, msg)
		http.Error(w, "Could not send mail", http.StatusInternalServerError)
		return
	}
	log.Infof(c, "Sent message id=%s", id)
	fmt.Fprint(w, "OK")
}

type Email struct {
	SenderName string
	SenderUserName string
	Subject string
	HTMLBody string
	Headers map [string]string
}

func sendEmail(email* Email, c context.Context) (msg string, id string, err error) {
	httpc := urlfetch.Client(c)
	mg := mailgun.NewMailgun(
		config.Domain,
		config.APIKey,
		config.PublicKey,
	)
	mg.SetClient(httpc)
	sender := fmt.Sprintf("%s <%s@%s>", email.SenderName, email.SenderUserName, config.Domain)
	message := mg.NewMessage(
		sender,
		email.Subject,
		email.HTMLBody,
		config.Recipient,
	)
	message.SetHtml(email.HTMLBody)
	for header, value := range email.Headers {
		message.AddHeader(header, value)
	}
	msg, id, err = mg.Send(message)
	if err != nil {
		log.Errorf(c, "Failed to send message: %v, ID %v, %+v", err, id, msg)
	} else {
		log.Infof(c, "Sent message: %s", id)
	}
	return msg, id, err
}

func handlePayload(eventType string, payloadReader io.Reader, c context.Context) (*Email, []DisplayCommit, error) {
	decoder := json.NewDecoder(payloadReader)
	if eventType == "push" {
		var payload PushPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, nil, err
		}
		return handlePushPayload(payload, c)
	} else if eventType == "commit_comment" {
		var payload CommitCommentPayload
		err := decoder.Decode(&payload)
		if err != nil {
			return nil, nil, err
		}
		email, err := handleCommitCommentPayload(payload, c)
		return email, nil, err
	}
	return nil, nil, nil
}

func handlePushPayload(payload PushPayload, c context.Context) (*Email, []DisplayCommit, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")

	displayCommits := make([]DisplayCommit, 0)
	for i := range payload.Commits {
		displayCommits = append(displayCommits, newDisplayCommit(&payload.Commits[i], payload.Sender, payload.Repo, location, c))
	}
	branchName := (*payload.Ref)[11:]
	branchUrl := fmt.Sprintf("https://github.com/%s/tree/%s", *payload.Repo.FullName, branchName)
	pushedDate := payload.Repo.PushedAt.In(location)
	// Last link is a link so that the GitHub Gmail extension
	// (https://github.com/muan/github-gmail) will open the diff view.
	extensionUrl := displayCommits[0].URL
	if len(displayCommits) > 1 {
		extensionUrl = *payload.Compare
	}
	var data = map[string]interface{}{
		"Payload":                  payload,
		"Commits":                  displayCommits,
		"BranchName":               branchName,
		"BranchURL":                branchUrl,
		"PushedDisplayDate":        safeFormattedDate(pushedDate.Format(DisplayDateFormat)),
		"PushedDisplayDateTooltip": pushedDate.Format(DisplayDateFullFormat),
		"ExtensionURL":             extensionUrl,
	}
	var mailHtml bytes.Buffer
	if err := templates["push"].Execute(&mailHtml, data); err != nil {
		return nil, nil, err
	}

	senderUserName := *payload.Pusher.Name
	senderName := senderUserName
	// We don't have the display name in the pusher, but usually it's one of the
	// commiters, so get it from there (without having to do any extra API
	// requests)
	for _, commit := range payload.Commits {
		if *commit.Author.Username == senderUserName {
			senderName = *commit.Author.Name
			break
		}
		if *commit.Committer.Username == senderUserName {
			senderName = *commit.Committer.Name
			break
		}
	}

	subjectCommit := displayCommits[0]
	subject := fmt.Sprintf("[%s] %s: %s", *payload.Repo.FullName, subjectCommit.ShortSHA, subjectCommit.Title)

	message := &Email{
		SenderName: senderName,
		SenderUserName: senderUserName,
		Subject:  subject,
		HTMLBody: mailHtml.String(),
	}
	return message, displayCommits, nil
}

func handleCommitCommentPayload(payload CommitCommentPayload, c context.Context) (*Email, error) {
	// TODO: allow location to be customized
	location, _ := time.LoadLocation("America/Los_Angeles")
	updatedDate := payload.Comment.UpdatedAt.In(location)

	commitSHA := *payload.Comment.CommitID
	commitShortSHA := commitSHA[:7]
	commitURL := *payload.Repo.HTMLURL + "/commit/" + commitSHA

	body := *payload.Comment.Body
	if len(body) > 0 {
		body = renderMessageMarkdown(body, payload.Repo, c)
	}

	var data = map[string]interface{}{
		"Payload":            payload,
		"Comment":            payload.Comment,
		"Sender":             payload.Sender,
		"Repo":               payload.Repo,
		"ShortSHA":           commitShortSHA,
		"Body":               body,
		"CommitURL":          commitURL,
		"UpdatedDisplayDate": safeFormattedDate(updatedDate.Format(DisplayDateFormat)),
	}

	var mailHtml bytes.Buffer
	if err := templates["commit-comment"].Execute(&mailHtml, data); err != nil {
		return nil, err
	}

	senderUserName := *payload.Sender.Login
	senderName := senderUserName

	thread := getEmailThreadForCommit(commitSHA, c)
	subject := fmt.Sprintf("[%s] %s", *payload.Repo.FullName, commitShortSHA)
	messageId := ""
	if thread != nil {
		subject = thread.Subject
		messageId = thread.MessageID
	}
	// We don't control the message ID, but hopefully subject-basic threading
	// wil work.
	subject = "Re: " + subject

	message := &Email{
		SenderName: senderName,
		SenderUserName: senderUserName,
		Subject: subject,
		HTMLBody: mailHtml.String(),
		Headers: make(map [string]string),
	}
	if len(messageId) > 0 {
		message.Headers["In-Reply-To"] = messageId
	}
	return message, nil
}

func hookTestHarnessHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["hook-test-harness"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		eventType := r.FormValue("event_type")
		payload := r.FormValue("payload")
		c := appengine.NewContext(r)

		message, _, err := handlePayload(eventType, strings.NewReader(payload), c)
		var data = map[string]interface{}{
			"EventType":  eventType,
			"Payload":    payload,
			"Message":    message,
			"MessageErr": err,
		}
		templates["hook-test-harness"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}

func bounceHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if b, err := ioutil.ReadAll(r.Body); err == nil {
		log.Warningf(c, "Bounce: %s", string(b))
	} else {
		log.Warningf(c, "Bounce: <unreadable body>")
	}
}

func testEmailThreadHandler(w http.ResponseWriter, r *http.Request) {
	if !appengine.IsDevAppServer() {
		http.Error(w, "", http.StatusMethodNotAllowed)
		return
	}
	values := r.URL.Query()
	sha, ok := values["sha"]
	if !ok || len(sha) < 1 {
		http.Error(w, "Need to specify sha param", http.StatusInternalServerError)
		return
	}
	c := appengine.NewContext(r)
	thread := getEmailThreadForCommit(sha[0], c)
	if thread == nil {
		http.Error(w, "No thread found", http.StatusInternalServerError)
	}
	fmt.Fprintf(w, "Subject: %s\n", thread.Subject)
	fmt.Fprintf(w, "MessageID: %s\n", thread.MessageID)
}

func testMailSendHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		templates["test-mail-send"].Execute(w, nil)
		return
	}
	if r.Method == "POST" {
		c := appengine.NewContext(r)
		email := &Email{
			SenderName:   r.FormValue("sender"),
			SenderUserName:   r.FormValue("sender"),
			Subject:  r.FormValue("subject"),
			HTMLBody: r.FormValue("html_body"),
		}
		_, id, err := sendEmail(email, c)
		var data = map[string]interface{}{
			"Message": email,
			"SendErr": err,
			"Id": id,
		}
		templates["test-mail-send"].Execute(w, data)
		return
	}
	http.Error(w, "", http.StatusMethodNotAllowed)
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

const (
	KEY_SCHEMA      = "$schema"
	KEY_ID          = "$id"
	KEY_REF         = "$ref"
	KEY_TITLE       = "title"
	KEY_DESCRIPTION = "description"
	KEY_TYPE        = "type"
	KEY_ITEMS       = "items"
	KEY_PROPERTIES  = "properties"

	STRING_STRING     = "string"
	STRING_SCHEMA     = "schema"
	STRING_PROPERTIES = "properties"

	ROOT_SCHEMA_PROPERTY = "(root)"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)
	d.pool = NewSchemaPool()

	spd, err := d.pool.GetPoolDocument(d.documentReference)
	if err != nil {
		return nil, err
	}

	err = d.parse(spd.Document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
	pool              *SchemaPool
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{property: ROOT_SCHEMA_PROPERTY}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_SCHEMA))
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, KEY_SCHEMA) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_SCHEMA))
		}
		if !isKind(m[KEY_SCHEMA], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_SCHEMA, STRING_STRING))
		}
		schemaRef := m[KEY_SCHEMA].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference

		if existsMapKey(m, KEY_REF) {
			return errors.New(fmt.Sprintf("No %s is allowed in root schema", KEY_REF))
		}

	}

	// ref
	if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_REF, STRING_STRING))
	}
	if k, ok := m[KEY_REF].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}

		if jsonReference.HasFullUrl {
			currentSchema.ref = &jsonReference
		} else {
			inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref, jsonReference)
			if err != nil {
				return err
			}
			currentSchema.ref = inheritedReference
		}

		dsp, err := d.pool.GetPoolDocument(*currentSchema.ref)
		if err != nil {
			return err
		}

		jsonPointer := currentSchema.ref.GetPointer()

		httpDocumentNode, err := jsonPointer.Get(dsp.Document)
		if err != nil {
			return err
		}

		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_SCHEMA))
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, KEY_ID) && !isKind(m[KEY_ID], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_ID, STRING_STRING))
	}
	if k, ok := m[KEY_ID].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_TITLE, STRING_STRING))
	}
	if k, ok := m[KEY_TITLE].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_DESCRIPTION, STRING_STRING))
	}
	if k, ok := m[KEY_DESCRIPTION].(string); ok {
		currentSchema.description = &k
	}

	// type
	if existsMapKey(m, KEY_TYPE) && !isKind(m[KEY_TYPE], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_TYPE, STRING_STRING))
	}
	if k, ok := m[KEY_TYPE].(string); ok {
		if !isStringInSlice(SCHEMA_TYPES, k) {
			return errors.New(fmt.Sprintf("schema %s - %s is invalid", currentSchema.property, KEY_TYPE))
		}
		currentSchema.etype = &k
	} else {
		return errors.New(fmt.Sprintf("schema %s - %s is required", currentSchema.property, KEY_TYPE))
	}

	// properties
	/*	if !existsMapKey(m, KEY_PROPERTIES) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_PROPERTIES))
		}
	*/
	for k := range m {
		if k == KEY_PROPERTIES {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	// items
	/*	if !existsMapKey(m, KEY_ITEMS) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_ITEMS))
		}
	*/
	for k := range m {
		if k == KEY_ITEMS {
			newSchema := &JsonSchema{parent: currentSchema}
			currentSchema.AddPropertiesChild(newSchema)
			err := d.parseSchema(m[k], newSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_PROPERTIES))
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>added type check and items/properties parsing<commit_after>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"errors"
	"fmt"
	"gojsonreference"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)
	d.pool = NewSchemaPool()

	spd, err := d.pool.GetPoolDocument(d.documentReference)
	if err != nil {
		return nil, err
	}

	err = d.parse(spd.Document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
	pool              *SchemaPool
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{property: ROOT_SCHEMA_PROPERTY}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_SCHEMA))
	}

	m := documentNode.(map[string]interface{})

	if currentSchema == d.rootSchema {
		if !existsMapKey(m, KEY_SCHEMA) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_SCHEMA))
		}
		if !isKind(m[KEY_SCHEMA], reflect.String) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_SCHEMA, STRING_STRING))
		}
		schemaRef := m[KEY_SCHEMA].(string)
		schemaReference, err := gojsonreference.NewJsonReference(schemaRef)
		currentSchema.schema = &schemaReference
		if err != nil {
			return err
		}

		currentSchema.ref = &d.documentReference

		if existsMapKey(m, KEY_REF) {
			return errors.New(fmt.Sprintf("No %s is allowed in root schema", KEY_REF))
		}

	}

	// ref
	if existsMapKey(m, KEY_REF) && !isKind(m[KEY_REF], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_REF, STRING_STRING))
	}
	if k, ok := m[KEY_REF].(string); ok {
		jsonReference, err := gojsonreference.NewJsonReference(k)
		if err != nil {
			return err
		}

		if jsonReference.HasFullUrl {
			currentSchema.ref = &jsonReference
		} else {
			inheritedReference, err := gojsonreference.Inherits(*currentSchema.ref, jsonReference)
			if err != nil {
				return err
			}
			currentSchema.ref = inheritedReference
		}

		dsp, err := d.pool.GetPoolDocument(*currentSchema.ref)
		if err != nil {
			return err
		}

		jsonPointer := currentSchema.ref.GetPointer()

		httpDocumentNode, err := jsonPointer.Get(dsp.Document)
		if err != nil {
			return err
		}

		if !isKind(httpDocumentNode, reflect.Map) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_SCHEMA))
		}
		m = httpDocumentNode.(map[string]interface{})
	}

	// id
	if existsMapKey(m, KEY_ID) && !isKind(m[KEY_ID], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_ID, STRING_STRING))
	}
	if k, ok := m[KEY_ID].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, KEY_TITLE) && !isKind(m[KEY_TITLE], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_TITLE, STRING_STRING))
	}
	if k, ok := m[KEY_TITLE].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, KEY_DESCRIPTION) && !isKind(m[KEY_DESCRIPTION], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_DESCRIPTION, STRING_STRING))
	}
	if k, ok := m[KEY_DESCRIPTION].(string); ok {
		currentSchema.description = &k
	}

	// type
	if existsMapKey(m, KEY_TYPE) && !isKind(m[KEY_TYPE], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_OF_TYPE_Y, KEY_TYPE, STRING_STRING))
	}
	if k, ok := m[KEY_TYPE].(string); ok {
		if !isStringInSlice(SCHEMA_TYPES, k) {
			return errors.New(fmt.Sprintf("schema %s - %s is invalid", currentSchema.property, KEY_TYPE))
		}
		currentSchema.etype = k
	} else {
		return errors.New(fmt.Sprintf("schema %s - %s is required", currentSchema.property, KEY_TYPE))
	}

	// properties
	if currentSchema.etype == "object" {
		if !existsMapKey(m, KEY_PROPERTIES) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_PROPERTIES))
		}

		for k := range m {
			if k == KEY_PROPERTIES {
				err := d.parseProperties(m[k], currentSchema)
				if err != nil {
					return err
				}
			}
		}
	}

	// items
	if currentSchema.etype == "array" {
		if !existsMapKey(m, KEY_ITEMS) {
			return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_IS_REQUIRED, KEY_ITEMS))
		}

		for k := range m {
			if k == KEY_ITEMS {
				newSchema := &JsonSchema{parent: currentSchema}
				currentSchema.AddPropertiesChild(newSchema)
				err := d.parseSchema(m[k], newSchema)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_X_MUST_BE_AN_OBJECT, STRING_PROPERTIES))
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		schemaProperty := k
		newSchema := &JsonSchema{property: schemaProperty, parent: currentSchema, ref: currentSchema.ref}
		currentSchema.AddPropertiesChild(newSchema)
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"encoding/json"
	"errors"
	"fmt"
	"gojsonreference"
	"io/ioutil"
	"net/http"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)

	resp, err := http.Get(documentReferenceString)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Could not access schema " + resp.Status)
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	err = d.parse(document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		currentSchema.ref = &k
	}

	// properties
	if !existsMapKey(m, "properties") {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
	}

	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		newSchema := &JsonSchema{property: &k}
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>disabled properties check<commit_after>// @author       sigu-399
// @description  An implementation of JSON Schema, draft v4 - Go language
// @created      26-02-2013

package gojsonschema

import (
	"encoding/json"
	"errors"
	"fmt"
	"gojsonreference"
	"io/ioutil"
	"net/http"
	"reflect"
)

func NewJsonSchemaDocument(documentReferenceString string) (*JsonSchemaDocument, error) {

	var err error

	d := JsonSchemaDocument{}
	d.documentReference, err = gojsonreference.NewJsonReference(documentReferenceString)

	resp, err := http.Get(documentReferenceString)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Could not access schema " + resp.Status)
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	err = d.parse(document)
	return &d, err
}

type JsonSchemaDocument struct {
	documentReference gojsonreference.JsonReference
	rootSchema        *JsonSchema
}

func (d *JsonSchemaDocument) parse(document interface{}) error {
	d.rootSchema = &JsonSchema{}
	return d.parseSchema(document, d.rootSchema)
}

func (d *JsonSchemaDocument) parseSchema(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Schema must be an object")
	}

	m := documentNode.(map[string]interface{})

	// id
	if existsMapKey(m, "id") && !isKind(m["id"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "id", "string"))
	}
	if k, ok := m["id"].(string); ok {
		currentSchema.id = &k
	}

	// title
	if existsMapKey(m, "title") && !isKind(m["title"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "title", "string"))
	}
	if k, ok := m["title"].(string); ok {
		currentSchema.title = &k
	}

	// description
	if existsMapKey(m, "description") && !isKind(m["description"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "description", "string"))
	}
	if k, ok := m["description"].(string); ok {
		currentSchema.description = &k
	}

	// ref
	if existsMapKey(m, "$ref") && !isKind(m["$ref"], reflect.String) {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_MUST_BE_OF_TYPE, "$ref", "string"))
	}
	if k, ok := m["$ref"].(string); ok {
		currentSchema.ref = &k
	}

	// properties
/*	if !existsMapKey(m, "properties") {
		return errors.New(fmt.Sprintf(ERROR_MESSAGE_IS_REQUIRED, "properties"))
	}
*/
	for k := range m {
		if k == "properties" {
			err := d.parseProperties(m[k], currentSchema)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *JsonSchemaDocument) parseProperties(documentNode interface{}, currentSchema *JsonSchema) error {

	if !isKind(documentNode, reflect.Map) {
		return errors.New("Properties must be an object")
	}

	m := documentNode.(map[string]interface{})
	for k := range m {
		newSchema := &JsonSchema{property: &k}
		err := d.parseSchema(m[k], newSchema)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// +build !windows

package exec

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"
)

func kill(c *exec.Cmd) error {
	return syscall.Kill(-c.Process.Pid, syscall.SIGKILL)
}

func interrupt(c *exec.Cmd) error {
	return syscall.Kill(-c.Process.Pid, syscall.SIGINT)
}

func setsid(c *exec.Cmd) {
	if c.SysProcAttr == nil {
		c.SysProcAttr = &syscall.SysProcAttr{}
	}
	c.SysProcAttr.Setsid = true
}

func maxRss(p *os.ProcessState) string {
	if p != nil {
		if ru, ok := p.SysUsage().(*syscall.Rusage); ok {
			// is this multiplier correct on OS X?
			return fmtBytes(ru.Maxrss * 1024)
		}
	}
	return ""
}

func fmtBytes(n int64) string {
	l := make([]string, 0, 4)
	pairs := []struct {
		s string
		n int64
	}{
		{"G", 1 << 30},
		{"M", 1 << 20},
		{"K", 1 << 10},
		{"B", 1},
	}

	for _, p := range pairs {
		if n >= p.n {
			l = append(l, fmt.Sprintf("%d%s", n/p.n, p.s))
			n %= p.n
		}
	}

	if len(l) > 0 {
		return strings.Join(l, ", ")
	}
	return "0B"
}
<commit_msg>* avoid dereferencing nil commands when attempting to send a signal<commit_after>// +build !windows

package exec

import (
	"fmt"
	"os"
	"os/exec"
	"strings"
	"syscall"
)

func killPG(c *exec.Cmd, sig syscall.Signal) error {
	if c == nil || c.Process == nil {
		return nil
	}
	return syscall.Kill(-c.Process.Pid, sig)
}

func kill(c *exec.Cmd) error {
	return killPG(c, syscall.SIGKILL)
}

func interrupt(c *exec.Cmd) error {
	return killPG(c, syscall.SIGINT)
}

func setsid(c *exec.Cmd) {
	if c.SysProcAttr == nil {
		c.SysProcAttr = &syscall.SysProcAttr{}
	}
	c.SysProcAttr.Setsid = true
}

func maxRss(p *os.ProcessState) string {
	if p != nil {
		if ru, ok := p.SysUsage().(*syscall.Rusage); ok {
			// is this multiplier correct on OS X?
			return fmtBytes(ru.Maxrss * 1024)
		}
	}
	return ""
}

func fmtBytes(n int64) string {
	l := make([]string, 0, 4)
	pairs := []struct {
		s string
		n int64
	}{
		{"G", 1 << 30},
		{"M", 1 << 20},
		{"K", 1 << 10},
		{"B", 1},
	}

	for _, p := range pairs {
		if n >= p.n {
			l = append(l, fmt.Sprintf("%d%s", n/p.n, p.s))
			n %= p.n
		}
	}

	if len(l) > 0 {
		return strings.Join(l, ", ")
	}
	return "0B"
}
<|endoftext|>


<commit_before>package rin

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/sqs"
)

var SQS *sqs.SQS
var config *Config
var Debug bool

var TrapSignals = []os.Signal{
	syscall.SIGHUP,
	syscall.SIGINT,
	syscall.SIGTERM,
	syscall.SIGQUIT,
}

func Run(configFile string) error {
	var err error
	log.Println("Loading config", configFile)
	config, err = LoadConfig(configFile)
	if err != nil {
		return err
	}
	for _, target := range config.Targets {
		log.Println("Define target", target)
	}

	auth := aws.Auth{
		AccessKey: config.Credentials.AWS_ACCESS_KEY_ID,
		SecretKey: config.Credentials.AWS_SECRET_ACCESS_KEY,
	}
	region := aws.GetRegion(config.Credentials.AWS_REGION)
	SQS = sqs.New(auth, region)

	shutdownCh := make(chan interface{})
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, TrapSignals...)

	// run worker
	var wg sync.WaitGroup
	wg.Add(1)
	go sqsWorker(&wg, shutdownCh)

	// wait for signal
	s := <-signalCh
	switch sig := s.(type) {
	case syscall.Signal:
		log.Printf("Got signal: %s(%d)", sig, sig)
	default:
	}
	log.Println("Shutting down worker...")
	close(shutdownCh) // notify shutdown to worker

	wg.Wait() // wait for worker completed
	log.Println("Shutdown successfully")
	return nil
}

func waitForRetry() {
	log.Println("Retry after 10 sec.")
	time.Sleep(10 * time.Second)
}

func runnable(ch chan interface{}) bool {
	select {
	case <-ch:
		// ch closed == shutdown
		return false
	default:
	}
	return true
}

func sqsWorker(wg *sync.WaitGroup, ch chan interface{}) {
	defer (*wg).Done()

	log.Printf("Starting up SQS Worker")
	defer log.Println("Shutdown SQS Worker")

	for runnable(ch) {
		log.Println("Connect to SQS:", config.QueueName)
		queue, err := SQS.GetQueue(config.QueueName)
		if err != nil {
			log.Println("Can't get queue:", err)
			waitForRetry()
			continue
		}
		quit, err := handleQueue(queue, ch)
		if err != nil {
			log.Println("Processing failed:", err)
			waitForRetry()
			continue
		}
		if quit {
			break
		}
	}
}

func handleQueue(queue *sqs.Queue, ch chan interface{}) (bool, error) {
	for runnable(ch) {
		err := handleMessage(queue)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

func handleMessage(queue *sqs.Queue) error {
	res, err := queue.ReceiveMessage(1)
	if err != nil {
		return err
	}
	if len(res.Messages) == 0 {
		return nil
	}
	msg := res.Messages[0]
	log.Printf("Starting Process message ID:%s", msg.MessageId)
	event, err := ParseEvent([]byte(msg.Body))
	if err != nil {
		log.Println("Can't parse event from Body.", err)
		return err
	}
	log.Println("Importing event:", event)
	n, err := Import(event)
	if err != nil {
		log.Println("Import failed.", err)
		return err
	}
	if n == 0 {
		log.Println("All events were not matched for any targets. Ignored.")
	} else {
		log.Printf("%d import action completed.", n)
	}
	_, err = queue.DeleteMessage(&msg)
	if err != nil {
		log.Println("Can't delete message.", err)
	}
	log.Printf("Completed message ID:%s", msg.MessageId)
	return nil
}
<commit_msg>cache runnable state to val.<commit_after>package rin

import (
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/sqs"
)

var SQS *sqs.SQS
var config *Config
var Debug bool
var Runnable bool

var TrapSignals = []os.Signal{
	syscall.SIGHUP,
	syscall.SIGINT,
	syscall.SIGTERM,
	syscall.SIGQUIT,
}

func Run(configFile string) error {
	Runnable = true
	var err error
	log.Println("Loading config", configFile)
	config, err = LoadConfig(configFile)
	if err != nil {
		return err
	}
	for _, target := range config.Targets {
		log.Println("Define target", target)
	}

	auth := aws.Auth{
		AccessKey: config.Credentials.AWS_ACCESS_KEY_ID,
		SecretKey: config.Credentials.AWS_SECRET_ACCESS_KEY,
	}
	region := aws.GetRegion(config.Credentials.AWS_REGION)
	SQS = sqs.New(auth, region)

	shutdownCh := make(chan interface{})
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, TrapSignals...)

	// run worker
	var wg sync.WaitGroup
	wg.Add(1)
	go sqsWorker(&wg, shutdownCh)

	// wait for signal
	s := <-signalCh
	switch sig := s.(type) {
	case syscall.Signal:
		log.Printf("Got signal: %s(%d)", sig, sig)
	default:
	}
	log.Println("Shutting down worker...")
	close(shutdownCh) // notify shutdown to worker

	wg.Wait() // wait for worker completed
	log.Println("Shutdown successfully")
	return nil
}

func waitForRetry() {
	log.Println("Retry after 10 sec.")
	time.Sleep(10 * time.Second)
}

func runnable(ch chan interface{}) bool {
	if !Runnable {
		return false
	}
	select {
	case <-ch:
		// ch closed == shutdown
		Runnable = false
		return false
	default:
	}
	return true
}

func sqsWorker(wg *sync.WaitGroup, ch chan interface{}) {
	defer (*wg).Done()

	log.Printf("Starting up SQS Worker")
	defer log.Println("Shutdown SQS Worker")

	for runnable(ch) {
		log.Println("Connect to SQS:", config.QueueName)
		queue, err := SQS.GetQueue(config.QueueName)
		if err != nil {
			log.Println("Can't get queue:", err)
			waitForRetry()
			continue
		}
		quit, err := handleQueue(queue, ch)
		if err != nil {
			log.Println("Processing failed:", err)
			waitForRetry()
			continue
		}
		if quit {
			break
		}
	}
}

func handleQueue(queue *sqs.Queue, ch chan interface{}) (bool, error) {
	for runnable(ch) {
		err := handleMessage(queue)
		if err != nil {
			return false, err
		}
	}
	return true, nil
}

func handleMessage(queue *sqs.Queue) error {
	res, err := queue.ReceiveMessage(1)
	if err != nil {
		return err
	}
	if len(res.Messages) == 0 {
		return nil
	}
	msg := res.Messages[0]
	log.Printf("Starting process message id:%s handle:%s", msg.MessageId, msg.ReceiptHandle)
	if Debug {
		log.Println("message body:", msg.Body)
	}
	event, err := ParseEvent([]byte(msg.Body))
	if err != nil {
		log.Println("Can't parse event from Body.", err)
		return err
	}
	log.Println("Importing event:", event)
	n, err := Import(event)
	if err != nil {
		log.Println("Import failed.", err)
		return err
	}
	if n == 0 {
		log.Println("All events were not matched for any targets. Ignored.")
	} else {
		log.Printf("%d import action completed.", n)
	}
	_, err = queue.DeleteMessage(&msg)
	if err != nil {
		log.Println("Can't delete message.", err)
	}
	log.Printf("Completed message ID:%s", msg.MessageId)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

const (
	NONE = iota
	DUP
	PRINT
	POP
	SWAP
	ZERO
)

func main() {
	for {
		ind := ZERO
		for ind == ZERO {
			fmt.Print("> ")
			ind, _, _ = run(false, 0)
		}
		fmt.Println("Invalid entry: bOpttom of stack reached")
	}
}

/*
	push indicates whether to push the passed value onto the stack or
	to wait for user input.

	Returns an operator indicator. This indicator can have values
	NONE, DUP, or PRINT

	NONE:
		The operator entered was a normal arithmetic operator

	DUP:
		The operator entered was the duplicate operator

	PRINT:
		The operator entered was the print operator

	POP:
		The operator entered was the pop operator

	SWAP:
		The operator entered was the swap operator

	ZERO:
		The operator entered was the zero operator

	If the indicator is equal to NONE, then one of the two returned
	functions will be the function corresponding to the operator entered.

*/
func run(push bool, n int) (int, unop, binop) {

	var s string
	var uOp unop
	var bOp binop

	// Operator indicator
	var ind int

	// If the duplicate operator was entered, then n
	// is already the equal to the value which should
	// be pushed onto the stack.
	if !push {
		for {
			fmt.Scan(&s)
			_, err := fmt.Sscanf(s, "%d", &n)

			// If it was not a number (ie, an operator)
			if err != nil {
				switch s {
				case "+":
					return NONE, nil, add
				case "-":
					return NONE, nil, subtract
				case "*":
					return NONE, nil, multiply
				case "/":
					return NONE, nil, divide
				case "|":
					return NONE, nil, or
				case "&":
					return NONE, nil, and
				case "c":
					return NONE, negate, nil
				case "~":
					return NONE, not, nil
				case "dup":
					return DUP, nil, nil
				case "print":
					return PRINT, nil, nil
				case "pop":
					return POP, nil, nil
				case "swap":
					return SWAP, nil, swap
				case "zero":
					return ZERO, nil, nil
				case "quit":
					os.Exit(0)
				}
				fmt.Printf("Unrecognized command: %s\n", s)
				fmt.Print("> ")
			} else {
				break
			}
		}
	}

	// Once control reaches this part of the function,
	// n is equal to the value on the top of the stack.

	push = false

	for {
		ind, uOp, bOp = run(push, n)

	TOP:

		push = false

		switch ind {
		case NONE:
			if uOp != nil {
				n = uOp(n)
			} else {
				return NONE, bOp(n), nil
			}
		case DUP:
			// Simply set push to true, since on next iteration of loop,
			// run(push, n) will be called, and n is already the correct value
			push = true
		case PRINT:
			fmt.Println(n)
			fmt.Print("> ")
		case POP:
			// Effectively letting the previous instance of run perform the call,
			// but way easier than having to pass back sentinal values, etc
			return run(false, 0)
		case SWAP:
			if uOp == nil {
				// bOp will return a function which, when given an argument,
				// will discard the argument and simply return this n
				return SWAP, bOp(n), nil
			} else {
				// uOp will discard its argument and return the argument which
				// was passed in the previous call
				m := uOp(0)
				ind, uOp, bOp = run(true, n)
				n = m

				// Necessary to avoid double-calling run
				goto TOP
			}
		case ZERO:
			return ZERO, nil, nil
		}
	}

	// Control should never reach this
	return NONE, nil, nil
}
<commit_msg>Fixed typo in output<commit_after>package main

import (
	"fmt"
	"os"
)

const (
	NONE = iota
	DUP
	PRINT
	POP
	SWAP
	ZERO
)

func main() {
	for {
		ind := ZERO
		for ind == ZERO {
			fmt.Print("> ")
			ind, _, _ = run(false, 0)
		}
		fmt.Println("Invalid entry: bottom of stack reached")
	}
}

/*
	push indicates whether to push the passed value onto the stack or
	to wait for user input.

	Returns an operator indicator. This indicator can have values
	NONE, DUP, or PRINT

	NONE:
		The operator entered was a normal arithmetic operator

	DUP:
		The operator entered was the duplicate operator

	PRINT:
		The operator entered was the print operator

	POP:
		The operator entered was the pop operator

	SWAP:
		The operator entered was the swap operator

	ZERO:
		The operator entered was the zero operator

	If the indicator is equal to NONE, then one of the two returned
	functions will be the function corresponding to the operator entered.

*/
func run(push bool, n int) (int, unop, binop) {

	var s string
	var uOp unop
	var bOp binop

	// Operator indicator
	var ind int

	// If the duplicate operator was entered, then n
	// is already the equal to the value which should
	// be pushed onto the stack.
	if !push {
		for {
			fmt.Scan(&s)
			_, err := fmt.Sscanf(s, "%d", &n)

			// If it was not a number (ie, an operator)
			if err != nil {
				switch s {
				case "+":
					return NONE, nil, add
				case "-":
					return NONE, nil, subtract
				case "*":
					return NONE, nil, multiply
				case "/":
					return NONE, nil, divide
				case "|":
					return NONE, nil, or
				case "&":
					return NONE, nil, and
				case "c":
					return NONE, negate, nil
				case "~":
					return NONE, not, nil
				case "dup":
					return DUP, nil, nil
				case "print":
					return PRINT, nil, nil
				case "pop":
					return POP, nil, nil
				case "swap":
					return SWAP, nil, swap
				case "zero":
					return ZERO, nil, nil
				case "quit":
					os.Exit(0)
				}
				fmt.Printf("Unrecognized command: %s\n", s)
				fmt.Print("> ")
			} else {
				break
			}
		}
	}

	// Once control reaches this part of the function,
	// n is equal to the value on the top of the stack.

	push = false

	for {
		ind, uOp, bOp = run(push, n)

	TOP:

		push = false

		switch ind {
		case NONE:
			if uOp != nil {
				n = uOp(n)
			} else {
				return NONE, bOp(n), nil
			}
		case DUP:
			// Simply set push to true, since on next iteration of loop,
			// run(push, n) will be called, and n is already the correct value
			push = true
		case PRINT:
			fmt.Println(n)
			fmt.Print("> ")
		case POP:
			// Effectively letting the previous instance of run perform the call,
			// but way easier than having to pass back sentinal values, etc
			return run(false, 0)
		case SWAP:
			if uOp == nil {
				// bOp will return a function which, when given an argument,
				// will discard the argument and simply return this n
				return SWAP, bOp(n), nil
			} else {
				// uOp will discard its argument and return the argument which
				// was passed in the previous call
				m := uOp(0)
				ind, uOp, bOp = run(true, n)
				n = m

				// Necessary to avoid double-calling run
				goto TOP
			}
		case ZERO:
			return ZERO, nil, nil
		}
	}

	// Control should never reach this
	return NONE, nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"io"
	"log"
	"net"
	"net/http/httputil"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/remind101/emp/Godeps/_workspace/src/github.com/bgentry/heroku-go"
	"github.com/remind101/emp/Godeps/_workspace/src/github.com/docker/docker/pkg/term"
)

var (
	detachedRun bool
	dynoSize    string
)

var cmdRun = &Command{
	Run:      runRun,
	Usage:    "run [-s <size>] [-d] <command> [<argument>...]",
	NeedsApp: true,
	Category: "dyno",
	Short:    "run a process in a dyno",
	Long: `
Run a process on Heroku. Flags such as` + " `-a` " + `may be parsed out of
the command unless the command is quoted or provided after a
double-dash (--).

Options:

    -s <size>  set the size for this dyno (e.g. 2X)
    -d         run in detached mode instead of attached to terminal

Examples:

    $ emp run echo "hello"
    Running ` + "`echo \"hello\"`" + ` on myapp as run.1234:
    "hello"

    $ emp run console
    Running ` + "`console`" + ` on myapp as run.5678:
    Loading production environment (Rails 3.2.14)
    irb(main):001:0> ...

    $ emp run -d -s 2X bin/my_worker
    Ran ` + "`bin/my_worker`" + ` on myapp as run.4321, detached.

    $ emp run -a myapp -- ls -a /
    Running ` + "`ls -a bin /`" + ` on myapp as run.8650:
    /:
    .  ..  app  bin  dev  etc  home  lib  lib64  lost+found  proc  sbin  tmp  usr  var
`,
}

func init() {
	cmdRun.Flag.BoolVarP(&detachedRun, "detached", "d", false, "detached")
	cmdRun.Flag.StringVarP(&dynoSize, "size", "s", "", "dyno size")
}

func runRun(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.PrintUsage()
		os.Exit(2)
	}
	appname := mustApp()

	w, err := term.GetWinsize(inFd)
	if err != nil {
		// If syscall.TIOCGWINSZ is not supported by the device, we're
		// probably trying to run tests. Set w to some sensible default.
		if err.Error() == "operation not supported by device" {
			w = &term.Winsize{
				Height: 20,
				Width:  80,
			}
		} else {
			printFatal(err.Error())
		}
	}

	attached := !detachedRun
	opts := heroku.DynoCreateOpts{Attach: &attached}
	if attached {
		env := map[string]string{
			"COLUMNS": strconv.Itoa(int(w.Width)),
			"LINES":   strconv.Itoa(int(w.Height)),
			"TERM":    os.Getenv("TERM"),
		}
		opts.Env = &env
	}
	if dynoSize != "" {
		if !strings.HasSuffix(dynoSize, "X") {
			cmd.PrintUsage()
			os.Exit(2)
		}
		opts.Size = &dynoSize
	}

	command := strings.Join(args, " ")
	if detachedRun {
		dyno, err := client.DynoCreate(appname, command, &opts)
		must(err)

		log.Printf("Ran `%s` on %s as %s, detached.", dyno.Command, appname, dyno.Name)
		return
	}

	params := struct {
		Command string             `json:"command"`
		Attach  *bool              `json:"attach,omitempty"`
		Env     *map[string]string `json:"env,omitempty"`
		Size    *string            `json:"size,omitempty"`
	}{
		Command: command,
		Attach:  opts.Attach,
		Env:     opts.Env,
		Size:    opts.Size,
	}
	req, err := client.NewRequest("POST", "/apps/"+appname+"/dynos", params)
	must(err)

	u, err := url.Parse(apiURL)
	must(err)

	protocol := u.Scheme
	address := u.Path
	if protocol != "unix" {
		protocol = "tcp"
		address = u.Host
	}

	if u.Scheme == "https" {
		address = address + ":443"
	}

	var dial net.Conn
	if u.Scheme == "https" {
		dial, err = tlsDial(protocol, address, &tls.Config{})
		if err != nil {
			printFatal(err.Error())
		}
	} else {
		dial, err = net.Dial(protocol, address)
		if err != nil {
			printFatal(err.Error())
		}
	}

	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()
	_, err = clientconn.Do(req)
	if err != nil && err != httputil.ErrPersistEOF {
		printFatal(err.Error())
	}
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	if isTerminalIn && isTerminalOut {
		state, err := term.SetRawTerminal(inFd)
		if err != nil {
			printFatal(err.Error())
		}
		defer term.RestoreTerminal(inFd, state)
	}

	errChanOut := make(chan error, 1)
	errChanIn := make(chan error, 1)
	exit := make(chan bool)
	go func() {
		defer close(exit)
		defer close(errChanOut)
		var err error
		_, err = io.Copy(os.Stdout, br)
		errChanOut <- err
	}()
	go func() {
		_, err := io.Copy(rwc, os.Stdin)
		errChanIn <- err
		rwc.(interface {
			CloseWrite() error
		}).CloseWrite()
	}()
	<-exit
	select {
	case err = <-errChanIn:
		must(err)
	case err = <-errChanOut:
		must(err)
	}
}
<commit_msg>Add missing port to fix run command<commit_after>package main

import (
	"crypto/tls"
	"io"
	"log"
	"net"
	"net/http/httputil"
	"net/url"
	"os"
	"strconv"
	"strings"

	"github.com/remind101/emp/Godeps/_workspace/src/github.com/bgentry/heroku-go"
	"github.com/remind101/emp/Godeps/_workspace/src/github.com/docker/docker/pkg/term"
)

var (
	detachedRun bool
	dynoSize    string
)

var cmdRun = &Command{
	Run:      runRun,
	Usage:    "run [-s <size>] [-d] <command> [<argument>...]",
	NeedsApp: true,
	Category: "dyno",
	Short:    "run a process in a dyno",
	Long: `
Run a process on Heroku. Flags such as` + " `-a` " + `may be parsed out of
the command unless the command is quoted or provided after a
double-dash (--).

Options:

    -s <size>  set the size for this dyno (e.g. 2X)
    -d         run in detached mode instead of attached to terminal

Examples:

    $ emp run echo "hello"
    Running ` + "`echo \"hello\"`" + ` on myapp as run.1234:
    "hello"

    $ emp run console
    Running ` + "`console`" + ` on myapp as run.5678:
    Loading production environment (Rails 3.2.14)
    irb(main):001:0> ...

    $ emp run -d -s 2X bin/my_worker
    Ran ` + "`bin/my_worker`" + ` on myapp as run.4321, detached.

    $ emp run -a myapp -- ls -a /
    Running ` + "`ls -a bin /`" + ` on myapp as run.8650:
    /:
    .  ..  app  bin  dev  etc  home  lib  lib64  lost+found  proc  sbin  tmp  usr  var
`,
}

func init() {
	cmdRun.Flag.BoolVarP(&detachedRun, "detached", "d", false, "detached")
	cmdRun.Flag.StringVarP(&dynoSize, "size", "s", "", "dyno size")
}

func runRun(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.PrintUsage()
		os.Exit(2)
	}
	appname := mustApp()

	w, err := term.GetWinsize(inFd)
	if err != nil {
		// If syscall.TIOCGWINSZ is not supported by the device, we're
		// probably trying to run tests. Set w to some sensible default.
		if err.Error() == "operation not supported by device" {
			w = &term.Winsize{
				Height: 20,
				Width:  80,
			}
		} else {
			printFatal(err.Error())
		}
	}

	attached := !detachedRun
	opts := heroku.DynoCreateOpts{Attach: &attached}
	if attached {
		env := map[string]string{
			"COLUMNS": strconv.Itoa(int(w.Width)),
			"LINES":   strconv.Itoa(int(w.Height)),
			"TERM":    os.Getenv("TERM"),
		}
		opts.Env = &env
	}
	if dynoSize != "" {
		if !strings.HasSuffix(dynoSize, "X") {
			cmd.PrintUsage()
			os.Exit(2)
		}
		opts.Size = &dynoSize
	}

	command := strings.Join(args, " ")
	if detachedRun {
		dyno, err := client.DynoCreate(appname, command, &opts)
		must(err)

		log.Printf("Ran `%s` on %s as %s, detached.", dyno.Command, appname, dyno.Name)
		return
	}

	params := struct {
		Command string             `json:"command"`
		Attach  *bool              `json:"attach,omitempty"`
		Env     *map[string]string `json:"env,omitempty"`
		Size    *string            `json:"size,omitempty"`
	}{
		Command: command,
		Attach:  opts.Attach,
		Env:     opts.Env,
		Size:    opts.Size,
	}
	req, err := client.NewRequest("POST", "/apps/"+appname+"/dynos", params)
	must(err)

	u, err := url.Parse(apiURL)
	must(err)

	protocol := u.Scheme
	address := u.Path
	if protocol != "unix" {
		protocol = "tcp"
		address = u.Host + ":80"
	}

	if u.Scheme == "https" {
		address = address + ":443"
	}

	var dial net.Conn
	if u.Scheme == "https" {
		dial, err = tlsDial(protocol, address, &tls.Config{})
		if err != nil {
			printFatal(err.Error())
		}
	} else {
		dial, err = net.Dial(protocol, address)
		if err != nil {
			printFatal(err.Error())
		}
	}

	clientconn := httputil.NewClientConn(dial, nil)
	defer clientconn.Close()
	_, err = clientconn.Do(req)
	if err != nil && err != httputil.ErrPersistEOF {
		printFatal(err.Error())
	}
	rwc, br := clientconn.Hijack()
	defer rwc.Close()

	if isTerminalIn && isTerminalOut {
		state, err := term.SetRawTerminal(inFd)
		if err != nil {
			printFatal(err.Error())
		}
		defer term.RestoreTerminal(inFd, state)
	}

	errChanOut := make(chan error, 1)
	errChanIn := make(chan error, 1)
	exit := make(chan bool)
	go func() {
		defer close(exit)
		defer close(errChanOut)
		var err error
		_, err = io.Copy(os.Stdout, br)
		errChanOut <- err
	}()
	go func() {
		_, err := io.Copy(rwc, os.Stdin)
		errChanIn <- err
		rwc.(interface {
			CloseWrite() error
		}).CloseWrite()
	}()
	<-exit
	select {
	case err = <-errChanIn:
		must(err)
	case err = <-errChanOut:
		must(err)
	}
}
<|endoftext|>


<commit_before>package sbf

// Scale Bloom Filte in Redis
// date format
// frame end padding

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"math"
	"math/rand"
	"sbf/internal/murmur"
	. "sbf/internal/utils"
	"time"
)

var (
	SBF_NAME    = "SBF"   // 3 bytes
	SBF_VERSION = "1.0.0" // 5 bytes
)

const (
	// header
	SBF_HEADER_SIZE = 18 // 18 bytes

	// frame
	SBF_FRAME_HEADER_SIZE = 6    // 6 bytes, empty now, reserve for future use
	SBF_FRAME_COUNT_LIMIT = 1024 // the frame count of the sbf
	SBF_FRAME_PADDING     = 1    // 1 byte, reserve for protect the data at the end of previus frame
	// slice
	SBF_DEFAULT_S_COUNT             = 10    // slice count
	SBF_DEFAULT_S_SIZE              = 65536 // slice size
	SBF_DEFAULT_S_ERROR_RATIO       = 0.5   // the percentage of a slice used
	SBF_DEFAULT_S_MIN_CAPACITY_SIZE = 2     // the min growth of the slice size
	SBF_DEFAULT_S_MAX_CAPACITY_SIZE = 4     // the max growth of the slice size
)

func init() {
	if len(SBF_NAME) != 3 || len(SBF_VERSION) != 5 {
		panic(errors.New("invalid sbf name or sbf version"))
	}
	rand.Seed(time.Now().UnixNano())
}

// header of SBF
type SBFHeader struct {
	Name       [3]byte // SBF                           0 -  3
	Version    [5]byte // 1.0.0                         3 -  8
	Count      uint16  // frame count                   8 - 10
	FullRate   uint16  // error ratio  =0.1 * 10000    10 - 12
	SliceCount uint16  // hash functions count         12 - 14
	SliceSize  uint32  // slice size                   14 - 18
	Refer      string
}

// sliceSize must be the multiple of 8
func NewHeader(conn redis.Conn, sliceRatio float32, sliceCount uint16, sliceSize uint32, refer string) (*SBFHeader, error) {
	header := new(SBFHeader)
	copy(header.Name[:], SBF_NAME)
	copy(header.Version[:], SBF_VERSION)
	header.Count = 1
	header.FullRate = uint16(sliceRatio * 10000)
	header.SliceCount = sliceCount
	header.SliceSize = sliceSize
	header.Refer = refer

	if header.SliceSize%8 != 0 {
		return nil, errors.New(fmt.Sprintf("%d NOT multiple of 8", sliceSize))
	}
	// save
	err := header.updateHeader(conn)
	return header, err
}

// load sbf header from redis
func LoadHeader(conn redis.Conn, refer string) (*SBFHeader, error) {
	if ret, err := redis.Bytes(conn.Do("GETRANGE", refer, 0, SBF_HEADER_SIZE-1)); err == nil {
		if len(ret) > 0 {
			header := new(SBFHeader)
			copy(header.Name[:], ret[0:3])
			copy(header.Version[:], ret[3:8])
			// from bytes to number
			if err := BytesToNumber(ret[8:10], &header.Count); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[10:12], &header.FullRate); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[12:14], &header.SliceCount); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[14:18], &header.SliceSize); err != nil {
				return nil, err
			}
			header.Refer = refer
			return header, nil
		} else {
			return nil, errors.New(fmt.Sprintf("SBF %s NOT FOUND.", refer))
		}
	} else {
		return nil, err
	}
}

func (s *SBFHeader) checkHeader() error {
	if bytes.Equal(s.Name[:], []byte(SBF_NAME)) {
		return errors.New("INVALID SBF header.")
	}
	if bytes.Compare(s.Version[:], []byte(SBF_VERSION)) > 0 {
		return errors.New("NOT supported version.")
	}
	return nil
}

func (s *SBFHeader) updateHeader(conn redis.Conn) error {
	// name
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.LittleEndian, s.Name)
	binary.Write(buf, binary.LittleEndian, s.Version)
	binary.Write(buf, binary.LittleEndian, s.Count)
	binary.Write(buf, binary.LittleEndian, s.FullRate)
	binary.Write(buf, binary.LittleEndian, s.SliceCount)
	binary.Write(buf, binary.LittleEndian, s.SliceSize)
	// write to redis
	_, err := conn.Do("SETRANGE", s.Refer, 0, buf.Bytes())
	return err
}

// update header info
// with big lock
func (s *SBFHeader) incrCount(conn redis.Conn) error {
	lockKey := fmt.Sprintf("lock:%s:count:%s", SBF_NAME, s.Refer)
	for i := 0; i < 10; i++ {
		if val, err := redis.Int(conn.Do("GET", lockKey)); err == nil && val > 0 {
			time.Sleep(time.Millisecond * 500)
		} else {
			if _, err := conn.Do("EXPIRE", lockKey, 5); err != nil {
				return err
			}
			break
		}
	}
	var count uint16
	if ret, err := redis.Bytes(conn.Do("GETRANGE", s.Refer, 8, 9)); err == nil {
		if err := BytesToNumber(ret, &count); err == nil {
			if count == s.Count {
				s.Count += 1
				if val, err := NumberToBytes(s.Count); err == nil {
					_, err := conn.Do("SETRANGE", s.Refer, 8, val)
					return err
				} else {
					return err
				}
			} else if count > s.Count {
				s.Count = count
				return nil
			} else {
				return nil
			}
		} else {
			return err
		}
	} else {
		return err
	}
}

// SBFFrame
type SBFFrame struct {
	SliceCount uint16  // frame hash functions count
	FullRate   float32 // frame error ratio
	SliceSize  uint32  // frame capacity
	StartIndex uint32  // frame start index (bit)
	EndIndex   uint32  // frame end index (bit)
	Count      uint32  // elements inserted
	Key        string  // key: sbf:refer:frameID
	Refer      string  // refer
}

func NewFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
	key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
	frame := new(SBFFrame)
	frame.Key = key
	frame.Refer = header.Refer
	frame.frameDataRange(header, id)
	conn.Send("MULTI")
	conn.Send("SETBIT", frame.Refer, frame.EndIndex, 0)
	conn.Send("SET", frame.Key, 0)
	_, err := conn.Do("EXEC")
	if err == nil {
		return frame, nil
	} else {
		return nil, err
	}
}

func LoadFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
	// key := SBF_NAME + ":count:" + header.Refer + ":1"
	key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
	frame := new(SBFFrame)
	frame.Key = key
	frame.Refer = header.Refer
	frame.Count = 0
	frame.frameDataRange(header, id)
	if count, err := redis.Uint64(conn.Do("GET", key)); err == nil {
		frame.Count = uint32(count)
	} else if err != redis.ErrNil {
		return nil, err
	}
	return frame, nil
}

// fullfill frame fields
// according to errorRate, capacity, can get the size one bloom filter.
func (s *SBFFrame) frameDataRange(header *SBFHeader, id uint16) {
	s.FullRate = float32(header.FullRate) / 10000
	for i := 1; i <= int(id); i++ {
		s.SliceCount = uint16(math.Ceil(float64(header.SliceCount) + float64(i-1)*math.Log2(1.0/float64(SBF_DEFAULT_S_ERROR_RATIO))))
		s.SliceSize = (uint32(float64(header.SliceSize)*math.Pow(SBF_DEFAULT_S_MIN_CAPACITY_SIZE, float64(i-1))) >> 3) << 3

		s.EndIndex += (s.SliceSize*uint32(s.SliceCount) + (SBF_FRAME_HEADER_SIZE+SBF_FRAME_PADDING)<<3)
	}
	s.EndIndex += SBF_HEADER_SIZE << 3
	s.StartIndex = s.EndIndex - uint32(s.SliceCount)*s.SliceSize - (SBF_FRAME_HEADER_SIZE+SBF_FRAME_PADDING)<<3
}

func (s *SBFFrame) IsFrameFull() bool {
	return float64(s.Count) >= float64(s.FullRate)*float64(s.SliceSize)
}

func (s *SBFFrame) Add(conn redis.Conn, element []byte) bool {
	hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
	// update bit val
	conn.Send("MULTI")
	for index, h := range hashes {
		pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
		conn.Send("SETBIT", s.Refer, pos, 1)
	}
	conn.Send("INCR", s.Key)
	_, err := conn.Do("EXEC")
	if err == nil {
		s.Count += 1
		return true
	} else {
		return false
	}
}

func (s *SBFFrame) Check(conn redis.Conn, element []byte) bool {
	var flag int = 1
	hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
	// check bit val
	conn.Send("MULTI")
	for index, h := range hashes {
		pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
		conn.Send("GETBIT", s.Refer, pos)
	}
	if data, err := redis.Ints(conn.Do("EXEC")); err == nil {
		for _, f := range data {
			flag = flag & f
			if flag != 1 {
				return false
			}
		}
		return (flag == 1)
	} else {
		return false
	}
}

// SBF struct
type SBF struct {
	Header *SBFHeader
	Conn   redis.Conn
}

// new SBF
func NewSBF(conn redis.Conn, sliceRatio float32, sliceCount uint16, sliceSize uint32, refer string) (*SBF, error) {
	// check if exist
	if flag, err := redis.Bool(conn.Do("EXISTS", refer)); err == nil && !flag {
		if header, err := NewHeader(conn, sliceRatio, sliceCount, sliceSize, refer); err == nil {
			if _, err = NewFrame(conn, header, 1); err == nil {
				sbf := new(SBF)
				sbf.Conn = conn
				sbf.Header = header
				return sbf, nil
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	} else {
		return nil, errors.New(fmt.Sprintf("sbf with key %s have exists.", refer))
	}
}

func LoadSBF(conn redis.Conn, refer string) (*SBF, error) {
	var err error
	sbf := new(SBF)
	if sbf.Header, err = LoadHeader(conn, refer); err != nil {
		// close
		conn.Close()
		return nil, err
	}
	sbf.Conn = conn

	return sbf, nil
}

func TruncateSBF(conn redis.Conn, refer string) error {
	if sbf, err := LoadSBF(conn, refer); err == nil {
		for i := 0; i < int(sbf.Header.Count); i++ {
			key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, refer, i)
			// ignore errors
			conn.Do("DEL", key)
		}
		if _, err := conn.Do("DEL", refer); err == nil {
			sbf.Header.Count = 1
			sbf.Header.updateHeader(conn)
			_, err = NewFrame(conn, sbf.Header, 1)
			return err
		} else {
			return err
		}
	} else {
		return err
	}
}

// Add element to sbf
// Steps:
//  * check if sbf exists.
//  * load or create sbf.
//  * load last frame.
//  * check if last frame is fullfilled.
//  * if frame is fullfilled, create a new frame.
//  * add to this frame.
func (s *SBF) Add(element []byte) bool {
	// if !s.Check(element) {
	if frame, err := LoadFrame(s.Conn, s.Header, s.Header.Count); err == nil {
		// check if frame is full
		if frame.IsFrameFull() {
			if s.Header.Count < SBF_FRAME_COUNT_LIMIT {
				// update header
				if err := s.Header.incrCount(s.Conn); err == nil {
					if frame, err = NewFrame(s.Conn, s.Header, s.Header.Count); err != nil {
						return false
					}
				} else {
					return false
				}
			} else {
				// frames have reached the limication, use old frames.
				// this may increase the error rate.
				id := uint16(rand.Uint32() % uint32(s.Header.Count))
				frame, err = LoadFrame(s.Conn, s.Header, id)
				if err != nil {
					return false
				}
			}
		}
		return frame.Add(s.Conn, element)
	} else {
		fmt.Println(err)
		return false
	}
	//}
	// return true
}

// Check if an element belongs to this sbf
// Steps:
//  * check if sbf exists.
//  * if not, return false
//  * if yes, check the first frame.
//  * if element in this frame, return true;
//  * else load next frame and check if element in this frame in loop, until find one in or not find in all frames.
func (s *SBF) Check(element []byte) bool {
	for i := 1; i <= int(s.Header.Count); i += 1 {
		if frame, err := LoadFrame(s.Conn, s.Header, uint16(i)); err == nil {
			if frame.Check(s.Conn, element) {
				return true
			}
		}
	}
	return false
}
<commit_msg>remove close<commit_after>package sbf

// Scale Bloom Filte in Redis
// date format
// frame end padding

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"math"
	"math/rand"
	"sbf/internal/murmur"
	. "sbf/internal/utils"
	"time"
)

var (
	SBF_NAME    = "SBF"   // 3 bytes
	SBF_VERSION = "1.0.0" // 5 bytes
)

const (
	// header
	SBF_HEADER_SIZE = 18 // 18 bytes

	// frame
	SBF_FRAME_HEADER_SIZE = 6    // 6 bytes, empty now, reserve for future use
	SBF_FRAME_COUNT_LIMIT = 1024 // the frame count of the sbf
	SBF_FRAME_PADDING     = 1    // 1 byte, reserve for protect the data at the end of previus frame
	// slice
	SBF_DEFAULT_S_COUNT             = 10    // slice count
	SBF_DEFAULT_S_SIZE              = 65536 // slice size
	SBF_DEFAULT_S_ERROR_RATIO       = 0.5   // the percentage of a slice used
	SBF_DEFAULT_S_MIN_CAPACITY_SIZE = 2     // the min growth of the slice size
	SBF_DEFAULT_S_MAX_CAPACITY_SIZE = 4     // the max growth of the slice size
)

func init() {
	if len(SBF_NAME) != 3 || len(SBF_VERSION) != 5 {
		panic(errors.New("invalid sbf name or sbf version"))
	}
	rand.Seed(time.Now().UnixNano())
}

// header of SBF
type SBFHeader struct {
	Name       [3]byte // SBF                           0 -  3
	Version    [5]byte // 1.0.0                         3 -  8
	Count      uint16  // frame count                   8 - 10
	FullRate   uint16  // error ratio  =0.1 * 10000    10 - 12
	SliceCount uint16  // hash functions count         12 - 14
	SliceSize  uint32  // slice size                   14 - 18
	Refer      string
}

// sliceSize must be the multiple of 8
func NewHeader(conn redis.Conn, sliceRatio float32, sliceCount uint16, sliceSize uint32, refer string) (*SBFHeader, error) {
	header := new(SBFHeader)
	copy(header.Name[:], SBF_NAME)
	copy(header.Version[:], SBF_VERSION)
	header.Count = 1
	header.FullRate = uint16(sliceRatio * 10000)
	header.SliceCount = sliceCount
	header.SliceSize = sliceSize
	header.Refer = refer

	if header.SliceSize%8 != 0 {
		return nil, errors.New(fmt.Sprintf("%d NOT multiple of 8", sliceSize))
	}
	// save
	err := header.updateHeader(conn)
	return header, err
}

// load sbf header from redis
func LoadHeader(conn redis.Conn, refer string) (*SBFHeader, error) {
	if ret, err := redis.Bytes(conn.Do("GETRANGE", refer, 0, SBF_HEADER_SIZE-1)); err == nil {
		if len(ret) > 0 {
			header := new(SBFHeader)
			copy(header.Name[:], ret[0:3])
			copy(header.Version[:], ret[3:8])
			// from bytes to number
			if err := BytesToNumber(ret[8:10], &header.Count); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[10:12], &header.FullRate); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[12:14], &header.SliceCount); err != nil {
				return nil, err
			}
			if err := BytesToNumber(ret[14:18], &header.SliceSize); err != nil {
				return nil, err
			}
			header.Refer = refer
			return header, nil
		} else {
			return nil, errors.New(fmt.Sprintf("SBF %s NOT FOUND.", refer))
		}
	} else {
		return nil, err
	}
}

func (s *SBFHeader) checkHeader() error {
	if bytes.Equal(s.Name[:], []byte(SBF_NAME)) {
		return errors.New("INVALID SBF header.")
	}
	if bytes.Compare(s.Version[:], []byte(SBF_VERSION)) > 0 {
		return errors.New("NOT supported version.")
	}
	return nil
}

func (s *SBFHeader) updateHeader(conn redis.Conn) error {
	// name
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.LittleEndian, s.Name)
	binary.Write(buf, binary.LittleEndian, s.Version)
	binary.Write(buf, binary.LittleEndian, s.Count)
	binary.Write(buf, binary.LittleEndian, s.FullRate)
	binary.Write(buf, binary.LittleEndian, s.SliceCount)
	binary.Write(buf, binary.LittleEndian, s.SliceSize)
	// write to redis
	_, err := conn.Do("SETRANGE", s.Refer, 0, buf.Bytes())
	return err
}

// update header info
// with big lock
func (s *SBFHeader) incrCount(conn redis.Conn) error {
	lockKey := fmt.Sprintf("lock:%s:count:%s", SBF_NAME, s.Refer)
	for i := 0; i < 10; i++ {
		if val, err := redis.Int(conn.Do("GET", lockKey)); err == nil && val > 0 {
			time.Sleep(time.Millisecond * 500)
		} else {
			if _, err := conn.Do("EXPIRE", lockKey, 5); err != nil {
				return err
			}
			break
		}
	}
	var count uint16
	if ret, err := redis.Bytes(conn.Do("GETRANGE", s.Refer, 8, 9)); err == nil {
		if err := BytesToNumber(ret, &count); err == nil {
			if count == s.Count {
				s.Count += 1
				if val, err := NumberToBytes(s.Count); err == nil {
					_, err := conn.Do("SETRANGE", s.Refer, 8, val)
					return err
				} else {
					return err
				}
			} else if count > s.Count {
				s.Count = count
				return nil
			} else {
				return nil
			}
		} else {
			return err
		}
	} else {
		return err
	}
}

// SBFFrame
type SBFFrame struct {
	SliceCount uint16  // frame hash functions count
	FullRate   float32 // frame error ratio
	SliceSize  uint32  // frame capacity
	StartIndex uint32  // frame start index (bit)
	EndIndex   uint32  // frame end index (bit)
	Count      uint32  // elements inserted
	Key        string  // key: sbf:refer:frameID
	Refer      string  // refer
}

func NewFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
	key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
	frame := new(SBFFrame)
	frame.Key = key
	frame.Refer = header.Refer
	frame.frameDataRange(header, id)
	conn.Send("MULTI")
	conn.Send("SETBIT", frame.Refer, frame.EndIndex, 0)
	conn.Send("SET", frame.Key, 0)
	_, err := conn.Do("EXEC")
	if err == nil {
		return frame, nil
	} else {
		return nil, err
	}
}

func LoadFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
	// key := SBF_NAME + ":count:" + header.Refer + ":1"
	key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
	frame := new(SBFFrame)
	frame.Key = key
	frame.Refer = header.Refer
	frame.Count = 0
	frame.frameDataRange(header, id)
	if count, err := redis.Uint64(conn.Do("GET", key)); err == nil {
		frame.Count = uint32(count)
	} else if err != redis.ErrNil {
		return nil, err
	}
	return frame, nil
}

// fullfill frame fields
// according to errorRate, capacity, can get the size one bloom filter.
func (s *SBFFrame) frameDataRange(header *SBFHeader, id uint16) {
	s.FullRate = float32(header.FullRate) / 10000
	for i := 1; i <= int(id); i++ {
		s.SliceCount = uint16(math.Ceil(float64(header.SliceCount) + float64(i-1)*math.Log2(1.0/float64(SBF_DEFAULT_S_ERROR_RATIO))))
		s.SliceSize = (uint32(float64(header.SliceSize)*math.Pow(SBF_DEFAULT_S_MIN_CAPACITY_SIZE, float64(i-1))) >> 3) << 3

		s.EndIndex += (s.SliceSize*uint32(s.SliceCount) + (SBF_FRAME_HEADER_SIZE+SBF_FRAME_PADDING)<<3)
	}
	s.EndIndex += SBF_HEADER_SIZE << 3
	s.StartIndex = s.EndIndex - uint32(s.SliceCount)*s.SliceSize - (SBF_FRAME_HEADER_SIZE+SBF_FRAME_PADDING)<<3
}

func (s *SBFFrame) IsFrameFull() bool {
	return float64(s.Count) >= float64(s.FullRate)*float64(s.SliceSize)
}

func (s *SBFFrame) Add(conn redis.Conn, element []byte) bool {
	hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
	// update bit val
	conn.Send("MULTI")
	for index, h := range hashes {
		pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
		conn.Send("SETBIT", s.Refer, pos, 1)
	}
	conn.Send("INCR", s.Key)
	_, err := conn.Do("EXEC")
	if err == nil {
		s.Count += 1
		return true
	} else {
		return false
	}
}

func (s *SBFFrame) Check(conn redis.Conn, element []byte) bool {
	var flag int = 1
	hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
	// check bit val
	conn.Send("MULTI")
	for index, h := range hashes {
		pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
		conn.Send("GETBIT", s.Refer, pos)
	}
	if data, err := redis.Ints(conn.Do("EXEC")); err == nil {
		for _, f := range data {
			flag = flag & f
			if flag != 1 {
				return false
			}
		}
		return (flag == 1)
	} else {
		return false
	}
}

// SBF struct
type SBF struct {
	Header *SBFHeader
	Conn   redis.Conn
}

// new SBF
func NewSBF(conn redis.Conn, sliceRatio float32, sliceCount uint16, sliceSize uint32, refer string) (*SBF, error) {
	// check if exist
	if flag, err := redis.Bool(conn.Do("EXISTS", refer)); err == nil && !flag {
		if header, err := NewHeader(conn, sliceRatio, sliceCount, sliceSize, refer); err == nil {
			if _, err = NewFrame(conn, header, 1); err == nil {
				sbf := new(SBF)
				sbf.Conn = conn
				sbf.Header = header
				return sbf, nil
			} else {
				return nil, err
			}
		} else {
			return nil, err
		}
	} else {
		return nil, errors.New(fmt.Sprintf("sbf with key %s have exists.", refer))
	}
}

func LoadSBF(conn redis.Conn, refer string) (*SBF, error) {
	var err error
	sbf := new(SBF)
	if sbf.Header, err = LoadHeader(conn, refer); err != nil {
		// close
		// conn.Close()
		return nil, err
	}
	sbf.Conn = conn

	return sbf, nil
}

func TruncateSBF(conn redis.Conn, refer string) error {
	if sbf, err := LoadSBF(conn, refer); err == nil {
		for i := 0; i < int(sbf.Header.Count); i++ {
			key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, refer, i)
			// ignore errors
			conn.Do("DEL", key)
		}
		if _, err := conn.Do("DEL", refer); err == nil {
			sbf.Header.Count = 1
			sbf.Header.updateHeader(conn)
			_, err = NewFrame(conn, sbf.Header, 1)
			return err
		} else {
			return err
		}
	} else {
		return err
	}
}

// Add element to sbf
// Steps:
//  * check if sbf exists.
//  * load or create sbf.
//  * load last frame.
//  * check if last frame is fullfilled.
//  * if frame is fullfilled, create a new frame.
//  * add to this frame.
func (s *SBF) Add(element []byte) bool {
	// if !s.Check(element) {
	if frame, err := LoadFrame(s.Conn, s.Header, s.Header.Count); err == nil {
		// check if frame is full
		if frame.IsFrameFull() {
			if s.Header.Count < SBF_FRAME_COUNT_LIMIT {
				// update header
				if err := s.Header.incrCount(s.Conn); err == nil {
					if frame, err = NewFrame(s.Conn, s.Header, s.Header.Count); err != nil {
						return false
					}
				} else {
					return false
				}
			} else {
				// frames have reached the limication, use old frames.
				// this may increase the error rate.
				id := uint16(rand.Uint32() % uint32(s.Header.Count))
				frame, err = LoadFrame(s.Conn, s.Header, id)
				if err != nil {
					return false
				}
			}
		}
		return frame.Add(s.Conn, element)
	} else {
		fmt.Println(err)
		return false
	}
	//}
	// return true
}

// Check if an element belongs to this sbf
// Steps:
//  * check if sbf exists.
//  * if not, return false
//  * if yes, check the first frame.
//  * if element in this frame, return true;
//  * else load next frame and check if element in this frame in loop, until find one in or not find in all frames.
func (s *SBF) Check(element []byte) bool {
	for i := 1; i <= int(s.Header.Count); i += 1 {
		if frame, err := LoadFrame(s.Conn, s.Header, uint16(i)); err == nil {
			if frame.Check(s.Conn, element) {
				return true
			}
		}
	}
	return false
}
<|endoftext|>