<commit_before>package server

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/meatballhat/negroni-logrus"
	"github.com/oxfeeefeee/appgo"
	"github.com/oxfeeefeee/appgo/auth"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/unrolled/render"
	"html/template"
	"net/http"
	"strings"
)

type Server struct {
	ts TokenStore
	*mux.Router
}
type TokenStore interface {
	Validate(token auth.Token) bool
}

func NewServer(ts TokenStore) *Server {
	return &Server{
		ts,
		mux.NewRouter(),
	}
}

func (s *Server) AddRest(path string, rests []interface{}) {
	renderer := render.New(render.Options{
		Directory:     "N/A",
		IndentJSON:    appgo.Conf.DevMode,
		IsDevelopment: appgo.Conf.DevMode,
	})
	for _, api := range rests {
		h := newHandler(api, HandlerTypeJson, s.ts, renderer)
		s.Handle(path+h.path, h).Methods(h.supports...)
	}
}

func (s *Server) AddHtml(path, layout string, htmls []interface{}, funcs template.FuncMap) {
	renderer := render.New(render.Options{
		Directory:     appgo.Conf.TemplatePath,
		Layout:        layout,
		Funcs:         []template.FuncMap{funcs},
		IsDevelopment: appgo.Conf.DevMode,
	})
	for _, api := range htmls {
		h := newHandler(api, HandlerTypeHtml, s.ts, renderer)
		s.Handle(path+h.path, h).Methods("GET")
	}
}

func (s *Server) AddProxy(path string, handler http.Handler) {
	s.PathPrefix(path).Handler(http.StripPrefix(path, handler))
}

func (s *Server) AddStatic(path, fileDir string) {
	s.AddProxy(path, http.FileServer(http.Dir(fileDir)))
}

func (s *Server) AddAppleAppSiteAsso(content []byte) {
	f := func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.Write(content)
	}
	s.HandleFunc("/apple-app-site-association", f)
}

func (s *Server) Serve() {
	n := negroni.New()
	n.Use(negroni.NewRecovery())
	n.Use(negronilogrus.NewCustomMiddleware(
		appgo.Conf.LogLevel, &log.TextFormatter{}, "appgo"))
	n.Use(cors.New(corsOptions()))
	if appgo.Conf.Negroni.GZip {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}
	n.UseHandler(s)
	n.Run(appgo.Conf.Negroni.Port)
}

func corsOptions() cors.Options {
	origins := strings.Split(appgo.Conf.Cors.AllowedOrigins, ",")
	methods := strings.Split(appgo.Conf.Cors.AllowedMethods, ",")
	headers := strings.Split(appgo.Conf.Cors.AllowedHeaders, ",")
	return cors.Options{
		AllowedOrigins:     origins,
		AllowedMethods:     methods,
		AllowedHeaders:     headers,
		OptionsPassthrough: appgo.Conf.Cors.OptionsPassthrough,
		Debug:              appgo.Conf.Cors.Debug,
	}
}
<commit_msg>custom middleware<commit_after>package server

import (
	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/gorilla/mux"
	"github.com/meatballhat/negroni-logrus"
	"github.com/oxfeeefeee/appgo"
	"github.com/oxfeeefeee/appgo/auth"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/unrolled/render"
	"html/template"
	"net/http"
	"strings"
)

type Server struct {
	ts          TokenStore
	middlewares []negroni.HandlerFunc
	*mux.Router
}
type TokenStore interface {
	Validate(token auth.Token) bool
}

func NewServer(ts TokenStore, middlewares []negroni.HandlerFunc) *Server {
	return &Server{
		ts,
		middlewares,
		mux.NewRouter(),
	}
}

func (s *Server) AddRest(path string, rests []interface{}) {
	renderer := render.New(render.Options{
		Directory:     "N/A",
		IndentJSON:    appgo.Conf.DevMode,
		IsDevelopment: appgo.Conf.DevMode,
	})
	for _, api := range rests {
		h := newHandler(api, HandlerTypeJson, s.ts, renderer)
		s.Handle(path+h.path, h).Methods(h.supports...)
	}
}

func (s *Server) AddHtml(path, layout string, htmls []interface{}, funcs template.FuncMap) {
	renderer := render.New(render.Options{
		Directory:     appgo.Conf.TemplatePath,
		Layout:        layout,
		Funcs:         []template.FuncMap{funcs},
		IsDevelopment: appgo.Conf.DevMode,
	})
	for _, api := range htmls {
		h := newHandler(api, HandlerTypeHtml, s.ts, renderer)
		s.Handle(path+h.path, h).Methods("GET")
	}
}

func (s *Server) AddProxy(path string, handler http.Handler) {
	s.PathPrefix(path).Handler(http.StripPrefix(path, handler))
}

func (s *Server) AddStatic(path, fileDir string) {
	s.AddProxy(path, http.FileServer(http.Dir(fileDir)))
}

func (s *Server) AddAppleAppSiteAsso(content []byte) {
	f := func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.Write(content)
	}
	s.HandleFunc("/apple-app-site-association", f)
}

func (s *Server) Serve() {
	n := negroni.New()
	n.Use(negroni.NewRecovery())
	n.Use(negronilogrus.NewCustomMiddleware(
		appgo.Conf.LogLevel, &log.TextFormatter{}, "appgo"))
	n.Use(cors.New(corsOptions()))
	if appgo.Conf.Negroni.GZip {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}
	for _, mw := range s.middlewares {
		n.Use(negroni.HandlerFunc(mw))
	}
	n.UseHandler(s)
	n.Run(appgo.Conf.Negroni.Port)
}

func corsOptions() cors.Options {
	origins := strings.Split(appgo.Conf.Cors.AllowedOrigins, ",")
	methods := strings.Split(appgo.Conf.Cors.AllowedMethods, ",")
	headers := strings.Split(appgo.Conf.Cors.AllowedHeaders, ",")
	return cors.Options{
		AllowedOrigins:     origins,
		AllowedMethods:     methods,
		AllowedHeaders:     headers,
		OptionsPassthrough: appgo.Conf.Cors.OptionsPassthrough,
		Debug:              appgo.Conf.Cors.Debug,
	}
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"text/template"
	"time"

	"github.com/TV4/graceful"
	"github.com/gogap/config"
	"github.com/gogap/go-wkhtmltox/wkhtmltox"
	"github.com/gorilla/mux"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/spf13/cast"
	"github.com/urfave/negroni"
)

const (
	defaultTemplateText = `{"code":{{.Code}},"message":"{{.Message}}"{{if .Result}},"result":{{.Result|jsonify}}{{end}}}`
)

var (
	htmlToX *wkhtmltox.WKHtmlToX

	renderTmpls = make(map[string]*template.Template)

	defaultTmpl *template.Template
)

type ConvertData struct {
	Data []byte `json:"data"`
}

type ConvertArgs struct {
	To        string                   `json:"to"`
	Fetcher   wkhtmltox.FetcherOptions `json:"fetcher"`
	Converter json.RawMessage          `json:"converter"`
	Template  string                   `json:"template"`
}

type TemplateArgs struct {
	To string
	ConvertResponse
	Response *RespHelper
}

type ConvertResponse struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Result  interface{} `json:"result"`
}

type serverWrapper struct {
	tls      bool
	certFile string
	keyFile  string

	reqNumber int64
	addr      string
	n         *negroni.Negroni

	timeout time.Duration
}

func (p *serverWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	atomic.AddInt64(&p.reqNumber, 1)
	defer atomic.AddInt64(&p.reqNumber, -1)

	p.n.ServeHTTP(w, r)
}

func (p *serverWrapper) ListenAndServe() (err error) {

	if p.tls {
		err = http.ListenAndServeTLS(p.addr, p.certFile, p.keyFile, p)
	} else {
		err = http.ListenAndServe(p.addr, p)
	}

	return
}

func (p *serverWrapper) Shutdown(ctx context.Context) error {
	num := atomic.LoadInt64(&p.reqNumber)

	schema := "HTTP"

	if p.tls {
		schema = "HTTPS"
	}

	beginTime := time.Now()

	for num > 0 {
		time.Sleep(time.Second)
		timeDiff := time.Now().Sub(beginTime)
		if timeDiff > p.timeout {
			break
		}
	}

	log.Printf("[%s] Shutdown finished, Address: %s\n", schema, p.addr)

	return nil
}

type WKHtmlToXServer struct {
	conf    config.Configuration
	servers []*serverWrapper
}

func New(conf config.Configuration) (srv *WKHtmlToXServer, err error) {

	serviceConf := conf.GetConfig("service")

	wkHtmlToXConf := conf.GetConfig("wkhtmltox")

	htmlToX, err = wkhtmltox.New(wkHtmlToXConf)

	if err != nil {
		return
	}

	// init templates

	defaultTmpl, err = template.New("default").Funcs(funcMap).Parse(defaultTemplateText)

	if err != nil {
		return
	}

	err = loadTemplates(
		serviceConf.GetConfig("templates"),
	)

	if err != nil {
		return
	}

	// init http server
	c := cors.New(
		cors.Options{
			AllowedOrigins:     serviceConf.GetStringList("cors.allowed-origins"),
			AllowedMethods:     serviceConf.GetStringList("cors.allowed-methods"),
			AllowedHeaders:     serviceConf.GetStringList("cors.allowed-headers"),
			ExposedHeaders:     serviceConf.GetStringList("cors.exposed-headers"),
			AllowCredentials:   serviceConf.GetBoolean("cors.allow-credentials"),
			MaxAge:             int(serviceConf.GetInt64("cors.max-age")),
			OptionsPassthrough: serviceConf.GetBoolean("cors.options-passthrough"),
			Debug:              serviceConf.GetBoolean("cors.debug"),
		},
	)

	r := mux.NewRouter()

	pathPrefix := serviceConf.GetString("path", "/")

	r.PathPrefix(pathPrefix).Path("/convert").
		Methods("POST").
		HandlerFunc(handleHtmlToX)

	n := negroni.Classic()

	n.Use(c) // use cors

	if serviceConf.GetBoolean("gzip-enabled", true) {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}

	n.UseHandler(r)

	gracefulTimeout := serviceConf.GetTimeDuration("graceful.timeout", time.Second*3)

	enableHTTP := serviceConf.GetBoolean("http.enabled", true)
	enableHTTPS := serviceConf.GetBoolean("https.enabled", false)

	var servers []*serverWrapper

	if enableHTTP {

		listenAddr := serviceConf.GetString("http.address", "127.0.0.1:8080")

		httpServer := &serverWrapper{
			n:       n,
			timeout: gracefulTimeout,
			addr:    listenAddr,
		}

		servers = append(servers, httpServer)
	}

	if enableHTTPS {

		listenAddr := serviceConf.GetString("http.address", "127.0.0.1:443")
		certFile := serviceConf.GetString("https.cert")
		keyFile := serviceConf.GetString("https.key")

		httpsServer := &serverWrapper{
			n:        n,
			timeout:  gracefulTimeout,
			addr:     listenAddr,
			tls:      true,
			certFile: certFile,
			keyFile:  keyFile,
		}

		servers = append(servers, httpsServer)
	}

	srv = &WKHtmlToXServer{
		conf:    conf,
		servers: servers,
	}

	return
}

func (p *WKHtmlToXServer) Run() (err error) {

	wg := sync.WaitGroup{}

	wg.Add(len(p.servers))

	for i := 0; i < len(p.servers); i++ {
		go func(srv *serverWrapper) {
			defer wg.Done()
			shcema := "HTTP"
			if srv.tls {
				shcema = "HTTPS"
			}
			log.Printf("[%s] Listening on %s\n", shcema, srv.addr)
			graceful.ListenAndServe(srv)
		}(p.servers[i])
	}

	wg.Wait()

	return
}

func writeResp(rw http.ResponseWriter, convertArgs ConvertArgs, resp ConvertResponse) {

	var tmpl *template.Template
	if len(convertArgs.Template) == 0 {
		tmpl = defaultTmpl
	} else {
		var exist bool

		tmpl, exist = renderTmpls[convertArgs.Template]
		if !exist {
			tmpl = defaultTmpl
		}
	}

	respHelper := newRespHelper(rw)

	args := TemplateArgs{
		To:              convertArgs.To,
		ConvertResponse: resp,
		Response:        respHelper,
	}

	buf := bytes.NewBuffer(nil)

	err := tmpl.Execute(buf, args)

	if err != nil {
		log.Println(err)
	}

	if !respHelper.Holding() {
		rw.Write(buf.Bytes())
	}
}

func handleHtmlToX(rw http.ResponseWriter, req *http.Request) {

	decoder := json.NewDecoder(req.Body)

	decoder.UseNumber()

	args := ConvertArgs{}

	err := decoder.Decode(&args)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	if len(args.Converter) == 0 {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, "converter is nil", nil})
		return
	}

	to := strings.ToUpper(args.To)

	var opts wkhtmltox.ConvertOptions

	if to == "IMAGE" {
		opts = &wkhtmltox.ToImageOptions{}
	} else if to == "PDF" {
		opts = &wkhtmltox.ToPDFOptions{}
	} else {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, "argument of to is illegal (image|pdf)", nil})
		return
	}

	err = json.Unmarshal(args.Converter, opts)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	var convData []byte

	convData, err = htmlToX.Convert(args.Fetcher, opts)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	writeResp(rw, args, ConvertResponse{0, "", ConvertData{Data: convData}})

	return
}

func loadTemplates(tmplsConf config.Configuration) (err error) {
	if tmplsConf == nil {
		return
	}

	tmpls := tmplsConf.Keys()

	for _, name := range tmpls {

		file := tmplsConf.GetString(name + ".template")

		tmpl := template.New(name).Funcs(funcMap)

		var data []byte
		data, err = ioutil.ReadFile(file)

		if err != nil {
			return
		}

		tmpl, err = tmpl.Parse(string(data))

		if err != nil {
			return
		}

		renderTmpls[name] = tmpl
	}

	return
}

type RespHelper struct {
	rw   http.ResponseWriter
	hold bool
}

func newRespHelper(rw http.ResponseWriter) *RespHelper {
	return &RespHelper{
		rw:   rw,
		hold: false,
	}
}

func (p *RespHelper) SetHeader(key, value interface{}) error {
	k := cast.ToString(key)
	v := cast.ToString(value)

	p.rw.Header().Set(k, v)

	return nil
}

func (p *RespHelper) Hold(v interface{}) error {
	h := cast.ToBool(v)
	p.hold = h

	return nil
}

func (p *RespHelper) Holding() bool {
	return p.hold
}

func (p *RespHelper) Write(data []byte) error {
	p.rw.Write(data)
	return nil
}

func (p *RespHelper) WriteHeader(code interface{}) error {
	c, err := cast.ToIntE(code)
	if err != nil {
		return err
	}

	p.rw.WriteHeader(c)

	return nil
}
<commit_msg>add ping pong support<commit_after>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"text/template"
	"time"

	"github.com/TV4/graceful"
	"github.com/gogap/config"
	"github.com/gogap/go-wkhtmltox/wkhtmltox"
	"github.com/gorilla/mux"
	"github.com/phyber/negroni-gzip/gzip"
	"github.com/rs/cors"
	"github.com/spf13/cast"
	"github.com/urfave/negroni"
)

const (
	defaultTemplateText = `{"code":{{.Code}},"message":"{{.Message}}"{{if .Result}},"result":{{.Result|jsonify}}{{end}}}`
)

var (
	htmlToX *wkhtmltox.WKHtmlToX

	renderTmpls = make(map[string]*template.Template)

	defaultTmpl *template.Template
)

type ConvertData struct {
	Data []byte `json:"data"`
}

type ConvertArgs struct {
	To        string                   `json:"to"`
	Fetcher   wkhtmltox.FetcherOptions `json:"fetcher"`
	Converter json.RawMessage          `json:"converter"`
	Template  string                   `json:"template"`
}

type TemplateArgs struct {
	To string
	ConvertResponse
	Response *RespHelper
}

type ConvertResponse struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Result  interface{} `json:"result"`
}

type serverWrapper struct {
	tls      bool
	certFile string
	keyFile  string

	reqNumber int64
	addr      string
	n         *negroni.Negroni

	timeout time.Duration
}

func (p *serverWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	atomic.AddInt64(&p.reqNumber, 1)
	defer atomic.AddInt64(&p.reqNumber, -1)

	p.n.ServeHTTP(w, r)
}

func (p *serverWrapper) ListenAndServe() (err error) {

	if p.tls {
		err = http.ListenAndServeTLS(p.addr, p.certFile, p.keyFile, p)
	} else {
		err = http.ListenAndServe(p.addr, p)
	}

	return
}

func (p *serverWrapper) Shutdown(ctx context.Context) error {
	num := atomic.LoadInt64(&p.reqNumber)

	schema := "HTTP"

	if p.tls {
		schema = "HTTPS"
	}

	beginTime := time.Now()

	for num > 0 {
		time.Sleep(time.Second)
		timeDiff := time.Now().Sub(beginTime)
		if timeDiff > p.timeout {
			break
		}
	}

	log.Printf("[%s] Shutdown finished, Address: %s\n", schema, p.addr)

	return nil
}

type WKHtmlToXServer struct {
	conf    config.Configuration
	servers []*serverWrapper
}

func New(conf config.Configuration) (srv *WKHtmlToXServer, err error) {

	serviceConf := conf.GetConfig("service")

	wkHtmlToXConf := conf.GetConfig("wkhtmltox")

	htmlToX, err = wkhtmltox.New(wkHtmlToXConf)

	if err != nil {
		return
	}

	// init templates

	defaultTmpl, err = template.New("default").Funcs(funcMap).Parse(defaultTemplateText)

	if err != nil {
		return
	}

	err = loadTemplates(
		serviceConf.GetConfig("templates"),
	)

	if err != nil {
		return
	}

	// init http server
	c := cors.New(
		cors.Options{
			AllowedOrigins:     serviceConf.GetStringList("cors.allowed-origins"),
			AllowedMethods:     serviceConf.GetStringList("cors.allowed-methods"),
			AllowedHeaders:     serviceConf.GetStringList("cors.allowed-headers"),
			ExposedHeaders:     serviceConf.GetStringList("cors.exposed-headers"),
			AllowCredentials:   serviceConf.GetBoolean("cors.allow-credentials"),
			MaxAge:             int(serviceConf.GetInt64("cors.max-age")),
			OptionsPassthrough: serviceConf.GetBoolean("cors.options-passthrough"),
			Debug:              serviceConf.GetBoolean("cors.debug"),
		},
	)

	r := mux.NewRouter()

	pathPrefix := serviceConf.GetString("path", "/")

	r.PathPrefix(pathPrefix).Path("/convert").
		Methods("POST").
		HandlerFunc(handleHtmlToX)

	r.PathPrefix(pathPrefix).Path("/ping").
		Methods("GET").HandlerFunc(
		func(rw http.ResponseWriter, req *http.Request) {
			rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
			rw.Write([]byte("pong"))
		},
	)

	n := negroni.Classic()

	n.Use(c) // use cors

	if serviceConf.GetBoolean("gzip-enabled", true) {
		n.Use(gzip.Gzip(gzip.DefaultCompression))
	}

	n.UseHandler(r)

	gracefulTimeout := serviceConf.GetTimeDuration("graceful.timeout", time.Second*3)

	enableHTTP := serviceConf.GetBoolean("http.enabled", true)
	enableHTTPS := serviceConf.GetBoolean("https.enabled", false)

	var servers []*serverWrapper

	if enableHTTP {

		listenAddr := serviceConf.GetString("http.address", "127.0.0.1:8080")

		httpServer := &serverWrapper{
			n:       n,
			timeout: gracefulTimeout,
			addr:    listenAddr,
		}

		servers = append(servers, httpServer)
	}

	if enableHTTPS {

		listenAddr := serviceConf.GetString("http.address", "127.0.0.1:443")
		certFile := serviceConf.GetString("https.cert")
		keyFile := serviceConf.GetString("https.key")

		httpsServer := &serverWrapper{
			n:        n,
			timeout:  gracefulTimeout,
			addr:     listenAddr,
			tls:      true,
			certFile: certFile,
			keyFile:  keyFile,
		}

		servers = append(servers, httpsServer)
	}

	srv = &WKHtmlToXServer{
		conf:    conf,
		servers: servers,
	}

	return
}

func (p *WKHtmlToXServer) Run() (err error) {

	wg := sync.WaitGroup{}

	wg.Add(len(p.servers))

	for i := 0; i < len(p.servers); i++ {
		go func(srv *serverWrapper) {
			defer wg.Done()
			shcema := "HTTP"
			if srv.tls {
				shcema = "HTTPS"
			}
			log.Printf("[%s] Listening on %s\n", shcema, srv.addr)
			graceful.ListenAndServe(srv)
		}(p.servers[i])
	}

	wg.Wait()

	return
}

func writeResp(rw http.ResponseWriter, convertArgs ConvertArgs, resp ConvertResponse) {

	var tmpl *template.Template
	if len(convertArgs.Template) == 0 {
		tmpl = defaultTmpl
	} else {
		var exist bool

		tmpl, exist = renderTmpls[convertArgs.Template]
		if !exist {
			tmpl = defaultTmpl
		}
	}

	respHelper := newRespHelper(rw)

	args := TemplateArgs{
		To:              convertArgs.To,
		ConvertResponse: resp,
		Response:        respHelper,
	}

	buf := bytes.NewBuffer(nil)

	err := tmpl.Execute(buf, args)

	if err != nil {
		log.Println(err)
	}

	if !respHelper.Holding() {
		rw.Write(buf.Bytes())
	}
}

func handleHtmlToX(rw http.ResponseWriter, req *http.Request) {

	decoder := json.NewDecoder(req.Body)

	decoder.UseNumber()

	args := ConvertArgs{}

	err := decoder.Decode(&args)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	if len(args.Converter) == 0 {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, "converter is nil", nil})
		return
	}

	to := strings.ToUpper(args.To)

	var opts wkhtmltox.ConvertOptions

	if to == "IMAGE" {
		opts = &wkhtmltox.ToImageOptions{}
	} else if to == "PDF" {
		opts = &wkhtmltox.ToPDFOptions{}
	} else {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, "argument of to is illegal (image|pdf)", nil})
		return
	}

	err = json.Unmarshal(args.Converter, opts)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	var convData []byte

	convData, err = htmlToX.Convert(args.Fetcher, opts)

	if err != nil {
		writeResp(rw, args, ConvertResponse{http.StatusBadRequest, err.Error(), nil})
		return
	}

	writeResp(rw, args, ConvertResponse{0, "", ConvertData{Data: convData}})

	return
}

func loadTemplates(tmplsConf config.Configuration) (err error) {
	if tmplsConf == nil {
		return
	}

	tmpls := tmplsConf.Keys()

	for _, name := range tmpls {

		file := tmplsConf.GetString(name + ".template")

		tmpl := template.New(name).Funcs(funcMap)

		var data []byte
		data, err = ioutil.ReadFile(file)

		if err != nil {
			return
		}

		tmpl, err = tmpl.Parse(string(data))

		if err != nil {
			return
		}

		renderTmpls[name] = tmpl
	}

	return
}

type RespHelper struct {
	rw   http.ResponseWriter
	hold bool
}

func newRespHelper(rw http.ResponseWriter) *RespHelper {
	return &RespHelper{
		rw:   rw,
		hold: false,
	}
}

func (p *RespHelper) SetHeader(key, value interface{}) error {
	k := cast.ToString(key)
	v := cast.ToString(value)

	p.rw.Header().Set(k, v)

	return nil
}

func (p *RespHelper) Hold(v interface{}) error {
	h := cast.ToBool(v)
	p.hold = h

	return nil
}

func (p *RespHelper) Holding() bool {
	return p.hold
}

func (p *RespHelper) Write(data []byte) error {
	p.rw.Write(data)
	return nil
}

func (p *RespHelper) WriteHeader(code interface{}) error {
	c, err := cast.ToIntE(code)
	if err != nil {
		return err
	}

	p.rw.WriteHeader(c)

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"

	"github.com/blacklabeldata/kappa/auth"
	"github.com/blacklabeldata/kappa/datamodel"
	"github.com/blacklabeldata/kappa/pkg/uuid"
	"github.com/blacklabeldata/sshh"
	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"golang.org/x/crypto/ssh"
	tomb "gopkg.in/tomb.v2"
)

const serfSnapshot = "serf/local.snapshot"

func NewServer(c *DatabaseConfig) (server *Server, err error) {

	// Create logger
	if c.LogOutput == nil {
		c.LogOutput = log.NewConcurrentWriter(os.Stdout)
	}
	logger := log.NewLogger(c.LogOutput, "kappa")

	// Create data directory
	if err = os.MkdirAll(c.DataPath, 0755); err != nil {
		logger.Warn("Could not create data directory", "err", err)
		// logger.Warn("Could not create data directory", "err", err.Error())
		return
	}

	// Connect to database
	cwd, err := os.Getwd()
	if err != nil {
		logger.Error("Could not get working directory", "error", err.Error())
		return
	}

	file := path.Join(cwd, c.DataPath, "meta.db")
	logger.Info("Connecting to database", "file", file)
	system, err := datamodel.NewSystem(file)
	if err != nil {
		logger.Error("Could not connect to database", "error", err.Error())
		return
	}

	// Get SSH Key file
	sshKeyFile := c.SSHPrivateKeyFile
	logger.Info("Reading private key", "file", sshKeyFile)

	privateKey, err := auth.ReadPrivateKey(logger, sshKeyFile)
	if err != nil {
		return
	}

	// Get admin certificate
	adminCertFile := c.AdminCertificateFile
	logger.Info("Reading admin public key", "file", adminCertFile)

	// Read admin certificate
	cert, err := ioutil.ReadFile(adminCertFile)
	if err != nil {
		logger.Error("admin certificate could not be read", "filename", c.AdminCertificateFile)
		return
	}

	// Add admin cert to key ring
	userStore, err := system.Users()
	if err != nil {
		logger.Error("could not get user store", "error", err.Error())
		return
	}

	// Create admin account
	admin, err := userStore.Create("admin")
	if err != nil {
		logger.Error("error creating admin account", "error", err.Error())
		return
	}

	// Add admin certificate
	keyRing := admin.KeyRing()
	fingerprint, err := keyRing.AddPublicKey(cert)
	if err != nil {
		logger.Error("admin certificate could not be added", "error", err.Error())
		return
	}
	logger.Info("Added admin certificate", "fingerprint", fingerprint)

	// Read root cert
	rootPem, err := ioutil.ReadFile(c.CACertificateFile)
	if err != nil {
		logger.Error("root certificate could not be read", "filename", c.CACertificateFile)
		return
	}

	// Create certificate pool
	roots := x509.NewCertPool()
	if ok := roots.AppendCertsFromPEM(rootPem); !ok {
		logger.Error("failed to parse root certificate")
		return
	}

	// Setup SSH Server
	sshLogger := log.NewLogger(c.LogOutput, "ssh")
	pubKeyCallback, err := PublicKeyCallback(system)
	if err != nil {
		logger.Error("failed to create PublicKeyCallback", err)
		return
	}

	// Setup server config
	config := sshh.Config{
		Deadline:          c.SSHConnectionDeadline,
		Logger:            sshLogger,
		Bind:              c.SSHBindAddress,
		PrivateKey:        privateKey,
		PublicKeyCallback: pubKeyCallback,
		AuthLogCallback: func(meta ssh.ConnMetadata, method string, err error) {
			if err == nil {
				sshLogger.Info("login success", "user", meta.User())
			} else if err != nil && method == "publickey" {
				sshLogger.Info("login failure", "user", meta.User(), "err", err.Error())
			}
		},
		Handlers: map[string]sshh.SSHHandler{
			"kappa-client": &EchoHandler{},
		},
	}

	// Create SSH server
	sshServer, err := sshh.NewSSHServer(&config)
	if err != nil {
		logger.Error("SSH Server could not be configured", "error", err.Error())
		return
	}

	// Create database server
	s := &Server{
		config:       c,
		logger:       logger,
		sshServer:    &sshServer,
		localKappas:  make(map[string]*NodeDetails),
		serfEventCh:  make(chan serf.Event, 256),
		kappaEventCh: make(chan serf.UserEvent, 256),
		reconcileCh:  make(chan serf.Member, 32),
	}

	// Create serf server
	s.serf, err = s.setupSerf()
	if err != nil {
		err = logger.Error("Failed to start serf: %v", err)
		return
	}

	return s, nil
}

type Server struct {
	config    *DatabaseConfig
	logger    log.Logger
	sshServer *sshh.SSHServer

	// localKappas is used to track the known kappas
	// in the cluster. Used to do leader forwarding.
	localKappas map[string]*NodeDetails
	localLock   sync.RWMutex

	serf         *serf.Serf
	serfEventCh  chan serf.Event
	kappaEventCh chan serf.UserEvent
	reconcileCh  chan serf.Member
	t            tomb.Tomb
}

func (s *Server) Start() error {
	s.sshServer.Start()

	// Start serf handler
	s.t.Go(s.serfEventHandler)

	// Join serf cluster
	// if !s.config.Bootstrap && len(s.config.ExistingNodes) > 0 {
	// addr, err := s.serf.Memberlist().LocalNode().Addr.MarshalText()
	s.logger.Info("local node", "port", s.serf.LocalMember().Port)
	s.logger.Info("Joining cluster", "nodes", s.config.ExistingNodes)

	n, err := s.serf.Join(s.config.ExistingNodes, true)
	if err != nil && !s.config.Bootstrap {
		err = s.logger.Error("Failed to join cluster", "err", err)
		return err
	}
	s.logger.Info("Joined cluster", "nodes", n)
	// }
	return nil
}

func (s *Server) Stop() {

	// Stop SSH server
	s.sshServer.Stop()

	// Shutdown serf
	s.serf.Leave()
	s.serf.Shutdown()
	<-s.serf.ShutdownCh()

	// Kill Serf handler
	s.t.Kill(nil)
	s.logger.Info("Shutting down Serf server...")
	s.t.Wait()
}

func (s *Server) setupSerf() (*serf.Serf, error) {
	conf := serf.DefaultConfig()

	// Generate NodeName if missing
	id, err := uuid.UUID4()
	if err != nil {
		return nil, err
	}

	// Get SSH server port
	port := s.sshServer.Addr.Port

	// Initialize serf
	conf.Init()
	// s.logger.Info("local port", "port", conf.MemberlistConfig.BindPort)

	conf.NodeName = s.config.NodeName
	conf.MemberlistConfig.BindAddr = s.config.GossipBindAddr
	conf.MemberlistConfig.BindPort = s.config.GossipBindPort
	conf.MemberlistConfig.AdvertiseAddr = s.config.GossipAdvertiseAddr
	conf.MemberlistConfig.AdvertisePort = s.config.GossipAdvertisePort
	s.logger.Info("Gossip", "BindAddr", conf.MemberlistConfig.BindAddr, "BindPort", conf.MemberlistConfig.BindPort, "AdvertiseAddr", conf.MemberlistConfig.AdvertiseAddr, "AdvertisePort", conf.MemberlistConfig.AdvertisePort)

	conf.Tags["id"] = id
	conf.Tags["role"] = "kappa"
	conf.Tags["cluster"] = s.config.ClusterName
	conf.Tags["build"] = s.config.Build
	conf.Tags["port"] = fmt.Sprintf("%d", port)
	if s.config.Bootstrap {
		conf.Tags["bootstrap"] = "1"
	}
	if s.config.BootstrapExpect != 0 {
		conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect)
	}

	conf.MemberlistConfig.LogOutput = s.config.LogOutput
	conf.LogOutput = s.config.LogOutput
	conf.EventCh = s.serfEventCh
	conf.SnapshotPath = filepath.Join(s.config.DataPath, serfSnapshot)
	conf.ProtocolVersion = conf.ProtocolVersion
	conf.RejoinAfterLeave = true
	conf.EnableNameConflictResolution = false

	conf.Merge = &mergeDelegate{name: s.config.ClusterName}
	if err := ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// Join is used to have Kappa join the cluster.
// The target address should be another node inside the cluster
// listening on the Serf address
func (s *Server) Join(addrs []string) (int, error) {
	return s.serf.Join(addrs, true)
}

// LocalMember is used to return the local node
func (c *Server) LocalMember() serf.Member {
	return c.serf.LocalMember()
}

// Members is used to return the members of the cluster
func (s *Server) Members() []serf.Member {
	return s.serf.Members()
}

// RemoveFailedNode is used to remove a failed node from the cluster
func (s *Server) RemoveFailedNode(node string) error {
	if err := s.serf.RemoveFailedNode(node); err != nil {
		return err
	}
	return nil
}

// IsLeader checks if this server is the cluster leader
func (s *Server) IsLeader() bool {
	return true
	// return s.raft.State() == raft.Leader
}

// KeyManager returns the Serf keyring manager
func (s *Server) KeyManager() *serf.KeyManager {
	return s.serf.KeyManager()
}

// Encrypted determines if gossip is encrypted
func (s *Server) Encrypted() bool {
	return s.serf.EncryptionEnabled()
}
<commit_msg>Add serfer to database server to handle Serf events<commit_after>package server

import (
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"

	"github.com/blacklabeldata/kappa/auth"
	"github.com/blacklabeldata/kappa/datamodel"
	"github.com/blacklabeldata/kappa/pkg/uuid"
	"github.com/blacklabeldata/serfer"
	"github.com/blacklabeldata/sshh"
	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"golang.org/x/crypto/ssh"
	tomb "gopkg.in/tomb.v2"
)

const serfSnapshot = "serf/local.snapshot"

func NewServer(c *DatabaseConfig) (server *Server, err error) {

	// Create logger
	if c.LogOutput == nil {
		c.LogOutput = log.NewConcurrentWriter(os.Stdout)
	}
	logger := log.NewLogger(c.LogOutput, "kappa")

	// Create data directory
	if err = os.MkdirAll(c.DataPath, 0755); err != nil {
		logger.Warn("Could not create data directory", "err", err)
		// logger.Warn("Could not create data directory", "err", err.Error())
		return
	}

	// Connect to database
	cwd, err := os.Getwd()
	if err != nil {
		logger.Error("Could not get working directory", "error", err.Error())
		return
	}

	file := path.Join(cwd, c.DataPath, "meta.db")
	logger.Info("Connecting to database", "file", file)
	system, err := datamodel.NewSystem(file)
	if err != nil {
		logger.Error("Could not connect to database", "error", err.Error())
		return
	}

	// Get SSH Key file
	sshKeyFile := c.SSHPrivateKeyFile
	logger.Info("Reading private key", "file", sshKeyFile)

	privateKey, err := auth.ReadPrivateKey(logger, sshKeyFile)
	if err != nil {
		return
	}

	// Get admin certificate
	adminCertFile := c.AdminCertificateFile
	logger.Info("Reading admin public key", "file", adminCertFile)

	// Read admin certificate
	cert, err := ioutil.ReadFile(adminCertFile)
	if err != nil {
		logger.Error("admin certificate could not be read", "filename", c.AdminCertificateFile)
		return
	}

	// Add admin cert to key ring
	userStore, err := system.Users()
	if err != nil {
		logger.Error("could not get user store", "error", err.Error())
		return
	}

	// Create admin account
	admin, err := userStore.Create("admin")
	if err != nil {
		logger.Error("error creating admin account", "error", err.Error())
		return
	}

	// Add admin certificate
	keyRing := admin.KeyRing()
	fingerprint, err := keyRing.AddPublicKey(cert)
	if err != nil {
		logger.Error("admin certificate could not be added", "error", err.Error())
		return
	}
	logger.Info("Added admin certificate", "fingerprint", fingerprint)

	// Read root cert
	rootPem, err := ioutil.ReadFile(c.CACertificateFile)
	if err != nil {
		logger.Error("root certificate could not be read", "filename", c.CACertificateFile)
		return
	}

	// Create certificate pool
	roots := x509.NewCertPool()
	if ok := roots.AppendCertsFromPEM(rootPem); !ok {
		logger.Error("failed to parse root certificate")
		return
	}

	// Setup SSH Server
	sshLogger := log.NewLogger(c.LogOutput, "ssh")
	pubKeyCallback, err := PublicKeyCallback(system)
	if err != nil {
		logger.Error("failed to create PublicKeyCallback", err)
		return
	}

	// Setup server config
	config := sshh.Config{
		Deadline:          c.SSHConnectionDeadline,
		Logger:            sshLogger,
		Bind:              c.SSHBindAddress,
		PrivateKey:        privateKey,
		PublicKeyCallback: pubKeyCallback,
		AuthLogCallback: func(meta ssh.ConnMetadata, method string, err error) {
			if err == nil {
				sshLogger.Info("login success", "user", meta.User())
			} else if err != nil && method == "publickey" {
				sshLogger.Info("login failure", "user", meta.User(), "err", err.Error())
			}
		},
		Handlers: map[string]sshh.SSHHandler{
			"kappa-client": &EchoHandler{},
		},
	}

	// Create SSH server
	sshServer, err := sshh.NewSSHServer(&config)
	if err != nil {
		logger.Error("SSH Server could not be configured", "error", err.Error())
		return
	}

	// Setup Serf handlers
	mgr := NewNodeManager()
	serfEventCh := make(chan serf.Event, 256)
	reconcilerCh := make(chan serf.Member, 32)
	userEventCh := make(chan serf.UserEvent, 256)
	serfer := serfer.NewSerfer(serfEventCh, serfer.SerfEventHandler{
		Logger:      log.NewLogger(c.LogOutput, "serf"),
		NodeJoined:  &SerfNodeJoinHandler{mgr, log.NewLogger(c.LogOutput, "serf:node-join")},
		NodeUpdated: &SerfNodeUpdateHandler{mgr, log.NewLogger(c.LogOutput, "serf:node-update")},
		NodeLeft:    &SerfNodeLeaveHandler{mgr, log.NewLogger(c.LogOutput, "serf:node-left")},
		NodeFailed:  &SerfNodeLeaveHandler{mgr, log.NewLogger(c.LogOutput, "serf:node-fail")},
		NodeReaped:  &SerfNodeLeaveHandler{mgr, log.NewLogger(c.LogOutput, "serf:node-reap")},
		UserEvent:   &SerfUserEventHandler{log.NewLogger(c.LogOutput, "serf:user-events"), userEventCh},
		Reconciler: &SerfReconciler{func() bool {

			// TODO: Replace with Raft IsLeader check
			return true
		}, reconcilerCh},
	})

	// Create database server
	s := &Server{
		config:       c,
		logger:       logger,
		sshServer:    &sshServer,
		serfer:       serfer,
		localKappas:  make(map[string]*NodeDetails),
		serfEventCh:  serfEventCh,
		kappaEventCh: userEventCh,
		reconcileCh:  reconcilerCh,
	}

	// Create serf server
	s.serf, err = s.setupSerf()
	if err != nil {
		err = logger.Error("Failed to start serf: %v", err)
		return
	}

	return s, nil
}

type Server struct {
	config    *DatabaseConfig
	logger    log.Logger
	sshServer *sshh.SSHServer

	serfer serfer.Serfer

	// localKappas is used to track the known kappas
	// in the cluster. Used to do leader forwarding.
	localKappas map[string]*NodeDetails
	localLock   sync.RWMutex

	serf         *serf.Serf
	serfEventCh  chan serf.Event
	kappaEventCh chan serf.UserEvent
	reconcileCh  chan serf.Member
	t            tomb.Tomb
}

func (s *Server) Start() error {
	s.sshServer.Start()

	// Start serf handler
	s.serfer.Start()

	// Join serf cluster
	s.logger.Info("Joining cluster", "nodes", s.config.ExistingNodes)

	n, err := s.serf.Join(s.config.ExistingNodes, true)
	if err != nil && !s.config.Bootstrap {
		err = s.logger.Error("Failed to join cluster", "err", err)
		return err
	}
	s.logger.Info("Joined cluster", "nodes", n)
	// }
	return nil
}

func (s *Server) Stop() {

	// Stop SSH server
	s.sshServer.Stop()

	// Shutdown serf
	s.logger.Info("Shutting down Serf server...")
	s.serf.Leave()
	s.serf.Shutdown()
	<-s.serf.ShutdownCh()

	// Stop serf event handlers
	if err := s.serfer.Stop(); err != nil {
		s.logger.Warn("error: stopping Serfer handlers", err.Error())
	}

	// Kill Serf handler
	// s.t.Kill(nil)
	// s.t.Wait()
}

func (s *Server) setupSerf() (*serf.Serf, error) {
	conf := serf.DefaultConfig()

	// Generate NodeName if missing
	id, err := uuid.UUID4()
	if err != nil {
		return nil, err
	}

	// Get SSH server port
	port := s.sshServer.Addr.Port

	// Initialize serf
	conf.Init()
	// s.logger.Info("local port", "port", conf.MemberlistConfig.BindPort)

	conf.NodeName = s.config.NodeName
	conf.MemberlistConfig.BindAddr = s.config.GossipBindAddr
	conf.MemberlistConfig.BindPort = s.config.GossipBindPort
	conf.MemberlistConfig.AdvertiseAddr = s.config.GossipAdvertiseAddr
	conf.MemberlistConfig.AdvertisePort = s.config.GossipAdvertisePort
	s.logger.Info("Gossip", "BindAddr", conf.MemberlistConfig.BindAddr, "BindPort", conf.MemberlistConfig.BindPort, "AdvertiseAddr", conf.MemberlistConfig.AdvertiseAddr, "AdvertisePort", conf.MemberlistConfig.AdvertisePort)

	conf.Tags["id"] = id
	conf.Tags["role"] = "kappa-server"
	conf.Tags["cluster"] = s.config.ClusterName
	conf.Tags["build"] = s.config.Build
	conf.Tags["port"] = fmt.Sprintf("%d", port)
	if s.config.Bootstrap {
		conf.Tags["bootstrap"] = "1"
	}
	if s.config.BootstrapExpect != 0 {
		conf.Tags["expect"] = fmt.Sprintf("%d", s.config.BootstrapExpect)
	}

	conf.MemberlistConfig.LogOutput = s.config.LogOutput
	conf.LogOutput = s.config.LogOutput
	conf.EventCh = s.serfEventCh
	conf.SnapshotPath = filepath.Join(s.config.DataPath, serfSnapshot)
	conf.ProtocolVersion = conf.ProtocolVersion
	conf.RejoinAfterLeave = true
	conf.EnableNameConflictResolution = false

	conf.Merge = &mergeDelegate{name: s.config.ClusterName}
	if err := ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// Join is used to have Kappa join the cluster.
// The target address should be another node inside the cluster
// listening on the Serf address
func (s *Server) Join(addrs []string) (int, error) {
	return s.serf.Join(addrs, true)
}

// LocalMember is used to return the local node
func (c *Server) LocalMember() serf.Member {
	return c.serf.LocalMember()
}

// Members is used to return the members of the cluster
func (s *Server) Members() []serf.Member {
	return s.serf.Members()
}

// RemoveFailedNode is used to remove a failed node from the cluster
func (s *Server) RemoveFailedNode(node string) error {
	if err := s.serf.RemoveFailedNode(node); err != nil {
		return err
	}
	return nil
}

// IsLeader checks if this server is the cluster leader
func (s *Server) IsLeader() bool {
	return true
	// return s.raft.State() == raft.Leader
}

// KeyManager returns the Serf keyring manager
func (s *Server) KeyManager() *serf.KeyManager {
	return s.serf.KeyManager()
}

// Encrypted determines if gossip is encrypted
func (s *Server) Encrypted() bool {
	return s.serf.EncryptionEnabled()
}
<|endoftext|>


<commit_before>package main

import "fmt"

const sang = "\x48\x65\x6e\x72\x69\x6b\x20\x41\x72\x6e\x6f\x6c\x64" +
"\x20\x57\x65\x72\x67\x65\x6c\x61\x6e\x64\x20\x28\x66\xf8\x64\x74\x20\x31\x37" +
"\x2e\x20\x6a\x75\x6e\x69\x20\x31\x38\x30\x38\x2c\x20\x64\xf8\x64\x20\x31\x32" +
"\x2e\x20\x6a\x75\x6c\x69\x20\x31\x38\x34\x35\x29\x0a\x56\x69\x20\x65\x72\x65\x20" +
"\x65\x6e\x20\x6e\x61\x73\x6a\x6f\x6e\x20\x76\x69\x20\x6d\x65\x64\x2c\x0a\x20\x76" +
"\x69\x20\x73\x6d\xe5\x20\x65\x6e\x20\x61\x6c\x65\x6e\x20\x6c\x61\x6e\x67\x65\x2c" +
"\x0a\x20\x65\x74\x20\x66\x65\x64\x72\x65\x6c\x61\x6e\x64\x20\x76\x69\x20\x66\x72" +
"\x79\x64\x65\x73\x20\x76\x65\x64\x2c\x0a\x20\x6f\x67\x20\x76\x69\x2c\x20\x76\x69" +
"\x20\x65\x72\x65\x20\x6d\x61\x6e\x67\x65\x2e\x0a\x20\x56\xe5\x72\x74\x20\x68\x6a" +
"\x65\x72\x74\x65\x20\x76\x65\x74\x2c\x20\x76\xe5\x72\x74\x20\xf8\x79\x65\x20\x73" +
"\x65\x72\x0a\x20\x68\x76\x6f\x72\x20\x67\x6f\x64\x74\x20\x6f\x67\x20\x76\x61\x6b" +
"\x6b\x65\x72\x74\x20\x4e\x6f\x72\x67\x65\x20\x65\x72\x2c\x0a\x20\x76\xe5\x72\x20" +
"\x74\x75\x6e\x67\x65\x20\x6b\x61\x6e\x20\x65\x6e\x20\x73\x61\x6e\x67\x20\x62\x6c" +
"\x61\x6e\x74\x20\x66\x6c\x65\x72\x0a\x20\x61\x76\x20\x4e\x6f\x72\x67\x65\x73\x20" +
"\xe6\x72\x65\x73\x2d\x73\x61\x6e\x67\x65\x2e\x0a\x0a\x20\x4d\x65\x72\x20\x67\x72" +
"\xf8\x6e\x74\x20\x65\x72\x20\x67\x72\x65\x73\x73\x65\x74\x20\x69\x6e\x67\x65\x6e" +
"\x73\x74\x65\x64\x73\x2c\x0a\x20\x6d\x65\x72\x20\x66\x75\x6c\x6c\x74\x20\x61\x76" +
"\x20\x62\x6c\x6f\x6d\x73\x74\x65\x72\x20\x76\x65\x76\x65\x74\x0a\x20\x65\x6e\x6e" +
"\x20\x69\x20\x64\x65\x74\x20\x6c\x61\x6e\x64\x20\x68\x76\x6f\x72\x20\x6a\x65\x67" +
"\x20\x74\x69\x6c\x66\x72\x65\x64\x73\x0a\x20\x6d\x65\x64\x20\x66\x61\x72\x20\x6f" +
"\x67\x20\x6d\x6f\x72\x20\x68\x61\x72\x20\x6c\x65\x76\x65\x74\x2e\x0a\x20\x4a\x65" +
"\x67\x20\x76\x69\x6c\x20\x64\x65\x74\x20\x65\x6c\x73\x6b\x65\x20\x74\x69\x6c\x20" +
"\x6d\x69\x6e\x20\x64\xf8\x64\x2c\x0a\x20\x65\x69\x20\x62\x79\x74\x74\x65\x20\x64" +
"\x65\x74\x20\x68\x76\x6f\x72\x20\x6a\x65\x67\x20\x65\x72\x20\x66\xf8\x64\x64\x2c" +
"\x0a\x20\x6f\x6d\x20\x6d\x61\x6e\x20\x65\x74\x20\x70\x61\x72\x61\x64\x69\x73\x20" +
"\x6d\x65\x67\x20\x62\xf8\x64\x0a\x20\x61\x76\x20\x70\x61\x6c\x6d\x65\x72\x20\x6f" +
"\x76\x65\x72\x73\x76\x65\x76\x65\x74\x2e\x0a"

func main(){
	fmt.Printf("%s", sang)
}
<commit_msg>endret så sangen viser æøå<commit_after>package main

import ("fmt"
	"bytes"
)

const sang = "\x48\x65\x6e\x72\x69\x6b\x20\x41\x72\x6e\x6f\x6c\x64" +
"\x20\x57\x65\x72\x67\x65\x6c\x61\x6e\x64\x20\x28\x66\xf8\x64\x74\x20\x31\x37" +
"\x2e\x20\x6a\x75\x6e\x69\x20\x31\x38\x30\x38\x2c\x20\x64\xf8\x64\x20\x31\x32" +
"\x2e\x20\x6a\x75\x6c\x69\x20\x31\x38\x34\x35\x29\x0a\x56\x69\x20\x65\x72\x65\x20" +
"\x65\x6e\x20\x6e\x61\x73\x6a\x6f\x6e\x20\x76\x69\x20\x6d\x65\x64\x2c\x0a\x20\x76" +
"\x69\x20\x73\x6d\xe5\x20\x65\x6e\x20\x61\x6c\x65\x6e\x20\x6c\x61\x6e\x67\x65\x2c" +
"\x0a\x20\x65\x74\x20\x66\x65\x64\x72\x65\x6c\x61\x6e\x64\x20\x76\x69\x20\x66\x72" +
"\x79\x64\x65\x73\x20\x76\x65\x64\x2c\x0a\x20\x6f\x67\x20\x76\x69\x2c\x20\x76\x69" +
"\x20\x65\x72\x65\x20\x6d\x61\x6e\x67\x65\x2e\x0a\x20\x56\xe5\x72\x74\x20\x68\x6a" +
"\x65\x72\x74\x65\x20\x76\x65\x74\x2c\x20\x76\xe5\x72\x74\x20\xf8\x79\x65\x20\x73" +
"\x65\x72\x0a\x20\x68\x76\x6f\x72\x20\x67\x6f\x64\x74\x20\x6f\x67\x20\x76\x61\x6b" +
"\x6b\x65\x72\x74\x20\x4e\x6f\x72\x67\x65\x20\x65\x72\x2c\x0a\x20\x76\xe5\x72\x20" +
"\x74\x75\x6e\x67\x65\x20\x6b\x61\x6e\x20\x65\x6e\x20\x73\x61\x6e\x67\x20\x62\x6c" +
"\x61\x6e\x74\x20\x66\x6c\x65\x72\x0a\x20\x61\x76\x20\x4e\x6f\x72\x67\x65\x73\x20" +
"\xe6\x72\x65\x73\x2d\x73\x61\x6e\x67\x65\x2e\x0a\x0a\x20\x4d\x65\x72\x20\x67\x72" +
"\xf8\x6e\x74\x20\x65\x72\x20\x67\x72\x65\x73\x73\x65\x74\x20\x69\x6e\x67\x65\x6e" +
"\x73\x74\x65\x64\x73\x2c\x0a\x20\x6d\x65\x72\x20\x66\x75\x6c\x6c\x74\x20\x61\x76" +
"\x20\x62\x6c\x6f\x6d\x73\x74\x65\x72\x20\x76\x65\x76\x65\x74\x0a\x20\x65\x6e\x6e" +
"\x20\x69\x20\x64\x65\x74\x20\x6c\x61\x6e\x64\x20\x68\x76\x6f\x72\x20\x6a\x65\x67" +
"\x20\x74\x69\x6c\x66\x72\x65\x64\x73\x0a\x20\x6d\x65\x64\x20\x66\x61\x72\x20\x6f" +
"\x67\x20\x6d\x6f\x72\x20\x68\x61\x72\x20\x6c\x65\x76\x65\x74\x2e\x0a\x20\x4a\x65" +
"\x67\x20\x76\x69\x6c\x20\x64\x65\x74\x20\x65\x6c\x73\x6b\x65\x20\x74\x69\x6c\x20" +
"\x6d\x69\x6e\x20\x64\xf8\x64\x2c\x0a\x20\x65\x69\x20\x62\x79\x74\x74\x65\x20\x64" +
"\x65\x74\x20\x68\x76\x6f\x72\x20\x6a\x65\x67\x20\x65\x72\x20\x66\xf8\x64\x64\x2c" +
"\x0a\x20\x6f\x6d\x20\x6d\x61\x6e\x20\x65\x74\x20\x70\x61\x72\x61\x64\x69\x73\x20" +
"\x6d\x65\x67\x20\x62\xf8\x64\x0a\x20\x61\x76\x20\x70\x61\x6c\x6d\x65\x72\x20\x6f" +
"\x76\x65\x72\x73\x76\x65\x76\x65\x74\x2e\x0a"

func main(){

	i := bytes.Replace([]byte(sang), []byte("\xF8"), []byte("ø"), 10)
	f := bytes.Replace([]byte(i), []byte("\xE5"), []byte("å"), 10)
	u := bytes.Replace([]byte(f), []byte("\xE6"), []byte("æ"), 10)

	fmt.Printf("%s", u)
}
<|endoftext|>


<commit_before>// Ported from fabioxgn/go-bot
package ported

import (
	"fmt"
	lazlo "github.com/djosephsen/lazlo/lib"
	"github.com/fabioxgn/go-bot/web"
	"net/url"
)

const (
	godocSiteURL    = "http://godoc.org"
	noPackagesFound = "No packages found."
)

type godocResults struct {
	Results []struct {
		Path     string `json:"path"`
		Synopsis string `json:"synopsis"`
	} `json:"results"`
}

var godocSearchURL = "http://api.godoc.org/search"

var GoDoc = &lazlo.Module{
	Name:  `GoDoc`,
	Usage: `godoc <package name>: Searchs godoc.org and displays the first result.`,
	Run:   goDocRun,
}

func goDocRun(b *lazlo.Broker) {
	cb := b.MessageCallback(`(?i)^godoc ([^\s]+)$`, false)
	for {
		pm := <-cb.Chan
		data := &godocResults{}

		url, _ := url.Parse(godocSearchURL)
		q := url.Query()
		q.Set("q", pm.Match[1])
		url.RawQuery = q.Encode()
		err := web.GetJSON(url.String(), data)
		if err != nil {
			pm.Event.Respond(fmt.Sprintf("Error: ", err.Error()))
		}

		if len(data.Results) == 0 {
			pm.Event.Respond(noPackagesFound)
		}
		a := []lazlo.Attachment{
			lazlo.Attachment{
				Color:      "#ff0000",
				Title:      fmt.Sprintf(`Results of "%v"`, pm.Match[1]),
				MarkdownIn: []string{"fields"},
				Fields:     []lazlo.AttachmentField{},
			},
		}

		for i := 0; i < len(data.Results); i++ {
			if i == 10 {
				break
			}
			if data.Results[i].Synopsis == "" {
				data.Results[i].Synopsis = "_No description_"
			}
			a[0].Fields = append(a[0].Fields, lazlo.AttachmentField{
				Title: fmt.Sprintf("%s", data.Results[i].Path),
				Value: fmt.Sprintf("%s\n<%s/%s|full GoDoc>", data.Results[i].Synopsis, godocSiteURL, data.Results[i].Path),
			})
		}

		pm.Event.RespondAttachments(a)
	}
}
<commit_msg>Update to godoc help output<commit_after>// Ported from fabioxgn/go-bot
package ported

import (
	"fmt"
	lazlo "github.com/djosephsen/lazlo/lib"
	"github.com/fabioxgn/go-bot/web"
	"net/url"
)

const (
	godocSiteURL    = "http://godoc.org"
	noPackagesFound = "No packages found."
)

type godocResults struct {
	Results []struct {
		Path     string `json:"path"`
		Synopsis string `json:"synopsis"`
	} `json:"results"`
}

var godocSearchURL = "http://api.godoc.org/search"

var GoDoc = &lazlo.Module{
	Name:  `GoDoc`,
	Usage: `godoc <package name>: Searchs godoc.org and displays the first 10 results.`,
	Run:   goDocRun,
}

func goDocRun(b *lazlo.Broker) {
	cb := b.MessageCallback(`(?i)^godoc ([^\s]+)$`, false)
	for {
		pm := <-cb.Chan
		data := &godocResults{}

		url, _ := url.Parse(godocSearchURL)
		q := url.Query()
		q.Set("q", pm.Match[1])
		url.RawQuery = q.Encode()
		err := web.GetJSON(url.String(), data)
		if err != nil {
			pm.Event.Respond(fmt.Sprintf("Error: ", err.Error()))
		}

		if len(data.Results) == 0 {
			pm.Event.Respond(noPackagesFound)
		}
		a := []lazlo.Attachment{
			lazlo.Attachment{
				Color:      "#ff0000",
				Title:      fmt.Sprintf(`Results of "%v"`, pm.Match[1]),
				MarkdownIn: []string{"fields"},
				Fields:     []lazlo.AttachmentField{},
			},
		}

		for i := 0; i < len(data.Results); i++ {
			if i == 10 {
				break
			}
			if data.Results[i].Synopsis == "" {
				data.Results[i].Synopsis = "_No description_"
			}
			a[0].Fields = append(a[0].Fields, lazlo.AttachmentField{
				Title: fmt.Sprintf("%s", data.Results[i].Path),
				Value: fmt.Sprintf("%s\n<%s/%s|full GoDoc>", data.Results[i].Synopsis, godocSiteURL, data.Results[i].Path),
			})
		}

		pm.Event.RespondAttachments(a)
	}
}
<|endoftext|>


<commit_before>package dns

import (
	"errors"
	"io"
)

const (
	MaxNameLen  = 255
	MaxLabelLen = 63

	headerLen = 12
)

var (
	ErrLabelEmpty          = errors.New("label cannot be empty")
	ErrLabelTooLong        = errors.New("label must be 63 octets or less")
	ErrLabelInvalid        = errors.New("label format is invalid")
	ErrLabelPointerIllegal = errors.New("label pointer is illegal")
	ErrNameTooLong         = errors.New("name must be 255 octets or less")
)

func UnpackMsg(b []byte, offset int) (msg *Msg, n int, err error) {
	initialOffset := offset

	h, n, err := unpackHeader(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	msg = &Msg{Header: *h}

	msg.Queries = make([]Query, msg.Header.QDCount)
	for i := range msg.Queries {
		q, n, err := unpackQuery(b, offset)
		if err != nil {
			return nil, 0, err
		}
		offset += n
		msg.Queries[i] = *q
	}

	msg.Responses = make([]RR, msg.Header.ANCount)
	for i := range msg.Responses {
		rr, n, err := unpackRR(b, offset)
		if err != nil {
			return nil, 0, err
		}
		offset += n
		msg.Responses[i] = *rr
	}
	return msg, offset - initialOffset, nil
}

func unpackHeader(b []byte, offset int) (h *Header, n int, err error) {
	iniOffset := offset
	h = &Header{}
	h.ID, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	var flags uint16
	flags, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.QR = (flags & maskQR >> maskQROffset) == 1
	h.OpCode = OpCode(flags & maskOpCode >> maskOpCodeOffset)
	h.AA = (flags & maskAA >> maskAAOffset) == 1
	h.TC = (flags & maskTC >> maskTCOffset) == 1
	h.RD = (flags & maskRD >> maskRDOffset) == 1
	h.RA = (flags & maskRA >> maskRAOffset) == 1
	h.Z = byte(flags & maskZ >> maskZOffset)
	h.RCode = byte(flags & maskRCode)

	h.QDCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.ANCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.NSCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.ARCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	return h, offset - iniOffset, nil

}

func unpackQuery(b []byte, offset int) (q *Query, n int, err error) {
	initialOffset := offset

	qName, n, err := unpackName(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	qtype, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	qclass, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	q = &Query{QName: qName, QType: Type(qtype), QClass: Class(qclass)}
	return q, offset - initialOffset, nil

}

func unpackRR(b []byte, offset int) (r *RR, n int, err error) {
	initialOffset := offset

	name, n, err := unpackName(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	rrtype, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	class, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	ttl, n, err := unpackUint32(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	rdlength, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	if len(b[offset:]) < int(rdlength) {
		return nil, 0, io.ErrShortBuffer
	}
	rdata := b[offset : offset+int(rdlength)]
	offset += int(rdlength)

	return &RR{
		Name:     name,
		Class:    Class(class),
		Type:     Type(rrtype),
		TTL:      ttl,
		RDLength: rdlength,
		RData:    rdata,
	}, offset - initialOffset, nil
}

func unpackName(b []byte, offset int) (name string, n int, err error) {
	var ln int
	var label string
	for {

		if !checkBounds(b, offset) {
			return "", 0, io.ErrShortBuffer
		}
		// Current byte indicates the length of the label.
		// If its a null byte, name is over.
		currentByte := b[offset]
		if currentByte == 0 {
			n++
			return name, n, nil
		}

		// Unpack a label and advance offset and read bytes.
		label, ln, err = unpackLabel(b, offset)
		if err != nil {
			return "", 0, err
		}

		// If successful, append label to the name.
		name += label + "."
		offset += ln
		n += ln
		if len(name) > MaxNameLen {
			return "", 0, ErrNameTooLong
		}
	}
}

func unpackUint16(b []byte, offset int) (r uint16, n int, err error) {
	end := offset + 1
	if !checkBounds(b, end) {
		return 0, 0, io.ErrShortBuffer
	}
	return uint16(b[offset])<<8 | uint16(b[end]), 2, nil
}

func unpackUint32(b []byte, offset int) (r uint32, n int, err error) {
	end := offset + 3
	if !checkBounds(b, end) {
		return 0, 0, io.ErrShortBuffer
	}
	return uint32(b[offset])<<24 | uint32(b[offset+1])<<16 | uint32(b[offset+2])<<8 | uint32(b[end]), 4, nil
}

// Check if begin and end are within bounds of a byte slice.
func checkBounds(b []byte, end int) bool {
	return end >= 0 && end < len(b)
}
<commit_msg>(wip) polishing on the unpacker<commit_after>package dns

import (
	"errors"
	"io"
)

const (
	MaxNameLen  = 255
	MaxLabelLen = 63

	headerLen = 12
)

var (
	ErrLabelEmpty          = errors.New("label cannot be empty")
	ErrLabelTooLong        = errors.New("label must be 63 octets or less")
	ErrLabelInvalid        = errors.New("label format is invalid")
	ErrLabelPointerIllegal = errors.New("label pointer is illegal")
	ErrNameTooLong         = errors.New("name must be 255 octets or less")
)

func UnpackMsg(b []byte, offset int) (msg *Msg, n int, err error) {
	initialOffset := offset

	h, n, err := unpackHeader(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	msg = &Msg{Header: *h}

	msg.Queries = make([]Query, msg.Header.QDCount)
	for i := range msg.Queries {
		q, n, err := unpackQuery(b, offset)
		if err != nil {
			return nil, 0, err
		}
		offset += n
		msg.Queries[i] = *q
	}

	msg.Responses = make([]RR, msg.Header.ANCount)
	for i := range msg.Responses {
		rr, n, err := unpackRR(b, offset)
		if err != nil {
			return nil, 0, err
		}
		offset += n
		msg.Responses[i] = *rr
	}
	return msg, offset - initialOffset, nil
}

func unpackHeader(b []byte, offset int) (h *Header, n int, err error) {
	iniOffset := offset
	h = &Header{}
	h.ID, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	var flags uint16
	flags, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.QR = (flags & maskQR >> maskQROffset) == 1
	h.OpCode = OpCode(flags & maskOpCode >> maskOpCodeOffset)
	h.AA = (flags & maskAA >> maskAAOffset) == 1
	h.TC = (flags & maskTC >> maskTCOffset) == 1
	h.RD = (flags & maskRD >> maskRDOffset) == 1
	h.RA = (flags & maskRA >> maskRAOffset) == 1
	h.Z = byte(flags & maskZ >> maskZOffset)
	h.RCode = byte(flags & maskRCode)

	h.QDCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.ANCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.NSCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	h.ARCount, n, err = unpackUint16(b, offset)
	offset += n
	if err != nil {
		return nil, 0, err
	}

	return h, offset - iniOffset, nil

}

func unpackQuery(b []byte, offset int) (q *Query, n int, err error) {
	initialOffset := offset

	qName, n, err := unpackName(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	qtype, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	qclass, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	q = &Query{QName: qName, QType: Type(qtype), QClass: Class(qclass)}
	return q, offset - initialOffset, nil

}

func unpackRR(b []byte, offset int) (r *RR, n int, err error) {
	initialOffset := offset

	name, n, err := unpackName(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	rrtype, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	class, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	ttl, n, err := unpackUint32(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	rdlength, n, err := unpackUint16(b, offset)
	if err != nil {
		return nil, 0, err
	}
	offset += n

	if len(b[offset:]) < int(rdlength) {
		return nil, 0, io.ErrShortBuffer
	}
	rdata := b[offset : offset+int(rdlength)]
	offset += int(rdlength)

	return &RR{
		Name:     name,
		Class:    Class(class),
		Type:     Type(rrtype),
		TTL:      ttl,
		RDLength: rdlength,
		RData:    rdata,
	}, offset - initialOffset, nil
}

func unpackName(b []byte, offset int) (name string, n int, err error) {
	var ln int
	var label string
	for {

		if !checkBounds(b, offset) {
			return "", 0, io.ErrShortBuffer
		} else if b[offset] == 0 {
			return name, n + 1, nil
		}

		label, ln, err = unpackLabel(b, offset)
		if err != nil {
			return "", 0, err
		}

		name += label + "."
		offset += ln
		n += ln
		if len(name) > MaxNameLen {
			return "", 0, ErrNameTooLong
		}
	}
}

func unpackUint16(b []byte, offset int) (r uint16, n int, err error) {
	end := offset + 1
	if !checkBounds(b, end) {
		return 0, 0, io.ErrShortBuffer
	}
	return uint16(b[offset])<<8 | uint16(b[end]), 2, nil
}

func unpackUint32(b []byte, offset int) (r uint32, n int, err error) {
	end := offset + 3
	if !checkBounds(b, end) {
		return 0, 0, io.ErrShortBuffer
	}
	return uint32(b[offset])<<24 | uint32(b[offset+1])<<16 | uint32(b[offset+2])<<8 | uint32(b[end]), 4, nil
}

// Check if begin and end are within bounds of a byte slice.
func checkBounds(b []byte, end int) bool {
	return end >= 0 && end < len(b)
}
<|endoftext|>


<commit_before>package tests

import (
	"context"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/bitfinexcom/bitfinex-api-go/v2/websocket"
)

// method of testing with mocked endpoints
func TestTicker(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()
	nonce := &IncrementingNonceGenerator{}

	// create client
	ws := websocket.NewWithAsyncFactoryNonce(newTestAsyncFactory(async), nonce)

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	// subscribe
	id, err := ws.SubscribeTicker(context.Background(), "tBTCUSD")
	if err != nil {
		t.Fatal(err)
	}

	// subscribe ack
	async.Publish(`{"event":"subscribed","channel":"ticker","chanId":5,"symbol":"tBTCUSD","subId":"nonce1","pair":"BTCUSD"}`)
	sub, err := listener.nextSubscriptionEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.SubscribeEvent{
		SubID:   "nonce1",
		Channel: "ticker",
		ChanID:  5,
		Symbol:  "tBTCUSD",
		Pair:    "BTCUSD",
	}, sub)

	// tick data
	async.Publish(`[5,[14957,68.17328796,14958,55.29588132,-659,-0.0422,14971,53723.08813995,16494,14454]]`)
	tick, err := listener.nextTick()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &bitfinex.Ticker{
		Symbol:          "tBTCUSD",
		Bid:             14957,
		Ask:             14958,
		BidSize:         68.17328796,
		AskSize:         55.29588132,
		DailyChange:     -659,
		DailyChangePerc: -0.0422,
		LastPrice:       14971,
		Volume:          53723.08813995,
		High:            16494,
		Low:             14454,
	}, tick)

	// unsubscribe
	err_unsub := ws.Unsubscribe(context.Background(), id)
	if err_unsub != nil {
		t.Fatal(err_unsub)
	}
	async.Publish(`{"event":"unsubscribed","chanId":5,"status":"OK"}`)
	unsub, err := listener.nextUnsubscriptionEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.UnsubscribeEvent{ChanID: 5, Status: "OK"}, unsub)
}

func TestOrderbook(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()

	// create client
	p := websocket.NewDefaultParameters()
	p.ManageOrderbook = true
	ws := websocket.NewWithParamsAsyncFactory(p, newTestAsyncFactory(async))

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	// we will use XRPBTC since that uses reallllyy small numbers
	bId, err_st := ws.SubscribeBook(context.Background(), bitfinex.TradingPrefix+bitfinex.XRPBTC, bitfinex.Precision0, bitfinex.FrequencyRealtime, 25)
	if err_st != nil {
		t.Fatal(err_st)
	}
	// checksum enabled ack
	async.Publish(`{"event":"conf","status":"OK","flags":131072}`)
	// subscribe ack
	async.Publish(`{"event":"subscribed","channel":"book","chanId":81757,"symbol":"tXRPBTC","prec":"P0","freq":"F0","len":"25","subId":"` + bId + `","pair":"XRPBTC"}`)

	// publish a snapshot
	async.Publish(`[81757,[[0.0000011,13,271510.49],[0.00000109,4,500793.10790141],[0.00000108,5,776367.43],[0.00000107,1,23329.54842056],[0.00000106,3,116868.87735849],[0.00000105,3,205000],[0.00000103,3,227308.25386407],[0.00000102,2,105000],[0.00000101,1,2970],[0.000001,2,21000],[7e-7,1,10000],[6.6e-7,1,10000],[6e-7,1,100000],[4.9e-7,1,10000],[2.5e-7,1,2000],[6e-8,1,100000],[5e-8,1,200000],[1e-8,4,640000],[0.00000111,1,-4847.13],[0.00000112,7,-528102.69042633],[0.00000113,5,-302397.07],[0.00000114,3,-339088.93],[0.00000126,4,-245944.06],[0.00000127,1,-5000],[0.0000013,1,-5000],[0.00000134,1,-8249.18938656],[0.00000136,1,-13161.25184766],[0.00000145,1,-2914],[0.0000015,3,-54448.5],[0.00000152,2,-5538.54849594],[0.00000153,1,-62691.75475079],[0.00000159,1,-2914],[0.0000016,1,-52631.10296831],[0.00000164,1,-4000],[0.00000166,1,-3831.46784605],[0.00000171,1,-14575.17730379],[0.00000174,1,-3124.81815395],[0.0000018,1,-18000],[0.00000182,1,-16000],[0.00000186,1,-4000],[0.00000189,1,-10000.686624],[0.00000191,1,-14500],[0.00000193,1,-2422]]]`)

	// publish new trade update
	async.Publish(`[81757,[0.0000011,12,266122.94]]`)

	// test that we can retrieve the orderbook
	ob, err_ob := ws.GetOrderbook("tXRPBTC")
	if err_ob != nil {
		t.Fatal(err_ob)
	}

	// test that changing the orderbook values will not invalidate the checksum
	// since they have been dereferenced
	ob.Bids()[0].Amount = 9999999

	// publish new checksum
	pre := async.SentCount()
	async.Publish(`[81757,"cs",-1175357890]`)

	// test that the new trade has been added to the orderbook
	newTrade := ob.Bids()[0]
	// check that it has overwritten the original trade in the book at that price
	if newTrade.PriceJsNum.String() != "0.0000011" {
		t.Fatal("Newly submitted trade did not update into orderbook")
	}
	if newTrade.AmountJsNum.String() != "266122.94" {
		t.Fatal("Newly submitted trade did not update into orderbook")
	}
	// check that we did not send an unsubscribe message
	// because that would mean the checksum was incorrect
	if err_unsub := async.waitForMessage(pre); err_unsub != nil {
		// no message sent
		return
	} else {
		t.Fatal("A new unsubscribe message was sent")
	}
}

func TestCreateNewSocket(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()

	// create client
	p := websocket.NewDefaultParameters()
	// lock the capacity to 10
	p.CapacityPerConnection = 10
	p.ManageOrderbook = true
	ws := websocket.NewWithParamsAsyncFactory(p, newTestAsyncFactory(async))

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	tickers := []string{"tBTCUSD", "tETHUSD", "tBTCUSD", "tVETUSD", "tDGBUSD", "tEOSUSD", "tTRXUSD", "tEOSETH", "tBTCETH",
		"tBTCEOS", "tXRPUSD", "tXRPBTC", "tTRXETH", "tTRXBTC", "tLTCUSD", "tLTCBTC", "tLTCETH"}
	for i, ticker := range tickers {
		id := i*10
		// subscribe to 15m candles
		id1, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.FifteenMinutes)
		if err != nil {
			t.Fatal(err)
		}
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":`+string(id)+`,"key":"trade:15m:`+ticker+`","subId":"`+id1+`"}`)
		// subscribe to 1hr candles
		id2, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.OneHour)
		if err != nil {
			t.Fatal(err)
		}
		// subscribe ack
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":`+string(id+1)+`,"key":"trade:1hr:`+ticker+`","subId":"`+id2+`"}`)
		// subscribe to 30min candles
		id3, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.OneHour)
		if err != nil {
			t.Fatal(err)
		}
		// subscribe ack
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":`+string(id+2)+`,"key":"trade:30m:`+ticker+`","subId":"`+id3+`"}`)
	}
	conCount := ws.ConnectionCount()
	if conCount != 6 {
		t.Fatal("Expected socket count to be 6 but got", conCount)
	}
}
<commit_msg>tests/integration/v2/mock_ws_public_test.go putting new Ticker package to work<commit_after>package tests

import (
	"context"
	"testing"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/ticker"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
	"github.com/bitfinexcom/bitfinex-api-go/v2/websocket"
)

// method of testing with mocked endpoints
func TestTicker(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()
	nonce := &IncrementingNonceGenerator{}

	// create client
	ws := websocket.NewWithAsyncFactoryNonce(newTestAsyncFactory(async), nonce)

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	// subscribe
	id, err := ws.SubscribeTicker(context.Background(), "tBTCUSD")
	if err != nil {
		t.Fatal(err)
	}

	// subscribe ack
	async.Publish(`{"event":"subscribed","channel":"ticker","chanId":5,"symbol":"tBTCUSD","subId":"nonce1","pair":"BTCUSD"}`)
	sub, err := listener.nextSubscriptionEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.SubscribeEvent{
		SubID:   "nonce1",
		Channel: "ticker",
		ChanID:  5,
		Symbol:  "tBTCUSD",
		Pair:    "BTCUSD",
	}, sub)

	// tick data
	async.Publish(`[5,[14957,68.17328796,14958,55.29588132,-659,-0.0422,14971,53723.08813995,16494,14454]]`)
	tick, err := listener.nextTick()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &ticker.Ticker{
		Symbol:          "tBTCUSD",
		Bid:             14957,
		Ask:             14958,
		BidSize:         68.17328796,
		AskSize:         55.29588132,
		DailyChange:     -659,
		DailyChangePerc: -0.0422,
		LastPrice:       14971,
		Volume:          53723.08813995,
		High:            16494,
		Low:             14454,
	}, tick)

	// unsubscribe
	err_unsub := ws.Unsubscribe(context.Background(), id)
	if err_unsub != nil {
		t.Fatal(err_unsub)
	}
	async.Publish(`{"event":"unsubscribed","chanId":5,"status":"OK"}`)
	unsub, err := listener.nextUnsubscriptionEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.UnsubscribeEvent{ChanID: 5, Status: "OK"}, unsub)
}

func TestOrderbook(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()

	// create client
	p := websocket.NewDefaultParameters()
	p.ManageOrderbook = true
	ws := websocket.NewWithParamsAsyncFactory(p, newTestAsyncFactory(async))

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	// we will use XRPBTC since that uses reallllyy small numbers
	bId, err_st := ws.SubscribeBook(context.Background(), bitfinex.TradingPrefix+bitfinex.XRPBTC, bitfinex.Precision0, bitfinex.FrequencyRealtime, 25)
	if err_st != nil {
		t.Fatal(err_st)
	}
	// checksum enabled ack
	async.Publish(`{"event":"conf","status":"OK","flags":131072}`)
	// subscribe ack
	async.Publish(`{"event":"subscribed","channel":"book","chanId":81757,"symbol":"tXRPBTC","prec":"P0","freq":"F0","len":"25","subId":"` + bId + `","pair":"XRPBTC"}`)

	// publish a snapshot
	async.Publish(`[81757,[[0.0000011,13,271510.49],[0.00000109,4,500793.10790141],[0.00000108,5,776367.43],[0.00000107,1,23329.54842056],[0.00000106,3,116868.87735849],[0.00000105,3,205000],[0.00000103,3,227308.25386407],[0.00000102,2,105000],[0.00000101,1,2970],[0.000001,2,21000],[7e-7,1,10000],[6.6e-7,1,10000],[6e-7,1,100000],[4.9e-7,1,10000],[2.5e-7,1,2000],[6e-8,1,100000],[5e-8,1,200000],[1e-8,4,640000],[0.00000111,1,-4847.13],[0.00000112,7,-528102.69042633],[0.00000113,5,-302397.07],[0.00000114,3,-339088.93],[0.00000126,4,-245944.06],[0.00000127,1,-5000],[0.0000013,1,-5000],[0.00000134,1,-8249.18938656],[0.00000136,1,-13161.25184766],[0.00000145,1,-2914],[0.0000015,3,-54448.5],[0.00000152,2,-5538.54849594],[0.00000153,1,-62691.75475079],[0.00000159,1,-2914],[0.0000016,1,-52631.10296831],[0.00000164,1,-4000],[0.00000166,1,-3831.46784605],[0.00000171,1,-14575.17730379],[0.00000174,1,-3124.81815395],[0.0000018,1,-18000],[0.00000182,1,-16000],[0.00000186,1,-4000],[0.00000189,1,-10000.686624],[0.00000191,1,-14500],[0.00000193,1,-2422]]]`)

	// publish new trade update
	async.Publish(`[81757,[0.0000011,12,266122.94]]`)

	// test that we can retrieve the orderbook
	ob, err_ob := ws.GetOrderbook("tXRPBTC")
	if err_ob != nil {
		t.Fatal(err_ob)
	}

	// test that changing the orderbook values will not invalidate the checksum
	// since they have been dereferenced
	ob.Bids()[0].Amount = 9999999

	// publish new checksum
	pre := async.SentCount()
	async.Publish(`[81757,"cs",-1175357890]`)

	// test that the new trade has been added to the orderbook
	newTrade := ob.Bids()[0]
	// check that it has overwritten the original trade in the book at that price
	if newTrade.PriceJsNum.String() != "0.0000011" {
		t.Fatal("Newly submitted trade did not update into orderbook")
	}
	if newTrade.AmountJsNum.String() != "266122.94" {
		t.Fatal("Newly submitted trade did not update into orderbook")
	}
	// check that we did not send an unsubscribe message
	// because that would mean the checksum was incorrect
	if err_unsub := async.waitForMessage(pre); err_unsub != nil {
		// no message sent
		return
	} else {
		t.Fatal("A new unsubscribe message was sent")
	}
}

func TestCreateNewSocket(t *testing.T) {
	// create transport & nonce mocks
	async := newTestAsync()

	// create client
	p := websocket.NewDefaultParameters()
	// lock the capacity to 10
	p.CapacityPerConnection = 10
	p.ManageOrderbook = true
	ws := websocket.NewWithParamsAsyncFactory(p, newTestAsyncFactory(async))

	// setup listener
	listener := newListener()
	listener.run(ws.Listen())

	// set ws options
	err_ws := ws.Connect()
	if err_ws != nil {
		t.Fatal(err_ws)
	}
	defer ws.Close()

	// info welcome msg
	async.Publish(`{"event":"info","version":2}`)
	ev, err := listener.nextInfoEvent()
	if err != nil {
		t.Fatal(err)
	}
	assert(t, &websocket.InfoEvent{Version: 2}, ev)

	tickers := []string{"tBTCUSD", "tETHUSD", "tBTCUSD", "tVETUSD", "tDGBUSD", "tEOSUSD", "tTRXUSD", "tEOSETH", "tBTCETH",
		"tBTCEOS", "tXRPUSD", "tXRPBTC", "tTRXETH", "tTRXBTC", "tLTCUSD", "tLTCBTC", "tLTCETH"}
	for i, ticker := range tickers {
		id := i * 10
		// subscribe to 15m candles
		id1, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.FifteenMinutes)
		if err != nil {
			t.Fatal(err)
		}
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":` + string(id) + `,"key":"trade:15m:` + ticker + `","subId":"` + id1 + `"}`)
		// subscribe to 1hr candles
		id2, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.OneHour)
		if err != nil {
			t.Fatal(err)
		}
		// subscribe ack
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":` + string(id+1) + `,"key":"trade:1hr:` + ticker + `","subId":"` + id2 + `"}`)
		// subscribe to 30min candles
		id3, err := ws.SubscribeCandles(context.Background(), ticker, bitfinex.OneHour)
		if err != nil {
			t.Fatal(err)
		}
		// subscribe ack
		async.Publish(`{"event":"subscribed","channel":"candles","chanId":` + string(id+2) + `,"key":"trade:30m:` + ticker + `","subId":"` + id3 + `"}`)
	}
	conCount := ws.ConnectionCount()
	if conCount != 6 {
		t.Fatal("Expected socket count to be 6 but got", conCount)
	}
}
<|endoftext|>


<commit_before>package main

// Enums for keys to be stored in a session context - this is how gorilla expects
// these to be implemented and is lifted pretty much from docs
const (
	SessionData     = "session_data"
	AuthHeaderValue = "auth_header"
)

// SessionState objects represent a current API session, mainly used for rate limiting.
type SessionState struct {
	ExpiresIn     int64  `json:"expires_in"`
	OauthClientID string `json:"oauth_client_id"`
	AccessToken   string `json:"access_token"`
	TokenType     string `json:"token_type"`
}
<commit_msg>Added the server id to the session state<commit_after>package main

import "gopkg.in/mgo.v2/bson"

// Enums for keys to be stored in a session context - this is how gorilla expects
// these to be implemented and is lifted pretty much from docs
const (
	SessionData     = "session_data"
	AuthHeaderValue = "auth_header"
)

// SessionState objects represent a current API session, mainly used for rate limiting.
type SessionState struct {
	OAuthServerID bson.ObjectId `json:"server_id"`
	ExpiresIn     int64         `json:"expires_in"`
	OauthClientID string        `json:"oauth_client_id"`
	AccessToken   string        `json:"access_token"`
	TokenType     string        `json:"token_type"`
}
<|endoftext|>


<commit_before>// +build windows

package chroot

import "errors"

func lockFile(*os.File) error {
	return errors.New("not supported on Windows")
}

func unlockFile(f *os.File) error {
	return nil
}
<commit_msg>builder/amazon/chroot: fix compilaton on Windows<commit_after>// +build windows

package chroot

import (
	"errors"
	"os"
)

func lockFile(*os.File) error {
	return errors.New("not supported on Windows")
}

func unlockFile(f *os.File) error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/google/go-github/github"
)

type contribCalendar struct {
	XMLName xml.Name       `xml:"svg"`
	Group   containerGroup `xml:"g"`
}

type containerGroup struct {
	Groups []contribGroup `xml:"g"`
}

type contribGroup struct {
	Rects []Contribution `xml:"rect"`
}

type Rect struct {
	Date string `xml:"data-date,attr"`
	Num  string `xml:"data-count,attr"`
}

type Contribution struct {
	Date time.Time
	Num  int
}

const dateFormat string = "2006-01-02"

func (c *Contribution) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var i Rect
	err := d.DecodeElement(&i, &start)
	if err != nil {
		return err
	}

	c.Date, err = time.Parse(dateFormat, i.Date)
	if err != nil {
		return err
	}

	c.Num, err = strconv.Atoi(i.Num)
	if err != nil {
		return err
	}

	return nil
}

func getContributions(user github.User) []Contribution {
	login := github.Stringify(user.Login)
	url := "https://github.com/users/" + login[1:len(login)-1] + "/contributions"
	var contribData contribCalendar

	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	if data == nil {
		log.Fatalln("No data returned.")
	}

	err = xml.Unmarshal(data, &contribData)
	if err != nil {
		log.Fatal(err)
	}
	var contributions []Contribution

	for _, group := range contribData.Group.Groups {
		contributions = append(contributions, group.Rects...)
	}

	return contributions
}

func getOrgMembers(orgName string) []github.User {
	client := github.NewClient(nil)

	users, _, err := client.Organizations.ListMembers(orgName, nil)
	if err != nil {
		log.Fatal(err)
	}

	return users
}

func main() {
	flag.Parse()
	var i interface{} = flag.Arg(0)
	var users []github.User

	var v, ok = i.(string)
	if ok == false || v == "" {
		fmt.Println("Usage: orgstreak <orgName>")
		os.Exit(1)
	} else {
		users = getOrgMembers(v)
	}

	orgContribs := make(map[time.Time]int)
	userContributions := make(chan []Contribution, len(users))

	for _, user := range users {
		go func(u github.User) {
			userContributions <- getContributions(u)
		}(user)
	}

	for y := 0; y < len(users); y++ {
		select {
		case userContribution := <-userContributions:
			for _, contrib := range userContribution {
				orgContribs[contrib.Date] += contrib.Num
			}
		}
	}

	contribData := []interface{}{}
	for k, v := range orgContribs {
		contribData = append(contribData, []interface{}{k.Format(dateFormat), v})
	}
	j, err := json.Marshal(contribData)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>pass pointers<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/google/go-github/github"
)

type contribCalendar struct {
	XMLName xml.Name       `xml:"svg"`
	Group   containerGroup `xml:"g"`
}

type containerGroup struct {
	Groups []contribGroup `xml:"g"`
}

type contribGroup struct {
	Rects []Contribution `xml:"rect"`
}

type Rect struct {
	Date string `xml:"data-date,attr"`
	Num  string `xml:"data-count,attr"`
}

type Contribution struct {
	Date time.Time
	Num  int
}

const dateFormat string = "2006-01-02"

func (c *Contribution) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var i Rect
	err := d.DecodeElement(&i, &start)
	if err != nil {
		return err
	}

	c.Date, err = time.Parse(dateFormat, i.Date)
	if err != nil {
		return err
	}

	c.Num, err = strconv.Atoi(i.Num)
	if err != nil {
		return err
	}

	return nil
}

func getContributions(user github.User) *[]Contribution {
	login := github.Stringify(user.Login)
	url := "https://github.com/users/" + login[1:len(login)-1] + "/contributions"
	var contribData contribCalendar

	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	if data == nil {
		log.Fatalln("No data returned.")
	}

	err = xml.Unmarshal(data, &contribData)
	if err != nil {
		log.Fatal(err)
	}
	var contributions []Contribution

	for _, group := range contribData.Group.Groups {
		contributions = append(contributions, group.Rects...)
	}

	return &contributions
}

func getOrgMembers(orgName string) *[]github.User {
	client := github.NewClient(nil)

	users, _, err := client.Organizations.ListMembers(orgName, nil)
	if err != nil {
		log.Fatal(err)
	}

	return &users
}

func main() {
	flag.Parse()
	var i interface{} = flag.Arg(0)
	var users *[]github.User

	var v, ok = i.(string)
	if ok == false || v == "" {
		fmt.Println("Usage: orgstreak <orgName>")
		os.Exit(1)
	} else {
		users = getOrgMembers(v)
	}

	orgContribs := make(map[time.Time]int)
	userContributions := make(chan *[]Contribution, len(*users))

	for _, user := range *users {
		go func(u github.User) {
			userContributions <- getContributions(u)
		}(user)
	}

	for y := 0; y < len(*users); y++ {
		select {
		case userContribution := <-userContributions:
			for _, contrib := range *userContribution {
				orgContribs[contrib.Date] += contrib.Num
			}
		}
	}

	contribData := []interface{}{}
	for k, v := range orgContribs {
		contribData = append(contribData, []interface{}{k.Format(dateFormat), v})
	}
	j, err := json.Marshal(contribData)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package carbonserver

import (
	"errors"
	"math"
	"os"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	"github.com/go-graphite/go-carbon/points"
	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type response struct {
	Name              string
	StartTime         int64
	StopTime          int64
	StepTime          int64
	Values            []float64
	PathExpression    string
	ConsolidationFunc string
	XFilesFactor      float32
	RequestStartTime  int64
	RequestStopTime   int64
}

func (r response) enrichFromCache(listener *CarbonserverListener, cacheData []points.Point) {
	if cacheData == nil {
		return
	}

	atomic.AddUint64(&listener.metrics.CacheRequestsTotal, 1)
	cacheStartTime := time.Now()
	pointsFetchedFromCache := 0
	for _, item := range cacheData {
		ts := int64(item.Timestamp) - int64(item.Timestamp)%r.StepTime
		if ts < r.StartTime || ts >= r.StopTime {
			continue
		}
		pointsFetchedFromCache++
		index := (ts - r.StartTime) / r.StepTime
		// TODO: log.debug such cases
		if index >= 0 && index < int64(len(r.Values)) {
			r.Values[index] = item.Value
		}
	}
	waitTime := time.Since(cacheStartTime)
	atomic.AddUint64(&listener.metrics.CacheWorkTimeNS, uint64(waitTime.Nanoseconds()))
	listener.prometheus.cacheDuration("work", waitTime)

	listener.prometheus.cacheRequest("metric", pointsFetchedFromCache > 0)
	if pointsFetchedFromCache > 0 {
		atomic.AddUint64(&listener.metrics.CacheHit, 1)
	} else {
		atomic.AddUint64(&listener.metrics.CacheMiss, 1)
	}
}

func (r response) proto2() *protov2.FetchResponse {
	resp := &protov2.FetchResponse{
		Name:      r.Name,
		StartTime: int32(r.StartTime),
		StopTime:  int32(r.StopTime),
		StepTime:  int32(r.StepTime),
		Values:    r.Values,
		IsAbsent:  make([]bool, len(r.Values)),
	}

	for i, p := range resp.Values {
		if math.IsNaN(p) {
			resp.Values[i] = 0
			resp.IsAbsent[i] = true
		}
	}

	return resp
}

func (r response) proto3() *protov3.FetchResponse {
	return &protov3.FetchResponse{
		Name:              r.Name,
		StartTime:         r.StartTime,
		StopTime:          r.StopTime,
		StepTime:          r.StepTime,
		Values:            r.Values,
		PathExpression:    r.PathExpression,
		ConsolidationFunc: r.ConsolidationFunc,
		XFilesFactor:      r.XFilesFactor,
		RequestStartTime:  r.RequestStartTime,
		RequestStopTime:   r.RequestStopTime,
	}
}

func (listener *CarbonserverListener) fetchSingleMetric(metric string, pathExpression string, fromTime, untilTime int32) (response, error) {
	logger := listener.logger.With(
		zap.String("metric", metric),
		zap.Int("fromTime", int(fromTime)),
		zap.Int("untilTime", int(untilTime)),
	)
	m, err := listener.fetchFromDisk(metric, fromTime, untilTime)
	switch {
	case err == nil:
		// Should never happen, because we have a check for proper archive now
		if m.Timeseries == nil {
			atomic.AddUint64(&listener.metrics.RenderErrors, 1)
			logger.Warn("metric time range not found")
			return response{}, errors.New("time range not found")
		}

		values := m.Timeseries.Values()
		from := int64(m.Timeseries.FromTime())
		until := int64(m.Timeseries.UntilTime())
		step := int64(m.Timeseries.Step())

		resp := response{
			Name:              metric,
			StartTime:         from,
			StopTime:          until,
			StepTime:          step,
			Values:            values,
			PathExpression:    pathExpression,
			ConsolidationFunc: m.Metadata.ConsolidationFunc,
			XFilesFactor:      m.Metadata.XFilesFactor,
			RequestStartTime:  int64(fromTime),
			RequestStopTime:   int64(untilTime),
		}

		resp.enrichFromCache(listener, m.CacheData)

		logger.Debug("fetched",
			zap.Any("response", resp),
		)

		return resp, nil
	case os.IsNotExist(err) && (listener.cacheGetRecentMetrics != nil || listener.realtimeIndex > 0):
		// Failed to fetch from disk, try to fetch from cache
		resp := response{
			Name:           metric,
			PathExpression: pathExpression,
		}
		cacheData, cerr := listener.fetchFromCache(metric, fromTime, untilTime, &resp)
		if cerr != nil {
			atomic.AddUint64(&listener.metrics.RenderErrors, 1)
			logger.Warn("metric not found even in Cache", zap.Error(err))
			// Metric has no Whisper file and/or has no datapoints in Cache
			return response{}, cerr
		}
		resp.enrichFromCache(listener, cacheData)
		logger.Debug("fetched cache-only",
			zap.Any("response", resp),
		)

		return resp, nil
	default:
		atomic.AddUint64(&listener.metrics.RenderErrors, 1)
		logger.Warn("failed to fetch points", zap.Error(err))
		return response{}, err
	}
}
<commit_msg>fix(go-carbon): proper fix for enrichFromCache panic<commit_after>package carbonserver

import (
	"errors"
	"math"
	"os"
	"sync/atomic"
	"time"

	"go.uber.org/zap"

	"github.com/go-graphite/go-carbon/points"
	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type response struct {
	Name              string
	StartTime         int64
	StopTime          int64
	StepTime          int64
	Values            []float64
	PathExpression    string
	ConsolidationFunc string
	XFilesFactor      float32
	RequestStartTime  int64
	RequestStopTime   int64
}

func (r response) enrichFromCache(listener *CarbonserverListener, cacheData []points.Point) {
	if cacheData == nil {
		return
	}

	atomic.AddUint64(&listener.metrics.CacheRequestsTotal, 1)
	cacheStartTime := time.Now()
	pointsFetchedFromCache := 0
	// extend values array if needed
	max_index := (r.StopTime - r.StartTime) / r.StepTime
	start_index := int64(len(r.Values))
	if start_index < max_index {
		for i := start_index; i < max_index; i++ {
			// math.NaN is not working here
			// values will be replaced when merging with cache
			r.Values = append(r.Values, 0)
		}
	}
	for _, item := range cacheData {
		ts := int64(item.Timestamp) - int64(item.Timestamp)%r.StepTime
		if ts < r.StartTime || ts >= r.StopTime {
			continue
		}
		pointsFetchedFromCache++
		index := (ts - r.StartTime) / r.StepTime
		r.Values[index] = item.Value
	}
	waitTime := time.Since(cacheStartTime)
	atomic.AddUint64(&listener.metrics.CacheWorkTimeNS, uint64(waitTime.Nanoseconds()))
	listener.prometheus.cacheDuration("work", waitTime)

	listener.prometheus.cacheRequest("metric", pointsFetchedFromCache > 0)
	if pointsFetchedFromCache > 0 {
		atomic.AddUint64(&listener.metrics.CacheHit, 1)
	} else {
		atomic.AddUint64(&listener.metrics.CacheMiss, 1)
	}
}

func (r response) proto2() *protov2.FetchResponse {
	resp := &protov2.FetchResponse{
		Name:      r.Name,
		StartTime: int32(r.StartTime),
		StopTime:  int32(r.StopTime),
		StepTime:  int32(r.StepTime),
		Values:    r.Values,
		IsAbsent:  make([]bool, len(r.Values)),
	}

	for i, p := range resp.Values {
		if math.IsNaN(p) {
			resp.Values[i] = 0
			resp.IsAbsent[i] = true
		}
	}

	return resp
}

func (r response) proto3() *protov3.FetchResponse {
	return &protov3.FetchResponse{
		Name:              r.Name,
		StartTime:         r.StartTime,
		StopTime:          r.StopTime,
		StepTime:          r.StepTime,
		Values:            r.Values,
		PathExpression:    r.PathExpression,
		ConsolidationFunc: r.ConsolidationFunc,
		XFilesFactor:      r.XFilesFactor,
		RequestStartTime:  r.RequestStartTime,
		RequestStopTime:   r.RequestStopTime,
	}
}

func (listener *CarbonserverListener) fetchSingleMetric(metric string, pathExpression string, fromTime, untilTime int32) (response, error) {
	logger := listener.logger.With(
		zap.String("metric", metric),
		zap.Int("fromTime", int(fromTime)),
		zap.Int("untilTime", int(untilTime)),
	)
	m, err := listener.fetchFromDisk(metric, fromTime, untilTime)
	switch {
	case err == nil:
		// Should never happen, because we have a check for proper archive now
		if m.Timeseries == nil {
			atomic.AddUint64(&listener.metrics.RenderErrors, 1)
			logger.Warn("metric time range not found")
			return response{}, errors.New("time range not found")
		}

		values := m.Timeseries.Values()
		from := int64(m.Timeseries.FromTime())
		until := int64(m.Timeseries.UntilTime())
		step := int64(m.Timeseries.Step())

		resp := response{
			Name:              metric,
			StartTime:         from,
			StopTime:          until,
			StepTime:          step,
			Values:            values,
			PathExpression:    pathExpression,
			ConsolidationFunc: m.Metadata.ConsolidationFunc,
			XFilesFactor:      m.Metadata.XFilesFactor,
			RequestStartTime:  int64(fromTime),
			RequestStopTime:   int64(untilTime),
		}

		resp.enrichFromCache(listener, m.CacheData)

		logger.Debug("fetched",
			zap.Any("response", resp),
		)

		return resp, nil
	case os.IsNotExist(err) && (listener.cacheGetRecentMetrics != nil || listener.realtimeIndex > 0):
		// Failed to fetch from disk, try to fetch from cache
		resp := response{
			Name:           metric,
			PathExpression: pathExpression,
		}
		cacheData, cerr := listener.fetchFromCache(metric, fromTime, untilTime, &resp)
		if cerr != nil {
			atomic.AddUint64(&listener.metrics.RenderErrors, 1)
			logger.Warn("metric not found even in Cache", zap.Error(err))
			// Metric has no Whisper file and/or has no datapoints in Cache
			return response{}, cerr
		}
		resp.enrichFromCache(listener, cacheData)
		logger.Debug("fetched cache-only",
			zap.Any("response", resp),
		)

		return resp, nil
	default:
		atomic.AddUint64(&listener.metrics.RenderErrors, 1)
		logger.Warn("failed to fetch points", zap.Error(err))
		return response{}, err
	}
}
<|endoftext|>