<commit_before>/*
	rulehunter - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

package html

import (
	"bytes"
	"fmt"
	"github.com/vlifesystems/rulehunter/config"
	"github.com/vlifesystems/rulehunter/html/cmd"
	"github.com/vlifesystems/rulehunter/logger"
	"github.com/vlifesystems/rulehunter/progress"
	"github.com/vlifesystems/rulehunter/quitter"
	"html/template"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// File mode permission used as standard for the html content:
// No special permission bits
// User: Read, Write Execute
// Group: Read
// Other: None
const modePerm = 0740

// This should be run as a goroutine and each time a command is passed to
// cmds the html will be generated
func Run(
	config *config.Config,
	pm *progress.ProgressMonitor,
	l logger.Logger,
	q *quitter.Quitter,
	cmds chan cmd.Cmd,
) {
	const minWaitSeconds = 4.0
	lastCmd := cmd.Flush
	lastTime := time.Now()
	if err := generate(cmd.All, config, pm); err != nil {
		l.Error(fmt.Sprintf("Couldn't generate report: %s", err))
	}

	q.Add()
	defer q.Done()
	go pulse(cmds)

	for c := range cmds {
		if q.ShouldQuit() {
			break
		}
		durationSinceLast := time.Since(lastTime)
		if c != lastCmd || durationSinceLast.Seconds() > minWaitSeconds {
			if c == cmd.Flush {
				c = lastCmd
				lastCmd = cmd.Flush
			} else {
				lastCmd = c
			}
			lastTime = time.Now()
			if err := generate(c, config, pm); err != nil {
				l.Error(fmt.Sprintf("Couldn't generate report: %s", err))
			}
		}
	}
}

// This is used where a command has been received such as when a report has
// finished, but the correct time hasn't elapsed before generating html.
// Therefore a pulse is periodically sent to flush the command.
func pulse(cmds chan cmd.Cmd) {
	sleepInSeconds := time.Duration(4)
	for {
		cmds <- cmd.Flush
		time.Sleep(sleepInSeconds * time.Second)
	}
}

func generate(
	c cmd.Cmd,
	config *config.Config,
	pm *progress.ProgressMonitor,
) error {
	switch c {
	case cmd.Progress:
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	case cmd.Reports:
		if err := generateReports(config, pm); err != nil {
			return err
		}
		if err := generateTagPages(config); err != nil {
			return err
		}
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	case cmd.All:
		if err := generateHomePage(config, pm); err != nil {
			return err
		}
		if err := generateReports(config, pm); err != nil {
			return err
		}
		if err := generateTagPages(config); err != nil {
			return err
		}
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	}
	return nil
}

var nonAlphaNumOrSpaceRegexp = regexp.MustCompile("[^[:alnum:] ]")
var spaceRegexp = regexp.MustCompile(" ")
var multipleHyphenRegexp = regexp.MustCompile("-+")

func escapeString(s string) string {
	newS := nonAlphaNumOrSpaceRegexp.ReplaceAllString(s, "")
	newS = spaceRegexp.ReplaceAllString(newS, "-")
	newS = multipleHyphenRegexp.ReplaceAllString(newS, "-")
	newS = strings.Trim(newS, " -")
	return strings.ToLower(newS)
}

func writeTemplate(
	filename string,
	tpl string,
	tplData interface{},
) error {
	t, err := template.New("webpage").Parse(tpl)
	if err != nil {
		return err
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	if err := t.Execute(f, tplData); err != nil {
		return err
	}
	return nil
}

func genReportFilename(wwwDir string, stamp time.Time, title string) string {
	localDir := genReportLocalDir(wwwDir, stamp, title)
	return filepath.Join(localDir, "index.html")
}

// This doesn't change below a second as if two or more reports had the same
// title and were made less than a second apart then you wouldn't be able to
// tell them apart anyway from the list of reports.  This should therfore
// be discouraged.
func genStampMagicString(stamp time.Time) string {
	sum := stamp.Hour()*3600 + stamp.Minute()*60 + stamp.Second()
	return strconv.FormatUint(uint64(sum), 36)
}

func genReportURLDir(
	stamp time.Time,
	title string,
) string {
	magicNumber := genStampMagicString(stamp)
	escapedTitle := escapeString(title)
	return fmt.Sprintf("/reports/%d/%02d/%02d/%s_%s/",
		stamp.Year(), stamp.Month(), stamp.Day(), magicNumber, escapedTitle)
}

func genReportLocalDir(
	wwwDir string,
	stamp time.Time,
	title string,
) string {
	magicNumber := genStampMagicString(stamp)
	escapedTitle := escapeString(title)
	return filepath.Join(wwwDir, "reports",
		fmt.Sprintf("%d", stamp.Year()),
		fmt.Sprintf("%02d", stamp.Month()),
		fmt.Sprintf("%02d", stamp.Day()),
		fmt.Sprintf("%s_%s", magicNumber, escapedTitle))
}

func makeReportURLDir(
	wwwDir string,
	stamp time.Time,
	title string,
) (string, error) {
	URLDir := genReportURLDir(stamp, title)
	localDir := genReportLocalDir(wwwDir, stamp, title)
	err := os.MkdirAll(localDir, modePerm)
	return URLDir, err
}

func countFiles(files []os.FileInfo) int {
	numFiles := 0
	for _, file := range files {
		if !file.IsDir() {
			numFiles++
		}
	}
	return numFiles
}

func makeHtmlNav(menuItem string) template.HTML {
	var doc bytes.Buffer
	validMenuItems := []string{
		"home",
		"reports",
		"tag",
		"progress",
	}

	foundValidItem := false
	for _, validMenuItem := range validMenuItems {
		if validMenuItem == menuItem {
			foundValidItem = true
		}
	}
	if !foundValidItem {
		panic(fmt.Sprintf("menuItem not valid: %s", menuItem))
	}

	t, err := template.New("webpage").Parse(navTpl)
	if err != nil {
		panic(fmt.Sprintf("Couldn't create nav html: %s", err))
	}

	tplData := struct{ MenuItem string }{menuItem}

	if err := t.Execute(&doc, tplData); err != nil {
		panic(fmt.Sprintf("Couldn't create nav html: %s", err))
	}
	return template.HTML(doc.String())
}

func makeHtml(menuItem string) map[string]template.HTML {
	r := make(map[string]template.HTML)
	r["head"] = template.HTML(headHtml)
	r["nav"] = makeHtmlNav(menuItem)
	r["bootstrapJS"] = template.HTML(bootstrapJSHtml)
	return r
}
<commit_msg>Ensure html generate has one more go if shouldQuit<commit_after>/*
	rulehunter - A server to find rules in data based on user specified goals
	Copyright (C) 2016 vLife Systems Ltd <http://vlifesystems.com>

	This program is free software: you can redistribute it and/or modify
	it under the terms of the GNU Affero General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU Affero General Public License for more details.

	You should have received a copy of the GNU Affero General Public License
	along with this program; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

package html

import (
	"bytes"
	"fmt"
	"github.com/vlifesystems/rulehunter/config"
	"github.com/vlifesystems/rulehunter/html/cmd"
	"github.com/vlifesystems/rulehunter/logger"
	"github.com/vlifesystems/rulehunter/progress"
	"github.com/vlifesystems/rulehunter/quitter"
	"html/template"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// File mode permission used as standard for the html content:
// No special permission bits
// User: Read, Write Execute
// Group: Read
// Other: None
const modePerm = 0740

// This should be run as a goroutine and each time a command is passed to
// cmds the html will be generated
func Run(
	config *config.Config,
	pm *progress.ProgressMonitor,
	l logger.Logger,
	q *quitter.Quitter,
	cmds chan cmd.Cmd,
) {
	const minWaitSeconds = 4.0
	lastCmd := cmd.Flush
	lastTime := time.Now()
	if err := generate(cmd.All, config, pm); err != nil {
		l.Error(fmt.Sprintf("Couldn't generate report: %s", err))
	}

	q.Add()
	defer q.Done()
	go pulse(cmds)

	for c := range cmds {
		durationSinceLast := time.Since(lastTime)
		if q.ShouldQuit() ||
			c != lastCmd ||
			durationSinceLast.Seconds() > minWaitSeconds {
			if c == cmd.Flush {
				c = lastCmd
				lastCmd = cmd.Flush
			} else {
				lastCmd = c
			}
			lastTime = time.Now()
			if err := generate(c, config, pm); err != nil {
				l.Error(fmt.Sprintf("Couldn't generate report: %s", err))
			}
		}
		if q.ShouldQuit() {
			break
		}
	}
}

// This is used where a command has been received such as when a report has
// finished, but the correct time hasn't elapsed before generating html.
// Therefore a pulse is periodically sent to flush the command.
func pulse(cmds chan cmd.Cmd) {
	sleepInSeconds := time.Duration(4)
	for {
		cmds <- cmd.Flush
		time.Sleep(sleepInSeconds * time.Second)
	}
}

func generate(
	c cmd.Cmd,
	config *config.Config,
	pm *progress.ProgressMonitor,
) error {
	switch c {
	case cmd.Progress:
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	case cmd.Reports:
		if err := generateReports(config, pm); err != nil {
			return err
		}
		if err := generateTagPages(config); err != nil {
			return err
		}
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	case cmd.All:
		if err := generateHomePage(config, pm); err != nil {
			return err
		}
		if err := generateReports(config, pm); err != nil {
			return err
		}
		if err := generateTagPages(config); err != nil {
			return err
		}
		if err := generateProgressPage(config, pm); err != nil {
			return err
		}
	}
	return nil
}

var nonAlphaNumOrSpaceRegexp = regexp.MustCompile("[^[:alnum:] ]")
var spaceRegexp = regexp.MustCompile(" ")
var multipleHyphenRegexp = regexp.MustCompile("-+")

func escapeString(s string) string {
	newS := nonAlphaNumOrSpaceRegexp.ReplaceAllString(s, "")
	newS = spaceRegexp.ReplaceAllString(newS, "-")
	newS = multipleHyphenRegexp.ReplaceAllString(newS, "-")
	newS = strings.Trim(newS, " -")
	return strings.ToLower(newS)
}

func writeTemplate(
	filename string,
	tpl string,
	tplData interface{},
) error {
	t, err := template.New("webpage").Parse(tpl)
	if err != nil {
		return err
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	if err := t.Execute(f, tplData); err != nil {
		return err
	}
	return nil
}

func genReportFilename(wwwDir string, stamp time.Time, title string) string {
	localDir := genReportLocalDir(wwwDir, stamp, title)
	return filepath.Join(localDir, "index.html")
}

// This doesn't change below a second as if two or more reports had the same
// title and were made less than a second apart then you wouldn't be able to
// tell them apart anyway from the list of reports.  This should therfore
// be discouraged.
func genStampMagicString(stamp time.Time) string {
	sum := stamp.Hour()*3600 + stamp.Minute()*60 + stamp.Second()
	return strconv.FormatUint(uint64(sum), 36)
}

func genReportURLDir(
	stamp time.Time,
	title string,
) string {
	magicNumber := genStampMagicString(stamp)
	escapedTitle := escapeString(title)
	return fmt.Sprintf("/reports/%d/%02d/%02d/%s_%s/",
		stamp.Year(), stamp.Month(), stamp.Day(), magicNumber, escapedTitle)
}

func genReportLocalDir(
	wwwDir string,
	stamp time.Time,
	title string,
) string {
	magicNumber := genStampMagicString(stamp)
	escapedTitle := escapeString(title)
	return filepath.Join(wwwDir, "reports",
		fmt.Sprintf("%d", stamp.Year()),
		fmt.Sprintf("%02d", stamp.Month()),
		fmt.Sprintf("%02d", stamp.Day()),
		fmt.Sprintf("%s_%s", magicNumber, escapedTitle))
}

func makeReportURLDir(
	wwwDir string,
	stamp time.Time,
	title string,
) (string, error) {
	URLDir := genReportURLDir(stamp, title)
	localDir := genReportLocalDir(wwwDir, stamp, title)
	err := os.MkdirAll(localDir, modePerm)
	return URLDir, err
}

func countFiles(files []os.FileInfo) int {
	numFiles := 0
	for _, file := range files {
		if !file.IsDir() {
			numFiles++
		}
	}
	return numFiles
}

func makeHtmlNav(menuItem string) template.HTML {
	var doc bytes.Buffer
	validMenuItems := []string{
		"home",
		"reports",
		"tag",
		"progress",
	}

	foundValidItem := false
	for _, validMenuItem := range validMenuItems {
		if validMenuItem == menuItem {
			foundValidItem = true
		}
	}
	if !foundValidItem {
		panic(fmt.Sprintf("menuItem not valid: %s", menuItem))
	}

	t, err := template.New("webpage").Parse(navTpl)
	if err != nil {
		panic(fmt.Sprintf("Couldn't create nav html: %s", err))
	}

	tplData := struct{ MenuItem string }{menuItem}

	if err := t.Execute(&doc, tplData); err != nil {
		panic(fmt.Sprintf("Couldn't create nav html: %s", err))
	}
	return template.HTML(doc.String())
}

func makeHtml(menuItem string) map[string]template.HTML {
	r := make(map[string]template.HTML)
	r["head"] = template.HTML(headHtml)
	r["nav"] = makeHtmlNav(menuItem)
	r["bootstrapJS"] = template.HTML(bootstrapJSHtml)
	return r
}
<|endoftext|>


<commit_before>// Package http provides HTTP handlers for drawing SVGs.
package http

//go:generate go-bindata -pkg $GOPACKAGE -prefix asset/ asset/...

import (
	"fmt"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/nochso/colourl/cache"
	"github.com/nochso/colourl/palette"
	"html/template"
	"net/http"
	"net/url"
	"strconv"
)

var scorer = &palette.SumScore{}

var tmpl *template.Template

func init() {
	b, err := Asset("index.html")
	if err != nil {
		panic(err)
	}
	tmpl = template.New("index.html")
	template.Must(tmpl.Parse(string(b)))
}

// SVGHandler returns a SVG based on GET parameters.
// See NewPaintJob() for parsing options.
// See NewPainter() for SVG style / Painter.
func SVGHandler(w http.ResponseWriter, req *http.Request) {
	v := req.URL.Query()
	url := v.Get("url")
	if url == "" {
		http.Error(w, "Missing parameter 'url'", http.StatusBadRequest)
		return
	}
	painter := NewPainter(v)
	job := NewPaintJob(v)
	// Look for a cached SVG
	key := svgKey(v.Get("style"), job)
	svg, err := cache.SVG.Get(key)
	if err == nil {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(svg.([]byte))
		return
	}
	p, err := palette.New(url, scorer)
	if err != nil {
		http.Error(w, "Unable to create a palette: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "image/svg+xml")
	b := p.Paint(painter, job)
	cache.SVG.Set(key, b)
	w.Write(b)
}

// svgKey creates a key for caching by combining all parameters of a drawing.
func svgKey(style string, job palette.PaintJob) string {
	return fmt.Sprintf("svg:%s %d %d %d", style, job.Width, job.Height, job.Max)
}

type IndexView struct {
	URL      string
	SVGURL   string
	Job      palette.PaintJob
	Painters map[string]palette.Painter
	Style    string
}

// IndexMux returns a ServeMux showing a wizard form for creating SVGs with all available options.
func IndexMux() (m *http.ServeMux) {
	m = http.NewServeMux()
	staticFs := http.FileServer(&assetfs.AssetFS{
		Asset:     Asset,
		AssetDir:  AssetDir,
		AssetInfo: AssetInfo,
		Prefix:    "",
	})
	m.Handle("/static/", staticFs)
	m.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		url := req.URL
		url.Path += "svg"
		url.Host = req.Host
		url.Scheme = "http"

		iv := IndexView{
			req.URL.Query().Get("url"),
			url.String(),
			NewPaintJob(req.URL.Query()),
			palette.Painters,
			req.URL.Query().Get("style"),
		}
		err := tmpl.ExecuteTemplate(w, "index.html", iv)
		if err != nil {
			http.Error(w, "Unable to render template: "+err.Error(), http.StatusInternalServerError)
		}
	})
	return m
}

// NewPaintJob creates a PaintJob based on GET parameters.
func NewPaintJob(v url.Values) palette.PaintJob {
	return palette.PaintJob{
		Max:    parseInt(v.Get("max"), 5, 1, 50),
		Width:  parseInt(v.Get("w"), 512, 8, 4096),
		Height: parseInt(v.Get("h"), 512, 8, 4096),
	}
}

// NewPainter picks a Painter based on the GET parameter "style".
func NewPainter(v url.Values) (p palette.Painter) {
	p, ok := palette.Painters[v.Get("style")]
	if !ok {
		p = &palette.BandPainter{}
	}
	return
}

func parseInt(s string, def, min, max int) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		return def
	}
	if i < min {
		i = min
	} else if i > max {
		i = max
	}
	return i
}
<commit_msg>Extract func NewIndexView<commit_after>// Package http provides HTTP handlers for drawing SVGs.
package http

//go:generate go-bindata -pkg $GOPACKAGE -prefix asset/ asset/...

import (
	"fmt"
	"github.com/elazarl/go-bindata-assetfs"
	"github.com/nochso/colourl/cache"
	"github.com/nochso/colourl/palette"
	"html/template"
	"net/http"
	"net/url"
	"strconv"
)

var scorer = &palette.SumScore{}

var tmpl *template.Template

func init() {
	b, err := Asset("index.html")
	if err != nil {
		panic(err)
	}
	tmpl = template.New("index.html")
	template.Must(tmpl.Parse(string(b)))
}

// SVGHandler returns a SVG based on GET parameters.
// See NewPaintJob() for parsing options.
// See NewPainter() for SVG style / Painter.
func SVGHandler(w http.ResponseWriter, req *http.Request) {
	v := req.URL.Query()
	url := v.Get("url")
	if url == "" {
		http.Error(w, "Missing parameter 'url'", http.StatusBadRequest)
		return
	}
	painter := NewPainter(v)
	job := NewPaintJob(v)
	// Look for a cached SVG
	key := svgKey(v.Get("style"), job)
	svg, err := cache.SVG.Get(key)
	if err == nil {
		w.Header().Set("Content-Type", "image/svg+xml")
		w.Write(svg.([]byte))
		return
	}
	p, err := palette.New(url, scorer)
	if err != nil {
		http.Error(w, "Unable to create a palette: "+err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "image/svg+xml")
	b := p.Paint(painter, job)
	cache.SVG.Set(key, b)
	w.Write(b)
}

// svgKey creates a key for caching by combining all parameters of a drawing.
func svgKey(style string, job palette.PaintJob) string {
	return fmt.Sprintf("svg:%s %d %d %d", style, job.Width, job.Height, job.Max)
}

type IndexView struct {
	URL      string
	SVGURL   string
	Job      palette.PaintJob
	Painters map[string]palette.Painter
	Style    string
}

func NewIndexView(req *http.Request) *IndexView {
	url := req.URL
	url.Path += "svg"
	url.Host = req.Host
	url.Scheme = "http"
	return &IndexView{
		req.URL.Query().Get("url"),
		url.String(),
		NewPaintJob(req.URL.Query()),
		palette.Painters,
		req.URL.Query().Get("style"),
	}
}

// IndexMux returns a ServeMux showing a wizard form for creating SVGs with all available options.
func IndexMux() (m *http.ServeMux) {
	m = http.NewServeMux()
	staticFs := http.FileServer(&assetfs.AssetFS{
		Asset:     Asset,
		AssetDir:  AssetDir,
		AssetInfo: AssetInfo,
		Prefix:    "",
	})
	m.Handle("/static/", staticFs)
	m.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		err := tmpl.ExecuteTemplate(w, "index.html", NewIndexView(req))
		if err != nil {
			http.Error(w, "Unable to render template: "+err.Error(), http.StatusInternalServerError)
		}
	})
	return m
}

// NewPaintJob creates a PaintJob based on GET parameters.
func NewPaintJob(v url.Values) palette.PaintJob {
	return palette.PaintJob{
		Max:    parseInt(v.Get("max"), 5, 1, 50),
		Width:  parseInt(v.Get("w"), 512, 8, 4096),
		Height: parseInt(v.Get("h"), 512, 8, 4096),
	}
}

// NewPainter picks a Painter based on the GET parameter "style".
func NewPainter(v url.Values) (p palette.Painter) {
	p, ok := palette.Painters[v.Get("style")]
	if !ok {
		p = &palette.BandPainter{}
	}
	return
}

func parseInt(s string, def, min, max int) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		return def
	}
	if i < min {
		i = min
	} else if i > max {
		i = max
	}
	return i
}
<|endoftext|>


<commit_before>// This is a reverse proxy that's useful for keeping persisten connections
// with the backend/load balancer.
// Configuration example:
//
// [[relayer]]
// protocol = "http"
// usebufferpool = true
// listen = ":9000"
// url = "http://getrates.aws.dotw.com"
// maxIdleConnections = 40
// timeout = 15
// compress = false

package http

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"time"

	"github.com/gallir/smart-relayer/lib"
)

const (
	defaultBytePoolSize = 128
)

// Pool is used, if enabled, as a pool for the reverse proxy
type Pool struct {
	pool sync.Pool
}

type HttpProxy struct {
	sync.Mutex
	target     *url.URL
	listen     string
	proxy      *httputil.ReverseProxy
	transport  *http.Transport
	server     *http.Server
	bufferPool Pool
	done       chan bool
}

// New returns a new http reverse proxy
func New(c lib.RelayerConfig, done chan bool) (*HttpProxy, error) {
	s := &HttpProxy{
		transport: &http.Transport{},
		done:      done,
	}

	s.proxy = &httputil.ReverseProxy{
		Director:  s.director,
		Transport: s.transport,
	}

	err := s.Reload(&c)
	if err != nil {
		log.Println("E: Error to", s.target, err)
		return nil, err
	}

	return s, nil
}

// Reload changes the configuration of the proxy
func (s *HttpProxy) Reload(c *lib.RelayerConfig) error {
	s.Lock()
	defer s.Unlock()

	s.listen = c.Listen

	if !c.Compress {
		s.transport.DisableCompression = true
	}

	if c.MaxIdleConnections > 0 {
		s.transport.MaxIdleConnsPerHost = c.MaxIdleConnections
		s.transport.MaxIdleConns = c.MaxIdleConnections
	} else {
		s.transport.DisableKeepAlives = true
	}

	if c.Timeout > 0 {
		s.transport.IdleConnTimeout = time.Duration(c.Timeout) * time.Second
	}

	if c.UseBufferPool {
		s.proxy.BufferPool = &s.bufferPool
	} else {
		s.proxy.BufferPool = nil
	}

	t, err := url.Parse(c.URL)
	if err != nil {
		return err
	}
	s.target = t

	if s.server != nil {
		s.transport.CloseIdleConnections()
	}

	return nil
}

// Start starts the http server in the local port/socket
func (s *HttpProxy) Start() error {
	if s.server == nil {
		s.server = &http.Server{
			Addr:           s.listen,
			Handler:        s.proxy,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}
	}
	go s.serve()
	return nil
}

// Exit closes the listener and send done to main
func (s *HttpProxy) Exit() {
	s.Lock()
	defer s.Unlock()
	s.server.Close()
	s.transport.CloseIdleConnections()
	s.done <- true
}

func (s *HttpProxy) serve() {
	err := s.server.ListenAndServe()
	log.Println("E: Error in http server", s.listen, err)
	s.Lock()
	s.server = nil
	s.Unlock()
}

func (s *HttpProxy) director(req *http.Request) {
	req.URL.Scheme = s.target.Scheme
	req.URL.Host = s.target.Host
	if s.target.Path != "" {
		req.URL.Path = s.target.Path + req.URL.Path
	}
	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

// Get returns a []byte from the Pool
func (p *Pool) Get() []byte {
	b := p.pool.Get()
	if b == nil {
		return make([]byte, 0, defaultBytePoolSize)
	}
	return b.([]byte)
}

// Put a []byte into the pool
func (p *Pool) Put(b []byte) {
	p.pool.Put(b)
}
<commit_msg>Added max connection throttler<commit_after>// This is a reverse proxy that's useful for keeping persisten connections
// with the backend/load balancer.
// Configuration example:
//
// [[relayer]]
// protocol = "http"
// usebufferpool = true
// listen = ":9000"
// url = "http://getrates.aws.dotw.com"
// maxIdleConnections = 40
// timeout = 15
// compress = false

package http

import (
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
	"sync"
	"time"

	"github.com/gallir/smart-relayer/lib"
)

const (
	defaultBytePoolSize = 128
)

// Pool is used, if enabled, as a pool for the reverse proxy
type Pool struct {
	pool sync.Pool
}

type HttpProxy struct {
	sync.Mutex
	target     *url.URL
	listen     string
	limiter    *limitHandler
	proxy      *httputil.ReverseProxy
	transport  *http.Transport
	server     *http.Server
	bufferPool Pool
	done       chan bool
}

// New returns a new http reverse proxy
func New(c lib.RelayerConfig, done chan bool) (*HttpProxy, error) {
	s := &HttpProxy{
		transport: &http.Transport{},
		done:      done,
	}

	s.proxy = &httputil.ReverseProxy{
		Director:  s.director,
		Transport: s.transport,
	}

	s.limiter = newLimitHandler(s.proxy)

	err := s.Reload(&c)
	if err != nil {
		log.Println("E: Error to", s.target, err)
		return nil, err
	}

	return s, nil
}

// Reload changes the configuration of the proxy
func (s *HttpProxy) Reload(c *lib.RelayerConfig) error {
	s.Lock()
	defer s.Unlock()

	s.listen = c.Listen

	s.limiter.MaxConns(c.MaxConnections)

	if !c.Compress {
		s.transport.DisableCompression = true
	}

	if c.MaxIdleConnections > 0 {
		s.transport.MaxIdleConnsPerHost = c.MaxIdleConnections
		s.transport.MaxIdleConns = c.MaxIdleConnections
	} else {
		s.transport.DisableKeepAlives = true
	}

	if c.Timeout > 0 {
		s.transport.IdleConnTimeout = time.Duration(c.Timeout) * time.Second
	}

	if c.UseBufferPool {
		s.proxy.BufferPool = &s.bufferPool
	} else {
		s.proxy.BufferPool = nil
	}

	t, err := url.Parse(c.URL)
	if err != nil {
		return err
	}
	s.target = t

	if s.server != nil {
		s.transport.CloseIdleConnections()
	}

	return nil
}

// Start starts the http server in the local port/socket
func (s *HttpProxy) Start() error {
	if s.server == nil {
		s.server = &http.Server{
			Addr:           s.listen,
			Handler:        s.limiter,
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}
	}
	go s.serve()
	return nil
}

// Exit closes the listener and send done to main
func (s *HttpProxy) Exit() {
	s.Lock()
	defer s.Unlock()
	s.server.Close()
	s.transport.CloseIdleConnections()
	s.done <- true
}

func (s *HttpProxy) serve() {
	err := s.server.ListenAndServe()
	log.Println("E: Error in http server", s.listen, err)
	s.Lock()
	s.server = nil
	s.Unlock()
}

func (s *HttpProxy) director(req *http.Request) {
	req.URL.Scheme = s.target.Scheme
	req.URL.Host = s.target.Host
	if s.target.Path != "" {
		req.URL.Path = s.target.Path + req.URL.Path
	}
	if _, ok := req.Header["User-Agent"]; !ok {
		// explicitly disable User-Agent so it's not set to default value
		req.Header.Set("User-Agent", "")
	}
}

// Get returns a []byte from the Pool
func (p *Pool) Get() []byte {
	b := p.pool.Get()
	if b == nil {
		return make([]byte, 0, defaultBytePoolSize)
	}
	return b.([]byte)
}

// Put a []byte into the pool
func (p *Pool) Put(b []byte) {
	p.pool.Put(b)
}

// Connection limit handler
type limitHandler struct {
	sync.Mutex
	sem     chan struct{}
	handler http.Handler
}

func newLimitHandler(handler http.Handler) *limitHandler {
	h := &limitHandler{
		handler: handler,
	}
	return h
}

func (h *limitHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	h.Lock()
	sem := h.sem
	h.Unlock()

	if sem == nil {
		h.handler.ServeHTTP(w, req)
		return
	}
	sem <- struct{}{}
	h.handler.ServeHTTP(w, req)
	<-sem
}

func (h *limitHandler) MaxConns(maxConns int) {
	h.Lock()
	defer h.Unlock()

	if h.sem != nil && maxConns == cap(h.sem) {
		return
	}

	if maxConns <= 0 {
		if h.sem != nil {
			h.sem = nil
		}
		return
	}

	h.sem = make(chan struct{}, maxConns)

}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatcher

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"golang.org/x/time/rate"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

func dummySendFn(*buffer.Batch) error { return nil }

func noDrop(dropped *buffer.Batch) { panic(fmt.Sprintf("dropping %+v", dropped)) }

func init() {
	verboseTests = testing.Verbose()
}

func TestChannelConstruction(t *testing.T) {
	Convey(`Channel`, t, func() {
		ctx, _ := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)

		Convey(`construction`, func() {

			Convey(`success`, func() {
				ch, err := NewChannel(ctx, nil, dummySendFn)
				So(err, ShouldBeNil)
				close(ch.C)
				<-ch.DrainC
			})

			Convey(`failure`, func() {
				Convey(`bad SendFn`, func() {
					_, err := NewChannel(ctx, nil, nil)
					So(err, ShouldErrLike, "send is required")
				})

				Convey(`bad Options`, func() {
					_, err := NewChannel(ctx, &Options{
						QPSLimit: rate.NewLimiter(100, 0),
					}, dummySendFn)
					So(err, ShouldErrLike, "normalizing dispatcher.Options")
				})

				Convey(`bad Options.Buffer`, func() {
					_, err := NewChannel(ctx, &Options{
						Buffer: buffer.Options{
							BatchSize: -3,
						},
					}, dummySendFn)
					So(err, ShouldErrLike, "allocating Buffer")
				})
			})

		})

	})

}

func TestSerialSenderWithoutDrops(t *testing.T) {
	Convey(`serial world-state sender without drops`, t, func(cvctx C) {
		ctx, tclock := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)

		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []string{}
		enableThisError := false

		ch, err := NewChannel(ctx, &Options{
			DropFn:   noDrop,
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 10},
			},
		}, func(batch *buffer.Batch) (err error) {
			cvctx.So(batch.Data, ShouldHaveLength, 1)
			str := batch.Data[0].(string)
			if enableThisError && str == "This" {
				enableThisError = false
				return errors.New("narp", transient.Tag)
			}
			sentBatches = append(sentBatches, str)
			if str == "test." {
				defaultRetryAmount := buffer.Defaults.Retry().Next(ctx, nil)
				tclock.Set(tclock.Now().Add(defaultRetryAmount))
			}
			return nil
		})
		So(err, ShouldBeNil)
		defer ch.CloseAndDrain(ctx)

		Convey(`no errors`, func() {
			ch.C <- "Hello"
			ch.C <- "World!"
			ch.C <- "This"
			ch.C <- "is"
			ch.C <- "a"
			ch.C <- "test."
			ch.CloseAndDrain(ctx)

			So(sentBatches, ShouldResemble, []string{
				"Hello", "World!",
				"This", "is", "a", "test.",
			})
		})

		Convey(`error and retry`, func() {
			enableThisError = true

			ch.C <- "Hello"
			ch.C <- "World!"
			ch.C <- "This"
			ch.C <- "is"
			ch.C <- "a"
			ch.C <- "test."
			ch.CloseAndDrain(ctx)

			So(sentBatches, ShouldResemble, []string{
				"Hello", "World!",
				"is", "a", "test.", "This",
			})
		})

	})
}

func TestContextShutdown(t *testing.T) {
	Convey(`context cancellation ends channel`, t, func(cvctx C) {
		ctx, _ := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}
		cctx, cancel := context.WithCancel(ctx)

		sentBatches := []string{}
		blockSend := make(chan struct{})

		ch, err := NewChannel(cctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 1},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(string))
			<-blockSend
			return
		})
		So(err, ShouldBeNil)

		ch.C <- "hey"
		select {
		case ch.C <- "blocked":
			panic("channel should have been blocked")
		case <-time.After(time.Millisecond):
			// OK
		}

		cancel()
		ch.CloseAndDrain(ctx)
	})
}

func TestQPSLimit(t *testing.T) {
	Convey(`QPS limited send`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Every(10*time.Millisecond), 1),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(int))
			return
		})
		So(err, ShouldBeNil)

		expected := []int{}

		start := time.Now()
		for i := 0; i < 20; i++ {
			ch.C <- i
			expected = append(expected, i)
		}
		ch.CloseAndDrain(ctx)
		end := time.Now()

		So(sentBatches, ShouldResemble, expected)

		// 20 batches, minus a batch because the QPSLimiter starts with full tokens.
		minThreshold := 19 * 10 * time.Millisecond
		So(end, ShouldHappenAfter, start.Add(minThreshold))
	})
}

func TestQPSLimitParallel(t *testing.T) {
	Convey(`QPS limited send (parallel)`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		var lock sync.Mutex
		sentBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Every(10*time.Millisecond), 10),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    4,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			lock.Lock()
			sentBatches = append(sentBatches, batch.Data[0].(int))
			lock.Unlock()
			return
		})
		So(err, ShouldBeNil)

		start := time.Now()
		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		ch.CloseAndDrain(ctx)
		end := time.Now()

		// We know it should have 20 things, but the order will be pseudo-random
		So(sentBatches, ShouldHaveLength, 20)

		// 20 batches across 4 workers, minus half a batch for sampling error.
		minThreshold := 5*10*time.Millisecond - 5*time.Millisecond

		So(end, ShouldHappenAfter, start.Add(minThreshold))
	})
}

func TestExplicitDrops(t *testing.T) {
	Convey(`explict drops with ErrorFn`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}
		droppedBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			DropFn: func(batch *buffer.Batch) {
				droppedBatches = append(droppedBatches, batch.Data[0].(int))
			},
			ErrorFn: func(batch *buffer.Batch, err error) (retry bool) {
				return false
			},
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			itm := batch.Data[0].(int)
			if itm%2 == 0 {
				err = errors.New("number is even")
			} else {
				sentBatches = append(sentBatches, itm)
			}
			return
		})
		So(err, ShouldBeNil)

		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		ch.CloseAndDrain(ctx)

		So(sentBatches, ShouldResemble, []int{1, 3, 5, 7, 9, 11, 13, 15, 17, 19})
		So(droppedBatches, ShouldResemble, []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18})
	})
}

func TestImplicitDrops(t *testing.T) {
	Convey(`implicit drops with DropOldestBatch`, t, func(cvctx C) {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}
		sendBlocker := make(chan struct{})

		limiter := rate.NewLimiter(rate.Every(10*time.Millisecond), 1)
		ch, err := NewChannel(ctx, &Options{
			QPSLimit: limiter,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.DropOldestBatch{MaxLiveItems: 1},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(int))
			<-sendBlocker
			return
		})
		So(err, ShouldBeNil)
		// Grab the first token; channel can't send until it recharges.
		limiter.Reserve()

		// Stuff a bunch of crap into the channel. We have 10ms to do this until the
		// channel is able to send something. Should be plenty of time (running this
		// on my laptop shows we have 6-7ms to spare with verbose logs).
		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		// At this point we can start draining the channel.
		close(ch.C)
		// then unblock the sender
		close(sendBlocker)
		// Then wait for the channel to drain
		<-ch.DrainC

		// We should only have seen one batch actually sent.
		So(sentBatches, ShouldHaveLength, 1)
	})
}
<commit_msg>[sync/dispatcher] Bump time threshold on TestImplicitDrops.<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatcher

import (
	"context"
	"fmt"
	"sync"
	"testing"
	"time"

	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"golang.org/x/time/rate"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

func dummySendFn(*buffer.Batch) error { return nil }

func noDrop(dropped *buffer.Batch) { panic(fmt.Sprintf("dropping %+v", dropped)) }

func init() {
	verboseTests = testing.Verbose()
}

func TestChannelConstruction(t *testing.T) {
	Convey(`Channel`, t, func() {
		ctx, _ := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)

		Convey(`construction`, func() {

			Convey(`success`, func() {
				ch, err := NewChannel(ctx, nil, dummySendFn)
				So(err, ShouldBeNil)
				close(ch.C)
				<-ch.DrainC
			})

			Convey(`failure`, func() {
				Convey(`bad SendFn`, func() {
					_, err := NewChannel(ctx, nil, nil)
					So(err, ShouldErrLike, "send is required")
				})

				Convey(`bad Options`, func() {
					_, err := NewChannel(ctx, &Options{
						QPSLimit: rate.NewLimiter(100, 0),
					}, dummySendFn)
					So(err, ShouldErrLike, "normalizing dispatcher.Options")
				})

				Convey(`bad Options.Buffer`, func() {
					_, err := NewChannel(ctx, &Options{
						Buffer: buffer.Options{
							BatchSize: -3,
						},
					}, dummySendFn)
					So(err, ShouldErrLike, "allocating Buffer")
				})
			})

		})

	})

}

func TestSerialSenderWithoutDrops(t *testing.T) {
	Convey(`serial world-state sender without drops`, t, func(cvctx C) {
		ctx, tclock := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)

		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []string{}
		enableThisError := false

		ch, err := NewChannel(ctx, &Options{
			DropFn:   noDrop,
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 10},
			},
		}, func(batch *buffer.Batch) (err error) {
			cvctx.So(batch.Data, ShouldHaveLength, 1)
			str := batch.Data[0].(string)
			if enableThisError && str == "This" {
				enableThisError = false
				return errors.New("narp", transient.Tag)
			}
			sentBatches = append(sentBatches, str)
			if str == "test." {
				defaultRetryAmount := buffer.Defaults.Retry().Next(ctx, nil)
				tclock.Set(tclock.Now().Add(defaultRetryAmount))
			}
			return nil
		})
		So(err, ShouldBeNil)
		defer ch.CloseAndDrain(ctx)

		Convey(`no errors`, func() {
			ch.C <- "Hello"
			ch.C <- "World!"
			ch.C <- "This"
			ch.C <- "is"
			ch.C <- "a"
			ch.C <- "test."
			ch.CloseAndDrain(ctx)

			So(sentBatches, ShouldResemble, []string{
				"Hello", "World!",
				"This", "is", "a", "test.",
			})
		})

		Convey(`error and retry`, func() {
			enableThisError = true

			ch.C <- "Hello"
			ch.C <- "World!"
			ch.C <- "This"
			ch.C <- "is"
			ch.C <- "a"
			ch.C <- "test."
			ch.CloseAndDrain(ctx)

			So(sentBatches, ShouldResemble, []string{
				"Hello", "World!",
				"is", "a", "test.", "This",
			})
		})

	})
}

func TestContextShutdown(t *testing.T) {
	Convey(`context cancellation ends channel`, t, func(cvctx C) {
		ctx, _ := testclock.UseTime(context.Background(), testclock.TestRecentTimeUTC)
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}
		cctx, cancel := context.WithCancel(ctx)

		sentBatches := []string{}
		blockSend := make(chan struct{})

		ch, err := NewChannel(cctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 1},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(string))
			<-blockSend
			return
		})
		So(err, ShouldBeNil)

		ch.C <- "hey"
		select {
		case ch.C <- "blocked":
			panic("channel should have been blocked")
		case <-time.After(time.Millisecond):
			// OK
		}

		cancel()
		ch.CloseAndDrain(ctx)
	})
}

func TestQPSLimit(t *testing.T) {
	Convey(`QPS limited send`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Every(10*time.Millisecond), 1),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(int))
			return
		})
		So(err, ShouldBeNil)

		expected := []int{}

		start := time.Now()
		for i := 0; i < 20; i++ {
			ch.C <- i
			expected = append(expected, i)
		}
		ch.CloseAndDrain(ctx)
		end := time.Now()

		So(sentBatches, ShouldResemble, expected)

		// 20 batches, minus a batch because the QPSLimiter starts with full tokens.
		minThreshold := 19 * 10 * time.Millisecond
		So(end, ShouldHappenAfter, start.Add(minThreshold))
	})
}

func TestQPSLimitParallel(t *testing.T) {
	Convey(`QPS limited send (parallel)`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		var lock sync.Mutex
		sentBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Every(10*time.Millisecond), 10),
			DropFn:   noDrop,
			Buffer: buffer.Options{
				MaxLeases:    4,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			lock.Lock()
			sentBatches = append(sentBatches, batch.Data[0].(int))
			lock.Unlock()
			return
		})
		So(err, ShouldBeNil)

		start := time.Now()
		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		ch.CloseAndDrain(ctx)
		end := time.Now()

		// We know it should have 20 things, but the order will be pseudo-random
		So(sentBatches, ShouldHaveLength, 20)

		// 20 batches across 4 workers, minus half a batch for sampling error.
		minThreshold := 5*10*time.Millisecond - 5*time.Millisecond

		So(end, ShouldHappenAfter, start.Add(minThreshold))
	})
}

func TestExplicitDrops(t *testing.T) {
	Convey(`explict drops with ErrorFn`, t, func() {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}
		droppedBatches := []int{}

		ch, err := NewChannel(ctx, &Options{
			QPSLimit: rate.NewLimiter(rate.Inf, 0),
			DropFn: func(batch *buffer.Batch) {
				droppedBatches = append(droppedBatches, batch.Data[0].(int))
			},
			ErrorFn: func(batch *buffer.Batch, err error) (retry bool) {
				return false
			},
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.BlockNewItems{MaxItems: 20},
			},
		}, func(batch *buffer.Batch) (err error) {
			itm := batch.Data[0].(int)
			if itm%2 == 0 {
				err = errors.New("number is even")
			} else {
				sentBatches = append(sentBatches, itm)
			}
			return
		})
		So(err, ShouldBeNil)

		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		ch.CloseAndDrain(ctx)

		So(sentBatches, ShouldResemble, []int{1, 3, 5, 7, 9, 11, 13, 15, 17, 19})
		So(droppedBatches, ShouldResemble, []int{0, 2, 4, 6, 8, 10, 12, 14, 16, 18})
	})
}

func TestImplicitDrops(t *testing.T) {
	Convey(`implicit drops with DropOldestBatch`, t, func(cvctx C) {
		ctx := context.Background() // uses real time!
		if testing.Verbose() {
			ctx = logging.SetLevel(gologger.StdConfig.Use(ctx), logging.Debug)
		}

		sentBatches := []int{}
		sendBlocker := make(chan struct{})

		limiter := rate.NewLimiter(rate.Every(100*time.Millisecond), 1)
		ch, err := NewChannel(ctx, &Options{
			QPSLimit: limiter,
			Buffer: buffer.Options{
				MaxLeases:    1,
				BatchSize:    1,
				FullBehavior: &buffer.DropOldestBatch{MaxLiveItems: 1},
			},
		}, func(batch *buffer.Batch) (err error) {
			sentBatches = append(sentBatches, batch.Data[0].(int))
			<-sendBlocker
			return
		})
		So(err, ShouldBeNil)
		// Grab the first token; channel can't send until it recharges.
		limiter.Reserve()

		// Stuff a bunch of crap into the channel. We have 100ms to do this until
		// the channel is able to send something. Should be plenty of time (running
		// this on my laptop takes 3-4ms with verbose logs).
		for i := 0; i < 20; i++ {
			ch.C <- i
		}
		// At this point we can start draining the channel.
		close(ch.C)
		// then unblock the sender
		close(sendBlocker)
		// Then wait for the channel to drain
		<-ch.DrainC

		// We should only have seen one batch actually sent.
		So(sentBatches, ShouldHaveLength, 1)
	})
}
<|endoftext|>


<commit_before>package presence

import (
	"errors"
	"fmt"
	"os"
	"testing"
	"time"
)

func initPresence() (*Session, error) {
	connStr := os.Getenv("REDIS_URI")
	if connStr == "" {
		connStr = "localhost:6379"
	}

	backend, err := NewRedis(connStr, 10, time.Second*1)
	if err != nil {
		return nil, err
	}

	ses, err := New(backend)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func withConn(f func(s *Session)) error {
	s, err := initPresence()
	if err != nil {
		return err
	}

	f(s)

	return s.Close()
}

func TestSinglePing(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id"); err != nil {
			t.Fatal(err)
		}
	})

	if err != nil {
		t.Fatal(err)
	}

}

func TestMultiPing(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id", "id2"); err != nil {
			t.Fatal(err)
		}
	})

	if err != nil {
		t.Fatal(err)
	}

}

func TestOnlineStatus(t *testing.T) {
	err := withConn(func(s *Session) {
		id := "id3"
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]

		if res.Status != Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOfflineStatus(t *testing.T) {
	err := withConn(func(s *Session) {

		id := "id4"
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status("id5")
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestMultiStatusAllOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id6", "id7"); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status([]string{"id6", "id7"}...)
		if err != nil {
			t.Fatal(err)
		}
		for _, res := range status {
			if res.Status != Online {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestMultiStatusAllOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id8", "id9"); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status([]string{"id10", "id11"}...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Offline {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusWithTimeout(t *testing.T) {
	err := withConn(func(s *Session) {
		id := "12"
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}
		time.Sleep(time.Second * 2)
		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status == Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestSubscriptions(t *testing.T) {
	t.Skip("Skipped to travis")
	s, err := initPresence()
	if err != nil {
		t.Fatal(err)
	}

	// wait for all keys to expire
	time.Sleep(time.Second * 1)

	id1 := "13"
	id2 := "14"
	id3 := "15"

	time.AfterFunc(time.Second*5, func() {
		err := s.Close()
		if err != nil {
			t.Fatal(err)
		}
	})

	time.AfterFunc(time.Second*1, func() {
		err := s.Online(id1, id2, id3)
		if err != nil {
			t.Fatal(err)
		}
		// err = s.Offline(id1, id2, id3)
		// if err != nil {
		//  t.Fatal(err)
		// }
	})

	onlineCount := 0
	offlineCount := 0
	for event := range s.ListenStatusChanges() {
		switch event.Status {
		case Online:
			onlineCount++
		case Offline:
			offlineCount++
		}
	}

	if onlineCount != 3 {
		t.Fatal(
			fmt.Errorf("online count should be 3 it is %d", onlineCount),
		)
	}

	if offlineCount != 3 {
		t.Fatal(
			fmt.Errorf("offline count should be 3 it is %d", offlineCount),
		)
	}
}

func TestJustMultiOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Offline("id16", "id17"); err != nil {
			t.Fatal(err)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestMultiOnlineAndOfflineTogether(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id18", "id19"); err != nil {
			t.Fatal(err)
		}
		if err := s.Offline("id18", "id19"); err != nil {
			t.Fatal(err)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestMultiOfflineWithMultiStatus(t *testing.T) {
	err := withConn(func(s *Session) {
		if err := s.Online("id20", "id21"); err != nil {
			t.Fatal(err)
		}
		if err := s.Offline("id20", "id21"); err != nil {
			t.Fatal(err)
		}
		status, err := s.Status([]string{"id20", "id21"}...)
		if err != nil {
			t.Fatal(err)
		}

		for _, st := range status {
			if st.Status != Offline {
				t.Fatal(errors.New("user should be offline"))
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>Redis: get ids from a singleton<commit_after>package presence

import (
	"fmt"
	"os"
	"strconv"
	"testing"
	"time"
)

var nextId chan string
var testTimeoutDuration = time.Second * 1

func init() {
	nextId = make(chan string)
	go func() {
		id := 0
		for {
			id++
			nextId <- "id" + strconv.Itoa(id)
		}
	}()
}

func initPresence() (*Session, error) {
	connStr := os.Getenv("REDIS_URI")
	if connStr == "" {
		connStr = "localhost:6379"
	}

	backend, err := NewRedis(connStr, 10, testTimeoutDuration)
	if err != nil {
		return nil, err
	}

	// adjust config for redis instance
	c := backend.(*Redis).redis.Pool().Get()
	if _, err := c.Do("CONFIG", "SET", "notify-keyspace-events", "Ex$"); err != nil {
		return nil, err
	}

	ses, err := New(backend)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func withConn(f func(s *Session)) error {
	s, err := initPresence()
	if err != nil {
		return err
	}

	f(s)

	return s.Close()
}

func TestOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatalf("non existing id can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(id); err != nil {
			t.Fatalf("existing id can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOnlineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Online(ids...); err != nil {
			t.Fatalf("non existing ids can be set as online, but got err: %s", err.Error())
		}

		if err := s.Online(ids...); err != nil {
			t.Fatalf("existing ids can be set as online again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatalf("non existing id can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(id); err != nil {
			t.Fatalf("existing id can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestOfflineMultiple(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		if err := s.Offline(ids...); err != nil {
			t.Fatalf("non existing ids can be set as offline, but got err: %s", err.Error())
		}

		if err := s.Offline(ids...); err != nil {
			t.Fatalf("existing ids can be set as offline again, but got err: %s", err.Error())
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]

		if res.Status != Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOffline(t *testing.T) {
	err := withConn(func(s *Session) {

		id := <-nextId
		if err := s.Offline(id); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOnline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}
		// mark all of them as online first
		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Online {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiAllOffline(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		// mark all of them as offline first
		if err := s.Offline(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for _, res := range status {
			if res.Status != Offline {
				t.Fatalf("%s should be %s, but it is %s", res.ID, Offline, res.Status)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusMultiMixed(t *testing.T) {
	err := withConn(func(s *Session) {
		onlineId := <-nextId
		offlineId := <-nextId

		ids := []string{onlineId, offlineId}

		if err := s.Online(onlineId); err != nil {
			t.Fatal(err)
		}

		if err := s.Offline(offlineId); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if status[0].Status != Online {
			t.Fatalf("%s should be %s, but it is %s", status[0].ID, Online, status[0].Status)
		}
		if status[1].Status != Offline {
			t.Fatalf("%s should be %s, but it is %s", status[1].ID, Offline, status[0].Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusOrder(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		for i := range status {
			if status[i].ID != ids[i] {
				t.Fatalf("%dth status should be %s, but it is %s", i, ids[i], status[i].ID)
			}
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusLen(t *testing.T) {
	err := withConn(func(s *Session) {
		ids := []string{<-nextId, <-nextId}

		if err := s.Online(ids...); err != nil {
			t.Fatal(err)
		}

		status, err := s.Status(ids...)
		if err != nil {
			t.Fatal(err)
		}

		if len(status) != len(ids) {
			t.Fatalf("Status response len should be: %d, but got: %d", len(ids), len(status))
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestStatusWithTimeout(t *testing.T) {
	err := withConn(func(s *Session) {
		id := <-nextId
		if err := s.Online(id); err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration)

		// get the status of the id
		status, err := s.Status(id)
		if err != nil {
			t.Fatal(err)
		}

		res := status[0]
		if res.Status == Online {
			t.Fatalf("%s should be %s, but it is %s", res.ID, Online, res.Status)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestSubscriptions(t *testing.T) {
	err := withConn(func(s *Session) {

		ids := []string{<-nextId, <-nextId, <-nextId}

		// sleep until expiration
		time.Sleep(testTimeoutDuration)

		onlineCount := 0
		offlineCount := 0
		go func() {
			for event := range s.ListenStatusChanges() {
				switch event.Status {
				case Online:
					onlineCount++
				case Offline:
					offlineCount++
				}
			}
		}()

		err := s.Online(ids...)
		if err != nil {
			t.Fatal(err)
		}

		// sleep until expiration
		time.Sleep(testTimeoutDuration * 2)

		if onlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("online count should be: %d, but it is: %d", len(ids), onlineCount),
			)
		}

		if offlineCount != len(ids) {
			t.Fatal(
				fmt.Errorf("offline count should be: %d, but it is: %d", len(ids), offlineCount),
			)
		}
	})

	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com

package scribe

import (
	"fmt"
	"strings"
)

// Describes the results of a test. The type can be marshaled into a JSON
// string as required.
type TestResult struct {
	Name       string   `json:"name"`       // The name of the test.
	Identifier string   `json:"identifier"` // The identifier for the test.
	Aliases    []string `json:"aliases"`    // Aliases for the test.

	IsError bool   `json:"iserror"` // True of error is encountered during evaluation.
	Error   string `json:"error"`   // Error associated with test.

	MasterResult   bool `json:"masterresult"`   // Master result of test.
	HasTrueResults bool `json:"hastrueresults"` // True if > 0 evaluations resulted in true.

	Results []TestSubResult `json:"results"` // The sub-results for the test.
}

// For a given test, a number of sources can be identified that match the
// criteria. For example, multiple files can be identifier with a given
// filename. Each test tracks individual results for these cases.
type TestSubResult struct {
	Result     bool   `json:"result"`     // The result of evaluation for an identifier source.
	Identifier string `json:"identifier"` // The identifier for the source.
}

// Return test results for a given test. Returns an error if an
// error occured during test preparation or execution.
func GetResults(d *Document, name string) (TestResult, error) {
	t, err := d.getTest(name)
	if err != nil {
		return TestResult{}, err
	}
	if t.err != nil {
		return TestResult{}, t.err
	}
	ret := TestResult{}
	ret.Name = t.Name
	ret.Aliases = t.Aliases
	ret.Identifier = t.Identifier
	if t.err != nil {
		ret.Error = fmt.Sprintf("%v", t.err)
	}
	ret.MasterResult = t.masterResult
	ret.HasTrueResults = t.hasTrueResults
	for _, x := range t.results {
		nr := TestSubResult{}
		nr.Result = x.result
		nr.Identifier = x.criteria.identifier
		ret.Results = append(ret.Results, nr)
	}
	return ret, nil
}

// A helper function to convert Testresult r into a slice of greppable single
// line results. Note that each line returned is not terminated with a line
// feed.
func (r *TestResult) SingleLineResults() []string {
	lns := make([]string, 0)

	rs := "[error]"
	if r.Error == "" {
		if r.MasterResult {
			rs = "[true]"
		} else {
			rs = "[false]"
		}
	}
	buf := fmt.Sprintf("master %v name:\"%v\" hastrue:%v error:\"%v\"", rs, r.Name, r.HasTrueResults, r.Error)
	lns = append(lns, buf)

	for _, x := range r.Results {
		if x.Result {
			rs = "[true]"
		} else {
			rs = "[false]"
		}
		buf := fmt.Sprintf("sub %v name:\"%v\" identifier:\"%v\"", rs, r.Name, x.Identifier)
		lns = append(lns, buf)
	}

	return lns
}

// A helper function to convert TestResult into a human readable result
// suitable for display.
func (r *TestResult) String() string {
	lns := make([]string, 0)
	lns = append(lns, fmt.Sprintf("result for \"%v\"", r.Name))
	if r.MasterResult {
		lns = append(lns, "\tmaster result: true")
	} else {
		buf := "\tmaster result: false"
		if r.HasTrueResults {
			buf = buf + ", has true results, failure caused by dependency"
		}
		lns = append(lns, buf)
	}
	if r.Error != "" {
		buf := fmt.Sprintf("[error] error: %v", r.Error)
		lns = append(lns, buf)
	}
	for _, x := range r.Results {
		buf := fmt.Sprintf("\t[%v] identifier: \"%v\"", x.Result, x.Identifier)
		lns = append(lns, buf)
	}
	return strings.Join(lns, "\n")
}
<commit_msg>change the way errors are returned for results.<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Aaron Meihm ameihm@mozilla.com

package scribe

import (
	"fmt"
	"strings"
)

// Describes the results of a test. The type can be marshaled into a JSON
// string as required.
type TestResult struct {
	Name       string   `json:"name"`       // The name of the test.
	Identifier string   `json:"identifier"` // The identifier for the test.
	Aliases    []string `json:"aliases"`    // Aliases for the test.

	IsError bool   `json:"iserror"` // True of error is encountered during evaluation.
	Error   string `json:"error"`   // Error associated with test.

	MasterResult   bool `json:"masterresult"`   // Master result of test.
	HasTrueResults bool `json:"hastrueresults"` // True if > 0 evaluations resulted in true.

	Results []TestSubResult `json:"results"` // The sub-results for the test.
}

// For a given test, a number of sources can be identified that match the
// criteria. For example, multiple files can be identifier with a given
// filename. Each test tracks individual results for these cases.
type TestSubResult struct {
	Result     bool   `json:"result"`     // The result of evaluation for an identifier source.
	Identifier string `json:"identifier"` // The identifier for the source.
}

// Return test results for a given test. Returns an error if for
// some reason the results can not be returned.
func GetResults(d *Document, name string) (TestResult, error) {
	t, err := d.getTest(name)
	if err != nil {
		return TestResult{}, err
	}
	ret := TestResult{}
	ret.Name = t.Name
	ret.Aliases = t.Aliases
	ret.Identifier = t.Identifier
	if t.err != nil {
		ret.Error = fmt.Sprintf("%v", t.err)
		ret.IsError = true
		return ret, nil
	}
	ret.MasterResult = t.masterResult
	ret.HasTrueResults = t.hasTrueResults
	for _, x := range t.results {
		nr := TestSubResult{}
		nr.Result = x.result
		nr.Identifier = x.criteria.identifier
		ret.Results = append(ret.Results, nr)
	}
	return ret, nil
}

// A helper function to convert Testresult r into a slice of greppable single
// line results. Note that each line returned is not terminated with a line
// feed.
func (r *TestResult) SingleLineResults() []string {
	lns := make([]string, 0)

	rs := "[error]"
	if !r.IsError {
		if r.MasterResult {
			rs = "[true]"
		} else {
			rs = "[false]"
		}
	}
	buf := fmt.Sprintf("master %v name:\"%v\" hastrue:%v error:\"%v\"", rs, r.Name, r.HasTrueResults, r.Error)
	lns = append(lns, buf)

	for _, x := range r.Results {
		if x.Result {
			rs = "[true]"
		} else {
			rs = "[false]"
		}
		buf := fmt.Sprintf("sub %v name:\"%v\" identifier:\"%v\"", rs, r.Name, x.Identifier)
		lns = append(lns, buf)
	}

	return lns
}

// A helper function to convert TestResult into a human readable result
// suitable for display.
func (r *TestResult) String() string {
	lns := make([]string, 0)
	lns = append(lns, fmt.Sprintf("result for \"%v\"", r.Name))
	if r.MasterResult {
		lns = append(lns, "\tmaster result: true")
	} else {
		buf := "\tmaster result: false"
		if r.HasTrueResults {
			buf = buf + ", has true results, failure caused by dependency"
		}
		lns = append(lns, buf)
	}
	if r.IsError {
		buf := fmt.Sprintf("\t[error] error: %v", r.Error)
		lns = append(lns, buf)
	}
	for _, x := range r.Results {
		buf := fmt.Sprintf("\t[%v] identifier: \"%v\"", x.Result, x.Identifier)
		lns = append(lns, buf)
	}
	return strings.Join(lns, "\n")
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) Mainflux
 *
 * Mainflux server is licensed under an Apache license, version 2.0.
 * All rights not explicitly granted in the Apache license, version 2.0 are reserved.
 * See the included LICENSE file for more details.
 */

package models

import (
	"github.com/krylovsk/gosenml"
)

type (
	Channel struct {
		Id      string `json:"id"`
		Device  string `json:"device,omitempty"`
		Created string `json:"created"`
		Updated string `json:"updated"`

		Values  []gosenml.Entry `json:"values"`

		Metadata  map[string]interface{} `json:"metadata"`
	}
)
<commit_msg>Add channel name and unit (used as bn and bu)<commit_after>/**
 * Copyright (c) Mainflux
 *
 * Mainflux server is licensed under an Apache license, version 2.0.
 * All rights not explicitly granted in the Apache license, version 2.0 are reserved.
 * See the included LICENSE file for more details.
 */

package models

import (
	"github.com/krylovsk/gosenml"
)

type (
	Channel struct {
		Id      string `json:"id"`
		Device  string `json:"device"`

		// Name is optional. If present, it is pre-pended to `bn` member of SenML.
		Name	string `json:"name"`
		// Unit is optional. If present, it is pre-pended to `bu` member of SenML.
		Unit	string `json:"unit"`

		Values  []gosenml.Entry `json:"values"`

		Created string `json:"created"`
		Updated string `json:"updated"`

		Metadata  map[string]interface{} `json:"metadata"`
	}
)
<|endoftext|>


<commit_before>package zip

import (
	"archive/zip"
	"bytes"
	"io"
	"io/ioutil"
	"strings"

	"github.com/hidez8891/albero-server/module"
)

func init() {
}

func files(r io.Reader) []string {
	buff, err := ioutil.ReadAll(r)
	if err != nil {
		return nil
	}

	arch, err := zip.NewReader(bytes.NewReader(buff), int64(len(buff)))
	if err != nil {
		return nil
	}

	files := make([]string, 0)
	for _, f := range arch.File {
		path := strings.Replace(f.Name, "\\", "/", -1)
		files = append(files, path)
	}

	return files
}

func read(r io.Reader, vpath string) *File {
	buff, err := ioutil.ReadAll(r)
	if err != nil {
		return nil
	}

	arch, err := zip.NewReader(bytes.NewReader(buff), int64(len(buff)))
	if err != nil {
		return nil
	}

	data := make([]byte, 0)
	files := make([]string, 0)
	for _, f := range arch.File {
		path := strings.Replace(f.Name, "\\", "/", -1)

		if path == vpath {
			rr, err := f.Open()
			if err != nil {
				return nil
			}
			defer rr.Close()

			data, err := ioutil.ReadAll(rr)
			if err != nil {
				return nil
			}

			break
		}
	}

	if len(data) == 0 {
		return nil
	}

	file := &module.File{
		Data: data,
		Mime: "",
		Size: len(data),
	}

	return file
}
<commit_msg>fix: zip.init() の実装忘れ<commit_after>package zip

import (
	"archive/zip"
	"bytes"
	"io"
	"io/ioutil"
	"strings"

	"github.com/hidez8891/albero-server/module"
)

func init() {
	module.RegisterArchModule([]string{"zip"}, files, read)
}

func files(r io.Reader) []string {
	buff, err := ioutil.ReadAll(r)
	if err != nil {
		return nil
	}

	arch, err := zip.NewReader(bytes.NewReader(buff), int64(len(buff)))
	if err != nil {
		return nil
	}

	files := make([]string, 0)
	for _, f := range arch.File {
		path := strings.Replace(f.Name, "\\", "/", -1)
		files = append(files, path)
	}

	return files
}

func read(r io.Reader, vpath string) *File {
	buff, err := ioutil.ReadAll(r)
	if err != nil {
		return nil
	}

	arch, err := zip.NewReader(bytes.NewReader(buff), int64(len(buff)))
	if err != nil {
		return nil
	}

	data := make([]byte, 0)
	files := make([]string, 0)
	for _, f := range arch.File {
		path := strings.Replace(f.Name, "\\", "/", -1)

		if path == vpath {
			rr, err := f.Open()
			if err != nil {
				return nil
			}
			defer rr.Close()

			data, err := ioutil.ReadAll(rr)
			if err != nil {
				return nil
			}

			break
		}
	}

	if len(data) == 0 {
		return nil
	}

	file := &module.File{
		Data: data,
		Mime: "",
		Size: len(data),
	}

	return file
}
<|endoftext|>


<commit_before>package weed_server

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	filenamePath "path"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  int64  `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, auth security.EncodedJwt, err error) {

	stats.FilerRequestCounter.WithLabelValues("assign").Inc()
	start := time.Now()
	defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }()

	ar := &operation.VolumeAssignRequest{
		Count:       1,
		Replication: replication,
		Collection:  collection,
		Ttl:         r.URL.Query().Get("ttl"),
		DataCenter:  dataCenter,
	}
	var altRequest *operation.VolumeAssignRequest
	if dataCenter != "" {
		altRequest = &operation.VolumeAssignRequest{
			Count:       1,
			Replication: replication,
			Collection:  collection,
			Ttl:         r.URL.Query().Get("ttl"),
			DataCenter:  "",
		}
	}

	assignResult, ae := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, ar, altRequest)
	if ae != nil {
		glog.Errorf("failing to assign a file id: %v", ae)
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		err = ae
		return
	}
	fileId = assignResult.Fid
	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
	auth = assignResult.Auth
	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	ctx := context.Background()

	query := r.URL.Query()
	collection, replication := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication"))
	dataCenter := query.Get("dataCenter")
	if dataCenter == "" {
		dataCenter = fs.option.DataCenter
	}

	if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter); autoChunked {
		return
	}

	if fs.option.Cipher {
		reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter)
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else if reply != nil {
			writeJsonQuiet(w, r, http.StatusCreated, reply)
		}

		return
	}

	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter)

	if err != nil || fileId == "" || urlLocation == "" {
		glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter))
		return
	}

	glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation)

	u, _ := url.Parse(urlLocation)

	// This allows a client to generate a chunk manifest and submit it to the filer -- it is a little off
	// because they need to provide FIDs instead of file paths...
	cm, _ := strconv.ParseBool(query.Get("cm"))
	if cm {
		q := u.Query()
		q.Set("cm", "true")
		u.RawQuery = q.Encode()
	}
	glog.V(4).Infoln("post to", u)

	ret, err := fs.uploadToVolumeServer(r, u, auth, w, fileId)
	if err != nil {
		return
	}

	if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, fileId); err != nil {
		return
	}

	// send back post result
	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  int64(ret.Size),
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	setEtag(w, ret.ETag)
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// update metadata in filer store
func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter,
	replication string, collection string, ret *operation.UploadResult, fileId string) (err error) {

	stats.FilerRequestCounter.WithLabelValues("postStoreWrite").Inc()
	start := time.Now()
	defer func() {
		stats.FilerRequestHistogram.WithLabelValues("postStoreWrite").Observe(time.Since(start).Seconds())
	}()

	modeStr := r.URL.Query().Get("mode")
	if modeStr == "" {
		modeStr = "0660"
	}
	mode, err := strconv.ParseUint(modeStr, 8, 32)
	if err != nil {
		glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
		mode = 0660
	}

	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		}
	}
	existingEntry, err := fs.filer.FindEntry(ctx, filer2.FullPath(path))
	crTime := time.Now()
	if err == nil && existingEntry != nil {
		crTime = existingEntry.Crtime
	}
	entry := &filer2.Entry{
		FullPath: filer2.FullPath(path),
		Attr: filer2.Attr{
			Mtime:       time.Now(),
			Crtime:      crTime,
			Mode:        os.FileMode(mode),
			Uid:         OS_UID,
			Gid:         OS_GID,
			Replication: replication,
			Collection:  collection,
			TtlSec:      int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)),
			Mime:        ret.Mime,
		},
		Chunks: []*filer_pb.FileChunk{{
			FileId: fileId,
			Size:   uint64(ret.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   ret.ETag,
		}},
	}
	if entry.Attr.Mime == "" {
		if ext := filenamePath.Ext(path); ext != "" {
			entry.Attr.Mime = mime.TypeByExtension(ext)
		}
	}
	// glog.V(4).Infof("saving %s => %+v", path, entry)
	if dbErr := fs.filer.CreateEntry(ctx, entry, false); dbErr != nil {
		fs.filer.DeleteChunks(entry.Chunks)
		glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
		writeJsonError(w, r, http.StatusInternalServerError, dbErr)
		err = dbErr
		return
	}

	return nil
}

// send request to volume server
func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth security.EncodedJwt, w http.ResponseWriter, fileId string) (ret *operation.UploadResult, err error) {

	stats.FilerRequestCounter.WithLabelValues("postUpload").Inc()
	start := time.Now()
	defer func() { stats.FilerRequestHistogram.WithLabelValues("postUpload").Observe(time.Since(start).Seconds()) }()

	ret = &operation.UploadResult{}
	hash := md5.New()
	var body = ioutil.NopCloser(io.TeeReader(r.Body, hash))

	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}

	if auth != "" {
		request.Header.Set("Authorization", "BEARER "+string(auth))
	}
	resp, doErr := util.Do(request)
	if doErr != nil {
		glog.Errorf("failing to connect to volume server %s: %v, %+v", r.RequestURI, doErr, r.Method)
		writeJsonError(w, r, http.StatusInternalServerError, doErr)
		err = doErr
		return
	}
	defer func() {
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}()

	respBody, raErr := ioutil.ReadAll(resp.Body)
	if raErr != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, raErr.Error())
		writeJsonError(w, r, http.StatusInternalServerError, raErr)
		err = raErr
		return
	}

	glog.V(4).Infoln("post result", string(respBody))
	unmarshalErr := json.Unmarshal(respBody, &ret)
	if unmarshalErr != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(respBody))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshalErr)
		err = unmarshalErr
		return
	}
	if ret.Error != "" {
		err = errors.New(ret.Error)
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}
	// find correct final path
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			err = fmt.Errorf("can not to write to folder %s without a file name", path)
			fs.filer.DeleteFileByFileId(fileId)
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	// use filer calculated md5 ETag, instead of the volume server crc ETag
	ret.ETag = fmt.Sprintf("%x", hash.Sum(nil))
	return
}

// curl -X DELETE http://localhost:8888/path/to
// curl -X DELETE http://localhost:8888/path/to?recursive=true
// curl -X DELETE http://localhost:8888/path/to?recursive=true&ignoreRecursiveError=true
// curl -X DELETE http://localhost:8888/path/to?recursive=true&skipChunkDeletion=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	isRecursive := r.FormValue("recursive") == "true"
	if !isRecursive && fs.option.recursiveDelete {
		if r.FormValue("recursive") != "false" {
			isRecursive = true
		}
	}
	ignoreRecursiveError := r.FormValue("ignoreRecursiveError") == "true"
	skipChunkDeletion := r.FormValue("skipChunkDeletion") == "true"

	err := fs.filer.DeleteEntryMetaAndData(context.Background(), filer2.FullPath(r.URL.Path), isRecursive, ignoreRecursiveError, !skipChunkDeletion)
	if err != nil {
		glog.V(1).Infoln("deleting", r.URL.Path, ":", err.Error())
		httpStatus := http.StatusInternalServerError
		if err == filer_pb.ErrNotFound {
			httpStatus = http.StatusNotFound
		}
		writeJsonError(w, r, httpStatus, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}

func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string) {
	// default
	collection = fs.option.Collection
	replication = fs.option.DefaultReplication

	// get default collection settings
	if qCollection != "" {
		collection = qCollection
	}
	if qReplication != "" {
		replication = qReplication
	}

	// required by buckets folder
	if strings.HasPrefix(requestURI, fs.filer.DirBucketsPath+"/") {
		bucketAndObjectKey := requestURI[len(fs.filer.DirBucketsPath)+1:]
		t := strings.Index(bucketAndObjectKey, "/")
		if t < 0 {
			collection = bucketAndObjectKey
		}
		if t > 0 {
			collection = bucketAndObjectKey[:t]
		}
		replication = fs.filer.ReadBucketOption(collection)
	}

	return
}
<commit_msg>filer: remove chunk manifest file support<commit_after>package weed_server

import (
	"context"
	"crypto/md5"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	filenamePath "path"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/security"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/util"
)

var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())
)

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  int64  `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
	Fid   string `json:"fid,omitempty"`
	Url   string `json:"url,omitempty"`
}

func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, auth security.EncodedJwt, err error) {

	stats.FilerRequestCounter.WithLabelValues("assign").Inc()
	start := time.Now()
	defer func() { stats.FilerRequestHistogram.WithLabelValues("assign").Observe(time.Since(start).Seconds()) }()

	ar := &operation.VolumeAssignRequest{
		Count:       1,
		Replication: replication,
		Collection:  collection,
		Ttl:         r.URL.Query().Get("ttl"),
		DataCenter:  dataCenter,
	}
	var altRequest *operation.VolumeAssignRequest
	if dataCenter != "" {
		altRequest = &operation.VolumeAssignRequest{
			Count:       1,
			Replication: replication,
			Collection:  collection,
			Ttl:         r.URL.Query().Get("ttl"),
			DataCenter:  "",
		}
	}

	assignResult, ae := operation.Assign(fs.filer.GetMaster(), fs.grpcDialOption, ar, altRequest)
	if ae != nil {
		glog.Errorf("failing to assign a file id: %v", ae)
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		err = ae
		return
	}
	fileId = assignResult.Fid
	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
	auth = assignResult.Auth
	return
}

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	ctx := context.Background()

	query := r.URL.Query()
	collection, replication := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication"))
	dataCenter := query.Get("dataCenter")
	if dataCenter == "" {
		dataCenter = fs.option.DataCenter
	}

	if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter); autoChunked {
		return
	}

	if fs.option.Cipher {
		reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter)
		if err != nil {
			writeJsonError(w, r, http.StatusInternalServerError, err)
		} else if reply != nil {
			writeJsonQuiet(w, r, http.StatusCreated, reply)
		}

		return
	}

	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter)

	if err != nil || fileId == "" || urlLocation == "" {
		glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter))
		return
	}

	glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation)

	u, _ := url.Parse(urlLocation)
	ret, err := fs.uploadToVolumeServer(r, u, auth, w, fileId)
	if err != nil {
		return
	}

	if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, fileId); err != nil {
		return
	}

	// send back post result
	reply := FilerPostResult{
		Name:  ret.Name,
		Size:  int64(ret.Size),
		Error: ret.Error,
		Fid:   fileId,
		Url:   urlLocation,
	}
	setEtag(w, ret.ETag)
	writeJsonQuiet(w, r, http.StatusCreated, reply)
}

// update metadata in filer store
func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter,
	replication string, collection string, ret *operation.UploadResult, fileId string) (err error) {

	stats.FilerRequestCounter.WithLabelValues("postStoreWrite").Inc()
	start := time.Now()
	defer func() {
		stats.FilerRequestHistogram.WithLabelValues("postStoreWrite").Observe(time.Since(start).Seconds())
	}()

	modeStr := r.URL.Query().Get("mode")
	if modeStr == "" {
		modeStr = "0660"
	}
	mode, err := strconv.ParseUint(modeStr, 8, 32)
	if err != nil {
		glog.Errorf("Invalid mode format: %s, use 0660 by default", modeStr)
		mode = 0660
	}

	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		}
	}
	existingEntry, err := fs.filer.FindEntry(ctx, filer2.FullPath(path))
	crTime := time.Now()
	if err == nil && existingEntry != nil {
		crTime = existingEntry.Crtime
	}
	entry := &filer2.Entry{
		FullPath: filer2.FullPath(path),
		Attr: filer2.Attr{
			Mtime:       time.Now(),
			Crtime:      crTime,
			Mode:        os.FileMode(mode),
			Uid:         OS_UID,
			Gid:         OS_GID,
			Replication: replication,
			Collection:  collection,
			TtlSec:      int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)),
			Mime:        ret.Mime,
		},
		Chunks: []*filer_pb.FileChunk{{
			FileId: fileId,
			Size:   uint64(ret.Size),
			Mtime:  time.Now().UnixNano(),
			ETag:   ret.ETag,
		}},
	}
	if entry.Attr.Mime == "" {
		if ext := filenamePath.Ext(path); ext != "" {
			entry.Attr.Mime = mime.TypeByExtension(ext)
		}
	}
	// glog.V(4).Infof("saving %s => %+v", path, entry)
	if dbErr := fs.filer.CreateEntry(ctx, entry, false); dbErr != nil {
		fs.filer.DeleteChunks(entry.Chunks)
		glog.V(0).Infof("failing to write %s to filer server : %v", path, dbErr)
		writeJsonError(w, r, http.StatusInternalServerError, dbErr)
		err = dbErr
		return
	}

	return nil
}

// send request to volume server
func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth security.EncodedJwt, w http.ResponseWriter, fileId string) (ret *operation.UploadResult, err error) {

	stats.FilerRequestCounter.WithLabelValues("postUpload").Inc()
	start := time.Now()
	defer func() { stats.FilerRequestHistogram.WithLabelValues("postUpload").Observe(time.Since(start).Seconds()) }()

	ret = &operation.UploadResult{}
	hash := md5.New()
	var body = ioutil.NopCloser(io.TeeReader(r.Body, hash))

	request := &http.Request{
		Method:        r.Method,
		URL:           u,
		Proto:         r.Proto,
		ProtoMajor:    r.ProtoMajor,
		ProtoMinor:    r.ProtoMinor,
		Header:        r.Header,
		Body:          body,
		Host:          r.Host,
		ContentLength: r.ContentLength,
	}

	if auth != "" {
		request.Header.Set("Authorization", "BEARER "+string(auth))
	}
	resp, doErr := util.Do(request)
	if doErr != nil {
		glog.Errorf("failing to connect to volume server %s: %v, %+v", r.RequestURI, doErr, r.Method)
		writeJsonError(w, r, http.StatusInternalServerError, doErr)
		err = doErr
		return
	}
	defer func() {
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}()

	respBody, raErr := ioutil.ReadAll(resp.Body)
	if raErr != nil {
		glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, raErr.Error())
		writeJsonError(w, r, http.StatusInternalServerError, raErr)
		err = raErr
		return
	}

	glog.V(4).Infoln("post result", string(respBody))
	unmarshalErr := json.Unmarshal(respBody, &ret)
	if unmarshalErr != nil {
		glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(respBody))
		writeJsonError(w, r, http.StatusInternalServerError, unmarshalErr)
		err = unmarshalErr
		return
	}
	if ret.Error != "" {
		err = errors.New(ret.Error)
		glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error)
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}
	// find correct final path
	path := r.URL.Path
	if strings.HasSuffix(path, "/") {
		if ret.Name != "" {
			path += ret.Name
		} else {
			err = fmt.Errorf("can not to write to folder %s without a file name", path)
			fs.filer.DeleteFileByFileId(fileId)
			glog.V(0).Infoln("Can not to write to folder", path, "without a file name!")
			writeJsonError(w, r, http.StatusInternalServerError, err)
			return
		}
	}
	// use filer calculated md5 ETag, instead of the volume server crc ETag
	ret.ETag = fmt.Sprintf("%x", hash.Sum(nil))
	return
}

// curl -X DELETE http://localhost:8888/path/to
// curl -X DELETE http://localhost:8888/path/to?recursive=true
// curl -X DELETE http://localhost:8888/path/to?recursive=true&ignoreRecursiveError=true
// curl -X DELETE http://localhost:8888/path/to?recursive=true&skipChunkDeletion=true
func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	isRecursive := r.FormValue("recursive") == "true"
	if !isRecursive && fs.option.recursiveDelete {
		if r.FormValue("recursive") != "false" {
			isRecursive = true
		}
	}
	ignoreRecursiveError := r.FormValue("ignoreRecursiveError") == "true"
	skipChunkDeletion := r.FormValue("skipChunkDeletion") == "true"

	err := fs.filer.DeleteEntryMetaAndData(context.Background(), filer2.FullPath(r.URL.Path), isRecursive, ignoreRecursiveError, !skipChunkDeletion)
	if err != nil {
		glog.V(1).Infoln("deleting", r.URL.Path, ":", err.Error())
		httpStatus := http.StatusInternalServerError
		if err == filer_pb.ErrNotFound {
			httpStatus = http.StatusNotFound
		}
		writeJsonError(w, r, httpStatus, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}

func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string) {
	// default
	collection = fs.option.Collection
	replication = fs.option.DefaultReplication

	// get default collection settings
	if qCollection != "" {
		collection = qCollection
	}
	if qReplication != "" {
		replication = qReplication
	}

	// required by buckets folder
	if strings.HasPrefix(requestURI, fs.filer.DirBucketsPath+"/") {
		bucketAndObjectKey := requestURI[len(fs.filer.DirBucketsPath)+1:]
		t := strings.Index(bucketAndObjectKey, "/")
		if t < 0 {
			collection = bucketAndObjectKey
		}
		if t > 0 {
			collection = bucketAndObjectKey[:t]
		}
		replication = fs.filer.ReadBucketOption(collection)
	}

	return
}
<|endoftext|>