<commit_before>package plt

import (
	"code.google.com/p/plotinum/vecgfx"
	"fmt"
	"image/color"
	"math"
)

var (
	Black = color.RGBA{A: 255}
	White = color.RGBA{R: 255, G: 255, B: 255, A: 255}
	Red   = color.RGBA{R: 255, A: 255}
	Green = color.RGBA{G: 255, A: 255}
	Blue  = color.RGBA{B: 255, A: 255}
)

// A DrawArea is a vector graphics canvas along with
// an associated Rect defining a section of the canvas
// to which drawing should take place.
type DrawArea struct {
	vecgfx.Canvas
	font vecgfx.Font
	Rect
}

// Center returns the center Point of the area
func (da *DrawArea) Center() Point {
	return Point{
		X: (da.Max().X-da.Min.X)/2 + da.Min.X,
		Y: (da.Max().Y-da.Min.Y)/2 + da.Min.Y,
	}
}

// crop returns a new DrawArea corresponding to the receiver
// area with the given number of inches added to each
// point of the area's Rect[.
func (da *DrawArea) crop(minx, miny, szx, szy float64) *DrawArea {
	return &DrawArea{
		vecgfx.Canvas: vecgfx.Canvas(da),
		font:          da.font,
		Rect: Rect{
			Min: Point{
				X: da.Rect.Min.X + minx*da.DPI(),
				Y: da.Rect.Min.Y + miny*da.DPI(),
			},
			Sz: Point{
				X: da.Rect.Sz.X + szx*da.DPI(),
				Y: da.Rect.Sz.Y + szy*da.DPI(),
			},
		},
	}
}

// SetTextStyle sets the current text style
func (da *DrawArea) SetTextStyle(sty TextStyle) {
	da.SetColor(sty.Color)
	da.font = sty.Font
}

// TextStyle describes what text will look like.
type TextStyle struct {
	// Color is the text color.
	Color color.Color

	// Font is the font description.
	Font vecgfx.Font
}

// MakeFont returns a font object.
// This function is merely included for convenience so that
// the user doesn't have to import the vecgfx package.
func MakeFont(name string, size float64) (vecgfx.Font, error) {
	return vecgfx.MakeFont(name, size)
}

// Text fills the text to the drawing area.  The string is created
// using the printf-style format specification and the text is
// located at x + width*fx, y + height*fy, where width and height
// are the width and height of the rendered string.
func (da *DrawArea) Text(x, y, fx, fy float64, f string, v ...interface{}) {
	if da.font.Font() == nil {
		panic("Drawing text without a current font set")
	}
	str := fmt.Sprintf(f, v...)
	w := da.font.Width(str) / vecgfx.PtInch * da.DPI()
	h := da.font.Extents().Ascent / vecgfx.PtInch * da.DPI()
	da.FillText(da.font, x+w*fx, y+h*fy, str)
}

// SetLineStyle sets the current line style
func (da *DrawArea) SetLineStyle(sty LineStyle) {
	da.SetColor(sty.Color)
	da.SetLineWidth(sty.Width * da.DPI())
	var dashDots []float64
	for _, dash := range sty.Dashes {
		dashDots = append(dashDots, dash*da.DPI())
	}
	da.SetLineDash(dashDots, sty.DashOffs*da.DPI())

}

// LineStyle describes what a line will look like.
type LineStyle struct {
	// Color is the color of the line.
	Color color.Color

	// Width is the width of the line in inches.
	Width float64

	Dashes   []float64
	DashOffs float64
}

// Line draws a line connecting the given points.
func (da *DrawArea) Line(pts []Point) {
	if len(pts) == 0 {
		return
	}

	var p vecgfx.Path
	p.Move(pts[0].X, pts[0].Y)
	for _, pt := range pts {
		p.Line(pt.X, pt.Y)
	}
	da.Stroke(p)
}

// ClippedLine draws a line that is clipped at the bounds
// the DrawArea.
func (da *DrawArea) ClippedLine(pts []Point) {
	// clip right
	lines0 := clip(isLeft, Point{da.Max().X, da.Min.Y}, Point{-1, 0}, pts)

	// clip bottom
	var lines1 [][]Point
	for _, line := range lines0 {
		ls := clip(isAbove, Point{da.Min.X, da.Min.Y}, Point{0, -1}, line)
		lines1 = append(lines1, ls...)
	}

	// clip left
	lines0 = lines0[:0]
	for _, line := range lines1 {
		ls := clip(isRight, Point{da.Min.X, da.Min.Y}, Point{1, 0}, line)
		lines0 = append(lines0, ls...)
	}

	// clip top
	lines1 = lines1[:0]
	for _, line := range lines0 {
		ls := clip(isBelow, Point{da.Min.X, da.Max().Y}, Point{0, 1}, line)
		lines1 = append(lines1, ls...)
	}

	for _, l := range lines1 {
		da.Line(l)
	}
	return
}

// clip performs clipping in a single clipping line specified
// by the norm, clip point, and in function.
func clip(in func(Point, Point) bool, clip, norm Point, pts []Point) (lines [][]Point) {
	var l []Point
	for i := 0; i < len(pts); i++ {
		cur, next := pts[i-1], pts[i]
		curIn, nextIn := in(cur, clip), in(next, clip)
		switch {
		case curIn && nextIn:
			l = append(l, cur)

		case curIn && !nextIn:
			l = append(l, cur, isect(cur, next, clip, norm))
			lines = append(lines, l)
			l = []Point{}

		case !curIn && !nextIn:
			// do nothing

		default: // !curIn && nextIn
			l = append(l, isect(cur, next, clip, norm))
		}
		if nextIn && i == len(pts)-1 {
			l = append(l, next)
		}
	}
	if len(l) > 1 {
		lines = append(lines, l)
	}
	return
}

// slop is some slop for floating point equality
const slop = 3e-8 // ≈ √1⁻¹⁵

func isLeft(p, clip Point) bool {
	return p.X <= clip.X+slop
}

func isRight(p, clip Point) bool {
	return p.X >= clip.X-slop
}

func isBelow(p, clip Point) bool {
	return p.Y <= clip.Y+slop
}

func isAbove(p, clip Point) bool {
	return p.Y >= clip.Y-slop
}

// isect returns the intersection of a line p0→p1 with the
// clipping line specified by the clip point and normal.
func isect(p0, p1, clip, norm Point) Point {
	// t = (norm · (p0 - clip)) / (norm · (p0 - p1))
	t := p0.minus(clip).dot(norm) / p0.minus(p1).dot(norm)

	// p = p0 + t*(p1 - p0)
	return p1.minus(p0).scale(t).plus(p0)
}

// CirclePath returns the path of a circle centered at x,y with
// radius r.
func CirclePath(x, y, r float64) (p vecgfx.Path) {
	p.Move(x+r, y)
	p.Arc(x, y, r, 0, 2*math.Pi)
	p.Close()
	return
}

// EqTrianglePath returns the path for an equilateral triangle
// that is circumscribed by a circle centered at x,y with
// radius r.  One point of the triangle is directly above the
// center point of the circle.
func EqTrianglePath(x, y, r float64) (p vecgfx.Path) {
	p.Move(x, y+r)
	p.Line(x+r*math.Cos(math.Pi/6), y-r*math.Sin(math.Pi/6))
	p.Line(x-r*math.Cos(math.Pi/6), y-r*math.Sin(math.Pi/6))
	p.Close()
	return
}

// RectPath returns the path of a rectangle specified by its
// upper left corner, width and height.
func RectPath(r Rect) (p vecgfx.Path) {
	p.Move(r.Min.X, r.Min.Y)
	p.Line(r.Max().X, r.Min.Y)
	p.Line(r.Max().X, r.Max().Y)
	p.Line(r.Min.X, r.Max().Y)
	p.Close()
	return
}

// A Rect represents a rectangular region of 2d space.
type Rect struct {
	Min, Sz Point
}

// Max returns the maxmium x and y values of a Rect.
func (r Rect) Max() Point {
	return Point{
		X: r.Min.X + r.Sz.X,
		Y: r.Min.Y + r.Sz.Y,
	}
}

// A Point is a location in 2d space.
type Point struct {
	X, Y float64
}

// Pt returns a Point with the given x and y values.
func Pt(x, y float64) Point {
	return Point{x, y}
}

// dot returns the dot product of two points.
func (p Point) dot(q Point) float64 {
	return p.X*q.X + p.Y*q.Y
}

// plus returns the component-wise sum of two points.
func (p Point) plus(q Point) Point {
	return Point{p.X + q.X, p.Y + q.Y}
}

// minus returns the component-wise differenec of two points.
func (p Point) minus(q Point) Point {
	return Point{p.X - q.X, p.Y - q.Y}
}

// scale returns the component-wise product of a point and a scalar.
func (p Point) scale(s float64) Point {
	return Point{p.X * s, p.Y * s}
}
<commit_msg>Spelling fix in a comment.<commit_after>package plt

import (
	"code.google.com/p/plotinum/vecgfx"
	"fmt"
	"image/color"
	"math"
)

var (
	Black = color.RGBA{A: 255}
	White = color.RGBA{R: 255, G: 255, B: 255, A: 255}
	Red   = color.RGBA{R: 255, A: 255}
	Green = color.RGBA{G: 255, A: 255}
	Blue  = color.RGBA{B: 255, A: 255}
)

// A DrawArea is a vector graphics canvas along with
// an associated Rect defining a section of the canvas
// to which drawing should take place.
type DrawArea struct {
	vecgfx.Canvas
	font vecgfx.Font
	Rect
}

// Center returns the center Point of the area
func (da *DrawArea) Center() Point {
	return Point{
		X: (da.Max().X-da.Min.X)/2 + da.Min.X,
		Y: (da.Max().Y-da.Min.Y)/2 + da.Min.Y,
	}
}

// crop returns a new DrawArea corresponding to the receiver
// area with the given number of inches added to each
// point of the area's Rect[.
func (da *DrawArea) crop(minx, miny, szx, szy float64) *DrawArea {
	return &DrawArea{
		vecgfx.Canvas: vecgfx.Canvas(da),
		font:          da.font,
		Rect: Rect{
			Min: Point{
				X: da.Rect.Min.X + minx*da.DPI(),
				Y: da.Rect.Min.Y + miny*da.DPI(),
			},
			Sz: Point{
				X: da.Rect.Sz.X + szx*da.DPI(),
				Y: da.Rect.Sz.Y + szy*da.DPI(),
			},
		},
	}
}

// SetTextStyle sets the current text style
func (da *DrawArea) SetTextStyle(sty TextStyle) {
	da.SetColor(sty.Color)
	da.font = sty.Font
}

// TextStyle describes what text will look like.
type TextStyle struct {
	// Color is the text color.
	Color color.Color

	// Font is the font description.
	Font vecgfx.Font
}

// MakeFont returns a font object.
// This function is merely included for convenience so that
// the user doesn't have to import the vecgfx package.
func MakeFont(name string, size float64) (vecgfx.Font, error) {
	return vecgfx.MakeFont(name, size)
}

// Text fills the text to the drawing area.  The string is created
// using the printf-style format specification and the text is
// located at x + width*fx, y + height*fy, where width and height
// are the width and height of the rendered string.
func (da *DrawArea) Text(x, y, fx, fy float64, f string, v ...interface{}) {
	if da.font.Font() == nil {
		panic("Drawing text without a current font set")
	}
	str := fmt.Sprintf(f, v...)
	w := da.font.Width(str) / vecgfx.PtInch * da.DPI()
	h := da.font.Extents().Ascent / vecgfx.PtInch * da.DPI()
	da.FillText(da.font, x+w*fx, y+h*fy, str)
}

// SetLineStyle sets the current line style
func (da *DrawArea) SetLineStyle(sty LineStyle) {
	da.SetColor(sty.Color)
	da.SetLineWidth(sty.Width * da.DPI())
	var dashDots []float64
	for _, dash := range sty.Dashes {
		dashDots = append(dashDots, dash*da.DPI())
	}
	da.SetLineDash(dashDots, sty.DashOffs*da.DPI())

}

// LineStyle describes what a line will look like.
type LineStyle struct {
	// Color is the color of the line.
	Color color.Color

	// Width is the width of the line in inches.
	Width float64

	Dashes   []float64
	DashOffs float64
}

// Line draws a line connecting the given points.
func (da *DrawArea) Line(pts []Point) {
	if len(pts) == 0 {
		return
	}

	var p vecgfx.Path
	p.Move(pts[0].X, pts[0].Y)
	for _, pt := range pts {
		p.Line(pt.X, pt.Y)
	}
	da.Stroke(p)
}

// ClippedLine draws a line that is clipped at the bounds
// the DrawArea.
func (da *DrawArea) ClippedLine(pts []Point) {
	// clip right
	lines0 := clip(isLeft, Point{da.Max().X, da.Min.Y}, Point{-1, 0}, pts)

	// clip bottom
	var lines1 [][]Point
	for _, line := range lines0 {
		ls := clip(isAbove, Point{da.Min.X, da.Min.Y}, Point{0, -1}, line)
		lines1 = append(lines1, ls...)
	}

	// clip left
	lines0 = lines0[:0]
	for _, line := range lines1 {
		ls := clip(isRight, Point{da.Min.X, da.Min.Y}, Point{1, 0}, line)
		lines0 = append(lines0, ls...)
	}

	// clip top
	lines1 = lines1[:0]
	for _, line := range lines0 {
		ls := clip(isBelow, Point{da.Min.X, da.Max().Y}, Point{0, 1}, line)
		lines1 = append(lines1, ls...)
	}

	for _, l := range lines1 {
		da.Line(l)
	}
	return
}

// clip performs clipping in a single clipping line specified
// by the norm, clip point, and in function.
func clip(in func(Point, Point) bool, clip, norm Point, pts []Point) (lines [][]Point) {
	var l []Point
	for i := 0; i < len(pts); i++ {
		cur, next := pts[i-1], pts[i]
		curIn, nextIn := in(cur, clip), in(next, clip)
		switch {
		case curIn && nextIn:
			l = append(l, cur)

		case curIn && !nextIn:
			l = append(l, cur, isect(cur, next, clip, norm))
			lines = append(lines, l)
			l = []Point{}

		case !curIn && !nextIn:
			// do nothing

		default: // !curIn && nextIn
			l = append(l, isect(cur, next, clip, norm))
		}
		if nextIn && i == len(pts)-1 {
			l = append(l, next)
		}
	}
	if len(l) > 1 {
		lines = append(lines, l)
	}
	return
}

// slop is some slop for floating point equality
const slop = 3e-8 // ≈ √1⁻¹⁵

func isLeft(p, clip Point) bool {
	return p.X <= clip.X+slop
}

func isRight(p, clip Point) bool {
	return p.X >= clip.X-slop
}

func isBelow(p, clip Point) bool {
	return p.Y <= clip.Y+slop
}

func isAbove(p, clip Point) bool {
	return p.Y >= clip.Y-slop
}

// isect returns the intersection of a line p0→p1 with the
// clipping line specified by the clip point and normal.
func isect(p0, p1, clip, norm Point) Point {
	// t = (norm · (p0 - clip)) / (norm · (p0 - p1))
	t := p0.minus(clip).dot(norm) / p0.minus(p1).dot(norm)

	// p = p0 + t*(p1 - p0)
	return p1.minus(p0).scale(t).plus(p0)
}

// CirclePath returns the path of a circle centered at x,y with
// radius r.
func CirclePath(x, y, r float64) (p vecgfx.Path) {
	p.Move(x+r, y)
	p.Arc(x, y, r, 0, 2*math.Pi)
	p.Close()
	return
}

// EqTrianglePath returns the path for an equilateral triangle
// that is circumscribed by a circle centered at x,y with
// radius r.  One point of the triangle is directly above the
// center point of the circle.
func EqTrianglePath(x, y, r float64) (p vecgfx.Path) {
	p.Move(x, y+r)
	p.Line(x+r*math.Cos(math.Pi/6), y-r*math.Sin(math.Pi/6))
	p.Line(x-r*math.Cos(math.Pi/6), y-r*math.Sin(math.Pi/6))
	p.Close()
	return
}

// RectPath returns the path of a rectangle specified by its
// upper left corner, width and height.
func RectPath(r Rect) (p vecgfx.Path) {
	p.Move(r.Min.X, r.Min.Y)
	p.Line(r.Max().X, r.Min.Y)
	p.Line(r.Max().X, r.Max().Y)
	p.Line(r.Min.X, r.Max().Y)
	p.Close()
	return
}

// A Rect represents a rectangular region of 2d space.
type Rect struct {
	Min, Sz Point
}

// Max returns the maxmium x and y values of a Rect.
func (r Rect) Max() Point {
	return Point{
		X: r.Min.X + r.Sz.X,
		Y: r.Min.Y + r.Sz.Y,
	}
}

// A Point is a location in 2d space.
type Point struct {
	X, Y float64
}

// Pt returns a Point with the given x and y values.
func Pt(x, y float64) Point {
	return Point{x, y}
}

// dot returns the dot product of two points.
func (p Point) dot(q Point) float64 {
	return p.X*q.X + p.Y*q.Y
}

// plus returns the component-wise sum of two points.
func (p Point) plus(q Point) Point {
	return Point{p.X + q.X, p.Y + q.Y}
}

// minus returns the component-wise difference of two points.
func (p Point) minus(q Point) Point {
	return Point{p.X - q.X, p.Y - q.Y}
}

// scale returns the component-wise product of a point and a scalar.
func (p Point) scale(s float64) Point {
	return Point{p.X * s, p.Y * s}
}
<|endoftext|>


<commit_before>package main

import (
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"
)

const formatValues = "values"
const formatTabSeparated = "tabseparated"

var regexFormat = regexp.MustCompile("(?i)format\\s\\S+(\\s+)")
var regexValues = regexp.MustCompile("(?i)\\svalues\\s")
var regexGetFormat = regexp.MustCompile("(?i)format\\s(\\S+)")

// Table - store query table info
type Table struct {
	Name          string
	Format        string
	Query         string
	Params        string
	Rows          []string
	count         int
	FlushCount    int
	FlushInterval int
	mu            sync.Mutex
	Sender        Sender
	TickerChan    *chan struct{}
	lastUpdate    time.Time
	// todo add Last Error
}

// Collector - query collector
type Collector struct {
	Tables        map[string]*Table
	mu            sync.RWMutex
	Count         int
	FlushInterval int
	CleanInterval int
	RemoveQueryID bool
	Sender        Sender
	TickerChan    *chan struct{}
}

// NewTable - default table constructor
func NewTable(name string, sender Sender, count int, interval int) (t *Table) {
	t = new(Table)
	t.Name = name
	t.Sender = sender
	t.FlushCount = count
	t.FlushInterval = interval
	return t
}

// NewCollector - default collector constructor
func NewCollector(sender Sender, count int, interval int, cleanInterval int, removeQueryID bool) (c *Collector) {
	c = new(Collector)
	c.Sender = sender
	c.Tables = make(map[string]*Table)
	c.Count = count
	c.FlushInterval = interval
	c.CleanInterval = cleanInterval
	c.RemoveQueryID = removeQueryID
	if cleanInterval > 0 {
		c.TickerChan = c.RunTimer()
	}
	return c
}

// Content - get text content of rowsfor query
func (t *Table) Content() string {
	rowDelimiter := "\n"
	if t.Format == "RowBinary" {
		rowDelimiter = ""
	}
	return t.Query + "\n" + strings.Join(t.Rows, rowDelimiter)
}

// Flush - sends collected data in table to clickhouse
func (t *Table) Flush() {
	req := ClickhouseRequest{
		Params:   t.Params,
		Query:    t.Query,
		Content:  t.Content(),
		Count:    len(t.Rows),
		isInsert: true,
	}
	t.Sender.Send(&req)
	t.Rows = make([]string, 0, t.FlushCount)
	t.count = 0
}

// CheckFlush - check if flush is need and sends data to clickhouse
func (t *Table) CheckFlush() bool {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.count > 0 {
		t.Flush()
		return true
	}
	return false
}

// Empty - Checks if table is empty
func (t *Table) Empty() bool {
	return t.GetCount() == 0
}

// GetCount - Checks if table is empty
func (t *Table) GetCount() int {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.count
}

// RunTimer - timer for periodical savings data
func (t *Table) RunTimer() *chan struct{} {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(time.Millisecond * time.Duration(t.FlushInterval))
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				t.CheckFlush()
			case <-done:
				return
			}
		}
	}()
	return &done
}

// Add - Adding query to table
func (t *Table) Add(text string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.count++
	if t.Format == "TabSeparated" {
		t.Rows = append(t.Rows, strings.Split(text, "\n")...)
	} else {
		t.Rows = append(t.Rows, text)
	}
	if len(t.Rows) >= t.FlushCount {
		t.Flush()
	}
	t.lastUpdate = time.Now()
}

// CleanTable - delete table from map
func (t *Table) CleanTable() {
	t.mu.Lock()
	close(*t.TickerChan)
	t = nil
}

// CleanTables - clean unsused tables
func (c *Collector) CleanTables() {
	c.mu.Lock()
	defer c.mu.Unlock()
	for k, t := range c.Tables {
		if t.lastUpdate.Add(time.Duration(c.CleanInterval) * time.Millisecond).Before(time.Now()) {
			// table was not updated for CleanInterval - delete that table - otherwise it can cause memLeak
			t.CleanTable()
			defer delete(c.Tables, k)
		}

	}
}

// RunTimer - timer for periodical cleaning unused tables
func (c *Collector) RunTimer() *chan struct{} {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(time.Duration(c.CleanInterval) * time.Millisecond)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				c.CleanTables()
			case <-done:
				return
			}
		}
	}()
	return &done
}

// Empty - check if all tables are empty
func (c *Collector) Empty() bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	for _, t := range c.Tables {
		if ok := t.Empty(); !ok {
			return false
		}
	}
	return true
}

// FlushAll - flush all tables to clickhouse
func (c *Collector) FlushAll() (count int) {
	c.mu.Lock()
	defer c.mu.Unlock()
	count = 0
	for _, t := range c.Tables {
		if ok := t.CheckFlush(); ok {
			count++
		}

	}
	return count
}

// WaitFlush - wait for flush all tables
func (c *Collector) WaitFlush() (err error) {
	return c.Sender.WaitFlush()
}

// AddTable - adding table to collector
func (c *Collector) AddTable(name string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.addTable(name)
}

func (c *Collector) separateQuery(name string) (query string, params string) {
	items := strings.Split(name, "&")
	for _, p := range items {
		if HasPrefix(p, "query=") {
			query = p[6:]
		} else {
			params += "&" + p
		}
	}
	if len(params) > 0 {
		params = strings.TrimSpace(params[1:])
	}
	q, err := url.QueryUnescape(query)
	if err != nil {
		return "", name
	}
	return q, params
}

func (c *Collector) getFormat(query string) (format string) {
	format = formatValues
	f := regexGetFormat.FindSubmatch([]byte(query))
	if len(f) > 1 {
		format = strings.TrimSpace(string(f[1]))
	}
	return format
}

func (c *Collector) addTable(name string) *Table {
	t := NewTable(name, c.Sender, c.Count, c.FlushInterval)
	query, params := c.separateQuery(name)
	t.Query = query
	t.Params = params
	t.Format = c.getFormat(query)
	c.Tables[name] = t
	t.TickerChan = t.RunTimer()
	t.lastUpdate = time.Now()
	return t
}

// Push - adding query to collector with query params (with query) and rows
func (c *Collector) Push(paramsIn string, content string) {
	// as we are using all params a table key, we have to remove query_id
	params := ""
	if c.RemoveQueryID {
		items := strings.Split(paramsIn, "&")
		for _, p := range items {
			if !HasPrefix(p, "query_id=") {
				//params = strings.ReplaceAll(params, p, "")
				params += "&" + p
			}
		}
		if len(params) > 0 {
			params = strings.TrimSpace(params[1:])
		}
	} else {
		params = paramsIn
	}
	c.mu.RLock()
	table, ok := c.Tables[params]
	if ok {
		table.Add(content)
		c.mu.RUnlock()
		pushCounter.Inc()
		return
	}
	c.mu.RUnlock()
	c.mu.Lock()
	table, ok = c.Tables[params]
	if !ok {
		table = c.addTable(params)
	}
	table.Add(content)
	c.mu.Unlock()
	pushCounter.Inc()
}

// ParseQuery - parsing inbound query to unified format (params/query), content (query data)
func (c *Collector) ParseQuery(queryString string, body string) (params string, content string, insert bool) {
	i := strings.Index(queryString, "query=")
	if i >= 0 {
		if HasPrefix(queryString[i+6:], "insert") {
			insert = true
		}
		var q string
		eoq := strings.Index(queryString[i+6:], "&")
		if eoq >= 0 {
			q = queryString[i+6 : eoq+6]
			params = queryString[:i] + queryString[eoq+7:]
		} else {
			q = queryString[i+6:]
			params = queryString[:i]
		}
		uq, err := url.QueryUnescape(q)
		if body != "" {
			uq += " " + body
		}
		if err != nil {
			return queryString, body, false
		}
		prefix, cnt := c.Parse(uq)
		if strings.HasSuffix(params, "&") || params == "" {
			params += "query=" + url.QueryEscape(strings.TrimSpace(prefix))
		} else {
			params += "&query=" + url.QueryEscape(strings.TrimSpace(prefix))
		}
		content = cnt
	} else {
		var q string
		q, content = c.Parse(body)
		q = strings.TrimSpace(q)
		if HasPrefix(q, "insert") {
			insert = true
		}
		if queryString != "" {
			params = queryString + "&query=" + url.QueryEscape(q)
		} else {
			params = "query=" + url.QueryEscape(q)
		}
	}
	return strings.TrimSpace(params), strings.TrimSpace(content), insert
}

// Parse - parsing text for query and data
func (c *Collector) Parse(text string) (prefix string, content string) {
	i := strings.Index(text, "FORMAT")
	k := strings.Index(text, "VALUES")
	if k == -1 {
		k = strings.Index(text, "values")
	}
	if i >= 0 && i < k {
		w := false
		off := -1
		for c := i + 7; c < len(text); c++ {
			if !w && text[c] != ' ' && text[c] != '\n' && text[c] != ';' {
				w = true
			}
			if w && (text[c] == ' ' || text[c] == '\n' || text[c] == ';') {
				off = c + 1
				break
			}
		}
		if off >= 0 {
			prefix = text[:off]
			content = text[off:]
		}
	} else {
		if k >= 0 {
			prefix = strings.TrimSpace(text[:k+6])
			content = strings.TrimSpace(text[k+6:])
		} else {
			off := regexFormat.FindStringSubmatchIndex(text)
			if len(off) > 3 {
				prefix = text[:off[3]]
				content = text[off[3]:]
			} else {
				off := regexValues.FindStringSubmatchIndex(text)
				if len(off) > 0 {
					prefix = text[:off[1]]
					content = text[off[1]:]
				} else {
					prefix = text
				}
			}
		}
	}
	return prefix, content
}
<commit_msg>add coment about removeQueryID<commit_after>package main

import (
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"
)

const formatValues = "values"
const formatTabSeparated = "tabseparated"

var regexFormat = regexp.MustCompile("(?i)format\\s\\S+(\\s+)")
var regexValues = regexp.MustCompile("(?i)\\svalues\\s")
var regexGetFormat = regexp.MustCompile("(?i)format\\s(\\S+)")

// Table - store query table info
type Table struct {
	Name          string
	Format        string
	Query         string
	Params        string
	Rows          []string
	count         int
	FlushCount    int
	FlushInterval int
	mu            sync.Mutex
	Sender        Sender
	TickerChan    *chan struct{}
	lastUpdate    time.Time
	// todo add Last Error
}

// Collector - query collector
type Collector struct {
	Tables        map[string]*Table
	mu            sync.RWMutex
	Count         int
	FlushInterval int
	CleanInterval int
	RemoveQueryID bool
	Sender        Sender
	TickerChan    *chan struct{}
}

// NewTable - default table constructor
func NewTable(name string, sender Sender, count int, interval int) (t *Table) {
	t = new(Table)
	t.Name = name
	t.Sender = sender
	t.FlushCount = count
	t.FlushInterval = interval
	return t
}

// NewCollector - default collector constructor
func NewCollector(sender Sender, count int, interval int, cleanInterval int, removeQueryID bool) (c *Collector) {
	c = new(Collector)
	c.Sender = sender
	c.Tables = make(map[string]*Table)
	c.Count = count
	c.FlushInterval = interval
	c.CleanInterval = cleanInterval
	c.RemoveQueryID = removeQueryID
	if cleanInterval > 0 {
		c.TickerChan = c.RunTimer()
	}
	return c
}

// Content - get text content of rowsfor query
func (t *Table) Content() string {
	rowDelimiter := "\n"
	if t.Format == "RowBinary" {
		rowDelimiter = ""
	}
	return t.Query + "\n" + strings.Join(t.Rows, rowDelimiter)
}

// Flush - sends collected data in table to clickhouse
func (t *Table) Flush() {
	req := ClickhouseRequest{
		Params:   t.Params,
		Query:    t.Query,
		Content:  t.Content(),
		Count:    len(t.Rows),
		isInsert: true,
	}
	t.Sender.Send(&req)
	t.Rows = make([]string, 0, t.FlushCount)
	t.count = 0
}

// CheckFlush - check if flush is need and sends data to clickhouse
func (t *Table) CheckFlush() bool {
	t.mu.Lock()
	defer t.mu.Unlock()
	if t.count > 0 {
		t.Flush()
		return true
	}
	return false
}

// Empty - Checks if table is empty
func (t *Table) Empty() bool {
	return t.GetCount() == 0
}

// GetCount - Checks if table is empty
func (t *Table) GetCount() int {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.count
}

// RunTimer - timer for periodical savings data
func (t *Table) RunTimer() *chan struct{} {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(time.Millisecond * time.Duration(t.FlushInterval))
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				t.CheckFlush()
			case <-done:
				return
			}
		}
	}()
	return &done
}

// Add - Adding query to table
func (t *Table) Add(text string) {
	t.mu.Lock()
	defer t.mu.Unlock()
	t.count++
	if t.Format == "TabSeparated" {
		t.Rows = append(t.Rows, strings.Split(text, "\n")...)
	} else {
		t.Rows = append(t.Rows, text)
	}
	if len(t.Rows) >= t.FlushCount {
		t.Flush()
	}
	t.lastUpdate = time.Now()
}

// CleanTable - delete table from map
func (t *Table) CleanTable() {
	t.mu.Lock()
	close(*t.TickerChan)
	t = nil
}

// CleanTables - clean unsused tables
func (c *Collector) CleanTables() {
	c.mu.Lock()
	defer c.mu.Unlock()
	for k, t := range c.Tables {
		if t.lastUpdate.Add(time.Duration(c.CleanInterval) * time.Millisecond).Before(time.Now()) {
			// table was not updated for CleanInterval - delete that table - otherwise it can cause memLeak
			t.CleanTable()
			defer delete(c.Tables, k)
		}

	}
}

// RunTimer - timer for periodical cleaning unused tables
func (c *Collector) RunTimer() *chan struct{} {
	done := make(chan struct{})
	go func() {
		ticker := time.NewTicker(time.Duration(c.CleanInterval) * time.Millisecond)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				c.CleanTables()
			case <-done:
				return
			}
		}
	}()
	return &done
}

// Empty - check if all tables are empty
func (c *Collector) Empty() bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	for _, t := range c.Tables {
		if ok := t.Empty(); !ok {
			return false
		}
	}
	return true
}

// FlushAll - flush all tables to clickhouse
func (c *Collector) FlushAll() (count int) {
	c.mu.Lock()
	defer c.mu.Unlock()
	count = 0
	for _, t := range c.Tables {
		if ok := t.CheckFlush(); ok {
			count++
		}

	}
	return count
}

// WaitFlush - wait for flush all tables
func (c *Collector) WaitFlush() (err error) {
	return c.Sender.WaitFlush()
}

// AddTable - adding table to collector
func (c *Collector) AddTable(name string) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.addTable(name)
}

func (c *Collector) separateQuery(name string) (query string, params string) {
	items := strings.Split(name, "&")
	for _, p := range items {
		if HasPrefix(p, "query=") {
			query = p[6:]
		} else {
			params += "&" + p
		}
	}
	if len(params) > 0 {
		params = strings.TrimSpace(params[1:])
	}
	q, err := url.QueryUnescape(query)
	if err != nil {
		return "", name
	}
	return q, params
}

func (c *Collector) getFormat(query string) (format string) {
	format = formatValues
	f := regexGetFormat.FindSubmatch([]byte(query))
	if len(f) > 1 {
		format = strings.TrimSpace(string(f[1]))
	}
	return format
}

func (c *Collector) addTable(name string) *Table {
	t := NewTable(name, c.Sender, c.Count, c.FlushInterval)
	query, params := c.separateQuery(name)
	t.Query = query
	t.Params = params
	t.Format = c.getFormat(query)
	c.Tables[name] = t
	t.TickerChan = t.RunTimer()
	t.lastUpdate = time.Now()
	return t
}

// Push - adding query to collector with query params (with query) and rows
func (c *Collector) Push(paramsIn string, content string) {
	// as we are using all params as a table key, we have to remove query_id
	// otherwise every query will be threated as unique thus it will consume more memory
	params := ""
	if c.RemoveQueryID {
		items := strings.Split(paramsIn, "&")
		for _, p := range items {
			if !HasPrefix(p, "query_id=") {
				params += "&" + p
			}
		}
		if len(params) > 0 {
			params = strings.TrimSpace(params[1:])
		}
	} else {
		params = paramsIn
	}
	c.mu.RLock()
	table, ok := c.Tables[params]
	if ok {
		table.Add(content)
		c.mu.RUnlock()
		pushCounter.Inc()
		return
	}
	c.mu.RUnlock()
	c.mu.Lock()
	table, ok = c.Tables[params]
	if !ok {
		table = c.addTable(params)
	}
	table.Add(content)
	c.mu.Unlock()
	pushCounter.Inc()
}

// ParseQuery - parsing inbound query to unified format (params/query), content (query data)
func (c *Collector) ParseQuery(queryString string, body string) (params string, content string, insert bool) {
	i := strings.Index(queryString, "query=")
	if i >= 0 {
		if HasPrefix(queryString[i+6:], "insert") {
			insert = true
		}
		var q string
		eoq := strings.Index(queryString[i+6:], "&")
		if eoq >= 0 {
			q = queryString[i+6 : eoq+6]
			params = queryString[:i] + queryString[eoq+7:]
		} else {
			q = queryString[i+6:]
			params = queryString[:i]
		}
		uq, err := url.QueryUnescape(q)
		if body != "" {
			uq += " " + body
		}
		if err != nil {
			return queryString, body, false
		}
		prefix, cnt := c.Parse(uq)
		if strings.HasSuffix(params, "&") || params == "" {
			params += "query=" + url.QueryEscape(strings.TrimSpace(prefix))
		} else {
			params += "&query=" + url.QueryEscape(strings.TrimSpace(prefix))
		}
		content = cnt
	} else {
		var q string
		q, content = c.Parse(body)
		q = strings.TrimSpace(q)
		if HasPrefix(q, "insert") {
			insert = true
		}
		if queryString != "" {
			params = queryString + "&query=" + url.QueryEscape(q)
		} else {
			params = "query=" + url.QueryEscape(q)
		}
	}
	return strings.TrimSpace(params), strings.TrimSpace(content), insert
}

// Parse - parsing text for query and data
func (c *Collector) Parse(text string) (prefix string, content string) {
	i := strings.Index(text, "FORMAT")
	k := strings.Index(text, "VALUES")
	if k == -1 {
		k = strings.Index(text, "values")
	}
	if i >= 0 && i < k {
		w := false
		off := -1
		for c := i + 7; c < len(text); c++ {
			if !w && text[c] != ' ' && text[c] != '\n' && text[c] != ';' {
				w = true
			}
			if w && (text[c] == ' ' || text[c] == '\n' || text[c] == ';') {
				off = c + 1
				break
			}
		}
		if off >= 0 {
			prefix = text[:off]
			content = text[off:]
		}
	} else {
		if k >= 0 {
			prefix = strings.TrimSpace(text[:k+6])
			content = strings.TrimSpace(text[k+6:])
		} else {
			off := regexFormat.FindStringSubmatchIndex(text)
			if len(off) > 3 {
				prefix = text[:off[3]]
				content = text[off[3]:]
			} else {
				off := regexValues.FindStringSubmatchIndex(text)
				if len(off) > 0 {
					prefix = text[:off[1]]
					content = text[off[1]:]
				} else {
					prefix = text
				}
			}
		}
	}
	return prefix, content
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"strings"

	"github.com/sendgrid/rest"
)

type configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func loadConfig(path string) configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}
	return conf
}

func indexHandler(response http.ResponseWriter, request *http.Request) {
	fmt.Fprintf(response, "%s", "Hello World")
}

func getBoundary(value string, contentType string) (string, *strings.Reader) {
	body := strings.NewReader(value)
	bodySplit := strings.Split(string(value), contentType)
	scanner := bufio.NewScanner(strings.NewReader(bodySplit[1]))
	var lines []string
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		break
	}
	boundary := lines[0][9:]
	return boundary, body
}

func inboundHandler(response http.ResponseWriter, request *http.Request) {
	mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(request.Body, params["boundary"])
		parsedEmail := make(map[string]string)
		emailHeader := make(map[string]string)
		binaryFiles := make(map[string][]byte)
		parsedRawEmail := make(map[string]string)
		rawFiles := make(map[string]string)
		for {
			p, err := mr.NextPart()
			// We have found an attachment with binary data
			if err == nil && p.FileName() != "" {
				contents, err := ioutil.ReadAll(p)
				if err != nil {
					log.Fatal(err)
				}
				binaryFiles[p.FileName()] = contents
			}
			if err == io.EOF {
				// We have finished parsing, do something with the parsed data
				printMap(parsedEmail, "")
				printMap(emailHeader, "e")

				for key, value := range binaryFiles {
					fmt.Println("bKey:", key, " bValue:", value)
				}

				printMap(parsedRawEmail, "r")
				printMap(rawFiles, "rf")
				// SendGrid needs a 200 OK response to stop POSTing
				response.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			value, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")
			if !strings.Contains(header, "filename") {
				header = header[17 : len(header)-1]
				parsedEmail[header] = string(value)
			} else {
				header = header[11:]
				f := strings.Split(header, "=")
				parsedEmail[f[1][1:len(f[1])-11]] = f[2][1 : len(f[2])-1]
			}

			if header == "headers" {
				handleHeaders(value, emailHeader)
			}
			// Since we have parsed the headers, we can delete the original
			delete(parsedEmail, "headers")

			// We have a raw message
			if header == "email" {
				handleRawEmail(value, parsedRawEmail, rawFiles)
			}
			// Since we've parsed this header, we can delete the original
			delete(parsedEmail, "email")
		}
	}
}
func handleHeaders(value []byte, emailHeader map[string]string) {
	s := strings.Split(string(value), "\n")
	var a []string
	for _, v := range s {
		t := strings.Split(string(v), ": ")
		a = append(a, t...)
	}
	for i := 0; i < len(a)-1; i += 2 {
		emailHeader[a[i]] = a[i+1]
	}
}

func printMap(inputMap map[string]string, prefix string) {
	for key, value := range inputMap {
		fmt.Println(prefix, "Key:", key, " ", prefix, "Value:", value)
	}
}
func handleRawEmail(value []byte, parsedRawEmail map[string]string, rawFiles map[string]string) {
	boundary, body := getBoundary(string(value), "Content-Type: multipart/mixed; ")
	raw := multipart.NewReader(body, boundary)
	for {
		next, err := raw.NextPart()
		if err == io.EOF {
			// We have finished parsing
			break
		}
		value, err := ioutil.ReadAll(next)
		if err != nil {
			log.Fatal(err)
		}
		header := next.Header.Get("Content-Type")

		// Parse the headers
		if strings.Contains(header, "multipart/alternative") {
			boundary, body := getBoundary(string(value), "Content-Type: multipart/alternative; ")
			raw := multipart.NewReader(body, boundary)
			for {
				next, err := raw.NextPart()
				if err == io.EOF {
					// We have finished parsing
					break
				}
				value, err := ioutil.ReadAll(next)
				if err != nil {
					log.Fatal(err)
				}
				header := next.Header.Get("Content-Type")
				parsedRawEmail[header] = string(value)
			}
		} else {
			// It's a base64 encoded attachment
			rawFiles[header] = string(value)
		}
	}
}

func main() {
	if len(os.Args) > 1 {
		// Test Sender
		path := os.Args[1]
		host := os.Args[2]
		file, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("Check your Filepath. ", err)
		}
		Headers := map[string]string{
			"User-Agent": 	"SendGrid-Test",
			"Content-Type": "multipart/form-data; boundary=xYzZY",
		}
		method := rest.Post
		request := rest.Request{
			Method:  method,
			BaseURL: host,
			Headers: Headers,
			Body:    file,
		}
		_, err = rest.API(request)
		if err != nil {
			log.Fatal("Check your Filepath. ", err)
		}
	} else {
		conf := loadConfig("./conf.json")
		http.HandleFunc("/", indexHandler)
		http.HandleFunc(conf.Endpoint, inboundHandler)
		port := os.Getenv("PORT")
		if port == "" {
			port = conf.Port
		}
		http.ListenAndServe(port, nil)
	}
}
<commit_msg>fix issues raised by gometalinter<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"os"
	"strings"

	"github.com/sendgrid/rest"
)

type configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func loadConfig(path string) configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}
	return conf
}

func indexHandler(response http.ResponseWriter, request *http.Request) {
	fmt.Fprintf(response, "%s", "Hello World")
}

func getBoundary(value string, contentType string) (string, *strings.Reader) {
	body := strings.NewReader(value)
	bodySplit := strings.Split(string(value), contentType)
	scanner := bufio.NewScanner(strings.NewReader(bodySplit[1]))
	var lines []string
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		break
	}
	boundary := lines[0][9:]
	return boundary, body
}

func inboundHandler(response http.ResponseWriter, request *http.Request) {
	mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(request.Body, params["boundary"])
		parsedEmail := make(map[string]string)
		emailHeader := make(map[string]string)
		binaryFiles := make(map[string][]byte)
		parsedRawEmail := make(map[string]string)
		rawFiles := make(map[string]string)
		for {
			p, err := mr.NextPart()
			// We have found an attachment with binary data
			if err == nil && p.FileName() != "" {
				contents, ioerr := ioutil.ReadAll(p)
				if ioerr != nil {
					log.Fatal(err)
				}
				binaryFiles[p.FileName()] = contents
			}
			if err == io.EOF {
				// We have finished parsing, do something with the parsed data
				printMap(parsedEmail, "")
				printMap(emailHeader, "e")

				for key, value := range binaryFiles {
					fmt.Println("bKey:", key, " bValue:", value)
				}

				printMap(parsedRawEmail, "r")
				printMap(rawFiles, "rf")
				// SendGrid needs a 200 OK response to stop POSTing
				response.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			value, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")
			if !strings.Contains(header, "filename") {
				header = header[17 : len(header)-1]
				parsedEmail[header] = string(value)
			} else {
				header = header[11:]
				f := strings.Split(header, "=")
				parsedEmail[f[1][1:len(f[1])-11]] = f[2][1 : len(f[2])-1]
			}

			if header == "headers" {
				handleHeaders(value, emailHeader)
			}
			// Since we have parsed the headers, we can delete the original
			delete(parsedEmail, "headers")

			// We have a raw message
			if header == "email" {
				handleRawEmail(value, parsedRawEmail, rawFiles)
			}
			// Since we've parsed this header, we can delete the original
			delete(parsedEmail, "email")
		}
	}
}
func handleHeaders(value []byte, emailHeader map[string]string) {
	s := strings.Split(string(value), "\n")
	var a []string
	for _, v := range s {
		t := strings.Split(string(v), ": ")
		a = append(a, t...)
	}
	for i := 0; i < len(a)-1; i += 2 {
		emailHeader[a[i]] = a[i+1]
	}
}

func printMap(inputMap map[string]string, prefix string) {
	for key, value := range inputMap {
		fmt.Println(prefix, "Key:", key, " ", prefix, "Value:", value)
	}
}
func handleRawEmail(value []byte, parsedRawEmail map[string]string, rawFiles map[string]string) {
	boundary, body := getBoundary(string(value), "Content-Type: multipart/mixed; ")
	raw := multipart.NewReader(body, boundary)
	for {
		next, err := raw.NextPart()
		if err == io.EOF {
			// We have finished parsing
			break
		}
		value, err := ioutil.ReadAll(next)
		if err != nil {
			log.Fatal(err)
		}
		header := next.Header.Get("Content-Type")

		// Parse the headers
		if strings.Contains(header, "multipart/alternative") {
			boundary, body := getBoundary(string(value), "Content-Type: multipart/alternative; ")
			raw := multipart.NewReader(body, boundary)
			for {
				next, err := raw.NextPart()
				if err == io.EOF {
					// We have finished parsing
					break
				}
				value, err = ioutil.ReadAll(next)
				if err != nil {
					log.Fatal(err)
				}
				header = next.Header.Get("Content-Type")
				parsedRawEmail[header] = string(value)
			}
		} else {
			// It's a base64 encoded attachment
			rawFiles[header] = string(value)
		}
	}
}

func main() {
	if len(os.Args) > 1 {
		// Test Sender
		path := os.Args[1]
		host := os.Args[2]
		file, err := ioutil.ReadFile(path)
		if err != nil {
			log.Fatal("Check your Filepath. ", err)
		}
		Headers := map[string]string{
			"User-Agent":   "SendGrid-Test",
			"Content-Type": "multipart/form-data; boundary=xYzZY",
		}
		method := rest.Post
		request := rest.Request{
			Method:  method,
			BaseURL: host,
			Headers: Headers,
			Body:    file,
		}
		_, err = rest.API(request)
		if err != nil {
			log.Fatal("Check your Filepath. ", err)
		}
	} else {
		conf := loadConfig("./conf.json")
		http.HandleFunc("/", indexHandler)
		http.HandleFunc(conf.Endpoint, inboundHandler)
		port := os.Getenv("PORT")
		if port == "" {
			port = conf.Port
		}
		if err := http.ListenAndServe(port, nil); err != nil {
			log.Fatalln("Error")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/c-bata/go-prompt"
	log "github.com/sirupsen/logrus"
	"sort"
	"strings"

	"github.com/piglei/lbssh/pkg/util"
	"github.com/renstrom/fuzzysearch/fuzzy"
)

// MainCompleter is the main completer
type MainCompleter struct {
	hostCompleter HostCompleter
}

func NewMainCompleter(hostCompleter HostCompleter) *MainCompleter {
	return &MainCompleter{
		hostCompleter: hostCompleter,
	}
}

func (cpl *MainCompleter) completer(d prompt.Document) []prompt.Suggest {
	if d.TextBeforeCursor() == "" {
		return []prompt.Suggest{}
	}
	args := strings.Split(d.CurrentLine(), " ")
	if args[0] == ActionGo {
		return cpl.hostCompleter.completer(d)
	}
	return []prompt.Suggest{}
}

// HostCompleter
type HostCompleter struct {
	entris []*HostEntry
}

func (cpl *HostCompleter) completer(d prompt.Document) []prompt.Suggest {
	key := d.GetWordBeforeCursor()
	// Only show suggestions when cursor not in first action word, which means there always will be
	// more than one space before cursor.
	if !strings.Contains(d.TextBeforeCursor(), " ") {
		return []prompt.Suggest{}
	}

	// Allow user to override user, "root@x.com" => "x.com"
	var textBeforekey string
	keySegments := strings.Split(key, "@")
	if len(keySegments) != 1 {
		key = keySegments[len(keySegments)-1]
		textBeforekey = strings.Join(keySegments[:len(keySegments)-1], "@") + "@"
	}

	var suggestions []prompt.Suggest
	for _, hostEntry := range FilterHostsByKeyword(cpl.entris, key) {
		suggestions = append(suggestions, prompt.Suggest{
			Text:        textBeforekey + hostEntry.Name,
			Description: fmt.Sprintf("%s", hostEntry.HostName),
		})
	}
	log.Debugf("%s matches found for key %s", len(suggestions), key)
	return suggestions
}

type MatchedItem struct {
	key       string
	host      *HostEntry
	weight    float64
	numFields int
	mGroups   []int
	editDistances []int
}

func (m *MatchedItem) SortmGroups() {
	sort.Sort(sort.Reverse(sort.IntSlice(m.mGroups)))
}

// FilterHostsByKeyword using fuzzy search to find the best matched hostEntried and sort the result
func FilterHostsByKeyword(entries []*HostEntry, key string) []*HostEntry {
	// Because we are using Levenshtein distance(aka. edit distance) algorithm to find the best matched result,
	// in order to perform an apples to apples comparison, we can:
	//
	// - Remove the leading and trailing characters, then count for edit distance
	var maxLengthName, maxLengthHostname int
	for _, hostEntry := range entries {
		if len(hostEntry.Name) > maxLengthName {
			maxLengthName = len(hostEntry.Name)
		}
		if len(hostEntry.HostName) > maxLengthHostname {
			maxLengthHostname = len(hostEntry.HostName)
		}
	}
	log.Debugf(
		"maxLengthName=%d, maxLengthHostname=%d, will extend all cadidates to this length",
		maxLengthName,
		maxLengthHostname,
	)

	key = strings.ToLower(key)
	var matchedItems []*MatchedItem
	for _, hostEntry := range entries {
		matched := &MatchedItem{
			key:  key,
			host: hostEntry,
		}
		for _, target := range []string{hostEntry.Name, hostEntry.HostName} {
			target = strings.ToLower(target)
			// Use fuzzy match to grep the result first for better performance
			if !fuzzy.Match(key, target) {
				continue
			}
			_, _, mGroups := util.LCSFuzzySearch(key, target)

			log.Debugf("Match field found: key=%s target=%s groups=%+v", key, target, mGroups)
			matched.numFields += 1
			for _, value := range mGroups {
				matched.mGroups = append(matched.mGroups, value)
			}
			// If mGroups is identical, levenshteinDistance will be use for sort
			matched.editDistances = append(matched.editDistances, fuzzy.LevenshteinDistance(key, target))
		}
		// Ignore non-matched found items
		if matched.numFields == 0 {
			continue
		}

		matched.SortmGroups()
		matchedItems = append(matchedItems, matched)
		log.Debugf("Match item found: key=%s groups=%+v", key, matched.mGroups)
	}

	// Sort results
	sort.SliceStable(matchedItems, func(i, j int) bool {
		i1, i2 := matchedItems[i], matchedItems[j]
		if i1.numFields > i2.numFields {
			return true
		}
		if len(i1.mGroups) != len(i2.mGroups) {
			return len(i1.mGroups) < len(i2.mGroups)
		}

		for i, v := range i1.mGroups {
			if v == i2.mGroups[i] {
				continue
			}
			return v > i2.mGroups[i]
		}

		for i, v := range i1.editDistances {
			if v == i2.editDistances[i] {
				continue
			}
			return v < i2.editDistances[i]
		}
		return true
	})

	var results []*HostEntry
	for _, item := range matchedItems {
		results = append(results, item.host)
	}
	return results
}
<commit_msg>Remove useless codes<commit_after>package main

import (
	"fmt"
	"github.com/c-bata/go-prompt"
	log "github.com/sirupsen/logrus"
	"sort"
	"strings"

	"github.com/piglei/lbssh/pkg/util"
	"github.com/renstrom/fuzzysearch/fuzzy"
)

// MainCompleter is the main completer
type MainCompleter struct {
	hostCompleter HostCompleter
}

func NewMainCompleter(hostCompleter HostCompleter) *MainCompleter {
	return &MainCompleter{
		hostCompleter: hostCompleter,
	}
}

func (cpl *MainCompleter) completer(d prompt.Document) []prompt.Suggest {
	if d.TextBeforeCursor() == "" {
		return []prompt.Suggest{}
	}
	args := strings.Split(d.CurrentLine(), " ")
	if args[0] == ActionGo {
		return cpl.hostCompleter.completer(d)
	}
	return []prompt.Suggest{}
}

// HostCompleter
type HostCompleter struct {
	entris []*HostEntry
}

func (cpl *HostCompleter) completer(d prompt.Document) []prompt.Suggest {
	key := d.GetWordBeforeCursor()
	// Only show suggestions when cursor not in first action word, which means there always will be
	// more than one space before cursor.
	if !strings.Contains(d.TextBeforeCursor(), " ") {
		return []prompt.Suggest{}
	}

	// Allow user to override user, "root@x.com" => "x.com"
	var textBeforekey string
	keySegments := strings.Split(key, "@")
	if len(keySegments) != 1 {
		key = keySegments[len(keySegments)-1]
		textBeforekey = strings.Join(keySegments[:len(keySegments)-1], "@") + "@"
	}

	var suggestions []prompt.Suggest
	for _, hostEntry := range FilterHostsByKeyword(cpl.entris, key) {
		suggestions = append(suggestions, prompt.Suggest{
			Text:        textBeforekey + hostEntry.Name,
			Description: fmt.Sprintf("%s", hostEntry.HostName),
		})
	}
	log.Debugf("%s matches found for key %s", len(suggestions), key)
	return suggestions
}

type MatchedItem struct {
	key       string
	host      *HostEntry
	weight    float64
	numFields int
	mGroups   []int
	editDistances []int
}

func (m *MatchedItem) SortmGroups() {
	sort.Sort(sort.Reverse(sort.IntSlice(m.mGroups)))
}

// FilterHostsByKeyword using fuzzy search to find the best matched hostEntried and sort the result
func FilterHostsByKeyword(entries []*HostEntry, key string) []*HostEntry {
	key = strings.ToLower(key)
	var matchedItems []*MatchedItem
	for _, hostEntry := range entries {
		matched := &MatchedItem{
			key:  key,
			host: hostEntry,
		}
		for _, target := range []string{hostEntry.Name, hostEntry.HostName} {
			target = strings.ToLower(target)
			// Use fuzzy match to grep the result first for better performance
			if !fuzzy.Match(key, target) {
				continue
			}
			_, _, mGroups := util.LCSFuzzySearch(key, target)

			log.Debugf("Match field found: key=%s target=%s groups=%+v", key, target, mGroups)
			matched.numFields += 1
			for _, value := range mGroups {
				matched.mGroups = append(matched.mGroups, value)
			}
			// If mGroups is identical, levenshteinDistance will be use for sort
			matched.editDistances = append(matched.editDistances, fuzzy.LevenshteinDistance(key, target))
		}
		// Ignore non-matched found items
		if matched.numFields == 0 {
			continue
		}

		matched.SortmGroups()
		matchedItems = append(matchedItems, matched)
		log.Debugf("Match item found: key=%s groups=%+v", key, matched.mGroups)
	}

	// Sort results
	sort.SliceStable(matchedItems, func(i, j int) bool {
		i1, i2 := matchedItems[i], matchedItems[j]
		if i1.numFields > i2.numFields {
			return true
		}
		if len(i1.mGroups) != len(i2.mGroups) {
			return len(i1.mGroups) < len(i2.mGroups)
		}

		for i, v := range i1.mGroups {
			if v == i2.mGroups[i] {
				continue
			}
			return v > i2.mGroups[i]
		}

		for i, v := range i1.editDistances {
			if v == i2.editDistances[i] {
				continue
			}
			return v < i2.editDistances[i]
		}
		return true
	})

	var results []*HostEntry
	for _, item := range matchedItems {
		results = append(results, item.host)
	}
	return results
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/gmail/v1"
)

func getClient(config *oauth2.Config) *http.Client {
	tokFile := "/home/teixeira/token.json"
	tok, err := tokenFromFile(tokFile)
	if err != nil {
		tok = getTokenFromWeb(config)
		saveToken(tokFile, tok)
	}
	return config.Client(context.Background(), tok)
}

func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go here: \n%v\n", authURL)

	var authCode string
	if _, err := fmt.Scan(&authCode); err != nil {
		log.Fatalf("Could not read auth code")
	}

	tok, err := config.Exchange(context.TODO(), authCode)
	if err != nil {
		log.Fatalf("Unable to retrieve token from web: %v", err)
	}
	return tok
}

func tokenFromFile(file string) (*oauth2.Token, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	tok := &oauth2.Token{}
	err = json.NewDecoder(f).Decode(tok)
	return tok, err
}

func saveToken(path string, token *oauth2.Token) {
	fmt.Printf("Saving credential file to %s\n", path)
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		log.Fatalf("Unable to cache auth token: %s\n", err)
	}
	defer f.Close()

	json.NewEncoder(f).Encode(token)
}

func main() {
	b, err := ioutil.ReadFile("/home/teixeira/credentials.json")
	if err != nil {
		log.Fatalf("Unable to read client secret: %v\n", err)
	}

	config, err := google.ConfigFromJSON(b, gmail.GmailReadonlyScope)
	if err != nil {
		log.Fatalf("Unable to parse client secret: %v\n", err)
	}
	client := getClient(config)

	srv, err := gmail.New(client)
	if err != nil {
		log.Fatalf("Unable to get Gmail client: %v\n", err)
	}

	user := "me"
	r, err := srv.Users.Labels.List(user).Do()
	if err != nil {
		log.Fatalf("Failed to retrive labels: %v\n", err)
	}
	if len(r.Labels) == 0 {
		fmt.Println("No labels found")
		return
	}

	fmt.Println("Labels:")
	for _, l := range r.Labels {
		fmt.Printf("- %s\n", l.Name)
	}
}
<commit_msg>It lives!<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"time"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/gmail/v1"
)

func getClient(config *oauth2.Config) *http.Client {
	tokFile := "/home/teixeira/token.json"
	tok, err := tokenFromFile(tokFile)
	if err != nil {
		tok = getTokenFromWeb(config)
		saveToken(tokFile, tok)
	}
	return config.Client(context.Background(), tok)
}

func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
	authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
	fmt.Printf("Go here: \n%v\n", authURL)

	var authCode string
	if _, err := fmt.Scan(&authCode); err != nil {
		log.Fatalf("Could not read auth code")
	}

	tok, err := config.Exchange(context.TODO(), authCode)
	if err != nil {
		log.Fatalf("Unable to retrieve token from web: %v", err)
	}
	return tok
}

func tokenFromFile(file string) (*oauth2.Token, error) {
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	tok := &oauth2.Token{}
	err = json.NewDecoder(f).Decode(tok)
	return tok, err
}

func saveToken(path string, token *oauth2.Token) {
	fmt.Printf("Saving credential file to %s\n", path)
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		log.Fatalf("Unable to cache auth token: %s\n", err)
	}
	defer f.Close()

	json.NewEncoder(f).Encode(token)
}

func getMessage(srv *gmail.Service, userId string, messageId string) (*gmail.Message, error) {
	return srv.Users.Messages.Get(userId, messageId).Do()
}

func getSubject(message *gmail.Message) string {
	payload := message.Payload
	if payload == nil {
		log.Fatalf("Retrieved message %v had no body\n", message.Raw)
	}

	headers := payload.Headers
	if headers == nil {
		log.Fatalf("Retrieved message %v had no body\n", message.Raw)
	}

	for _, header := range headers {
		if header.Name == "Subject" {
			return header.Value
		}
	}

	return ""
}

func getMostRecentHistoryId(srv *gmail.Service, userId string) uint64 {
	r, err := srv.Users.Messages.List(userId).MaxResults(1).Do()
	if err != nil {
		log.Fatalf("Failed to retrieve recent messages %v\n", err)
	}

	for _, m := range r.Messages {
		message, err := getMessage(srv, userId, m.Id)
		if err != nil {
			log.Fatalf("Failed to retrieve message %v due to %v\n", m.Id, err)
		}

		return message.HistoryId
	}

	log.Fatalf("Could not get most recent history ID")
	return 0
}

// First result = has new messages since
// second result = most recently seen history ID
func hasMessagesSince(srv *gmail.Service, userId string, historyId uint64) (bool, uint64) {
	r, err := srv.Users.History.List(userId).HistoryTypes("messageAdded").StartHistoryId(historyId).Do()
	if err != nil {
		log.Fatalf("Could not get message history due to %v\n", err)
	}

	newHistoryId := r.HistoryId
	hasResults := (len(r.History) > 0)
	return hasResults, newHistoryId
}

func main() {
	b, err := ioutil.ReadFile("/home/teixeira/credentials.json")
	if err != nil {
		log.Fatalf("Unable to read client secret: %v\n", err)
	}

	config, err := google.ConfigFromJSON(b, gmail.GmailReadonlyScope)
	if err != nil {
		log.Fatalf("Unable to parse client secret: %v\n", err)
	}
	client := getClient(config)

	srv, err := gmail.New(client)
	if err != nil {
		log.Fatalf("Unable to get Gmail client: %v\n", err)
	}

	user := "me"
	historyId := getMostRecentHistoryId(srv, user)
	hasMessages := false

	ticker := time.NewTicker(30 * time.Second)
  for range ticker.C {
    hasMessages, historyId = hasMessagesSince(srv, user, historyId)
    fmt.Printf("Has messages since %v: %v\n", historyId, hasMessages)
  }
	ticker.Stop()
}
<|endoftext|>


<commit_before>package cassandra

import (
	"crypto/rand"
	"fmt"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/raintank/met/helper"
	"github.com/raintank/metrictank/idx"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/raintank/schema.v1"
)

func init() {
	keyspace = "raintank"
	hosts = ""
	consistency = "one"
	timeout = time.Second
	numConns = 1
	writeQueueSize = 1000
	protoVer = 4
}

func getSeriesNames(depth, count int, prefix string) []string {
	series := make([]string, count)
	for i := 0; i < count; i++ {
		ns := make([]string, depth)
		for j := 0; j < depth; j++ {
			ns[j] = getRandomString(4)
		}
		series[i] = prefix + "." + strings.Join(ns, ".")
	}
	return series
}

// source: https://github.com/gogits/gogs/blob/9ee80e3e5426821f03a4e99fad34418f5c736413/modules/base/tool.go#L58
func getRandomString(n int, alphabets ...byte) string {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		if len(alphabets) == 0 {
			bytes[i] = alphanum[b%byte(len(alphanum))]
		} else {
			bytes[i] = alphabets[b%byte(len(alphabets))]
		}
	}
	return string(bytes)
}

func getMetricData(orgId, depth, count, interval int, prefix string) []*schema.MetricData {
	data := make([]*schema.MetricData, count)
	series := getSeriesNames(depth, count, prefix)
	for i, s := range series {

		data[i] = &schema.MetricData{
			Name:     s,
			Metric:   s,
			OrgId:    orgId,
			Interval: interval,
		}
		data[i].SetId()
	}
	return data
}

func TestGetAddKey(t *testing.T) {
	ix := New()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	ix.Init(stats)

	publicSeries := getMetricData(-1, 2, 5, 10, "metric.public")
	org1Series := getMetricData(1, 2, 5, 10, "metric.org1")
	org2Series := getMetricData(2, 2, 5, 10, "metric.org2")

	for _, series := range [][]*schema.MetricData{publicSeries, org1Series, org2Series} {
		orgId := series[0].OrgId
		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
			for _, s := range series {
				ix.Add(s)
			}
			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
				defs := ix.List(orgId)
				numSeries := len(series)
				if orgId != -1 {
					numSeries += 5
				}
				So(defs, ShouldHaveLength, numSeries)

			})
		})
	}

	Convey("When adding metricDefs with the same series name as existing metricDefs", t, func() {
		for _, series := range org1Series {
			series.Interval = 60
			series.SetId()
			ix.Add(series)
		}
		Convey("then listing metrics", func() {
			defs := ix.List(1)
			So(defs, ShouldHaveLength, 15)
		})
	})
}

func TestFind(t *testing.T) {
	ix := New()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	ix.Init(stats)
	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
		ix.Add(s)
	}
	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
		ix.Add(s)
	}
	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
		ix.Add(s)
		s.Interval = 60
		s.SetId()
		ix.Add(s)
	}
	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
		ix.Add(s)
	}

	Convey("When listing root nodes", t, func() {
		Convey("root nodes for orgId 1", func() {
			nodes, err := ix.Find(1, "*")
			So(err, ShouldBeNil)
			So(nodes, ShouldHaveLength, 2)
			So(nodes[0].Path, ShouldBeIn, "metric", "foo")
			So(nodes[1].Path, ShouldBeIn, "metric", "foo")
			So(nodes[0].Leaf, ShouldBeFalse)
		})
		Convey("root nodes for orgId 2", func() {
			nodes, err := ix.Find(2, "*")
			So(err, ShouldBeNil)
			So(nodes, ShouldHaveLength, 1)
			So(nodes[0].Path, ShouldEqual, "metric")
			So(nodes[0].Leaf, ShouldBeFalse)
		})
	})

	Convey("When searching with GLOB", t, func() {
		nodes, err := ix.Find(2, "metric.{f*,demo}.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 10)
		for _, n := range nodes {
			So(n.Leaf, ShouldBeFalse)
		}
	})

	Convey("When searching with multiple wildcards", t, func() {
		nodes, err := ix.Find(1, "*.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 2)
		for _, n := range nodes {
			So(n.Leaf, ShouldBeFalse)
		}
	})

	Convey("When searching nodes not in public series", t, func() {
		nodes, err := ix.Find(1, "foo.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 5)
		Convey("When searching for specific series", func() {
			found, err := ix.Find(1, nodes[0].Path)
			So(err, ShouldBeNil)
			So(found, ShouldHaveLength, 1)
			So(found[0].Path, ShouldEqual, nodes[0].Path)
		})
		Convey("When searching nodes that are children of a leaf", func() {
			found, err := ix.Find(1, nodes[0].Path+".*")
			So(err, ShouldBeNil)
			So(found, ShouldHaveLength, 0)
		})
	})

	Convey("When searching with multiple wildcards mixed leaf/branch", t, func() {
		nodes, err := ix.Find(1, "*.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 15)
		for _, n := range nodes {
			if strings.HasPrefix(n.Path, "foo.demo") {
				So(n.Leaf, ShouldBeTrue)
				So(n.Defs, ShouldHaveLength, 2)
			} else {
				So(n.Leaf, ShouldBeFalse)
			}
		}
	})
	Convey("When searching nodes for unkown orgId", t, func() {
		nodes, err := ix.Find(4, "foo.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 0)
	})

	Convey("When searching nodes that dont exist", t, func() {
		nodes, err := ix.Find(1, "foo.demo.blah.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 0)
	})

}

func BenchmarkIndexing(b *testing.B) {
	keyspace = "raintank"
	hosts = "localhost:9042"
	consistency = "one"
	timeout = time.Second
	numConns = 10
	writeQueueSize = 10
	protoVer = 4
	ix := New()
	tmpSession, _ := ix.cluster.CreateSession()
	tmpSession.Query("TRUNCATE raintank.metric_def_idx").Exec()
	tmpSession.Close()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	pre := time.Now()
	ix.Init(stats)
	loadTime := time.Since(pre)
	currentMetrics := ix.MemoryIdx.List(-1)
	b.Logf("there are %d metrics already in the index. Loaded in %s", len(currentMetrics), loadTime.String())

	b.ReportAllocs()
	b.ResetTimer()
	insertDefs(ix, b.N)
	ix.Stop()
}

func insertDefs(ix idx.MetricIndex, i int) {
	var series string
	var data *schema.MetricData
	for n := 0; n < i; n++ {
		series = "some.metric." + strconv.Itoa(n)
		data = &schema.MetricData{
			Name:     series,
			Metric:   series,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.Add(data)
	}
}

func BenchmarkLoad(b *testing.B) {
	keyspace = "raintank"
	hosts = "localhost:9042"
	consistency = "one"
	timeout = time.Second
	numConns = 10
	writeQueueSize = 10
	protoVer = 4
	ix := New()

	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	tmpSession, _ := ix.cluster.CreateSession()
	tmpSession.Query("TRUNCATE raintank.metric_def_idx").Exec()
	tmpSession.Close()
	ix.Init(stats)
	insertDefs(ix, b.N)
	ix.Stop()

	b.ReportAllocs()
	b.ResetTimer()
	ix = New()
	ix.Init(stats)
	ix.Stop()
}
<commit_msg>remove verbose logging from benchmark<commit_after>package cassandra

import (
	"crypto/rand"
	"fmt"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/raintank/met/helper"
	"github.com/raintank/metrictank/idx"
	. "github.com/smartystreets/goconvey/convey"
	"gopkg.in/raintank/schema.v1"
)

func init() {
	keyspace = "raintank"
	hosts = ""
	consistency = "one"
	timeout = time.Second
	numConns = 1
	writeQueueSize = 1000
	protoVer = 4
}

func getSeriesNames(depth, count int, prefix string) []string {
	series := make([]string, count)
	for i := 0; i < count; i++ {
		ns := make([]string, depth)
		for j := 0; j < depth; j++ {
			ns[j] = getRandomString(4)
		}
		series[i] = prefix + "." + strings.Join(ns, ".")
	}
	return series
}

// source: https://github.com/gogits/gogs/blob/9ee80e3e5426821f03a4e99fad34418f5c736413/modules/base/tool.go#L58
func getRandomString(n int, alphabets ...byte) string {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		if len(alphabets) == 0 {
			bytes[i] = alphanum[b%byte(len(alphanum))]
		} else {
			bytes[i] = alphabets[b%byte(len(alphabets))]
		}
	}
	return string(bytes)
}

func getMetricData(orgId, depth, count, interval int, prefix string) []*schema.MetricData {
	data := make([]*schema.MetricData, count)
	series := getSeriesNames(depth, count, prefix)
	for i, s := range series {

		data[i] = &schema.MetricData{
			Name:     s,
			Metric:   s,
			OrgId:    orgId,
			Interval: interval,
		}
		data[i].SetId()
	}
	return data
}

func TestGetAddKey(t *testing.T) {
	ix := New()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	ix.Init(stats)

	publicSeries := getMetricData(-1, 2, 5, 10, "metric.public")
	org1Series := getMetricData(1, 2, 5, 10, "metric.org1")
	org2Series := getMetricData(2, 2, 5, 10, "metric.org2")

	for _, series := range [][]*schema.MetricData{publicSeries, org1Series, org2Series} {
		orgId := series[0].OrgId
		Convey(fmt.Sprintf("When indexing metrics for orgId %d", orgId), t, func() {
			for _, s := range series {
				ix.Add(s)
			}
			Convey(fmt.Sprintf("Then listing metrics for OrgId %d", orgId), func() {
				defs := ix.List(orgId)
				numSeries := len(series)
				if orgId != -1 {
					numSeries += 5
				}
				So(defs, ShouldHaveLength, numSeries)

			})
		})
	}

	Convey("When adding metricDefs with the same series name as existing metricDefs", t, func() {
		for _, series := range org1Series {
			series.Interval = 60
			series.SetId()
			ix.Add(series)
		}
		Convey("then listing metrics", func() {
			defs := ix.List(1)
			So(defs, ShouldHaveLength, 15)
		})
	})
}

func TestFind(t *testing.T) {
	ix := New()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	ix.Init(stats)
	for _, s := range getMetricData(-1, 2, 5, 10, "metric.demo") {
		ix.Add(s)
	}
	for _, s := range getMetricData(1, 2, 5, 10, "metric.demo") {
		ix.Add(s)
	}
	for _, s := range getMetricData(1, 1, 5, 10, "foo.demo") {
		ix.Add(s)
		s.Interval = 60
		s.SetId()
		ix.Add(s)
	}
	for _, s := range getMetricData(2, 2, 5, 10, "metric.foo") {
		ix.Add(s)
	}

	Convey("When listing root nodes", t, func() {
		Convey("root nodes for orgId 1", func() {
			nodes, err := ix.Find(1, "*")
			So(err, ShouldBeNil)
			So(nodes, ShouldHaveLength, 2)
			So(nodes[0].Path, ShouldBeIn, "metric", "foo")
			So(nodes[1].Path, ShouldBeIn, "metric", "foo")
			So(nodes[0].Leaf, ShouldBeFalse)
		})
		Convey("root nodes for orgId 2", func() {
			nodes, err := ix.Find(2, "*")
			So(err, ShouldBeNil)
			So(nodes, ShouldHaveLength, 1)
			So(nodes[0].Path, ShouldEqual, "metric")
			So(nodes[0].Leaf, ShouldBeFalse)
		})
	})

	Convey("When searching with GLOB", t, func() {
		nodes, err := ix.Find(2, "metric.{f*,demo}.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 10)
		for _, n := range nodes {
			So(n.Leaf, ShouldBeFalse)
		}
	})

	Convey("When searching with multiple wildcards", t, func() {
		nodes, err := ix.Find(1, "*.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 2)
		for _, n := range nodes {
			So(n.Leaf, ShouldBeFalse)
		}
	})

	Convey("When searching nodes not in public series", t, func() {
		nodes, err := ix.Find(1, "foo.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 5)
		Convey("When searching for specific series", func() {
			found, err := ix.Find(1, nodes[0].Path)
			So(err, ShouldBeNil)
			So(found, ShouldHaveLength, 1)
			So(found[0].Path, ShouldEqual, nodes[0].Path)
		})
		Convey("When searching nodes that are children of a leaf", func() {
			found, err := ix.Find(1, nodes[0].Path+".*")
			So(err, ShouldBeNil)
			So(found, ShouldHaveLength, 0)
		})
	})

	Convey("When searching with multiple wildcards mixed leaf/branch", t, func() {
		nodes, err := ix.Find(1, "*.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 15)
		for _, n := range nodes {
			if strings.HasPrefix(n.Path, "foo.demo") {
				So(n.Leaf, ShouldBeTrue)
				So(n.Defs, ShouldHaveLength, 2)
			} else {
				So(n.Leaf, ShouldBeFalse)
			}
		}
	})
	Convey("When searching nodes for unkown orgId", t, func() {
		nodes, err := ix.Find(4, "foo.demo.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 0)
	})

	Convey("When searching nodes that dont exist", t, func() {
		nodes, err := ix.Find(1, "foo.demo.blah.*")
		So(err, ShouldBeNil)
		So(nodes, ShouldHaveLength, 0)
	})

}

func BenchmarkIndexing(b *testing.B) {
	keyspace = "raintank"
	hosts = "localhost:9042"
	consistency = "one"
	timeout = time.Second
	numConns = 10
	writeQueueSize = 10
	protoVer = 4
	ix := New()
	tmpSession, _ := ix.cluster.CreateSession()
	tmpSession.Query("TRUNCATE raintank.metric_def_idx").Exec()
	tmpSession.Close()
	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	ix.Init(stats)

	b.ReportAllocs()
	b.ResetTimer()
	insertDefs(ix, b.N)
	ix.Stop()
}

func insertDefs(ix idx.MetricIndex, i int) {
	var series string
	var data *schema.MetricData
	for n := 0; n < i; n++ {
		series = "some.metric." + strconv.Itoa(n)
		data = &schema.MetricData{
			Name:     series,
			Metric:   series,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.Add(data)
	}
}

func BenchmarkLoad(b *testing.B) {
	keyspace = "raintank"
	hosts = "localhost:9042"
	consistency = "one"
	timeout = time.Second
	numConns = 10
	writeQueueSize = 10
	protoVer = 4
	ix := New()

	stats, _ := helper.New(false, "", "standard", "metrictank", "")
	tmpSession, _ := ix.cluster.CreateSession()
	tmpSession.Query("TRUNCATE raintank.metric_def_idx").Exec()
	tmpSession.Close()
	ix.Init(stats)
	insertDefs(ix, b.N)
	ix.Stop()

	b.ReportAllocs()
	b.ResetTimer()
	ix = New()
	ix.Init(stats)
	ix.Stop()
}
<|endoftext|>


<commit_before>package system_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"syscall"

	"github.com/cloudfoundry-incubator/garden-linux/containerizer/system"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Mount", func() {
	var (
		dest string
		src  string
	)

	BeforeEach(func() {
		var err error
		src, err = ioutil.TempDir("", "")
		Expect(err).ToNot(HaveOccurred())
		dest = fmt.Sprintf("/tmp/mount-dest-temp-%d", GinkgoParallelNode())
	})

	AfterEach(func() {
		Expect(os.RemoveAll(src)).To(Succeed())
		Expect(os.RemoveAll(dest)).To(Succeed())
	})

	var testMount = func(privileged bool) {
		Context("with an invalid mount type", func() {
			It("returns an informative error", func() {
				stderr := gbytes.NewBuffer()
				Expect(
					runInContainer(GinkgoWriter, io.MultiWriter(stderr, GinkgoWriter),
						privileged, "fake_mounter", "-type=not-a-mount-type", "-targetPath="+dest, "-flags=0", "cat", "/proc/mounts"),
				).To(HaveOccurred())

				Expect(stderr).To(gbytes.Say("error: system: mount not-a-mount-type on %s: no such device", dest))
			})
		})

		It("can mount tmpfs", func() {
			stdout := gbytes.NewBuffer()
			Expect(
				runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
					privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+dest, "-flags=0", "cat", "/proc/mounts"),
			).To(Succeed())

			Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s tmpfs", dest)))
		})

		Context("when flags are supplied", func() {
			It("mounts using the flags", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+dest, fmt.Sprintf("-flags=%d", syscall.MS_NODEV), "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s tmpfs rw,nodev", dest)))
			})
		})

		Context("when data is provided", func() {
			It("mounts using the data", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Devpts), "-targetPath="+dest, "-flags=0", "-data=newinstance,ptmxmode=0666", "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("devpts %s devpts rw,relatime,mode=600,ptmxmode=666", dest)))
			})
		})

		Context("when the destination does not already exist", func() {
			It("creates the directory before mounting", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+filepath.Join(dest, "foo"), "-flags=0", "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s/foo tmpfs", dest)))
			})
		})

		Context("when the destination cannot be created", func() {
			BeforeEach(func() {
				var err error
				dest, err = ioutil.TempDir("", "")
				Expect(err).ToNot(HaveOccurred())
			})

			It("returns an informative error", func() {
				ioutil.WriteFile(filepath.Join(dest, "foo"), []byte("block"), 0700)
				stderr := gbytes.NewBuffer()
				Expect(
					runInContainer(GinkgoWriter, io.MultiWriter(stderr, GinkgoWriter),
						privileged, "fake_mounter", "-type=tmpfs", "-targetPath="+filepath.Join(dest, "foo"), "-flags=0", "cat", "/proc/mounts"),
				).To(HaveOccurred())

				Expect(stderr).To(gbytes.Say("error: system: create mount point directory %s/foo: ", dest))
			})
		})

		Context("when the sourcePath is provided", func() {
			var matcher *regexp.Regexp

			JustBeforeEach(func() {
				matcher = regexp.MustCompile(fmt.Sprintf("%s (aufs|ext4|btrfs) rw,relatime", dest))
			})

			It("mounts using the sourcePath", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type=bind", "-sourcePath="+src, "-targetPath="+dest, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(matcher.Find(stdout.Contents())).ToNot(BeNil())
			})

			Context("when file is mounted", func() {
				var (
					srcFile string
					dstFile string
				)

				BeforeEach(func() {
					file, err := ioutil.TempFile("", "")
					Expect(err).ToNot(HaveOccurred())
					fmt.Fprintf(file, "MountMe")
					srcFile = file.Name()
					dstFile = fmt.Sprintf("/tmp/fake-mount-file-%d", GinkgoParallelNode())
				})

				JustBeforeEach(func() {
					matcher = regexp.MustCompile(fmt.Sprintf("%s (aufs|ext4|btrfs) rw,relatime", dstFile))
				})

				AfterEach(func() {
					Expect(os.Remove(srcFile)).To(Succeed())
				})

				It("mounts a file using the sourcePath", func() {
					stdout := gbytes.NewBuffer()
					Expect(
						runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
							privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath="+dstFile, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
					).To(Succeed())

					info, err := os.Stat(dstFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(info.IsDir()).ToNot(BeTrue())
					Expect(matcher.Find(stdout.Contents())).ToNot(BeNil())
					Expect(os.Remove(dstFile)).To(Succeed())
				})

				Context("when destination file is created in non-existing directory", func() {
					BeforeEach(func() {
						dstFile = fmt.Sprintf("/tmp/non-existing-dir-%d/fake-mount-file-%d", GinkgoParallelNode(), GinkgoParallelNode())
					})

					It("mounts a file using the sourcePath", func() {
						stdout := gbytes.NewBuffer()
						Expect(
							runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
								privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath="+dstFile, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
						).To(Succeed())

						info, err := os.Stat(dstFile)
						Expect(err).ToNot(HaveOccurred())
						Expect(info.IsDir()).ToNot(BeTrue())
						Expect(matcher.Find(stdout.Contents())).ToNot(BeNil())
						Expect(os.Remove(dstFile)).To(Succeed())
					})
				})

				Context("when destination file cannot be created", func() {
					It("returns an informative error", func() {
						stderr := gbytes.NewBuffer()
						Expect(
							runInContainer(GinkgoWriter, io.MultiWriter(GinkgoWriter, stderr),
								privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath=/tmp", fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
						).To(HaveOccurred())

						Expect(stderr).To(gbytes.Say("error: system: create mount point file /tmp: "))
					})
				})
			})
		})
	}

	Context("in an unprivileged container", func() {
		testMount(false)
	})

	Context("in an privileged container", func() {
		testMount(true)
	})
})
<commit_msg>Use gbytes.Say rather than explicit regex<commit_after>package system_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"

	"github.com/cloudfoundry-incubator/garden-linux/containerizer/system"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Mount", func() {
	var (
		dest string
		src  string
	)

	BeforeEach(func() {
		var err error
		src, err = ioutil.TempDir("", "")
		Expect(err).ToNot(HaveOccurred())
		dest = fmt.Sprintf("/tmp/mount-dest-temp-%d", GinkgoParallelNode())
	})

	AfterEach(func() {
		Expect(os.RemoveAll(src)).To(Succeed())
		Expect(os.RemoveAll(dest)).To(Succeed())
	})

	var testMount = func(privileged bool) {
		Context("with an invalid mount type", func() {
			It("returns an informative error", func() {
				stderr := gbytes.NewBuffer()
				Expect(
					runInContainer(GinkgoWriter, io.MultiWriter(stderr, GinkgoWriter),
						privileged, "fake_mounter", "-type=not-a-mount-type", "-targetPath="+dest, "-flags=0", "cat", "/proc/mounts"),
				).To(HaveOccurred())

				Expect(stderr).To(gbytes.Say("error: system: mount not-a-mount-type on %s: no such device", dest))
			})
		})

		It("can mount tmpfs", func() {
			stdout := gbytes.NewBuffer()
			Expect(
				runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
					privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+dest, "-flags=0", "cat", "/proc/mounts"),
			).To(Succeed())

			Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s tmpfs", dest)))
		})

		Context("when flags are supplied", func() {
			It("mounts using the flags", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+dest, fmt.Sprintf("-flags=%d", syscall.MS_NODEV), "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s tmpfs rw,nodev", dest)))
			})
		})

		Context("when data is provided", func() {
			It("mounts using the data", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Devpts), "-targetPath="+dest, "-flags=0", "-data=newinstance,ptmxmode=0666", "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("devpts %s devpts rw,relatime,mode=600,ptmxmode=666", dest)))
			})
		})

		Context("when the destination does not already exist", func() {
			It("creates the directory before mounting", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type="+string(system.Tmpfs), "-targetPath="+filepath.Join(dest, "foo"), "-flags=0", "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say(fmt.Sprintf("tmpfs %s/foo tmpfs", dest)))
			})
		})

		Context("when the destination cannot be created", func() {
			BeforeEach(func() {
				var err error
				dest, err = ioutil.TempDir("", "")
				Expect(err).ToNot(HaveOccurred())
			})

			It("returns an informative error", func() {
				ioutil.WriteFile(filepath.Join(dest, "foo"), []byte("block"), 0700)
				stderr := gbytes.NewBuffer()
				Expect(
					runInContainer(GinkgoWriter, io.MultiWriter(stderr, GinkgoWriter),
						privileged, "fake_mounter", "-type=tmpfs", "-targetPath="+filepath.Join(dest, "foo"), "-flags=0", "cat", "/proc/mounts"),
				).To(HaveOccurred())

				Expect(stderr).To(gbytes.Say("error: system: create mount point directory %s/foo: ", dest))
			})
		})

		Context("when the sourcePath is provided", func() {
			It("mounts using the sourcePath", func() {
				stdout := gbytes.NewBuffer()
				Expect(
					runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
						privileged, "fake_mounter", "-type=bind", "-sourcePath="+src, "-targetPath="+dest, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
				).To(Succeed())

				Expect(stdout).To(gbytes.Say("%s (aufs|ext4|btrfs) rw,relatime", dest))
			})

			Context("when file is mounted", func() {
				var (
					srcFile string
					dstFile string
				)

				BeforeEach(func() {
					file, err := ioutil.TempFile("", "")
					Expect(err).ToNot(HaveOccurred())
					fmt.Fprintf(file, "MountMe")
					srcFile = file.Name()
					dstFile = fmt.Sprintf("/tmp/fake-mount-file-%d", GinkgoParallelNode())
				})

				AfterEach(func() {
					Expect(os.Remove(srcFile)).To(Succeed())
				})

				It("mounts a file using the sourcePath", func() {
					stdout := gbytes.NewBuffer()
					Expect(
						runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
							privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath="+dstFile, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
					).To(Succeed())

					info, err := os.Stat(dstFile)
					Expect(err).ToNot(HaveOccurred())
					Expect(info.IsDir()).ToNot(BeTrue())
					Expect(stdout).To(gbytes.Say("%s (aufs|ext4|btrfs) rw,relatime", dstFile))
					Expect(os.Remove(dstFile)).To(Succeed())
				})

				Context("when destination file is created in non-existing directory", func() {
					BeforeEach(func() {
						dstFile = fmt.Sprintf("/tmp/non-existing-dir-%d/fake-mount-file-%d", GinkgoParallelNode(), GinkgoParallelNode())
					})

					It("mounts a file using the sourcePath", func() {
						stdout := gbytes.NewBuffer()
						Expect(
							runInContainer(io.MultiWriter(stdout, GinkgoWriter), GinkgoWriter,
								privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath="+dstFile, fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
						).To(Succeed())

						info, err := os.Stat(dstFile)
						Expect(err).ToNot(HaveOccurred())
						Expect(info.IsDir()).ToNot(BeTrue())
						Expect(stdout).To(gbytes.Say("%s (aufs|ext4|btrfs) rw,relatime", dstFile))
						Expect(os.Remove(dstFile)).To(Succeed())
					})
				})

				Context("when destination file cannot be created", func() {
					It("returns an informative error", func() {
						stderr := gbytes.NewBuffer()
						Expect(
							runInContainer(GinkgoWriter, io.MultiWriter(GinkgoWriter, stderr),
								privileged, "fake_mounter", "-type=bind", "-sourcePath="+srcFile, "-targetPath=/tmp", fmt.Sprintf("-flags=%d", syscall.MS_BIND), "cat", "/proc/mounts"),
						).To(HaveOccurred())

						Expect(stderr).To(gbytes.Say("error: system: create mount point file /tmp: "))
					})
				})
			})
		})
	}

	Context("in an unprivileged container", func() {
		testMount(false)
	})

	Context("in an privileged container", func() {
		testMount(true)
	})
})
<|endoftext|>


<commit_before>package sudoku

import (
	"testing"
)

func TestForcingChains(t *testing.T) {

	//Steps to test this:
	//* In the forcing chain helper, calculate the steps once, then
	//pass them in each time in a list of ~10 calls to solveTechniqueTEstHelper that we know are valid here.
	//* VERIFY MANUALLY that each step that is returned is actually a valid application of forcingchains.

	options := solveTechniqueTestHelperOptions{
		checkAllSteps: true,
	}

	grid, solver, steps := humanSolveTechniqueTestHelperStepGenerator(t,
		"forcingchain_test1.sdk", "Forcing Chain", options)

	options.stepsToCheck.grid = grid
	options.stepsToCheck.solver = solver
	options.stepsToCheck.steps = steps

	//OK, now we'll walk through all of the options in a loop and make sure they all show
	//up in the solve steps.

	type loopOptions struct {
		targetCells  []cellRef
		targetNums   IntSlice
		pointerCells []cellRef
		pointerNums  IntSlice
		description  string
	}

	//Tester puzzle: http://www.komoroske.com/sudoku/index.php?puzzle=Q6Ur5iYGINSUFcyocqaY6G91DpttiqYzs

	tests := []loopOptions{
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
			description:  "cell (1,0) only has two options, 1 and 2, and if you put either one in and see the chain of implications it leads to, both ones end up with 7 in cell (0,1), so we can just fill that number in",
		},
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
			//Explicitly don't test description after the first one.
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		//Skipping 0,1 / 7 / 5,7 / 1,3 because I think implications force it wrong.
		//The reason is because there is an inconsistency down that branch... just one implication
		//step beyond when it finds a match. Hmmmm... interesting test case.
		//Of course, the number being filled _IS_ right... is that a coincidence? I wondder
		//if other ones in this set that I'm considering valid would ahve the same problem...
		//... I think they would, right? By definition one of hte branches leads to invalidity. This
		//technique is about finding a universal before you find that invalidity.
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//This next one's particularly long implication chain
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
	}

	if len(tests) != len(steps) {
		t.Error("We didn't have enough tests for all of the steps that forcing chains returned. Got", len(tests), "expected", len(steps))
	}

	for _, test := range tests {

		options.targetCells = test.targetCells
		options.targetNums = test.targetNums
		options.pointerCells = test.pointerCells
		options.pointerNums = test.pointerNums
		options.description = test.description

		humanSolveTechniqueTestHelper(t, "forcingchain_test1.sdk", "Forcing Chain", options)
	}

	//TODO: test all other valid steps that could be found at this grid state for this technique.

}
<commit_msg>TESTS FAIL. Changed how we print. Manually going through and checking which ones are missing creates an... inconsistency, which I marked down. Change the len test to after checking validity of all of them. The odd run hasn't reproduced yet...<commit_after>package sudoku

import (
	"log"
	"testing"
)

func TestForcingChains(t *testing.T) {

	//Steps to test this:
	//* In the forcing chain helper, calculate the steps once, then
	//pass them in each time in a list of ~10 calls to solveTechniqueTEstHelper that we know are valid here.
	//* VERIFY MANUALLY that each step that is returned is actually a valid application of forcingchains.

	options := solveTechniqueTestHelperOptions{
		checkAllSteps: true,
	}

	grid, solver, steps := humanSolveTechniqueTestHelperStepGenerator(t,
		"forcingchain_test1.sdk", "Forcing Chain", options)

	options.stepsToCheck.grid = grid
	options.stepsToCheck.solver = solver
	options.stepsToCheck.steps = steps

	//OK, now we'll walk through all of the options in a loop and make sure they all show
	//up in the solve steps.

	type loopOptions struct {
		targetCells  []cellRef
		targetNums   IntSlice
		pointerCells []cellRef
		pointerNums  IntSlice
		description  string
	}

	//Tester puzzle: http://www.komoroske.com/sudoku/index.php?puzzle=Q6Ur5iYGINSUFcyocqaY6G91DpttiqYzs

	tests := []loopOptions{
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
			description:  "cell (1,0) only has two options, 1 and 2, and if you put either one in and see the chain of implications it leads to, both ones end up with 7 in cell (0,1), so we can just fill that number in",
		},
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
			//Explicitly don't test description after the first one.
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		//Skipping 0,1 / 7 / 5,7 / 1,3 because I think implications force it wrong.
		//The reason is because there is an inconsistency down that branch... just one implication
		//step beyond when it finds a match. Hmmmm... interesting test case.
		//Of course, the number being filled _IS_ right... is that a coincidence? I wondder
		//if other ones in this set that I'm considering valid would ahve the same problem...
		//... I think they would, right? By definition one of hte branches leads to invalidity. This
		//technique is about finding a universal before you find that invalidity.
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//This next one's particularly long implication chain
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},

		//All of these are missing... what?
		//Oh, we fail as soon as we notice they don't all match.
		// 0,6 /3 / 1,0 / 1,2
		// 0,1 / 7 / 5,1 / 1,2
		// 0,6 / 3 / 5,1 / 1,2
		//8,3 / 7 / 8,7 / 1,2
		// 0,1 /7 / 5,4 / 2,3
		// 8,3 / 7 / 5,4 / 2,3

		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
	}

	for _, step := range steps {
		log.Println(step)
	}

	for _, test := range tests {

		options.targetCells = test.targetCells
		options.targetNums = test.targetNums
		options.pointerCells = test.pointerCells
		options.pointerNums = test.pointerNums
		options.description = test.description

		humanSolveTechniqueTestHelper(t, "forcingchain_test1.sdk", "Forcing Chain", options)
	}

	if len(tests) != len(steps) {
		t.Error("We didn't have enough tests for all of the steps that forcing chains returned. Got", len(tests), "expected", len(steps))
	}

	//TODO: test all other valid steps that could be found at this grid state for this technique.

}
<|endoftext|>


<commit_before>package web

import (
	"log"
	"net/http"
	"sync"

	"code.google.com/p/go.net/websocket"
)

type Message map[string]interface{}

type Hub struct {
	In   chan Message
	outs map[chan Message]bool
	sync.Mutex
}

func NewHub() *Hub {
	h := &Hub{outs: make(map[chan Message]bool)}
	h.In = make(chan Message)
	go func() {
		for m := range h.In {
			for out, _ := range h.outs {
				select {
				case out <- m:
				default:
					log.Println("could not broadcast tweet:", m)
				}
			}
		}
	}()

	return h
}

func (h *Hub) Add(out chan Message) {
	h.Lock()
	h.outs[out] = true
	h.Unlock()
}

func (h *Hub) Remove(out chan Message) {
	h.Lock()
	delete(h.outs, out)
	h.Unlock()
}

func (h *Hub) Handler() http.Handler {
	return websocket.Handler(func(ws *websocket.Conn) {
		in := make(chan Message)
		h.Add(in)
		go func() {
			for {
				var m Message
				if err := websocket.JSON.Receive(ws, &m); err == nil {
					//out <- m
				} else {
					log.Println("Message Websocket receive err:", err)
					return
				}
			}
		}()

		for m := range in {
			if err := websocket.JSON.Send(ws, &m); err != nil {
				log.Println("Message Websocket send err:", err)
				break
			}
		}
		h.Remove(in)
		ws.Close()
	})
}
<commit_msg>Changed Message to be more generic.<commit_after>package web

import (
	"log"
	"net/http"
	"sync"

	"code.google.com/p/go.net/websocket"
)

type Message interface{}

type Hub struct {
	In   chan Message
	outs map[chan Message]bool
	sync.Mutex
}

func NewHub() *Hub {
	h := &Hub{outs: make(map[chan Message]bool)}
	h.In = make(chan Message)
	go func() {
		for m := range h.In {
			for out, _ := range h.outs {
				select {
				case out <- m:
				default:
					log.Println("could not broadcast tweet:", m)
				}
			}
		}
	}()

	return h
}

func (h *Hub) Add(out chan Message) {
	h.Lock()
	h.outs[out] = true
	h.Unlock()
}

func (h *Hub) Remove(out chan Message) {
	h.Lock()
	delete(h.outs, out)
	h.Unlock()
}

func (h *Hub) Handler() http.Handler {
	return websocket.Handler(func(ws *websocket.Conn) {
		in := make(chan Message)
		h.Add(in)
		go func() {
			for {
				var m Message
				if err := websocket.JSON.Receive(ws, &m); err == nil {
					//out <- m
				} else {
					log.Println("Message Websocket receive err:", err)
					return
				}
			}
		}()

		for m := range in {
			if err := websocket.JSON.Send(ws, &m); err != nil {
				log.Println("Message Websocket send err:", err)
				break
			}
		}
		h.Remove(in)
		ws.Close()
	})
}
<|endoftext|>