<commit_before>package common

import (
	"bytes"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/microcosm-cc/bluemonday"
	"github.com/webx-top/com"
)

func NewUGCPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	allowMedia(p)
	return p
}

func NewStrictPolicy() *bluemonday.Policy {
	p := bluemonday.StrictPolicy()
	return p
}

var (
	secureStrictPolicy                = NewStrictPolicy()
	secureUGCPolicy                   = NewUGCPolicy()
	secureUGCPolicyAllowDataURIImages *bluemonday.Policy
	secureUGCPolicyNoLink             = NoLink()
)

func init() {
	secureUGCPolicyAllowDataURIImages = NewUGCPolicy()
	secureUGCPolicyAllowDataURIImages.AllowDataURIImages()
}

// ClearHTML 清除所有HTML标签及其属性，一般用处理文章标题等不含HTML标签的字符串
func ClearHTML(title string) string {
	return secureStrictPolicy.Sanitize(title)
}

// RemoveXSS 清除不安全的HTML标签和属性，一般用于处理文章内容
func RemoveXSS(content string, noLinks ...bool) string {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.Sanitize(content)
	}
	return secureUGCPolicy.Sanitize(content)
}

func NoLink() *bluemonday.Policy {
	p := HTMLFilter()
	p.AllowStandardAttributes()

	////////////////////////////////
	// Declarations and structure //
	////////////////////////////////

	// "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are
	// expecting user generated content to be a fragment of HTML and not a full
	// document.

	//////////////////////////
	// Sectioning root tags //
	//////////////////////////

	// "article" and "aside" are permitted and takes no attributes
	p.AllowElements("article", "aside")

	// "body" is not permitted as we are expecting user generated content to be a fragment
	// of HTML and not a full document.

	// "details" is permitted, including the "open" attribute which can either
	// be blank or the value "open".
	p.AllowAttrs(
		"open",
	).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details")

	// "fieldset" is not permitted as we are not allowing forms to be created.

	// "figure" is permitted and takes no attributes
	p.AllowElements("figure")

	// "nav" is not permitted as it is assumed that the site (and not the user)
	// has defined navigation elements

	// "section" is permitted and takes no attributes
	p.AllowElements("section")

	// "summary" is permitted and takes no attributes
	p.AllowElements("summary")

	//////////////////////////
	// Headings and footers //
	//////////////////////////

	// "footer" is not permitted as we expect user content to be a fragment and
	// not structural to this extent

	// "h1" through "h6" are permitted and take no attributes
	p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6")

	// "header" is not permitted as we expect user content to be a fragment and
	// not structural to this extent

	// "hgroup" is permitted and takes no attributes
	p.AllowElements("hgroup")

	/////////////////////////////////////
	// Content grouping and separating //
	/////////////////////////////////////

	// "blockquote" is permitted, including the "cite" attribute which must be
	// a standard URL.
	p.AllowAttrs("cite").OnElements("blockquote")

	// "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes
	p.AllowElements("br", "div", "hr", "p", "span", "wbr")

	// "area" is permitted along with the attributes that map image maps work
	p.AllowAttrs("name").Matching(
		regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`),
	).OnElements("map")
	p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area")
	p.AllowAttrs("coords").Matching(
		regexp.MustCompile(`^([0-9]+,)+[0-9]+$`),
	).OnElements("area")
	p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area")
	p.AllowAttrs("shape").Matching(
		regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`),
	).OnElements("area")
	p.AllowAttrs("usemap").Matching(
		regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`),
	).OnElements("img")

	// "link" is not permitted

	/////////////////////
	// Phrase elements //
	/////////////////////

	// The following are all inline phrasing elements
	p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em",
		"figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var")

	// "q" is permitted and "cite" is a URL and handled by URL policies
	p.AllowAttrs("cite").OnElements("q")

	// "time" is permitted
	p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time")

	////////////////////
	// Style elements //
	////////////////////

	// block and inline elements that impart no semantic meaning but style the
	// document
	p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u")

	// "style" is not permitted as we are not yet sanitising CSS and it is an
	// XSS attack vector

	//////////////////////
	// HTML5 Formatting //
	//////////////////////

	// "bdi" "bdo" are permitted
	p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo")

	// "rp" "rt" "ruby" are permitted
	p.AllowElements("rp", "rt", "ruby")

	///////////////////////////
	// HTML5 Change tracking //
	///////////////////////////

	// "del" "ins" are permitted
	p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins")
	p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins")

	///////////
	// Lists //
	///////////

	p.AllowLists()

	////////////
	// Tables //
	////////////

	p.AllowTables()

	///////////
	// Forms //
	///////////

	// By and large, forms are not permitted. However there are some form
	// elements that can be used to present data, and we do permit those
	//
	// "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist"
	// "textarea" "optgroup" "option" are all not permitted

	// "meter" is permitted
	p.AllowAttrs(
		"value",
		"min",
		"max",
		"low",
		"high",
		"optimum",
	).Matching(bluemonday.Number).OnElements("meter")

	// "progress" is permitted
	p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress")

	//////////////////////
	// Embedded content //
	//////////////////////

	// Vast majority not permitted
	// "audio" "canvas" "embed" "iframe" "object" "param" "source" "svg" "track"
	// "video" are all not permitted
	allowMedia(p)

	// "img" is permitted
	p.AllowAttrs("align").Matching(bluemonday.ImageAlign).OnElements("img")
	p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("img")
	p.AllowAttrs("height", "width").Matching(bluemonday.NumberOrPercent).OnElements("img")
	p.AllowAttrs("src").OnElements("img")

	return p
}

func allowMedia(p *bluemonday.Policy) {
	p.AllowElements("picture")
	//<video webkit-playsinline="true" x-webkit-airplay="true" playsinline="true" x5-video-player-type="h5" x5-video-orientation="h5" x5-video-player-fullscreen="true" preload="auto" class="evaluate-video" src="'+source+'" poster="'+source+'?vframe/jpg/offset/1"></video>
	p.AllowAttrs("src", "controls", "width", "height", "autoplay", "muted", "loop", "poster", "preload", "playsinline", "webkit-playsinline", "x-webkit-airplay", "x5-video-player-type", "x5-video-orientation", "x5-video-player-fullscreen").OnElements("video")
	p.AllowAttrs("src", "controls", "width", "height", "autoplay", "muted", "loop", "preload").OnElements("audio")
	p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
}

func RemoveBytesXSS(content []byte, noLinks ...bool) []byte {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.SanitizeBytes(content)
	}
	return secureUGCPolicy.SanitizeBytes(content)
}

func RemoveReaderXSS(reader io.Reader, noLinks ...bool) *bytes.Buffer {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.SanitizeReader(reader)
	}
	return secureUGCPolicy.SanitizeReader(reader)
}

// HTMLFilter 构建自定义的HTML标签过滤器
func HTMLFilter() *bluemonday.Policy {
	return bluemonday.NewPolicy()
}

func MyRemoveXSS(content string) string {
	return com.RemoveXSS(content)
}

func MyCleanText(value string) string {
	value = com.StripTags(value)
	value = com.RemoveEOL(value)
	return value
}

func MyCleanTags(value string) string {
	value = com.StripTags(value)
	return value
}

var (
	q                           = rune('`')
	markdownLinkWithDoubleQuote = regexp.MustCompile(`(\]\([^ \)]+ )"([^"\)]+)"(\))`)
	markdownLinkWithSingleQuote = regexp.MustCompile(`(\]\([^ \)]+ )'([^'\)]+)'(\))`)
	markdownLinkWithScript      = regexp.MustCompile(`(?i)(\]\()(javascript):([^\)]*\))`)
	markdownQuoteTag            = regexp.MustCompile("((\n|^)[ ]{0,3})>")
	markdownCodeBlock           = regexp.MustCompile("(?s)([\r\n]|^)```([\\w]*[\r\n].*?[\r\n])```([\r\n]|$)")
)

func MarkdownPickoutCodeblock(content string) (repl []string, newContent string) {
	newContent = markdownCodeBlock.ReplaceAllStringFunc(content, func(found string) string {
		placeholder := `{codeblock(` + strconv.Itoa(len(repl)) + `)}`
		leftIndex := strings.Index(found, "```")
		rightIndex := strings.LastIndex(found, "```")
		repl = append(repl, found[leftIndex+3:rightIndex])
		return found[0:leftIndex+3] + placeholder + found[rightIndex:]
	})
	//echo.Dump([]interface{}{repl, newContent, content})
	return
}

func MarkdownRestorePickout(repl []string, content string) string {
	for i, r := range repl {
		if strings.Count(r, "\n") < 2 {
			r = strings.TrimLeft(r, "\r")
			if !strings.HasPrefix(r, "\n") {
				r = "\n" + r
			}
		}
		if !strings.HasSuffix(r, "\n") {
			r += "\n"
		}
		find := "```{codeblock(" + strconv.Itoa(i) + ")}```"
		content = strings.Replace(content, find, "```"+r+"```", 1)
	}
	return content
}

func ContentEncode(content string, contypes ...string) string {
	if len(content) == 0 {
		return content
	}
	var contype string
	if len(contypes) > 0 {
		contype = contypes[0]
	}
	switch contype {
	case `html`:
		content = RemoveXSS(content)

	case `url`, `image`, `video`, `audio`, `file`, `id`:
		content = MyCleanText(content)

	case `text`:
		content = com.StripTags(content)

	case `json`:
		// pass

	case `markdown`:
		// 提取代码块
		var pick []string
		pick, content = MarkdownPickoutCodeblock(content)

		// - 删除XSS

		// 删除HTML中的XSS代码
		content = RemoveXSS(content)
		// 拦截Markdown链接中的“javascript:”
		content = markdownLinkWithScript.ReplaceAllString(content, `${1}-${2}-${3}`)

		// - 还原

		// 还原双引号
		content = markdownLinkWithDoubleQuote.ReplaceAllString(content, `${1}"${2}"${3}`)
		// 还原单引号
		content = markdownLinkWithSingleQuote.ReplaceAllString(content, `${1}'${2}'${3}`)
		// 还原引用标识
		content = markdownQuoteTag.ReplaceAllString(content, `${1}>`)
		// 还原代码块
		content = MarkdownRestorePickout(pick, content)

	case `list`:
		content = MyCleanText(content)
		content = strings.TrimSpace(content)
		content = strings.Trim(content, `,`)

	default:
		content = com.StripTags(content)
	}
	content = strings.TrimSpace(content)
	return content
}
<commit_msg>update<commit_after>package common

import (
	"bytes"
	"io"
	"regexp"
	"strconv"
	"strings"

	"github.com/microcosm-cc/bluemonday"
	"github.com/webx-top/com"
)

func NewUGCPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	allowMedia(p)
	return p
}

func NewStrictPolicy() *bluemonday.Policy {
	p := bluemonday.StrictPolicy()
	return p
}

var (
	secureStrictPolicy                = NewStrictPolicy()
	secureUGCPolicy                   = NewUGCPolicy()
	secureUGCPolicyAllowDataURIImages *bluemonday.Policy
	secureUGCPolicyNoLink             = NoLink()
)

func init() {
	secureUGCPolicyAllowDataURIImages = NewUGCPolicy()
	secureUGCPolicyAllowDataURIImages.AllowDataURIImages()
}

// ClearHTML 清除所有HTML标签及其属性，一般用处理文章标题等不含HTML标签的字符串
func ClearHTML(title string) string {
	return secureStrictPolicy.Sanitize(title)
}

// RemoveXSS 清除不安全的HTML标签和属性，一般用于处理文章内容
func RemoveXSS(content string, noLinks ...bool) string {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.Sanitize(content)
	}
	return secureUGCPolicy.Sanitize(content)
}

func NoLink() *bluemonday.Policy {
	p := HTMLFilter()
	p.AllowStandardAttributes()

	////////////////////////////////
	// Declarations and structure //
	////////////////////////////////

	// "xml" "xslt" "DOCTYPE" "html" "head" are not permitted as we are
	// expecting user generated content to be a fragment of HTML and not a full
	// document.

	//////////////////////////
	// Sectioning root tags //
	//////////////////////////

	// "article" and "aside" are permitted and takes no attributes
	p.AllowElements("article", "aside")

	// "body" is not permitted as we are expecting user generated content to be a fragment
	// of HTML and not a full document.

	// "details" is permitted, including the "open" attribute which can either
	// be blank or the value "open".
	p.AllowAttrs(
		"open",
	).Matching(regexp.MustCompile(`(?i)^(|open)$`)).OnElements("details")

	// "fieldset" is not permitted as we are not allowing forms to be created.

	// "figure" is permitted and takes no attributes
	p.AllowElements("figure")

	// "nav" is not permitted as it is assumed that the site (and not the user)
	// has defined navigation elements

	// "section" is permitted and takes no attributes
	p.AllowElements("section")

	// "summary" is permitted and takes no attributes
	p.AllowElements("summary")

	//////////////////////////
	// Headings and footers //
	//////////////////////////

	// "footer" is not permitted as we expect user content to be a fragment and
	// not structural to this extent

	// "h1" through "h6" are permitted and take no attributes
	p.AllowElements("h1", "h2", "h3", "h4", "h5", "h6")

	// "header" is not permitted as we expect user content to be a fragment and
	// not structural to this extent

	// "hgroup" is permitted and takes no attributes
	p.AllowElements("hgroup")

	/////////////////////////////////////
	// Content grouping and separating //
	/////////////////////////////////////

	// "blockquote" is permitted, including the "cite" attribute which must be
	// a standard URL.
	p.AllowAttrs("cite").OnElements("blockquote")

	// "br" "div" "hr" "p" "span" "wbr" are permitted and take no attributes
	p.AllowElements("br", "div", "hr", "p", "span", "wbr")

	// "area" is permitted along with the attributes that map image maps work
	p.AllowAttrs("name").Matching(
		regexp.MustCompile(`^([\p{L}\p{N}_-]+)$`),
	).OnElements("map")
	p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("area")
	p.AllowAttrs("coords").Matching(
		regexp.MustCompile(`^([0-9]+,)+[0-9]+$`),
	).OnElements("area")
	p.AllowAttrs("rel").Matching(bluemonday.SpaceSeparatedTokens).OnElements("area")
	p.AllowAttrs("shape").Matching(
		regexp.MustCompile(`(?i)^(default|circle|rect|poly)$`),
	).OnElements("area")
	p.AllowAttrs("usemap").Matching(
		regexp.MustCompile(`(?i)^#[\p{L}\p{N}_-]+$`),
	).OnElements("img")

	// "link" is not permitted

	/////////////////////
	// Phrase elements //
	/////////////////////

	// The following are all inline phrasing elements
	p.AllowElements("abbr", "acronym", "cite", "code", "dfn", "em",
		"figcaption", "mark", "s", "samp", "strong", "sub", "sup", "var")

	// "q" is permitted and "cite" is a URL and handled by URL policies
	p.AllowAttrs("cite").OnElements("q")

	// "time" is permitted
	p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("time")

	////////////////////
	// Style elements //
	////////////////////

	// block and inline elements that impart no semantic meaning but style the
	// document
	p.AllowElements("b", "i", "pre", "small", "strike", "tt", "u")

	// "style" is not permitted as we are not yet sanitising CSS and it is an
	// XSS attack vector

	//////////////////////
	// HTML5 Formatting //
	//////////////////////

	// "bdi" "bdo" are permitted
	p.AllowAttrs("dir").Matching(bluemonday.Direction).OnElements("bdi", "bdo")

	// "rp" "rt" "ruby" are permitted
	p.AllowElements("rp", "rt", "ruby")

	///////////////////////////
	// HTML5 Change tracking //
	///////////////////////////

	// "del" "ins" are permitted
	p.AllowAttrs("cite").Matching(bluemonday.Paragraph).OnElements("del", "ins")
	p.AllowAttrs("datetime").Matching(bluemonday.ISO8601).OnElements("del", "ins")

	///////////
	// Lists //
	///////////

	p.AllowLists()

	////////////
	// Tables //
	////////////

	p.AllowTables()

	///////////
	// Forms //
	///////////

	// By and large, forms are not permitted. However there are some form
	// elements that can be used to present data, and we do permit those
	//
	// "button" "fieldset" "input" "keygen" "label" "output" "select" "datalist"
	// "textarea" "optgroup" "option" are all not permitted

	// "meter" is permitted
	p.AllowAttrs(
		"value",
		"min",
		"max",
		"low",
		"high",
		"optimum",
	).Matching(bluemonday.Number).OnElements("meter")

	// "progress" is permitted
	p.AllowAttrs("value", "max").Matching(bluemonday.Number).OnElements("progress")

	//////////////////////
	// Embedded content //
	//////////////////////

	// Vast majority not permitted
	// "audio" "canvas" "embed" "iframe" "object" "param" "source" "svg" "track"
	// "video" are all not permitted
	allowMedia(p)

	// "img" is permitted
	p.AllowAttrs("align").Matching(bluemonday.ImageAlign).OnElements("img")
	p.AllowAttrs("alt").Matching(bluemonday.Paragraph).OnElements("img")
	p.AllowAttrs("height", "width").Matching(bluemonday.NumberOrPercent).OnElements("img")
	p.AllowAttrs("src").OnElements("img")

	return p
}

func allowMedia(p *bluemonday.Policy) {
	p.AllowElements("picture")
	//<video webkit-playsinline="true" x-webkit-airplay="true" playsinline="true" x5-video-player-type="h5" x5-video-orientation="h5" x5-video-player-fullscreen="true" preload="auto" class="evaluate-video" src="'+source+'" poster="'+source+'?vframe/jpg/offset/1"></video>
	p.AllowAttrs(
		"src", "controls", "width", "height", "autoplay", "muted", "loop", "poster", "preload", "playsinline",
		"webkit-playsinline", "x-webkit-airplay",
		"x5-video-player-type", "x5-video-orientation", "x5-video-player-fullscreen",
	).OnElements("video")
	p.AllowAttrs("src", "controls", "width", "height", "autoplay", "muted", "loop", "preload").OnElements("audio")
	p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
}

func RemoveBytesXSS(content []byte, noLinks ...bool) []byte {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.SanitizeBytes(content)
	}
	return secureUGCPolicy.SanitizeBytes(content)
}

func RemoveReaderXSS(reader io.Reader, noLinks ...bool) *bytes.Buffer {
	if len(noLinks) > 0 && noLinks[0] {
		return secureUGCPolicyNoLink.SanitizeReader(reader)
	}
	return secureUGCPolicy.SanitizeReader(reader)
}

// HTMLFilter 构建自定义的HTML标签过滤器
func HTMLFilter() *bluemonday.Policy {
	return bluemonday.NewPolicy()
}

func MyRemoveXSS(content string) string {
	return com.RemoveXSS(content)
}

func MyCleanText(value string) string {
	value = com.StripTags(value)
	value = com.RemoveEOL(value)
	return value
}

func MyCleanTags(value string) string {
	value = com.StripTags(value)
	return value
}

var (
	q                           = rune('`')
	markdownLinkWithDoubleQuote = regexp.MustCompile(`(\]\([^ \)]+ )"([^"\)]+)"(\))`)
	markdownLinkWithSingleQuote = regexp.MustCompile(`(\]\([^ \)]+ )'([^'\)]+)'(\))`)
	markdownLinkWithScript      = regexp.MustCompile(`(?i)(\]\()(javascript):([^\)]*\))`)
	markdownQuoteTag            = regexp.MustCompile("((\n|^)[ ]{0,3})>")
	markdownCodeBlock           = regexp.MustCompile("(?s)([\r\n]|^)```([\\w]*[\r\n].*?[\r\n])```([\r\n]|$)")
)

func MarkdownPickoutCodeblock(content string) (repl []string, newContent string) {
	newContent = markdownCodeBlock.ReplaceAllStringFunc(content, func(found string) string {
		placeholder := `{codeblock(` + strconv.Itoa(len(repl)) + `)}`
		leftIndex := strings.Index(found, "```")
		rightIndex := strings.LastIndex(found, "```")
		repl = append(repl, found[leftIndex+3:rightIndex])
		return found[0:leftIndex+3] + placeholder + found[rightIndex:]
	})
	//echo.Dump([]interface{}{repl, newContent, content})
	return
}

func MarkdownRestorePickout(repl []string, content string) string {
	for i, r := range repl {
		if strings.Count(r, "\n") < 2 {
			r = strings.TrimLeft(r, "\r")
			if !strings.HasPrefix(r, "\n") {
				r = "\n" + r
			}
		}
		if !strings.HasSuffix(r, "\n") {
			r += "\n"
		}
		find := "```{codeblock(" + strconv.Itoa(i) + ")}```"
		content = strings.Replace(content, find, "```"+r+"```", 1)
	}
	return content
}

func ContentEncode(content string, contypes ...string) string {
	if len(content) == 0 {
		return content
	}
	var contype string
	if len(contypes) > 0 {
		contype = contypes[0]
	}
	switch contype {
	case `html`:
		content = RemoveXSS(content)

	case `url`, `image`, `video`, `audio`, `file`, `id`:
		content = MyCleanText(content)

	case `text`:
		content = com.StripTags(content)

	case `json`:
		// pass

	case `markdown`:
		// 提取代码块
		var pick []string
		pick, content = MarkdownPickoutCodeblock(content)

		// - 删除XSS

		// 删除HTML中的XSS代码
		content = RemoveXSS(content)
		// 拦截Markdown链接中的“javascript:”
		content = markdownLinkWithScript.ReplaceAllString(content, `${1}-${2}-${3}`)

		// - 还原

		// 还原双引号
		content = markdownLinkWithDoubleQuote.ReplaceAllString(content, `${1}"${2}"${3}`)
		// 还原单引号
		content = markdownLinkWithSingleQuote.ReplaceAllString(content, `${1}'${2}'${3}`)
		// 还原引用标识
		content = markdownQuoteTag.ReplaceAllString(content, `${1}>`)
		// 还原代码块
		content = MarkdownRestorePickout(pick, content)

	case `list`:
		content = MyCleanText(content)
		content = strings.TrimSpace(content)
		content = strings.Trim(content, `,`)

	default:
		content = com.StripTags(content)
	}
	content = strings.TrimSpace(content)
	return content
}
<|endoftext|>


<commit_before>package text

import (
	"testing"
)

func TestWord_Match(t *testing.T) {
	cases := []struct {
		s   string
		q   word
		exp int
	}{
		{
			s:   "Alligators eat mattresses",
			q:   Word("Alligators"),
			exp: 0,
		},
		{
			s:   "Alligators eat mattresses",
			q:   Word("mattresses"),
			exp: 15,
		},
		{
			s:   "Alligators eat mattresses",
			q:   Word("gators"),
			exp: -1,
		},
		{
			s:   "Alli\ngators eat mattresses",
			q:   Word("gators"),
			exp: 5,
		},
		{
			s:   "Alligators eat meat",
			q:   Word("eat"),
			exp: 11,
		},
	}

	for _, c := range cases {
		act := c.q.Match(c.s)

		if act != c.exp {
			t.Errorf("Expected Word(%q).Match(%q) to return %v, got %v", c.q.W, c.s, c.exp, act)
		}
	}
}

func BenchmarkWord_Match(b *testing.B) {
	w := Word("foo bar baz biz boz bem boos bick dale biz buul hum dirk hass tukk murr")
	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		w.Match("biz")
	}
}
<commit_msg>Increase message size in Word.Match() benchmark<commit_after>package text

import (
	"testing"
)

func TestWord_Match(t *testing.T) {
	cases := []struct {
		s   string
		q   word
		exp int
	}{
		{
			s:   "Alligators eat mattresses",
			q:   Word("Alligators"),
			exp: 0,
		},
		{
			s:   "Alligators eat mattresses",
			q:   Word("mattresses"),
			exp: 15,
		},
		{
			s:   "Alligators eat mattresses",
			q:   Word("gators"),
			exp: -1,
		},
		{
			s:   "Alli\ngators eat mattresses",
			q:   Word("gators"),
			exp: 5,
		},
		{
			s:   "Alligators eat meat",
			q:   Word("eat"),
			exp: 11,
		},
	}

	for _, c := range cases {
		act := c.q.Match(c.s)

		if act != c.exp {
			t.Errorf("Expected Word(%q).Match(%q) to return %v, got %v", c.q.W, c.s, c.exp, act)
		}
	}
}

func BenchmarkWord_MatchExist(b *testing.B) {
	w := Word("foobar")

	txt := `Lorem ipsum dolor sit amet, an cum vero soleat concludaturque, te purto vero reprimique vis.
	Ignota mediocritatem ut sea. Cetero deserunt pericula te vel. Omnis legendos no per.
	Sale illum pertinax no sed, est posse putent minimum foobar no. Pri et vitae mentitum eligendi,
	no ius reque fugit libris, eos ad quaeque pericula mediocrem. Habemus corpora an mea,
	inermis partiendo per et, at nemore dolorem iudicabit eos. At est mucius docendi. Sed et nisl facilisi.
	Idque suavitate argumentum eu eam, vis putant insolens dissentiunt id. Dictas labitur in mei, duo omnium assentior scripserit cu.`

	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		w.Match(txt)
	}
}

func BenchmarkWord_MatchNotExist(b *testing.B) {
	w := Word("foobar")

	txt := `Lorem ipsum dolor sit amet, an cum vero soleat concludaturque, te purto vero reprimique vis.
	Ignota mediocritatem ut sea. Cetero deserunt pericula te vel. Omnis legendos no per.
	Sale illum pertinax no sed, est posse putent minimum no. Pri et vitae mentitum eligendi,
	no ius reque fugit libris, eos ad quaeque pericula mediocrem. Habemus corpora an mea,
	inermis partiendo per et, at nemore dolorem iudicabit eos. At est mucius docendi. Sed et nisl facilisi.
	Idque suavitate argumentum eu eam, vis putant insolens dissentiunt id. Dictas labitur in mei, duo omnium assentior scripserit cu.`

	b.ResetTimer()

	for n := 0; n < b.N; n++ {
		w.Match(txt)
	}
}
<|endoftext|>


<commit_before>// A library to Manage Memset snapshots
package snapshot

// FIXME return the total bytes from putChunked and adjust the manifest appropriately with source size (if .raw) or returned size (if .raw.gz)

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/ncw/swift"
)

const (
	// Name of the container with the shapshots
	DefaultContainer = "miniserver-snapshots"
	// Date format for the snapshots directory names
	DirectoryDate = "2006-01-02-15-04-05"
	// Python date format as used in the README.txt
	ReadmeDateFormat = "2006-01-02T15:04:05.999999999"
)

// Describes a snapshot
type Snapshot struct {
	Manager    *Manager
	Name       string
	Path       string
	Comment    string
	Date       time.Time
	ReadMe     string
	Broken     bool
	Miniserver string
	ImageType  string
	ImageLeaf  string
	Md5        string
	DiskSize   int64
}

// Return whether the snapshot exists
func (s *Snapshot) Exists() (bool, error) {
	objects, err := s.Manager.Swift.Objects(s.Manager.Container, &swift.ObjectsOpts{
		Prefix:    s.Name + "/",
		Delimiter: '/',
	})
	if err == swift.ContainerNotFound || err == swift.ObjectNotFound {
		return false, nil
	}
	if err != nil {
		return false, fmt.Errorf("failed to list snapshots: %v", err)
	}
	return len(objects) != 0, nil
}

// Lists the snapshot to stdout
func (s *Snapshot) List() {
	fmt.Printf("%s\n", s.Name)
	if s.Comment != "" {
		fmt.Printf("  Comment    - %s\n", s.Comment)
	}
	if s.Path != "" {
		fmt.Printf("  Path       - %s\n", s.Path)
	}
	if !s.Date.IsZero() {
		fmt.Printf("  Date       - %s\n", s.Date)
	}
	fmt.Printf("  Broken     - %v\n", s.Broken)
	if s.Miniserver != "" {
		fmt.Printf("  Miniserver - %s\n", s.Miniserver)
	}
	if s.ImageType != "" {
		fmt.Printf("  ImageType  - %s\n", s.ImageType)
	}
	if s.ImageLeaf != "" {
		fmt.Printf("  ImageLeaf  - %s\n", s.ImageLeaf)
	}
	if s.Md5 != "" {
		fmt.Printf("  Md5        - %s\n", s.Md5)
	}
	if s.DiskSize != 0 {
		fmt.Printf("  DiskSize   - %d\n", s.DiskSize)
	}
}

// Parses the README.txt
func (s *Snapshot) ParseReadme(readme string) {
	var err error
	s.ReadMe = readme
	for _, line := range strings.Split(readme, "\n") {
		if !strings.Contains(line, "=") {
			continue
		}
		tokens := strings.SplitN(line, "=", 2)
		token := strings.ToLower(strings.TrimSpace(tokens[0]))
		value := strings.TrimSpace(tokens[1])
		switch token {
		case "user_comment":
			s.Comment = value
		case "date": // 2015-01-08T15:44:16.695676
			s.Date, err = time.Parse(ReadmeDateFormat, value)
			if err != nil {
				log.Printf("Failed to parse date from %q: %v", value, err)
			}
		case "miniserver": // myaccaa1
			s.Miniserver = value
		case "image_type": // Tarball file
			s.ImageType = value
		case "snapshot_image": // myacaa1.tar
			s.ImageLeaf = value
		case "md5(snapshot_image)": // 09e29a798ec4f3e4273981cc176adc32
			s.Md5 = value
		case "disk_size": // 42949672960
			s.DiskSize, err = strconv.ParseInt(value, 10, 64)
			if err != nil {
				log.Printf("Failed to parse disk size from %q: %v", value, err)
			}
		}
	}
}

// Creates the README from the Snapshot
func (s *Snapshot) CreateReadme() {
	out := new(bytes.Buffer)
	fmt.Fprintf(out, `; This directory contains a virtual machine disk image snapshot.
; The files in this directory are described below.
; For more information see: http://www.memset.com/docs/
;
; Uploaded by snapshot-manager on %v to %q
;
`, time.Now(), s.Name)
	if !s.Date.IsZero() {
		fmt.Fprintf(out, "date = %s\n", s.Date.Format(ReadmeDateFormat))
	}
	if s.Miniserver != "" {
		fmt.Fprintf(out, "miniserver = %s\n", s.Miniserver)
	}
	if s.Comment != "" {
		fmt.Fprintf(out, "user_comment = %s\n", s.Comment)
	}
	if s.ImageType != "" {
		fmt.Fprintf(out, "image_type = %s\n", s.ImageType)
	}
	if s.ImageLeaf != "" {
		fmt.Fprintf(out, "snapshot_image = %s\n", s.ImageLeaf)
	}
	if s.Md5 != "" {
		fmt.Fprintf(out, "md5(snapshot_image) = %s\n", s.Md5)
	}
	if s.DiskSize != 0 {
		fmt.Fprintf(out, "disk_size = %d\n", s.DiskSize)
	}
	s.ReadMe = out.String()
}

// putChunkedFile puts in to continer/obectPath storing the chunks in
// chunksContainer/chunksPath.  It returns the number of bytes
// uploaded and an error
func (s *Snapshot) putChunkedFile(in io.Reader, container, objectPath string, chunksContainer, chunksPath string, mimeType string) (int64, error) {
	// Read chunks from the file
	buf := make([]byte, s.Manager.ChunkSize)
	size := int64(0)
	for finished, chunk := false, 1; !finished; chunk++ {
		n, err := io.ReadFull(in, buf)
		size += int64(n)
		if err == io.EOF {
			break
		} else if err == io.ErrUnexpectedEOF {
			finished = true
		} else if err != io.ErrUnexpectedEOF && err != nil {
			return size, fmt.Errorf("error reading %v", err)
		}
		chunkPath := fmt.Sprintf("%s/%04d", chunksPath, chunk)
		// FIXME retry
		log.Printf("Uploading chunk %q", chunkPath)
		err = s.Manager.Swift.ObjectPutBytes(container, chunkPath, buf[:n], mimeType)
		if err != nil {
			return size, fmt.Errorf("failed to upload chunk %q: %v", chunkPath, err)
		}
	}

	// Put the manifest if all was successful
	log.Printf("Uploading manifest %q", objectPath)
	contents := strings.NewReader("")
	headers := swift.Headers{
		"X-Object-Manifest": chunksContainer + "/" + chunksPath,
	}
	_, err := s.Manager.Swift.ObjectPut(container, objectPath, contents, true, "", "application/octet-stream", headers)
	return size, err
}

// Download a snapshot into outputDirectory
func (s *Snapshot) Get(outputDirectory string) error {
	objects, err := s.Manager.Objects(s.Name)
	if len(objects) == 0 {
		log.Fatal("Snapshot or snapshot objects not found")
	}
	err = os.MkdirAll(outputDirectory, 0755)
	if err != nil {
		return fmt.Errorf("failed to make output directory %q", outputDirectory)
	}
	err = os.Chdir(outputDirectory)
	if err != nil {
		return fmt.Errorf("failed chdir output directory %q", outputDirectory)
	}
	for _, object := range objects {
		if object.PseudoDirectory {
			continue
		}
		objectPath := object.Name
		leaf := path.Base(objectPath)
		fmt.Printf("Downloading %s\n", objectPath)
		out, err := os.Create(leaf)
		if err != nil {
			return fmt.Errorf("failed to open output file %q: %v", leaf, err)
		}
		_, err = s.Manager.Swift.ObjectGet(s.Manager.Container, objectPath, out, false, nil) // don't check MD5 because they are wrong for chunked files
		if err != nil {
			return fmt.Errorf("failed to download %q: %v", s.Name, err)
		}
		err = out.Close()
		if err != nil {
			return fmt.Errorf("failed to close %q: %v", s.Name, err)
		}
	}
	return nil
}

// checkClose is used to check the return from Close in a defer
// statement.
func checkClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil {
		*err = cerr
	}
}

// countWriter acts as an io.Writer counting the output
type countWriter int64

// Write counts up the data and ignores it
func (c *countWriter) Write(p []byte) (int, error) {
	*c += countWriter(len(p))
	return len(p), nil
}

// Puts a snapshot
func (s *Snapshot) Put(file string) (err error) {
	// Work out where to put things
	leaf := s.ImageLeaf
	Type := Types.Find(file)
	if Type == nil {
		return fmt.Errorf("unknown snapshot type %q - use types command to see available", leaf)
	}
	if !Type.Upload {
		return fmt.Errorf("can't upload snapshot type %q - use types command to see available", leaf)
	}
	s.ImageType = Type.ImageType
	chunksPath := s.Name + "/" + leaf[:len(leaf)-len(Type.Suffix)]
	objectPath := s.Path

	// Get file stat
	fi, err := os.Stat(file)
	if err != nil {
		return fmt.Errorf("failed to stat %q: %v", file, err)
	}
	if fi.IsDir() {
		return fmt.Errorf("%q is a directory", file)
	}
	s.Date = fi.ModTime()

	// Check file doesn't exist and container does
	ok, err := s.Exists()
	if err != nil {
		return err
	}
	if ok {
		return fmt.Errorf("snapshot %q already exists - delete it first", s.Name)
	}
	err = s.Manager.CreateContainer()
	if err != nil {
		return err
	}

	// Upload the file with chunks
	var in io.Reader
	fileIn, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("failed to open %q: %v", file, err)
	}
	in = fileIn
	defer checkClose(fileIn, &err)

	// If we need to read the size from the ungzipped data then do
	// it as we go along
	var gzipCounter *GzipCounter
	if Type.DiskSizeFrom == DiskSizeFromGzip {
		log.Printf("Gunzipping on the fly to count size")
		gzipCounter, err = NewGzipCounter()
		if err != nil {
			return fmt.Errorf("failed to make gzip counter: %v", err)
		}
		in = io.TeeReader(in, gzipCounter)
	}

	// Check if needs gunzip
	if Type.NeedsGunzip {
		log.Printf("Gunzipping on the fly")
		objectPath = objectPath[:len(objectPath)-3]
		var gzipRd io.ReadCloser
		gzipRd, err = gzip.NewReader(in)
		if err != nil {
			return fmt.Errorf("failed to make gzip decompressor: %v", err)
		}
		defer checkClose(gzipRd, &err)
		in = gzipRd
	}

	// Check if needs gzip
	if Type.NeedsGzip {
		log.Printf("Gzipping on the fly")
		objectPath += ".gz"
		var gzipRd io.ReadCloser
		gzipRd, err = NewGzipReader(in)
		if err != nil {
			return fmt.Errorf("failed to make gzip compressor: %v", err)
		}
		defer checkClose(gzipRd, &err)
		in = gzipRd
	}

	// Put the file in chunks
	size, err := s.putChunkedFile(in, s.Manager.Container, objectPath, s.Manager.Container, chunksPath, Type.MimeType)
	if err != nil {
		return err
	}

	// Set the DiskSize to the raw size of the upload
	switch Type.DiskSizeFrom {
	case DiskSizeFromUpload:
		// .tar.gz -> .tar
		s.DiskSize = size
	case DiskSizeFromFile:
		// .raw -> raw.gz
		// .tar
		s.DiskSize = fi.Size()
	case DiskSizeFromGzip:
		// .raw.gz
		err = gzipCounter.Close()
		if err != nil {
			return fmt.Errorf("error closing gzip counter: %v", err)
		}
		s.DiskSize = gzipCounter.Size()
	default:
		log.Printf("Can't figure out the disk size for %q - using the file size", Type.Suffix)
		s.DiskSize = fi.Size()
	}
	log.Printf("Using %d as disk_size in README.txt", s.DiskSize)

	// Write the README.txt
	s.CreateReadme()
	err = s.Manager.Swift.ObjectPutString(s.Manager.Container, s.Name+"/README.txt", s.ReadMe, "text/plain")
	if err != nil {
		return fmt.Errorf("failed to create README.txt: %v", err)
	}
	return nil
}

// Delete all the objects in the snapshot
func (s *Snapshot) Delete() error {
	objects, err := s.Manager.Swift.Objects(s.Manager.Container, &swift.ObjectsOpts{
		Prefix: s.Name + "/",
	})
	if err != nil {
		return fmt.Errorf("failed to read snapshot %q: %v", s.Name, err)
	}
	if len(objects) == 0 {
		return fmt.Errorf("snapshot or snapshot objects not found")
	}

	errors := 0
	for _, object := range objects {
		if object.PseudoDirectory {
			continue
		}
		log.Printf("Deleting %q", object.Name)
		err = s.Manager.Swift.ObjectDelete(s.Manager.Container, object.Name)
		if err != nil {
			errors += 1
			log.Printf("Failed to delete %q: %v", object.Name, err)
		}
	}
	if errors != 0 {
		return fmt.Errorf("failed to delete %d objects", errors)
	}
	return nil
}
<commit_msg>Add Md5 to README.txt and fix image leaf name<commit_after>// A library to Manage Memset snapshots
package snapshot

// FIXME return the total bytes from putChunked and adjust the manifest appropriately with source size (if .raw) or returned size (if .raw.gz)

import (
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/ncw/swift"
)

const (
	// Name of the container with the shapshots
	DefaultContainer = "miniserver-snapshots"
	// Date format for the snapshots directory names
	DirectoryDate = "2006-01-02-15-04-05"
	// Python date format as used in the README.txt
	ReadmeDateFormat = "2006-01-02T15:04:05.999999999"
)

// Describes a snapshot
type Snapshot struct {
	Manager    *Manager
	Name       string
	Path       string
	Comment    string
	Date       time.Time
	ReadMe     string
	Broken     bool
	Miniserver string
	ImageType  string
	ImageLeaf  string
	Md5        string
	DiskSize   int64
}

// Return whether the snapshot exists
func (s *Snapshot) Exists() (bool, error) {
	objects, err := s.Manager.Swift.Objects(s.Manager.Container, &swift.ObjectsOpts{
		Prefix:    s.Name + "/",
		Delimiter: '/',
	})
	if err == swift.ContainerNotFound || err == swift.ObjectNotFound {
		return false, nil
	}
	if err != nil {
		return false, fmt.Errorf("failed to list snapshots: %v", err)
	}
	return len(objects) != 0, nil
}

// Lists the snapshot to stdout
func (s *Snapshot) List() {
	fmt.Printf("%s\n", s.Name)
	if s.Comment != "" {
		fmt.Printf("  Comment    - %s\n", s.Comment)
	}
	if s.Path != "" {
		fmt.Printf("  Path       - %s\n", s.Path)
	}
	if !s.Date.IsZero() {
		fmt.Printf("  Date       - %s\n", s.Date)
	}
	fmt.Printf("  Broken     - %v\n", s.Broken)
	if s.Miniserver != "" {
		fmt.Printf("  Miniserver - %s\n", s.Miniserver)
	}
	if s.ImageType != "" {
		fmt.Printf("  ImageType  - %s\n", s.ImageType)
	}
	if s.ImageLeaf != "" {
		fmt.Printf("  ImageLeaf  - %s\n", s.ImageLeaf)
	}
	if s.Md5 != "" {
		fmt.Printf("  Md5        - %s\n", s.Md5)
	}
	if s.DiskSize != 0 {
		fmt.Printf("  DiskSize   - %d\n", s.DiskSize)
	}
}

// Parses the README.txt
func (s *Snapshot) ParseReadme(readme string) {
	var err error
	s.ReadMe = readme
	for _, line := range strings.Split(readme, "\n") {
		if !strings.Contains(line, "=") {
			continue
		}
		tokens := strings.SplitN(line, "=", 2)
		token := strings.ToLower(strings.TrimSpace(tokens[0]))
		value := strings.TrimSpace(tokens[1])
		switch token {
		case "user_comment":
			s.Comment = value
		case "date": // 2015-01-08T15:44:16.695676
			s.Date, err = time.Parse(ReadmeDateFormat, value)
			if err != nil {
				log.Printf("Failed to parse date from %q: %v", value, err)
			}
		case "miniserver": // myaccaa1
			s.Miniserver = value
		case "image_type": // Tarball file
			s.ImageType = value
		case "snapshot_image": // myacaa1.tar
			s.ImageLeaf = value
		case "md5(snapshot_image)": // 09e29a798ec4f3e4273981cc176adc32
			s.Md5 = value
		case "disk_size": // 42949672960
			s.DiskSize, err = strconv.ParseInt(value, 10, 64)
			if err != nil {
				log.Printf("Failed to parse disk size from %q: %v", value, err)
			}
		}
	}
}

// Creates the README from the Snapshot
func (s *Snapshot) CreateReadme() {
	out := new(bytes.Buffer)
	fmt.Fprintf(out, `; This directory contains a virtual machine disk image snapshot.
; The files in this directory are described below.
; For more information see: http://www.memset.com/docs/
;
; Uploaded by snapshot-manager on %v to %q
;
`, time.Now(), s.Name)
	if !s.Date.IsZero() {
		fmt.Fprintf(out, "date = %s\n", s.Date.Format(ReadmeDateFormat))
	}
	if s.Miniserver != "" {
		fmt.Fprintf(out, "miniserver = %s\n", s.Miniserver)
	}
	if s.Comment != "" {
		fmt.Fprintf(out, "user_comment = %s\n", s.Comment)
	}
	if s.ImageType != "" {
		fmt.Fprintf(out, "image_type = %s\n", s.ImageType)
	}
	if s.ImageLeaf != "" {
		fmt.Fprintf(out, "snapshot_image = %s\n", s.ImageLeaf)
	}
	if s.Md5 != "" {
		fmt.Fprintf(out, "md5(snapshot_image) = %s\n", s.Md5)
	}
	if s.DiskSize != 0 {
		fmt.Fprintf(out, "disk_size = %d\n", s.DiskSize)
	}
	s.ReadMe = out.String()
}

// putChunkedFile puts in to continer/obectPath storing the chunks in
// chunksContainer/chunksPath.  It returns the number of bytes
// uploaded and an error
func (s *Snapshot) putChunkedFile(in io.Reader, container, objectPath string, chunksContainer, chunksPath string, mimeType string) (int64, error) {
	// Read chunks from the file
	buf := make([]byte, s.Manager.ChunkSize)
	size := int64(0)
	for finished, chunk := false, 1; !finished; chunk++ {
		n, err := io.ReadFull(in, buf)
		size += int64(n)
		if err == io.EOF {
			break
		} else if err == io.ErrUnexpectedEOF {
			finished = true
		} else if err != io.ErrUnexpectedEOF && err != nil {
			return size, fmt.Errorf("error reading %v", err)
		}
		chunkPath := fmt.Sprintf("%s/%04d", chunksPath, chunk)
		// FIXME retry
		log.Printf("Uploading chunk %q", chunkPath)
		err = s.Manager.Swift.ObjectPutBytes(container, chunkPath, buf[:n], mimeType)
		if err != nil {
			return size, fmt.Errorf("failed to upload chunk %q: %v", chunkPath, err)
		}
	}

	// Put the manifest if all was successful
	log.Printf("Uploading manifest %q", objectPath)
	contents := strings.NewReader("")
	headers := swift.Headers{
		"X-Object-Manifest": chunksContainer + "/" + chunksPath,
	}
	_, err := s.Manager.Swift.ObjectPut(container, objectPath, contents, true, "", "application/octet-stream", headers)
	return size, err
}

// Download a snapshot into outputDirectory
func (s *Snapshot) Get(outputDirectory string) error {
	objects, err := s.Manager.Objects(s.Name)
	if len(objects) == 0 {
		log.Fatal("Snapshot or snapshot objects not found")
	}
	err = os.MkdirAll(outputDirectory, 0755)
	if err != nil {
		return fmt.Errorf("failed to make output directory %q", outputDirectory)
	}
	err = os.Chdir(outputDirectory)
	if err != nil {
		return fmt.Errorf("failed chdir output directory %q", outputDirectory)
	}
	for _, object := range objects {
		if object.PseudoDirectory {
			continue
		}
		objectPath := object.Name
		leaf := path.Base(objectPath)
		fmt.Printf("Downloading %s\n", objectPath)
		out, err := os.Create(leaf)
		if err != nil {
			return fmt.Errorf("failed to open output file %q: %v", leaf, err)
		}
		_, err = s.Manager.Swift.ObjectGet(s.Manager.Container, objectPath, out, false, nil) // don't check MD5 because they are wrong for chunked files
		if err != nil {
			return fmt.Errorf("failed to download %q: %v", s.Name, err)
		}
		err = out.Close()
		if err != nil {
			return fmt.Errorf("failed to close %q: %v", s.Name, err)
		}
	}
	return nil
}

// checkClose is used to check the return from Close in a defer
// statement.
func checkClose(c io.Closer, err *error) {
	cerr := c.Close()
	if *err == nil {
		*err = cerr
	}
}

// countWriter acts as an io.Writer counting the output
type countWriter int64

// Write counts up the data and ignores it
func (c *countWriter) Write(p []byte) (int, error) {
	*c += countWriter(len(p))
	return len(p), nil
}

// Puts a snapshot
func (s *Snapshot) Put(file string) (err error) {
	// Work out where to put things
	leaf := s.ImageLeaf
	Type := Types.Find(file)
	if Type == nil {
		return fmt.Errorf("unknown snapshot type %q - use types command to see available", leaf)
	}
	if !Type.Upload {
		return fmt.Errorf("can't upload snapshot type %q - use types command to see available", leaf)
	}
	s.ImageType = Type.ImageType
	chunksPath := s.Name + "/" + leaf[:len(leaf)-len(Type.Suffix)]
	objectPath := s.Path

	// Get file stat
	fi, err := os.Stat(file)
	if err != nil {
		return fmt.Errorf("failed to stat %q: %v", file, err)
	}
	if fi.IsDir() {
		return fmt.Errorf("%q is a directory", file)
	}
	s.Date = fi.ModTime()

	// Check file doesn't exist and container does
	ok, err := s.Exists()
	if err != nil {
		return err
	}
	if ok {
		return fmt.Errorf("snapshot %q already exists - delete it first", s.Name)
	}
	err = s.Manager.CreateContainer()
	if err != nil {
		return err
	}

	// Upload the file with chunks
	var in io.Reader
	fileIn, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("failed to open %q: %v", file, err)
	}
	in = fileIn
	defer checkClose(fileIn, &err)

	// If we need to read the size from the ungzipped data then do
	// it as we go along
	var gzipCounter *GzipCounter
	if Type.DiskSizeFrom == DiskSizeFromGzip {
		log.Printf("Gunzipping on the fly to count size")
		gzipCounter, err = NewGzipCounter()
		if err != nil {
			return fmt.Errorf("failed to make gzip counter: %v", err)
		}
		in = io.TeeReader(in, gzipCounter)
	}

	// Check if needs gunzip
	if Type.NeedsGunzip {
		log.Printf("Gunzipping on the fly")
		objectPath = objectPath[:len(objectPath)-3]
		s.ImageLeaf = s.ImageLeaf[:len(s.ImageLeaf)-3]
		var gzipRd io.ReadCloser
		gzipRd, err = gzip.NewReader(in)
		if err != nil {
			return fmt.Errorf("failed to make gzip decompressor: %v", err)
		}
		defer checkClose(gzipRd, &err)
		in = gzipRd
	}

	// Check if needs gzip
	if Type.NeedsGzip {
		log.Printf("Gzipping on the fly")
		objectPath += ".gz"
		s.ImageLeaf += ".gz"
		var gzipRd io.ReadCloser
		gzipRd, err = NewGzipReader(in)
		if err != nil {
			return fmt.Errorf("failed to make gzip compressor: %v", err)
		}
		defer checkClose(gzipRd, &err)
		in = gzipRd
	}

	// Calculate the MD5 of the uploaded object on the fly
	hash := md5.New()
	in = io.TeeReader(in, hash)

	// Put the file in chunks
	size, err := s.putChunkedFile(in, s.Manager.Container, objectPath, s.Manager.Container, chunksPath, Type.MimeType)
	if err != nil {
		return err
	}

	// Set the Md5
	s.Md5 = fmt.Sprintf("%x", hash.Sum(nil))

	// Set the DiskSize to the raw size of the upload
	switch Type.DiskSizeFrom {
	case DiskSizeFromUpload:
		// .tar.gz -> .tar
		s.DiskSize = size
	case DiskSizeFromFile:
		// .raw -> raw.gz
		// .tar
		s.DiskSize = fi.Size()
	case DiskSizeFromGzip:
		// .raw.gz
		err = gzipCounter.Close()
		if err != nil {
			return fmt.Errorf("error closing gzip counter: %v", err)
		}
		s.DiskSize = gzipCounter.Size()
	default:
		log.Printf("Can't figure out the disk size for %q - using the file size", Type.Suffix)
		s.DiskSize = fi.Size()
	}

	// Write the README.txt
	s.CreateReadme()
	log.Printf("Uploading README.txt\n%s\n", s.ReadMe)
	err = s.Manager.Swift.ObjectPutString(s.Manager.Container, s.Name+"/README.txt", s.ReadMe, "text/plain")
	if err != nil {
		return fmt.Errorf("failed to create README.txt: %v", err)
	}
	return nil
}

// Delete all the objects in the snapshot
func (s *Snapshot) Delete() error {
	objects, err := s.Manager.Swift.Objects(s.Manager.Container, &swift.ObjectsOpts{
		Prefix: s.Name + "/",
	})
	if err != nil {
		return fmt.Errorf("failed to read snapshot %q: %v", s.Name, err)
	}
	if len(objects) == 0 {
		return fmt.Errorf("snapshot or snapshot objects not found")
	}

	errors := 0
	for _, object := range objects {
		if object.PseudoDirectory {
			continue
		}
		log.Printf("Deleting %q", object.Name)
		err = s.Manager.Swift.ObjectDelete(s.Manager.Container, object.Name)
		if err != nil {
			errors += 1
			log.Printf("Failed to delete %q: %v", object.Name, err)
		}
	}
	if errors != 0 {
		return fmt.Errorf("failed to delete %d objects", errors)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bug implements the “go bug” command.
package bug

import (
	"bytes"
	"context"
	"fmt"
	"io"
	urlpkg "net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"cmd/go/internal/base"
	"cmd/go/internal/cfg"
	"cmd/go/internal/envcmd"
	"cmd/go/internal/web"
	"cmd/go/internal/work"
)

var CmdBug = &base.Command{
	Run:       runBug,
	UsageLine: "go bug",
	Short:     "start a bug report",
	Long: `
Bug opens the default browser and starts a new bug report.
The report includes useful system information.
	`,
}

func init() {
	CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "")
}

func runBug(ctx context.Context, cmd *base.Command, args []string) {
	if len(args) > 0 {
		base.Fatalf("go: bug takes no arguments")
	}
	work.BuildInit()

	var buf strings.Builder
	buf.WriteString(bugHeader)
	printGoVersion(&buf)
	buf.WriteString("### Does this issue reproduce with the latest release?\n\n\n")
	printEnvDetails(&buf)
	buf.WriteString(bugFooter)

	body := buf.String()
	url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body)
	if !web.OpenBrowser(url) {
		fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n")
		fmt.Print(body)
	}
}

const bugHeader = `

`
const bugFooter = `### What did you do?

### What did you expect to see?

### What did you see instead?

`

func printGoVersion(w io.Writer) {
	fmt.Fprintf(w, "### What version of Go are you using (`go version`)?\n\n")
	fmt.Fprintf(w, "
\n")
	fmt.Fprintf(w, "$ go version\n")
	fmt.Fprintf(w, "go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH)
	fmt.Fprintf(w, "
\n")
	fmt.Fprintf(w, "\n")
}

func printEnvDetails(w io.Writer) {
	fmt.Fprintf(w, "### What operating system and processor architecture are you using (`go env`)?\n\n")
	fmt.Fprintf(w, "go env Output

\n")
	fmt.Fprintf(w, "$ go env\n")
	printGoEnv(w)
	printGoDetails(w)
	printOSDetails(w)
	printCDetails(w)
	fmt.Fprintf(w, "
\n\n")
}

func printGoEnv(w io.Writer) {
	env := envcmd.MkEnv()
	env = append(env, envcmd.ExtraEnvVars()...)
	env = append(env, envcmd.ExtraEnvVarsCostly()...)
	envcmd.PrintEnv(w, env)
}

func printGoDetails(w io.Writer) {
	gocmd := filepath.Join(runtime.GOROOT(), "bin/go")
	printCmdOut(w, "GOROOT/bin/go version: ", gocmd, "version")
	printCmdOut(w, "GOROOT/bin/go tool compile -V: ", gocmd, "tool", "compile", "-V")
}

func printOSDetails(w io.Writer) {
	switch runtime.GOOS {
	case "darwin", "ios":
		printCmdOut(w, "uname -v: ", "uname", "-v")
		printCmdOut(w, "", "sw_vers")
	case "linux":
		printCmdOut(w, "uname -sr: ", "uname", "-sr")
		printCmdOut(w, "", "lsb_release", "-a")
		printGlibcVersion(w)
	case "openbsd", "netbsd", "freebsd", "dragonfly":
		printCmdOut(w, "uname -v: ", "uname", "-v")
	case "illumos", "solaris":
		// Be sure to use the OS-supplied uname, in "/usr/bin":
		printCmdOut(w, "uname -srv: ", "/usr/bin/uname", "-srv")
		out, err := os.ReadFile("/etc/release")
		if err == nil {
			fmt.Fprintf(w, "/etc/release: %s\n", out)
		} else {
			if cfg.BuildV {
				fmt.Printf("failed to read /etc/release: %v\n", err)
			}
		}
	}
}

func printCDetails(w io.Writer) {
	printCmdOut(w, "lldb --version: ", "lldb", "--version")
	cmd := exec.Command("gdb", "--version")
	out, err := cmd.Output()
	if err == nil {
		// There's apparently no combination of command line flags
		// to get gdb to spit out its version without the license and warranty.
		// Print up to the first newline.
		fmt.Fprintf(w, "gdb --version: %s\n", firstLine(out))
	} else {
		if cfg.BuildV {
			fmt.Printf("failed to run gdb --version: %v\n", err)
		}
	}
}

// printCmdOut prints the output of running the given command.
// It ignores failures; 'go bug' is best effort.
func printCmdOut(w io.Writer, prefix, path string, args ...string) {
	cmd := exec.Command(path, args...)
	out, err := cmd.Output()
	if err != nil {
		if cfg.BuildV {
			fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err)
		}
		return
	}
	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
}

// firstLine returns the first line of a given byte slice.
func firstLine(buf []byte) []byte {
	idx := bytes.IndexByte(buf, '\n')
	if idx > 0 {
		buf = buf[:idx]
	}
	return bytes.TrimSpace(buf)
}

// printGlibcVersion prints information about the glibc version.
// It ignores failures.
func printGlibcVersion(w io.Writer) {
	tempdir := os.TempDir()
	if tempdir == "" {
		return
	}
	src := []byte(`int main() {}`)
	srcfile := filepath.Join(tempdir, "go-bug.c")
	outfile := filepath.Join(tempdir, "go-bug")
	err := os.WriteFile(srcfile, src, 0644)
	if err != nil {
		return
	}
	defer os.Remove(srcfile)
	cmd := exec.Command("gcc", "-o", outfile, srcfile)
	if _, err = cmd.CombinedOutput(); err != nil {
		return
	}
	defer os.Remove(outfile)

	cmd = exec.Command("ldd", outfile)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return
	}
	re := regexp.MustCompile(`libc\.so[^ ]* => ([^ ]+)`)
	m := re.FindStringSubmatch(string(out))
	if m == nil {
		return
	}
	cmd = exec.Command(m[1])
	out, err = cmd.Output()
	if err != nil {
		return
	}
	fmt.Fprintf(w, "%s: %s\n", m[1], firstLine(out))

	// print another line (the one containing version string) in case of musl libc
	if idx := bytes.IndexByte(out, '\n'); bytes.Index(out, []byte("musl")) != -1 && idx > -1 {
		fmt.Fprintf(w, "%s\n", firstLine(out[idx+1:]))
	}
}
<commit_msg>cmd/go/internal/bug: use bytes.Contains<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bug implements the “go bug” command.
package bug

import (
	"bytes"
	"context"
	"fmt"
	"io"
	urlpkg "net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"cmd/go/internal/base"
	"cmd/go/internal/cfg"
	"cmd/go/internal/envcmd"
	"cmd/go/internal/web"
	"cmd/go/internal/work"
)

var CmdBug = &base.Command{
	Run:       runBug,
	UsageLine: "go bug",
	Short:     "start a bug report",
	Long: `
Bug opens the default browser and starts a new bug report.
The report includes useful system information.
	`,
}

func init() {
	CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "")
}

func runBug(ctx context.Context, cmd *base.Command, args []string) {
	if len(args) > 0 {
		base.Fatalf("go: bug takes no arguments")
	}
	work.BuildInit()

	var buf strings.Builder
	buf.WriteString(bugHeader)
	printGoVersion(&buf)
	buf.WriteString("### Does this issue reproduce with the latest release?\n\n\n")
	printEnvDetails(&buf)
	buf.WriteString(bugFooter)

	body := buf.String()
	url := "https://github.com/golang/go/issues/new?body=" + urlpkg.QueryEscape(body)
	if !web.OpenBrowser(url) {
		fmt.Print("Please file a new issue at golang.org/issue/new using this template:\n\n")
		fmt.Print(body)
	}
}

const bugHeader = `

`
const bugFooter = `### What did you do?

### What did you expect to see?

### What did you see instead?

`

func printGoVersion(w io.Writer) {
	fmt.Fprintf(w, "### What version of Go are you using (`go version`)?\n\n")
	fmt.Fprintf(w, "
\n")
	fmt.Fprintf(w, "$ go version\n")
	fmt.Fprintf(w, "go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH)
	fmt.Fprintf(w, "
\n")
	fmt.Fprintf(w, "\n")
}

func printEnvDetails(w io.Writer) {
	fmt.Fprintf(w, "### What operating system and processor architecture are you using (`go env`)?\n\n")
	fmt.Fprintf(w, "go env Output

\n")
	fmt.Fprintf(w, "$ go env\n")
	printGoEnv(w)
	printGoDetails(w)
	printOSDetails(w)
	printCDetails(w)
	fmt.Fprintf(w, "
\n\n")
}

func printGoEnv(w io.Writer) {
	env := envcmd.MkEnv()
	env = append(env, envcmd.ExtraEnvVars()...)
	env = append(env, envcmd.ExtraEnvVarsCostly()...)
	envcmd.PrintEnv(w, env)
}

func printGoDetails(w io.Writer) {
	gocmd := filepath.Join(runtime.GOROOT(), "bin/go")
	printCmdOut(w, "GOROOT/bin/go version: ", gocmd, "version")
	printCmdOut(w, "GOROOT/bin/go tool compile -V: ", gocmd, "tool", "compile", "-V")
}

func printOSDetails(w io.Writer) {
	switch runtime.GOOS {
	case "darwin", "ios":
		printCmdOut(w, "uname -v: ", "uname", "-v")
		printCmdOut(w, "", "sw_vers")
	case "linux":
		printCmdOut(w, "uname -sr: ", "uname", "-sr")
		printCmdOut(w, "", "lsb_release", "-a")
		printGlibcVersion(w)
	case "openbsd", "netbsd", "freebsd", "dragonfly":
		printCmdOut(w, "uname -v: ", "uname", "-v")
	case "illumos", "solaris":
		// Be sure to use the OS-supplied uname, in "/usr/bin":
		printCmdOut(w, "uname -srv: ", "/usr/bin/uname", "-srv")
		out, err := os.ReadFile("/etc/release")
		if err == nil {
			fmt.Fprintf(w, "/etc/release: %s\n", out)
		} else {
			if cfg.BuildV {
				fmt.Printf("failed to read /etc/release: %v\n", err)
			}
		}
	}
}

func printCDetails(w io.Writer) {
	printCmdOut(w, "lldb --version: ", "lldb", "--version")
	cmd := exec.Command("gdb", "--version")
	out, err := cmd.Output()
	if err == nil {
		// There's apparently no combination of command line flags
		// to get gdb to spit out its version without the license and warranty.
		// Print up to the first newline.
		fmt.Fprintf(w, "gdb --version: %s\n", firstLine(out))
	} else {
		if cfg.BuildV {
			fmt.Printf("failed to run gdb --version: %v\n", err)
		}
	}
}

// printCmdOut prints the output of running the given command.
// It ignores failures; 'go bug' is best effort.
func printCmdOut(w io.Writer, prefix, path string, args ...string) {
	cmd := exec.Command(path, args...)
	out, err := cmd.Output()
	if err != nil {
		if cfg.BuildV {
			fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err)
		}
		return
	}
	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
}

// firstLine returns the first line of a given byte slice.
func firstLine(buf []byte) []byte {
	idx := bytes.IndexByte(buf, '\n')
	if idx > 0 {
		buf = buf[:idx]
	}
	return bytes.TrimSpace(buf)
}

// printGlibcVersion prints information about the glibc version.
// It ignores failures.
func printGlibcVersion(w io.Writer) {
	tempdir := os.TempDir()
	if tempdir == "" {
		return
	}
	src := []byte(`int main() {}`)
	srcfile := filepath.Join(tempdir, "go-bug.c")
	outfile := filepath.Join(tempdir, "go-bug")
	err := os.WriteFile(srcfile, src, 0644)
	if err != nil {
		return
	}
	defer os.Remove(srcfile)
	cmd := exec.Command("gcc", "-o", outfile, srcfile)
	if _, err = cmd.CombinedOutput(); err != nil {
		return
	}
	defer os.Remove(outfile)

	cmd = exec.Command("ldd", outfile)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return
	}
	re := regexp.MustCompile(`libc\.so[^ ]* => ([^ ]+)`)
	m := re.FindStringSubmatch(string(out))
	if m == nil {
		return
	}
	cmd = exec.Command(m[1])
	out, err = cmd.Output()
	if err != nil {
		return
	}
	fmt.Fprintf(w, "%s: %s\n", m[1], firstLine(out))

	// print another line (the one containing version string) in case of musl libc
	if idx := bytes.IndexByte(out, '\n'); bytes.Contains(out, []byte("musl")) && idx > -1 {
		fmt.Fprintf(w, "%s\n", firstLine(out[idx+1:]))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

func init() {
	scenarios = append(scenarios, RunValidScenario)
}

func RunValidScenario(url, email string, nodes NodeSlice, client *http.Client) (errors []error) {
	/*  - Generate a bunch of requests
	*   - Interpret the requests, pulling out the node id
	*   - Count the number of requests per id
	*		- Find the ratio of responses
	 */
	nodes.Enable()
	time.Sleep(6 * time.Second)
	var loadBlancingResults = map[string]int{}
	var validRequestResponse validResponse

	for _, node := range nodes {
		loadBlancingResults[node.Port] = 0
	}

	for i := 0; i < 100; i++ {
		resp, err := client.Get(url)
		if err != nil {
			errors = append(errors, err)
			return
		}
		//defer resp.Body.Close()
		if resp.StatusCode != 200 {
			errors = append(errors, fmt.Errorf("Expected status code 200, but got %d", resp.StatusCode))
		}

		err = json.NewDecoder(resp.Body).Decode(&validRequestResponse)
		if err != nil {
			errors = append(errors, fmt.Errorf("Cannot parse json: %s", err))
		}
		loadBlancingResults[validRequestResponse.Id] = loadBlancingResults[validRequestResponse.Id] + 1
		resp.Body.Close()
	}
	if loadBlancingResults[nodes[0].Port] != loadBlancingResults[nodes[1].Port] {
		errors = append(errors, fmt.Errorf("Load wasn't balanced enough, node0 received %d requests and node1 %d", loadBlancingResults[nodes[0].Port], loadBlancingResults[nodes[1].Port]))
	}
	return
}
<commit_msg>Extracting testing the LB<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"
)

func init() {
	scenarios = append(scenarios, RunValidScenario)
}

func TestUrl(url string, client *http.Client) (id string, errors []error) {
	var validRequestResponse validResponse

	resp, err := client.Get(url)
	if err != nil {
		errors = append(errors, err)
		return "", errors
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		errors = append(errors, fmt.Errorf("Expected status code 200, but got %d", resp.StatusCode))
	}

	err = json.NewDecoder(resp.Body).Decode(&validRequestResponse)
	if err != nil {
		errors = append(errors, fmt.Errorf("Cannot parse json: %s", err))
	}

	return validRequestResponse.Id, errors
}

func RunValidScenario(url, email string, nodes NodeSlice, client *http.Client) (errors []error) {
	/*  - Generate a bunch of requests
	*   - Interpret the requests, pulling out the node id
	*   - Count the number of requests per id
	*		- Find the ratio of responses
	 */
	nodes.Enable()
	time.Sleep(6 * time.Second)
	var loadBlancingResults = map[string]int{}

	for _, node := range nodes {
		loadBlancingResults[node.Port] = 0
	}

	for i := 0; i < 100; i++ {
		id, err := TestUrl(url, client)
		if err != nil {
			errors = append(errors, err...)
		}
		if id != "" {
			loadBlancingResults[id] = loadBlancingResults[id] + 1
		}
	}
	if loadBlancingResults[nodes[0].Port] != loadBlancingResults[nodes[1].Port] {
		errors = append(errors, fmt.Errorf("Load wasn't balanced enough, node0 received %d requests and node1 %d", loadBlancingResults[nodes[0].Port], loadBlancingResults[nodes[1].Port]))
	}
	return
}
<|endoftext|>


<commit_before>package query

import (
	"reflect"
	"regexp"
	"strings"

	"github.com/rs/rest-layer/schema"
)

const (
	opAnd            = "$and"
	opOr             = "$or"
	opExists         = "$exists"
	opIn             = "$in"
	opNotIn          = "$nin"
	opNotEqual       = "$ne"
	opLowerThan      = "$lt"
	opLowerOrEqual   = "$lte"
	opGreaterThan    = "$gt"
	opGreaterOrEqual = "$gte"
	opRegex          = "$regex"
)

// Predicate defines an expression against a schema to perform a match on schema's data.
type Predicate []Expression

// Match implements Expression interface.
func (e Predicate) Match(payload map[string]interface{}) bool {
	if e == nil || len(e) == 0 {
		// nil or empty predicates always match
		return true
	}
	// Run each sub queries like a root query, stop/pass on first match
	for _, subQuery := range e {
		if !subQuery.Match(payload) {
			return false
		}
	}
	return true
}

// String implements Expression interface.
func (e Predicate) String() string {
	if len(e) == 0 {
		return "{}"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, subQuery.String())
	}
	return "{" + strings.Join(s, ", ") + "}"
}

// Validate implements Expression interface.
func (e Predicate) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// Expression is a query or query component that can be matched against a payload.
type Expression interface {
	Match(payload map[string]interface{}) bool
	Validate(validator schema.Validator) error
	String() string
}

// Value represents any kind of value to use in query.
type Value interface{}

// And joins query clauses with a logical AND, returns all documents that match
// the conditions of both clauses.
type And []Expression

// Match implements Expression interface.
func (e And) Match(payload map[string]interface{}) bool {
	// Run each sub queries like a root query, stop/pass on first match.
	for _, subQuery := range e {
		if !subQuery.Match(payload) {
			return false
		}
	}
	return true
}

// Validate implements Expression interface.
func (e And) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// String implements Expression interface.
func (e And) String() string {
	if len(e) == 0 {
		return opAnd + ": []"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, "{"+subQuery.String()+"}")
	}
	return opAnd + ": [" + strings.Join(s, ", ") + "]"
}

// Or joins query clauses with a logical OR, returns all documents that
// match the conditions of either clause.
type Or []Expression

// Match implements Expression interface.
func (e Or) Match(payload map[string]interface{}) bool {
	// Run each sub queries like a root query, stop/pass on first match
	for _, subQuery := range e {
		if subQuery.Match(payload) {
			return true
		}
	}
	return false
}

// Validate implements Expression interface.
func (e Or) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// String implements Expression interface.
func (e Or) String() string {
	if len(e) == 0 {
		return opOr + ": []"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, "{"+subQuery.String()+"}")
	}
	return opOr + ": [" + strings.Join(s, ", ") + "]"
}

// In matches any of the values specified in an array.
type In struct {
	Field  string
	Values []Value
}

// Match implements Expression interface.
func (e In) Match(payload map[string]interface{}) bool {
	value := getField(payload, e.Field)
	for _, v := range e.Values {
		if reflect.DeepEqual(v, value) {
			return true
		}
	}
	return false
}

// Validate implements Expression interface.
func (e In) Validate(validator schema.Validator) error {
	return validateValues(e.Field, e.Values, validator)
}

// String implements Expression interface.
func (e In) String() string {
	s := make([]string, 0, len(e.Values))
	for _, v := range e.Values {
		s = append(s, valueString(v))
	}
	return quoteField(e.Field) + ": {" + opIn + ": [" + strings.Join(s, ", ") + "]}"
}

// NotIn matches none of the values specified in an array.
type NotIn struct {
	Field  string
	Values []Value
}

// Match implements Expression interface.
func (e NotIn) Match(payload map[string]interface{}) bool {
	value := getField(payload, e.Field)
	for _, v := range e.Values {
		if reflect.DeepEqual(v, value) {
			return false
		}
	}
	return true
}

// String implements Expression interface.
func (e NotIn) String() string {
	s := make([]string, 0, len(e.Values))
	for _, v := range e.Values {
		s = append(s, valueString(v))
	}
	return quoteField(e.Field) + ": {" + opNotIn + ": [" + strings.Join(s, ", ") + "]}"
}

// Validate implements Expression interface.
func (e NotIn) Validate(validator schema.Validator) error {
	return validateValues(e.Field, e.Values, validator)
}

// Equal matches all values that are equal to a specified value.
type Equal struct {
	Field string
	Value Value
}

// Match implements Expression interface.
func (e Equal) Match(payload map[string]interface{}) bool {
	return reflect.DeepEqual(getField(payload, e.Field), e.Value)
}

// Validate implements Expression interface.
func (e Equal) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value, validator)
}

// String implements Expression interface.
func (e Equal) String() string {
	return quoteField(e.Field) + ": " + valueString(e.Value)
}

// NotEqual matches all values that are not equal to a specified value.
type NotEqual struct {
	Field string
	Value Value
}

// Match implements Expression interface.
func (e NotEqual) Match(payload map[string]interface{}) bool {
	return !reflect.DeepEqual(getField(payload, e.Field), e.Value)
}

// Validate implements Expression interface.
func (e NotEqual) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value, validator)
}

// String implements Expression interface.
func (e NotEqual) String() string {
	return quoteField(e.Field) + ": {" + opNotEqual + ": " + valueString(e.Value) + "}"
}

// Exist matches all values which are present, even if nil
type Exist struct {
	Field string
}

// Match implements Expression interface.
func (e Exist) Match(payload map[string]interface{}) bool {
	_, found := getFieldExist(payload, e.Field)
	return found
}

// Validate implements Expression interface.
func (e Exist) Validate(validator schema.Validator) error {
	return validateField(e.Field, validator)
}

// String implements Expression interface.
func (e Exist) String() string {
	return quoteField(e.Field) + ": {" + opExists + ": true}"
}

// NotExist matches all values which are absent
type NotExist struct {
	Field string
}

// Match implements Expression interface.
func (e NotExist) Match(payload map[string]interface{}) bool {
	_, found := getFieldExist(payload, e.Field)
	return !found
}

// Validate implements Expression interface.
func (e NotExist) Validate(validator schema.Validator) error {
	return validateField(e.Field, validator)
}

// String implements Expression interface.
func (e NotExist) String() string {
	return quoteField(e.Field) + ": {" + opExists + ": false}"
}

// GreaterThan matches values that are greater than a specified value.
type GreaterThan struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e GreaterThan) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n > e.Value)
}

// Validate implements Expression interface.
func (e GreaterThan) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opGreaterThan, validator)
}

// String implements Expression interface.
func (e GreaterThan) String() string {
	return quoteField(e.Field) + ": {" + opGreaterThan + ": " + valueString(e.Value) + "}"
}

// GreaterOrEqual matches values that are greater than or equal to a specified value.
type GreaterOrEqual struct {
	Field string
	Value float64
}

// Match implements Expression interface
func (e GreaterOrEqual) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n >= e.Value)
}

// Validate implements Expression interface.
func (e GreaterOrEqual) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opGreaterOrEqual, validator)
}

// String implements Expression interface.
func (e GreaterOrEqual) String() string {
	return quoteField(e.Field) + ": {" + opGreaterOrEqual + ": " + valueString(e.Value) + "}"
}

// LowerThan matches values that are less than a specified value.
type LowerThan struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e LowerThan) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n < e.Value)
}

// Validate implements Expression interface.
func (e LowerThan) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opLowerThan, validator)
}

// String implements Expression interface.
func (e LowerThan) String() string {
	return quoteField(e.Field) + ": {" + opLowerThan + ": " + valueString(e.Value) + "}"
}

// LowerOrEqual matches values that are less than or equal to a specified value.
type LowerOrEqual struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e LowerOrEqual) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n <= e.Value)
}

// Validate implements Expression interface.
func (e LowerOrEqual) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opLowerOrEqual, validator)
}

// String implements Expression interface.
func (e LowerOrEqual) String() string {
	return quoteField(e.Field) + ": {" + opLowerOrEqual + ": " + valueString(e.Value) + "}"
}

// Regex matches values that match to a specified regular expression.
type Regex struct {
	Field string
	Value *regexp.Regexp
}

// Match implements Expression interface.
func (e Regex) Match(payload map[string]interface{}) bool {
	return e.Value.MatchString(payload[e.Field].(string))
}

// Validate implements Expression interface.
func (e Regex) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value, validator)
}

// String implements Expression interface.
func (e Regex) String() string {
	return quoteField(e.Field) + ": {" + opRegex + ": " + valueString(e.Value) + "}"
}
<commit_msg>Fix $regex filter operator<commit_after>package query

import (
	"reflect"
	"regexp"
	"strings"

	"github.com/rs/rest-layer/schema"
)

const (
	opAnd            = "$and"
	opOr             = "$or"
	opExists         = "$exists"
	opIn             = "$in"
	opNotIn          = "$nin"
	opNotEqual       = "$ne"
	opLowerThan      = "$lt"
	opLowerOrEqual   = "$lte"
	opGreaterThan    = "$gt"
	opGreaterOrEqual = "$gte"
	opRegex          = "$regex"
)

// Predicate defines an expression against a schema to perform a match on schema's data.
type Predicate []Expression

// Match implements Expression interface.
func (e Predicate) Match(payload map[string]interface{}) bool {
	if e == nil || len(e) == 0 {
		// nil or empty predicates always match
		return true
	}
	// Run each sub queries like a root query, stop/pass on first match
	for _, subQuery := range e {
		if !subQuery.Match(payload) {
			return false
		}
	}
	return true
}

// String implements Expression interface.
func (e Predicate) String() string {
	if len(e) == 0 {
		return "{}"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, subQuery.String())
	}
	return "{" + strings.Join(s, ", ") + "}"
}

// Validate implements Expression interface.
func (e Predicate) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// Expression is a query or query component that can be matched against a payload.
type Expression interface {
	Match(payload map[string]interface{}) bool
	Validate(validator schema.Validator) error
	String() string
}

// Value represents any kind of value to use in query.
type Value interface{}

// And joins query clauses with a logical AND, returns all documents that match
// the conditions of both clauses.
type And []Expression

// Match implements Expression interface.
func (e And) Match(payload map[string]interface{}) bool {
	// Run each sub queries like a root query, stop/pass on first match.
	for _, subQuery := range e {
		if !subQuery.Match(payload) {
			return false
		}
	}
	return true
}

// Validate implements Expression interface.
func (e And) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// String implements Expression interface.
func (e And) String() string {
	if len(e) == 0 {
		return opAnd + ": []"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, "{"+subQuery.String()+"}")
	}
	return opAnd + ": [" + strings.Join(s, ", ") + "]"
}

// Or joins query clauses with a logical OR, returns all documents that
// match the conditions of either clause.
type Or []Expression

// Match implements Expression interface.
func (e Or) Match(payload map[string]interface{}) bool {
	// Run each sub queries like a root query, stop/pass on first match
	for _, subQuery := range e {
		if subQuery.Match(payload) {
			return true
		}
	}
	return false
}

// Validate implements Expression interface.
func (e Or) Validate(validator schema.Validator) error {
	return validateExpressions(e, validator)
}

// String implements Expression interface.
func (e Or) String() string {
	if len(e) == 0 {
		return opOr + ": []"
	}
	s := make([]string, 0, len(e))
	for _, subQuery := range e {
		s = append(s, "{"+subQuery.String()+"}")
	}
	return opOr + ": [" + strings.Join(s, ", ") + "]"
}

// In matches any of the values specified in an array.
type In struct {
	Field  string
	Values []Value
}

// Match implements Expression interface.
func (e In) Match(payload map[string]interface{}) bool {
	value := getField(payload, e.Field)
	for _, v := range e.Values {
		if reflect.DeepEqual(v, value) {
			return true
		}
	}
	return false
}

// Validate implements Expression interface.
func (e In) Validate(validator schema.Validator) error {
	return validateValues(e.Field, e.Values, validator)
}

// String implements Expression interface.
func (e In) String() string {
	s := make([]string, 0, len(e.Values))
	for _, v := range e.Values {
		s = append(s, valueString(v))
	}
	return quoteField(e.Field) + ": {" + opIn + ": [" + strings.Join(s, ", ") + "]}"
}

// NotIn matches none of the values specified in an array.
type NotIn struct {
	Field  string
	Values []Value
}

// Match implements Expression interface.
func (e NotIn) Match(payload map[string]interface{}) bool {
	value := getField(payload, e.Field)
	for _, v := range e.Values {
		if reflect.DeepEqual(v, value) {
			return false
		}
	}
	return true
}

// String implements Expression interface.
func (e NotIn) String() string {
	s := make([]string, 0, len(e.Values))
	for _, v := range e.Values {
		s = append(s, valueString(v))
	}
	return quoteField(e.Field) + ": {" + opNotIn + ": [" + strings.Join(s, ", ") + "]}"
}

// Validate implements Expression interface.
func (e NotIn) Validate(validator schema.Validator) error {
	return validateValues(e.Field, e.Values, validator)
}

// Equal matches all values that are equal to a specified value.
type Equal struct {
	Field string
	Value Value
}

// Match implements Expression interface.
func (e Equal) Match(payload map[string]interface{}) bool {
	return reflect.DeepEqual(getField(payload, e.Field), e.Value)
}

// Validate implements Expression interface.
func (e Equal) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value, validator)
}

// String implements Expression interface.
func (e Equal) String() string {
	return quoteField(e.Field) + ": " + valueString(e.Value)
}

// NotEqual matches all values that are not equal to a specified value.
type NotEqual struct {
	Field string
	Value Value
}

// Match implements Expression interface.
func (e NotEqual) Match(payload map[string]interface{}) bool {
	return !reflect.DeepEqual(getField(payload, e.Field), e.Value)
}

// Validate implements Expression interface.
func (e NotEqual) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value, validator)
}

// String implements Expression interface.
func (e NotEqual) String() string {
	return quoteField(e.Field) + ": {" + opNotEqual + ": " + valueString(e.Value) + "}"
}

// Exist matches all values which are present, even if nil
type Exist struct {
	Field string
}

// Match implements Expression interface.
func (e Exist) Match(payload map[string]interface{}) bool {
	_, found := getFieldExist(payload, e.Field)
	return found
}

// Validate implements Expression interface.
func (e Exist) Validate(validator schema.Validator) error {
	return validateField(e.Field, validator)
}

// String implements Expression interface.
func (e Exist) String() string {
	return quoteField(e.Field) + ": {" + opExists + ": true}"
}

// NotExist matches all values which are absent
type NotExist struct {
	Field string
}

// Match implements Expression interface.
func (e NotExist) Match(payload map[string]interface{}) bool {
	_, found := getFieldExist(payload, e.Field)
	return !found
}

// Validate implements Expression interface.
func (e NotExist) Validate(validator schema.Validator) error {
	return validateField(e.Field, validator)
}

// String implements Expression interface.
func (e NotExist) String() string {
	return quoteField(e.Field) + ": {" + opExists + ": false}"
}

// GreaterThan matches values that are greater than a specified value.
type GreaterThan struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e GreaterThan) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n > e.Value)
}

// Validate implements Expression interface.
func (e GreaterThan) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opGreaterThan, validator)
}

// String implements Expression interface.
func (e GreaterThan) String() string {
	return quoteField(e.Field) + ": {" + opGreaterThan + ": " + valueString(e.Value) + "}"
}

// GreaterOrEqual matches values that are greater than or equal to a specified value.
type GreaterOrEqual struct {
	Field string
	Value float64
}

// Match implements Expression interface
func (e GreaterOrEqual) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n >= e.Value)
}

// Validate implements Expression interface.
func (e GreaterOrEqual) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opGreaterOrEqual, validator)
}

// String implements Expression interface.
func (e GreaterOrEqual) String() string {
	return quoteField(e.Field) + ": {" + opGreaterOrEqual + ": " + valueString(e.Value) + "}"
}

// LowerThan matches values that are less than a specified value.
type LowerThan struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e LowerThan) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n < e.Value)
}

// Validate implements Expression interface.
func (e LowerThan) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opLowerThan, validator)
}

// String implements Expression interface.
func (e LowerThan) String() string {
	return quoteField(e.Field) + ": {" + opLowerThan + ": " + valueString(e.Value) + "}"
}

// LowerOrEqual matches values that are less than or equal to a specified value.
type LowerOrEqual struct {
	Field string
	Value float64
}

// Match implements Expression interface.
func (e LowerOrEqual) Match(payload map[string]interface{}) bool {
	n, ok := isNumber(getField(payload, e.Field))
	return ok && (n <= e.Value)
}

// Validate implements Expression interface.
func (e LowerOrEqual) Validate(validator schema.Validator) error {
	return validateNumericValue(e.Field, e.Value, opLowerOrEqual, validator)
}

// String implements Expression interface.
func (e LowerOrEqual) String() string {
	return quoteField(e.Field) + ": {" + opLowerOrEqual + ": " + valueString(e.Value) + "}"
}

// Regex matches values that match to a specified regular expression.
type Regex struct {
	Field string
	Value *regexp.Regexp
}

// Match implements Expression interface.
func (e Regex) Match(payload map[string]interface{}) bool {
	return e.Value.MatchString(payload[e.Field].(string))
}

// Validate implements Expression interface.
func (e Regex) Validate(validator schema.Validator) error {
	return validateValue(e.Field, e.Value.String(), validator)
}

// String implements Expression interface.
func (e Regex) String() string {
	return quoteField(e.Field) + ": {" + opRegex + ": " + valueString(e.Value) + "}"
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package logger

import (
	"launchpad.net/loggo"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/state/api/logger"
	"launchpad.net/juju-core/state/api/watcher"
	"launchpad.net/juju-core/worker"
)

var log = loggo.GetLogger("juju.worker.logger")

// Cleaner is responsible for cleaning up the state.
type Logger struct {
	api         *logger.State
	agentConfig agent.Config
	lastConfig  string
}

var _ worker.NotifyWatchHandler = (*Logger)(nil)

// NewLogger returns a worker.Worker that runs state.Cleanup()
// if the CleanupWatcher signals documents marked for deletion.
func NewLogger(api *logger.State, agentConfig agent.Config) worker.Worker {
	logger := &Logger{
		api:         api,
		agentConfig: agentConfig,
		lastConfig:  loggo.LoggerInfo(),
	}
	log.Debugf("initial log config: %q", logger.lastConfig)
	return worker.NewNotifyWorker(logger)
}

func (logger *Logger) setLogging() {
	loggingConfig, err := logger.api.LoggingConfig(logger.agentConfig.Tag())
	if err != nil {
		log.Errorf("%v", err)
	} else {
		if loggingConfig != logger.lastConfig {
			log.Debugf("reconfigurint logging from %q to %q", logger.lastConfig, loggingConfig)
			loggo.ResetLoggers()
			if err := loggo.ConfigureLoggers(loggingConfig); err != nil {
				// This shouldn't occur as the loggingConfig should be
				// validated by the original Config before it gets here.
				log.Warningf("configure loggers failed: %v", err)
				// Try to reset to what we had before
				loggo.ConfigureLoggers(logger.lastConfig)
			}
			logger.lastConfig = loggingConfig
			// Set the value in the agent config and write it out.
			logger.agentConfig.SetValue(agent.LoggingConfig, loggingConfig)
			if err := logger.agentConfig.Write(); err != nil {
				// Just log the error, this isn't fatal.
				log.Errorf("failed to write out agent config: %v", err)
			}
		}
	}
}

func (logger *Logger) SetUp() (watcher.NotifyWatcher, error) {
	log.Debugf("logger setup")
	// We need to set this up initially as the NotifyWorker sucks up the first
	// event.
	logger.setLogging()
	return logger.api.WatchLoggingConfig(logger.agentConfig.Tag())
}

func (logger *Logger) Handle() error {
	logger.setLogging()
	return nil
}

func (logger *Logger) TearDown() error {
	// Nothing to cleanup, only state is the watcher
	return nil
}
<commit_msg>Comment updates.<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package logger

import (
	"launchpad.net/loggo"

	"launchpad.net/juju-core/agent"
	"launchpad.net/juju-core/state/api/logger"
	"launchpad.net/juju-core/state/api/watcher"
	"launchpad.net/juju-core/worker"
)

var log = loggo.GetLogger("juju.worker.logger")

// Logger is responsible for updating the loggo configuration when the
// environment watcher tells the agent that the value has changed.
type Logger struct {
	api         *logger.State
	agentConfig agent.Config
	lastConfig  string
}

var _ worker.NotifyWatchHandler = (*Logger)(nil)

// NewLogger returns a worker.Worker that uses the notify watcher returned
// from the setup.
func NewLogger(api *logger.State, agentConfig agent.Config) worker.Worker {
	logger := &Logger{
		api:         api,
		agentConfig: agentConfig,
		lastConfig:  loggo.LoggerInfo(),
	}
	log.Debugf("initial log config: %q", logger.lastConfig)
	return worker.NewNotifyWorker(logger)
}

func (logger *Logger) setLogging() {
	loggingConfig, err := logger.api.LoggingConfig(logger.agentConfig.Tag())
	if err != nil {
		log.Errorf("%v", err)
	} else {
		if loggingConfig != logger.lastConfig {
			log.Debugf("reconfigurint logging from %q to %q", logger.lastConfig, loggingConfig)
			loggo.ResetLoggers()
			if err := loggo.ConfigureLoggers(loggingConfig); err != nil {
				// This shouldn't occur as the loggingConfig should be
				// validated by the original Config before it gets here.
				log.Warningf("configure loggers failed: %v", err)
				// Try to reset to what we had before
				loggo.ConfigureLoggers(logger.lastConfig)
			}
			logger.lastConfig = loggingConfig
			// Set the value in the agent config and write it out.
			logger.agentConfig.SetValue(agent.LoggingConfig, loggingConfig)
			if err := logger.agentConfig.Write(); err != nil {
				// Just log the error, this isn't fatal.
				log.Errorf("failed to write out agent config: %v", err)
			}
		}
	}
}

func (logger *Logger) SetUp() (watcher.NotifyWatcher, error) {
	log.Debugf("logger setup")
	// We need to set this up initially as the NotifyWorker sucks up the first
	// event.
	logger.setLogging()
	return logger.api.WatchLoggingConfig(logger.agentConfig.Tag())
}

func (logger *Logger) Handle() error {
	logger.setLogging()
	return nil
}

func (logger *Logger) TearDown() error {
	// Nothing to cleanup, only state is the watcher
	return nil
}
<|endoftext|>