<commit_before>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|caption|description|credit|foot|blq-dotcom|story-feature|related")
)

type Document struct {
	Title  *Chunk   // the <title>...</title> text
	Chunks []*Chunk // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// No title found? The title plays a crucial role in detecting
	// the article content. We need it and every page should contain it.
	// So skip parsing in case we could not find a title.
	if doc.Title == nil {
		return errors.New("Document missing <title>.")
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<commit_msg>ignore community<commit_after>package html

import (
	"code.google.com/p/go.net/html"
	"errors"
	"io"
	"regexp"
	"unicode"
)

const (
	// We remember a few special node types when descending into their
	// children.
	AncestorArticle = 1 << iota
	AncestorAside
	AncestorBlockquote
	AncestorList
)

var (
	ignorePattern = regexp.MustCompile("(?i)comment|community|caption|description|credit|foot|blq-dotcom|story-feature|related")
)

type Document struct {
	Title  *Chunk   // the <title>...</title> text
	Chunks []*Chunk // list of all chunks found in this document

	// Unexported fields.
	root *html.Node // the ... part
	head *html.Node // the ... part
	body *html.Node // the ... part

	// State variables used when collectiong chunks.
	ancestors int // bitmask which stores ancestor of the current node

	// Number of non-space characters inside link tags / normal tags
	// per html.ElementNode.
	linkText map[*html.Node]int // length of text inside  tags
	normText map[*html.Node]int // length of text outside  tags
}

func NewDocument(r io.Reader) (*Document, error) {
	doc := new(Document)
	doc.Chunks = make([]*Chunk, 0, 512)
	if err := doc.Parse(r); err != nil {
		return nil, err
	}
	return doc, nil
}

func (doc *Document) Parse(r io.Reader) error {
	root, err := html.Parse(r)
	if err != nil {
		return err
	}
	// Assign the fields root, head and body from the HTML page.
	doc.setNodes(root)

	// Check if ,  and  nodes were found.
	if doc.root == nil || doc.head == nil || doc.body == nil {
		return errors.New("Document missing ,  or .")
	}

	doc.parseHead(doc.head)

	// No title found? The title plays a crucial role in detecting
	// the article content. We need it and every page should contain it.
	// So skip parsing in case we could not find a title.
	if doc.Title == nil {
		return errors.New("Document missing <title>.")
	}
	doc.linkText = make(map[*html.Node]int)
	doc.normText = make(map[*html.Node]int)

	doc.cleanBody(doc.body, 0)
	doc.countText(doc.body, false)
	doc.parseBody(doc.body)

	// Now link the chunks.
	for i := range doc.Chunks {
		if i > 0 {
			doc.Chunks[i].Prev = doc.Chunks[i-1]
		}
		if i < len(doc.Chunks)-1 {
			doc.Chunks[i].Next = doc.Chunks[i+1]
		}
	}
	return nil
}

// Assign the struct fields root, head and body from the HTML tree of node n.
//  doc.root ->
//  doc.head ->
//  doc.body ->
func (doc *Document) setNodes(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		switch c.Data {
		case "html":
			doc.root = c
			doc.setNodes(c)
		case "body":
			doc.body = c
		case "head":
			doc.head = c
		}
	}
}

// parseHead parses the ... part of the HTML page. Right now it
// only detects the <title>...</title>.
func (doc *Document) parseHead(n *html.Node) {
	if n.Type == html.ElementNode && n.Data == "title" {
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Title = chunk
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		doc.parseHead(c)
	}
}

// countText counts the link text and the normal text per html.Node.
// "Link text" is text inside  tags and "normal text" is text inside
// anything but  tags. Of course, counting is done cumulative, so the
// numbers of a parent node include the numbers of it's child nodes.
func (doc *Document) countText(n *html.Node, insideLink bool) (linkText int, normText int) {
	linkText = 0
	normText = 0
	if n.Type == html.ElementNode && n.Data == "a" {
		insideLink = true
	}
	for s := n.FirstChild; s != nil; s = s.NextSibling {
		linkTextChild, normTextChild := doc.countText(s, insideLink)
		linkText += linkTextChild
		normText += normTextChild
	}
	if n.Type == html.TextNode {
		count := 0
		for _, rune := range n.Data {
			if unicode.IsLetter(rune) {
				count += 1
			}
		}
		if insideLink {
			linkText += count
		} else {
			normText += count
		}
	}
	doc.linkText[n] = linkText
	doc.normText[n] = normText
	return
}

// cleanBody removes unwanted HTML elements from the HTML body.
func (doc *Document) cleanBody(n *html.Node, level int) {

	// removeNode returns true if a node should be removed from HTML document.
	removeNode := func(c *html.Node, level int) bool {
		switch c.Data {
		// Elements save to ignore.
		case "address", "audio", "button", "canvas", "caption", "fieldset",
			"figcaption", "figure", "footer", "form", "frame", "iframe",
			"map", "menu", "nav", "noscript", "object", "option", "output",
			"script", "select", "style", "svg", "textarea", "video":
			return true
		// High-level tables might be used to layout the document, so we better
		// not ignore them.
		case "table":
			return level > 5
		}
		return false
	}

	var curr *html.Node = n.FirstChild
	var next *html.Node = nil
	for ; curr != nil; curr = next {
		// We have to remember the next sibling here becase calling RemoveChild
		// sets curr's NextSibling pointer to nil and we would quit the loop
		// prematurely.
		next = curr.NextSibling
		if curr.Type == html.ElementNode {
			if removeNode(curr, level) {
				n.RemoveChild(curr)
			} else {
				doc.cleanBody(curr, level+1)
			}
		}
	}
}

// parseBody parses the ... part of the HTML page. It creates
// Chunks for every html.TextNode found in the body.
func (doc *Document) parseBody(n *html.Node) {
	switch n.Type {
	case html.ElementNode:
		// We ignore the node if it has some nasty classes/ids/itemprobs.
		for _, attr := range n.Attr {
			switch attr.Key {
			case "id", "class", "itemprop":
				if ignorePattern.FindStringIndex(attr.Val) != nil {
					return
				}
			}
		}
		ancestorMask := 0
		switch n.Data {
		// We convert headings and links to text immediately. This is easier
		// and feasible because headings and links don't contain many children.
		// Descending into these children and handling every TextNode separately
		// would make things unnecessary complicated and our results noisy.
		case "h1", "h2", "h3", "h4", "h5", "h6", "a":
			if chunk, err := NewChunk(doc, n); err == nil {
				doc.Chunks = append(doc.Chunks, chunk)
			}
			return
		// Now mask the element type, but only if it isn't already set.
		// If we mask a bit which was already set by one of our callers, we'd also
		// clear it at the end of this function, though it actually should be cleared
		// by the caller.
		case "article":
			ancestorMask = AncestorArticle &^ doc.ancestors
		case "aside":
			ancestorMask = AncestorAside &^ doc.ancestors
		case "blockquote":
			ancestorMask = AncestorBlockquote &^ doc.ancestors
		case "ul", "ol":
			ancestorMask = AncestorList &^ doc.ancestors
		}
		// Add our mask to the ancestor bitmask.
		doc.ancestors |= ancestorMask
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			doc.parseBody(c)
		}
		// Remove our mask from the ancestor bitmask.
		doc.ancestors &^= ancestorMask
	case html.TextNode:
		if chunk, err := NewChunk(doc, n); err == nil {
			doc.Chunks = append(doc.Chunks, chunk)
		}
	}
}

type TextStat struct {
	Words     int
	Sentences int
	Count     int
}

// GetClassStats groups the document chunks by their classes (defined by the
// class attribute of HTML nodes) and calculates TextStats for each class.
func (doc *Document) GetClassStats() map[string]*TextStat {
	result := make(map[string]*TextStat)
	for _, chunk := range doc.Chunks {
		for _, class := range chunk.Classes {
			if stat, ok := result[class]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				result[class] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
		}
	}
	return result
}

// GetClusterStats groups the document chunks by common ancestors and
// calculates TextStats for each group of chunks.
func (doc *Document) GetClusterStats() map[*Chunk]*TextStat {
	// Don't ascend further than this constant.
	const maxAncestors = 3

	// Count TextStats for Chunk ancestors.
	ancestorStat := make(map[*html.Node]*TextStat)
	for _, chunk := range doc.Chunks {
		node, count := chunk.Block, 0
		for node != nil && count < maxAncestors {
			if stat, ok := ancestorStat[node]; ok {
				stat.Words += chunk.Text.Words
				stat.Sentences += chunk.Text.Sentences
				stat.Count += 1
			} else {
				ancestorStat[node] = &TextStat{chunk.Text.Words, chunk.Text.Sentences, 1}
			}
			node, count = node.Parent, count+1
		}
	}

	// Generate result.
	result := make(map[*Chunk]*TextStat)
	for _, chunk := range doc.Chunks {
		node := chunk.Block
		if node == nil {
			continue
		}
		// Start with the parent's TextStat. Then ascend and check if the
		// current chunk has an ancestor with better stats. Use the best stat
		// as result.
		stat := ancestorStat[node]
		for {
			if node = node.Parent; node == nil {
				break
			}
			if statPrev, ok := ancestorStat[node]; ok {
				if stat.Count < statPrev.Count {
					stat = statPrev
				}
			} else {
				break
			}
		}
		result[chunk] = stat
	}
	return result
}
<|endoftext|>


<commit_before>package embeddedmongo

import (
	"distribution/uuid"
	"embeddedmongo/env"
	"fmt"
	"os"
)

type (
	Command int
	Version string
)

const (
	Mongod Command = iota
)

const (
	V3_4_1 Version = "3.4.1"
)

type Distribution struct {
	Configuration
	Url       string
	Os        string
	Platform  string
	Extension string
	Tmp       string
}

type Configuration struct {
	Version Version
	Dir     string
}

func NewDistribution(configuration Configuration) *Distribution {
	return &Distribution{
		Configuration: configuration,
		Url:           env.MONGO_URL,
		Os:            env.MONGO_OS,
		Platform:      env.MONGO_BITSIZE,
		Extension:     env.MONGO_EXT,
		Tmp:           uuid.Generate().String(),
	}
}

func GetDistributionName(d *Distribution) string {
	return fmt.Sprintf("%v-%v-%v-%v.%v", "mongodb", d.Os, d.Platform, d.Version, d.Extension)
}

func GetDistributionUrl(d *Distribution) string {
	return fmt.Sprintf("%v%s/%v", d.Url, d.Os, GetDistributionName(d))
}

func GetWorkDir(d *Distribution) string {
	return fmt.Sprintf("%v%v/", d.Dir, d.Os)
}

func GetTmpDir(d *Distribution) string {
	return fmt.Sprintf("%v%v/%v/", d.Dir, d.Os, d.Tmp)
}

func CreateDir(path string) error {
	err := os.MkdirAll(path, 0755)
	return err
}
<commit_msg>Set full path to inner package<commit_after>package embeddedmongo

import (
	"distribution/uuid"
	"fmt"
	"os"
	"github.com/MartenCatcher/embeddedmongo/env"
)

type (
	Command int
	Version string
)

const (
	Mongod Command = iota
)

const (
	V3_4_1 Version = "3.4.1"
)

type Distribution struct {
	Configuration
	Url       string
	Os        string
	Platform  string
	Extension string
	Tmp       string
}

type Configuration struct {
	Version Version
	Dir     string
}

func NewDistribution(configuration Configuration) *Distribution {
	return &Distribution{
		Configuration: configuration,
		Url:           env.MONGO_URL,
		Os:            env.MONGO_OS,
		Platform:      env.MONGO_BITSIZE,
		Extension:     env.MONGO_EXT,
		Tmp:           uuid.Generate().String(),
	}
}

func GetDistributionName(d *Distribution) string {
	return fmt.Sprintf("%v-%v-%v-%v.%v", "mongodb", d.Os, d.Platform, d.Version, d.Extension)
}

func GetDistributionUrl(d *Distribution) string {
	return fmt.Sprintf("%v%s/%v", d.Url, d.Os, GetDistributionName(d))
}

func GetWorkDir(d *Distribution) string {
	return fmt.Sprintf("%v%v/", d.Dir, d.Os)
}

func GetTmpDir(d *Distribution) string {
	return fmt.Sprintf("%v%v/%v/", d.Dir, d.Os, d.Tmp)
}

func CreateDir(path string) error {
	err := os.MkdirAll(path, 0755)
	return err
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2019 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"github.com/shopspring/decimal"
	"math"
	"math/big"
)

func dpdBitToInt(dpd uint, mask uint) int {
	if (dpd & mask) != 0 {
		return 1
	} else {
		return 0
	}
}

func dpdToInt(dpd uint) int {
	// Convert DPD encodined value to int (0-999)
	// dpd: DPD encoded value. 10bit unsigned int

	b := make([]int, 10)
	b[9] = dpdBitToInt(dpd, 0x0200)
	b[8] = dpdBitToInt(dpd, 0x0100)
	b[7] = dpdBitToInt(dpd, 0x0080)
	b[6] = dpdBitToInt(dpd, 0x0040)
	b[5] = dpdBitToInt(dpd, 0x0020)
	b[4] = dpdBitToInt(dpd, 0x0010)
	b[3] = dpdBitToInt(dpd, 0x0008)
	b[2] = dpdBitToInt(dpd, 0x0004)
	b[1] = dpdBitToInt(dpd, 0x0002)
	b[0] = dpdBitToInt(dpd, 0x0001)

	d := make([]int, 3)
	if b[3] == 0 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = b[2]*4 + b[1]*2 + b[0]
	} else if b[3] == 1 && b[2] == 0 && b[1] == 0 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = 8 + b[0]
	} else if b[3] == 1 && b[2] == 0 && b[1] == 1 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = 8 + b[4]
		d[0] = b[6]*4 + b[5]*2 + b[0]
	} else if b[3] == 1 && b[2] == 1 && b[1] == 0 {
		d[2] = 8 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = b[9]*4 + b[8]*2 + b[0]
	} else if b[6] == 0 && b[5] == 0 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = 8 + b[4]
		d[0] = b[9]*4 + b[8]*2 + b[0]
	} else if b[6] == 0 && b[5] == 1 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = b[9]*4 + b[8]*2 + b[4]
		d[0] = 8 + b[0]
	} else if b[6] == 1 && b[5] == 0 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = 8 + b[4]
		d[0] = 8 + b[0]
	} else if b[6] == 1 && b[5] == 1 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = 8 + b[4]
		d[0] = 8 + b[0]
	} else {
		panic("Invalid DPD encoding")
	}

	return d[2]*100 + d[1]*10 + d[0]
}

func calcSignificand(prefix int64, dpdBits *big.Int, numBits int) *big.Int {
	// prefix: High bits integer value
	// dpdBits: dpd encoded bits
	// numBits: bit length of dpd_bits
	// https://en.wikipedia.org/wiki/Decimal128_floating-point_format#Densely_packed_decimal_significand_field
	numSegments := numBits / 10
	segments := make([]uint, numSegments)
	bi1024 := big.NewInt(1024)

	for i := 0; i < numSegments; i++ {
		var work big.Int
		work = *dpdBits
		segments[numSegments-i-1] = uint(dpdBits.Mod(&work, bi1024).Int64())
		dpdBits.Rsh(dpdBits, 10)
	}

	v := big.NewInt(prefix)
	bi1000 := big.NewInt(1000)
	for _, dpd := range segments {
		v.Mul(v, bi1000)
		v.Add(v, big.NewInt(int64(dpd)))
	}

	return v
}

func decimal128ToSignDigitsExponent(b []byte) (v *decimal.Decimal, sign int, digits *big.Int, exponent int32) {
	// https://en.wikipedia.org/wiki/Decimal128_floating-point_format

	var prefix int64
	if (b[0] & 0x80) == 0x80 {
		sign = 1
	}
	cf := (uint32(b[0]&0x7f) << 10) + uint32(b[1]<<2) + uint32(b[2]>>6)
	if (cf & 0x1F000) == 0x1F000 {
		var d decimal.Decimal
		if sign == 1 {
			// Is there -NaN ?
			d = decimal.NewFromFloat(math.NaN())
		} else {
			d = decimal.NewFromFloat(math.NaN())
		}
		v = &d
		return
	} else if (cf & 0x1F000) == 0x1E000 {
		var d decimal.Decimal
		if sign == 1 {
			d = decimal.NewFromFloat(math.Inf(-1))
		} else {
			d = decimal.NewFromFloat(math.Inf(1))
		}
		v = &d
		return
	} else if (cf & 0x18000) == 0x00000 {
		exponent = int32(0x0000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x18000) == 0x08000 {
		exponent = int32(0x1000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x18000) == 0x10000 {
		exponent = int32(0x2000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x1e000) == 0x18000 {
		exponent = int32(0x0000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else if (cf & 0x1e000) == 0x1a000 {
		exponent = int32(0x1000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else if (cf & 0x1e000) == 0x1c000 {
		exponent = int32(0x2000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else {
		panic("decimal128 value error")
	}
	exponent -= 6176

	dpdBits := bytesToBig(b)
	mask := bigFromHexString("3fffffffffffffffffffffffffff")
	dpdBits.Add(dpdBits, mask)
	digits = calcSignificand(prefix, dpdBits, 110)

	return
}

func decimalFixedToDecimal(b []byte, scale int32) decimal.Decimal {
	v, sign, digits, _ := decimal128ToSignDigitsExponent(b)
	if v != nil {
		return *v
	}
	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, scale)
}

func decimal64ToDecimal(b []byte) decimal.Decimal {
	// https://en.wikipedia.org/wiki/Decimal64_floating-point_format
	var prefix int64
	var sign int
	if (b[0] & 0x80) == 0x80 {
		sign = 1
	}

	cf := (uint32(b[0]) >> 2) & 0x1f
	exponent := ((int32(b[0]) & 3) << 6) + ((int32(b[1]) >> 2) & 0x3f)

	dpdBits := bytesToBig(b)
	mask := bigFromHexString("3ffffffffffff")
	dpdBits.Add(dpdBits, mask)

	if cf == 0x1f {
		if sign == 1 {
			// Is there -NaN ?
			return decimal.NewFromFloat(math.NaN())
		} else {
			return decimal.NewFromFloat(math.NaN())
		}
	} else if cf == 0x1e {
		if sign == 1 {
			return decimal.NewFromFloat(math.Inf(-1))
		} else {
			return decimal.NewFromFloat(math.Inf(1))
		}
	} else if (cf & 0x18) == 0x00 {
		exponent = 0x000 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x18) == 0x08 {
		exponent = 0x100 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x18) == 0x10 {
		exponent = 0x200 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x1e) == 0x18 {
		exponent = 0x000 + exponent
		prefix = int64(8 + cf&1)
	} else if (cf & 0x1e) == 0x1a {
		exponent = 0x100 + exponent
		prefix = int64(8 + cf&1)
	} else if (cf & 0x1e) == 0x1c {
		exponent = 0x200 + exponent
		prefix = int64(8 + cf&1)
	} else {
		panic("decimal64 value error")
	}
	digits := calcSignificand(prefix, dpdBits, 50)
	exponent -= 398

	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, exponent)
}

func decimal128ToDecimal(b []byte) decimal.Decimal {
	// https://en.wikipedia.org/wiki/Decimal64_floating-point_format
	v, sign, digits, exponent := decimal128ToSignDigitsExponent(b)
	if v != nil {
		return *v
	}
	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, exponent)
}
<commit_msg>fix decimal128ToSignDigitsExponent()<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2019 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
	"github.com/shopspring/decimal"
	"math"
	"math/big"
)

func dpdBitToInt(dpd uint, mask uint) int {
	if (dpd & mask) != 0 {
		return 1
	} else {
		return 0
	}
}

func dpdToInt(dpd uint) int {
	// Convert DPD encodined value to int (0-999)
	// dpd: DPD encoded value. 10bit unsigned int

	b := make([]int, 10)
	b[9] = dpdBitToInt(dpd, 0x0200)
	b[8] = dpdBitToInt(dpd, 0x0100)
	b[7] = dpdBitToInt(dpd, 0x0080)
	b[6] = dpdBitToInt(dpd, 0x0040)
	b[5] = dpdBitToInt(dpd, 0x0020)
	b[4] = dpdBitToInt(dpd, 0x0010)
	b[3] = dpdBitToInt(dpd, 0x0008)
	b[2] = dpdBitToInt(dpd, 0x0004)
	b[1] = dpdBitToInt(dpd, 0x0002)
	b[0] = dpdBitToInt(dpd, 0x0001)

	d := make([]int, 3)
	if b[3] == 0 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = b[2]*4 + b[1]*2 + b[0]
	} else if b[3] == 1 && b[2] == 0 && b[1] == 0 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = 8 + b[0]
	} else if b[3] == 1 && b[2] == 0 && b[1] == 1 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = 8 + b[4]
		d[0] = b[6]*4 + b[5]*2 + b[0]
	} else if b[3] == 1 && b[2] == 1 && b[1] == 0 {
		d[2] = 8 + b[7]
		d[1] = b[6]*4 + b[5]*2 + b[4]
		d[0] = b[9]*4 + b[8]*2 + b[0]
	} else if b[6] == 0 && b[5] == 0 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = 8 + b[4]
		d[0] = b[9]*4 + b[8]*2 + b[0]
	} else if b[6] == 0 && b[5] == 1 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = b[9]*4 + b[8]*2 + b[4]
		d[0] = 8 + b[0]
	} else if b[6] == 1 && b[5] == 0 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = b[9]*4 + b[8]*2 + b[7]
		d[1] = 8 + b[4]
		d[0] = 8 + b[0]
	} else if b[6] == 1 && b[5] == 1 && b[3] == 1 && b[2] == 1 && b[1] == 1 {
		d[2] = 8 + b[7]
		d[1] = 8 + b[4]
		d[0] = 8 + b[0]
	} else {
		panic("Invalid DPD encoding")
	}

	return d[2]*100 + d[1]*10 + d[0]
}

func calcSignificand(prefix int64, dpdBits *big.Int, numBits int) *big.Int {
	// prefix: High bits integer value
	// dpdBits: dpd encoded bits
	// numBits: bit length of dpd_bits
	// https://en.wikipedia.org/wiki/Decimal128_floating-point_format#Densely_packed_decimal_significand_field
	numSegments := numBits / 10
	segments := make([]uint, numSegments)
	bi1024 := big.NewInt(1024)

	for i := 0; i < numSegments; i++ {
		var work big.Int
		work = *dpdBits
		segments[numSegments-i-1] = uint(dpdBits.Mod(&work, bi1024).Int64())
		dpdBits.Rsh(dpdBits, 10)
	}

	v := big.NewInt(prefix)
	bi1000 := big.NewInt(1000)
	for _, dpd := range segments {
		v.Mul(v, bi1000)
		v.Add(v, big.NewInt(int64(dpd)))
	}

	return v
}

func decimal128ToSignDigitsExponent(b []byte) (v *decimal.Decimal, sign int, digits *big.Int, exponent int32) {
	// https://en.wikipedia.org/wiki/Decimal128_floating-point_format

	var prefix int64
	if (b[0] & 0x80) == 0x80 {
		sign = 1
	}
	cf := (uint32(b[0]&0x7f) << 10) + uint32(b[1]<<2) + uint32(b[2]>>6)
	if (cf & 0x1F000) == 0x1F000 {
		var d decimal.Decimal
		if sign == 1 {
			// Is there -NaN ?
			d = decimal.NewFromFloat(math.NaN())
		} else {
			d = decimal.NewFromFloat(math.NaN())
		}
		v = &d
		return
	} else if (cf & 0x1F000) == 0x1E000 {
		var d decimal.Decimal
		if sign == 1 {
			d = decimal.NewFromFloat(math.Inf(-1))
		} else {
			d = decimal.NewFromFloat(math.Inf(1))
		}
		v = &d
		return
	} else if (cf & 0x18000) == 0x00000 {
		exponent = int32(0x0000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x18000) == 0x08000 {
		exponent = int32(0x1000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x18000) == 0x10000 {
		exponent = int32(0x2000 + (cf & 0x00fff))
		prefix = int64((cf >> 12) & 0x07)
	} else if (cf & 0x1e000) == 0x18000 {
		exponent = int32(0x0000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else if (cf & 0x1e000) == 0x1a000 {
		exponent = int32(0x1000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else if (cf & 0x1e000) == 0x1c000 {
		exponent = int32(0x2000 + (cf & 0x00fff))
		prefix = int64(8 + (cf>>12)&0x01)
	} else {
		panic("decimal128 value error")
	}
	exponent -= 6176

	dpdBits := bytesToBig(b)
	mask := bigFromHexString("3fffffffffffffffffffffffffff")
	dpdBits.And(dpdBits, mask)
	digits = calcSignificand(prefix, dpdBits, 110)

	return
}

func decimalFixedToDecimal(b []byte, scale int32) decimal.Decimal {
	v, sign, digits, _ := decimal128ToSignDigitsExponent(b)
	if v != nil {
		return *v
	}
	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, scale)
}

func decimal64ToDecimal(b []byte) decimal.Decimal {
	// https://en.wikipedia.org/wiki/Decimal64_floating-point_format
	var prefix int64
	var sign int
	if (b[0] & 0x80) == 0x80 {
		sign = 1
	}

	cf := (uint32(b[0]) >> 2) & 0x1f
	exponent := ((int32(b[0]) & 3) << 6) + ((int32(b[1]) >> 2) & 0x3f)

	dpdBits := bytesToBig(b)
	mask := bigFromHexString("3ffffffffffff")
	dpdBits.Add(dpdBits, mask)

	if cf == 0x1f {
		if sign == 1 {
			// Is there -NaN ?
			return decimal.NewFromFloat(math.NaN())
		} else {
			return decimal.NewFromFloat(math.NaN())
		}
	} else if cf == 0x1e {
		if sign == 1 {
			return decimal.NewFromFloat(math.Inf(-1))
		} else {
			return decimal.NewFromFloat(math.Inf(1))
		}
	} else if (cf & 0x18) == 0x00 {
		exponent = 0x000 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x18) == 0x08 {
		exponent = 0x100 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x18) == 0x10 {
		exponent = 0x200 + exponent
		prefix = int64(cf & 0x07)
	} else if (cf & 0x1e) == 0x18 {
		exponent = 0x000 + exponent
		prefix = int64(8 + cf&1)
	} else if (cf & 0x1e) == 0x1a {
		exponent = 0x100 + exponent
		prefix = int64(8 + cf&1)
	} else if (cf & 0x1e) == 0x1c {
		exponent = 0x200 + exponent
		prefix = int64(8 + cf&1)
	} else {
		panic("decimal64 value error")
	}
	digits := calcSignificand(prefix, dpdBits, 50)
	exponent -= 398

	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, exponent)
}

func decimal128ToDecimal(b []byte) decimal.Decimal {
	// https://en.wikipedia.org/wiki/Decimal64_floating-point_format
	v, sign, digits, exponent := decimal128ToSignDigitsExponent(b)
	if v != nil {
		return *v
	}
	if sign != 0 {
		digits.Mul(digits, big.NewInt(-1))
	}
	return decimal.NewFromBigInt(digits, exponent)
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Jason Ish. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above
//    copyright notice, this list of conditions and the following
//    disclaimer in the documentation and/or other materials provided
//    with the distribution.
//
// THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
// OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"time"
)

var (
	snortFastTimestampPattern = regexp.MustCompile("^(?P<month>\\d\\d)\\/(?P<day>\\d\\d)(?:\\/)?(?P<year>\\d{4})?-(?P<hour>\\d\\d):(?P<minute>\\d\\d):(?P<seconds>\\d\\d).(?P<microseconds>\\d+)")
	snortFastEventRegexp = regexp.MustCompile("{(?P<protocol>\\d+|\\w+)}\\s([\\d\\.]+):?(\\d+)?\\s..\\s([\\d\\.]+):?(\\d+)?")
)

type SuricataEveFlow struct {
	Start string `json:"start"`
	End   string `json:"end"`
}

type SuricataJsonEvent struct {
	Timestamp  string `json:"timestamp"`
	Protocol   string `json:"proto"`
	SourceAddr string `json:"src_ip"`
	SourcePort uint16 `json:"src_port"`
	DestAddr   string `json:"dest_ip"`
	DestPort   uint16 `json:"dest_port"`
	EventType  string `json:"event_type"`
	Alert SuricataJsonAlert `json:"alert"`
	Flow       SuricataEveFlow `json:"flow"`
}

type SuricataJsonAlert struct {
	SignatureId uint32 `json:"signature_id"`
}

type Event struct {
	Timestamp  string
	Protocol   string
	SourceAddr string
	SourcePort uint16
	DestAddr   string
	DestPort   uint16
	EventType  string
	SignatureId uint32
	Flow       SuricataEveFlow

	// The original event.
	Original   string
}

func (e *Event) ToPcapFilter() string {

	if e.SourcePort > 0 && e.DestPort > 0 {
		return fmt.Sprintf(
			"proto %s and ((host %s and port %d) and (host %s and port %d))",
			e.Protocol,
			e.SourceAddr, e.SourcePort,
			e.DestAddr, e.DestPort)
	} else {
		return fmt.Sprintf(
			"proto %s and host %s and host %s",
			e.Protocol, e.SourceAddr, e.DestAddr)
	}
}

func DecodeSnortFastEventTimestamp(buf string) string {

	matches := snortFastTimestampPattern.FindStringSubmatch(buf)
	if matches == nil {
		return ""
	}
	year := matches[3]
	if year == "" {
		year = strconv.Itoa(time.Now().Year())
	}
	return fmt.Sprintf("%s-%s-%sT%s:%s:%s.%s",
		year, matches[1], matches[2],
		matches[4], matches[5], matches[6], matches[7])
}

// DecodeSnortFastEvent will decode Snort and Suricata "fast" style
// events.
func DecodeSnortFastEvent(buf string) *Event {

	eventMatches := snortFastEventRegexp.FindStringSubmatch(buf)
	if eventMatches == nil {
		return nil
	}
	timestamp := DecodeSnortFastEventTimestamp(buf)
	if timestamp == "" {
		return nil
	}
	sourcePort, err := strconv.ParseUint(eventMatches[3], 10, 16)
	if err != nil {
		return nil
	}
	destPort, err := strconv.ParseUint(eventMatches[5], 10, 16)
	if err != nil {
		return nil
	}
	return &Event{
		Timestamp:  timestamp,
		Protocol:   eventMatches[1],
		SourceAddr: eventMatches[2],
		SourcePort: (uint16)(sourcePort),
		DestAddr:   eventMatches[4],
		DestPort:   (uint16)(destPort),
		Original:   buf,
	}
}

// DecodeSuricataJsonEvent decodes a Suricata style JSON event.
func DecodeSuricataJsonEvent(buf string) *Event {

	suricataJsonEvent := SuricataJsonEvent{}

	if err := json.Unmarshal(([]byte)(buf), &suricataJsonEvent); err != nil {
		return nil
	}

	return &Event{
		Timestamp:  suricataJsonEvent.Timestamp,
		Protocol:   suricataJsonEvent.Protocol,
		SourceAddr: suricataJsonEvent.SourceAddr,
		SourcePort: suricataJsonEvent.SourcePort,
		DestAddr:   suricataJsonEvent.DestAddr,
		DestPort:   suricataJsonEvent.DestPort,
		EventType: suricataJsonEvent.EventType,
		SignatureId: suricataJsonEvent.Alert.SignatureId,
		Original:   buf,
	}
}

func DecodeEvent(buf string) (*Event, error) {
	if event := DecodeSuricataJsonEvent(buf); event != nil {
		return event, nil
	}
	if event := DecodeSnortFastEvent(buf); event != nil {
		return event, nil
	}
	return nil, fmt.Errorf("error: event not recognized by any decoders")
}
<commit_msg>add flow event; missed in merge<commit_after>// Copyright (c) 2014 Jason Ish. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above
//    copyright notice, this list of conditions and the following
//    disclaimer in the documentation and/or other materials provided
//    with the distribution.
//
// THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
// OF THE POSSIBILITY OF SUCH DAMAGE.

package main

import (
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"time"
)

var (
	snortFastTimestampPattern = regexp.MustCompile("^(?P<month>\\d\\d)\\/(?P<day>\\d\\d)(?:\\/)?(?P<year>\\d{4})?-(?P<hour>\\d\\d):(?P<minute>\\d\\d):(?P<seconds>\\d\\d).(?P<microseconds>\\d+)")
	snortFastEventRegexp = regexp.MustCompile("{(?P<protocol>\\d+|\\w+)}\\s([\\d\\.]+):?(\\d+)?\\s..\\s([\\d\\.]+):?(\\d+)?")
)

type SuricataEveFlow struct {
	Start string `json:"start"`
	End   string `json:"end"`
}

type SuricataJsonEvent struct {
	Timestamp  string `json:"timestamp"`
	Protocol   string `json:"proto"`
	SourceAddr string `json:"src_ip"`
	SourcePort uint16 `json:"src_port"`
	DestAddr   string `json:"dest_ip"`
	DestPort   uint16 `json:"dest_port"`
	EventType  string `json:"event_type"`
	Alert      SuricataJsonAlert `json:"alert"`
	Flow       SuricataEveFlow `json:"flow"`
}

type SuricataJsonAlert struct {
	SignatureId uint32 `json:"signature_id"`
}

type Event struct {
	Timestamp   string
	Protocol    string
	SourceAddr  string
	SourcePort  uint16
	DestAddr    string
	DestPort    uint16
	EventType   string
	SignatureId uint32
	Flow        SuricataEveFlow

	// The original event.
	Original    string
}

func (e *Event) ToPcapFilter() string {

	if e.SourcePort > 0 && e.DestPort > 0 {
		return fmt.Sprintf(
			"proto %s and ((host %s and port %d) and (host %s and port %d))",
			e.Protocol,
			e.SourceAddr, e.SourcePort,
			e.DestAddr, e.DestPort)
	} else {
		return fmt.Sprintf(
			"proto %s and host %s and host %s",
			e.Protocol, e.SourceAddr, e.DestAddr)
	}
}

func DecodeSnortFastEventTimestamp(buf string) string {

	matches := snortFastTimestampPattern.FindStringSubmatch(buf)
	if matches == nil {
		return ""
	}
	year := matches[3]
	if year == "" {
		year = strconv.Itoa(time.Now().Year())
	}
	return fmt.Sprintf("%s-%s-%sT%s:%s:%s.%s",
		year, matches[1], matches[2],
		matches[4], matches[5], matches[6], matches[7])
}

// DecodeSnortFastEvent will decode Snort and Suricata "fast" style
// events.
func DecodeSnortFastEvent(buf string) *Event {

	eventMatches := snortFastEventRegexp.FindStringSubmatch(buf)
	if eventMatches == nil {
		return nil
	}
	timestamp := DecodeSnortFastEventTimestamp(buf)
	if timestamp == "" {
		return nil
	}
	sourcePort, err := strconv.ParseUint(eventMatches[3], 10, 16)
	if err != nil {
		return nil
	}
	destPort, err := strconv.ParseUint(eventMatches[5], 10, 16)
	if err != nil {
		return nil
	}
	return &Event{
		Timestamp:  timestamp,
		Protocol:   eventMatches[1],
		SourceAddr: eventMatches[2],
		SourcePort: (uint16)(sourcePort),
		DestAddr:   eventMatches[4],
		DestPort:   (uint16)(destPort),
		Original:   buf,
	}
}

// DecodeSuricataJsonEvent decodes a Suricata style JSON event.
func DecodeSuricataJsonEvent(buf string) *Event {

	suricataJsonEvent := SuricataJsonEvent{}

	if err := json.Unmarshal(([]byte)(buf), &suricataJsonEvent); err != nil {
		return nil
	}

	return &Event{
		Timestamp:  suricataJsonEvent.Timestamp,
		Protocol:   suricataJsonEvent.Protocol,
		SourceAddr: suricataJsonEvent.SourceAddr,
		SourcePort: suricataJsonEvent.SourcePort,
		DestAddr:   suricataJsonEvent.DestAddr,
		DestPort:   suricataJsonEvent.DestPort,
		EventType: suricataJsonEvent.EventType,
		SignatureId: suricataJsonEvent.Alert.SignatureId,
		Flow: suricataJsonEvent.Flow,
		Original:   buf,
	}
}

func DecodeEvent(buf string) (*Event, error) {
	if event := DecodeSuricataJsonEvent(buf); event != nil {
		return event, nil
	}
	if event := DecodeSnortFastEvent(buf); event != nil {
		return event, nil
	}
	return nil, fmt.Errorf("error: event not recognized by any decoders")
}
<|endoftext|>


<commit_before>package httptreemux

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestEmptyGroupAndMapping(t *testing.T) {
	defer func() {
		if err := recover(); err != nil {
			//everything is good, it paniced
		} else {
			t.Error(`Expected NewGroup("")`)
		}
	}()
	New().GET("", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {})
}
func TestSubGroupSlashMapping(t *testing.T) {
	r := New()
	r.NewGroup("/foo").GET("/", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {
		w.WriteHeader(200)
	})

	var req *http.Request
	var recorder *httptest.ResponseRecorder

	req, _ = http.NewRequest("GET", "/foo", nil)
	recorder = httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 301 { //should get redirected
		t.Error(`/foo on NewGroup("/foo").GET("/") should result in 301 response, got:`, recorder.Code)
	}

	req, _ = http.NewRequest("GET", "/foo/", nil)
	recorder = httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 200 {
		t.Error(`/foo/ on NewGroup("/foo").GET("/"") should result in 200 response, got:`, recorder.Code)
	}
}

func TestSubGroupEmptyMapping(t *testing.T) {
	r := New()
	r.NewGroup("/foo").GET("", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {
		w.WriteHeader(200)
	})
	req, _ := http.NewRequest("GET", "/foo", nil)
	recorder := httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 200 {
		t.Error(`/foo on NewGroup("/foo").GET("") should result in 200 response, got:`, recorder.Code)
	}
}

func TestGroupMethods(t *testing.T) {
	for _, scenario := range scenarios {
		t.Log(scenario.description)
		testGroupMethods(t, scenario.RequestCreator, false)
		testGroupMethods(t, scenario.RequestCreator, true)
	}
}

func TestInvalidHandle(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad handle path should have caused a panic")
		}
	}()
	New().NewGroup("/foo").GET("bar", nil)
}

func TestInvalidSubPath(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad sub-path should have caused a panic")
		}
	}()
	New().NewGroup("/foo").NewGroup("bar")
}

func TestInvalidPath(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad path should have caused a panic")
		}
	}()
	New().NewGroup("foo")
}

//Liberally borrowed from router_test
func testGroupMethods(t *testing.T, reqGen RequestCreator, headCanUseGet bool) {
	var result string
	makeHandler := func(method string) HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request, params map[string]string) {
			result = method
		}
	}
	router := New()
	router.HeadCanUseGet = headCanUseGet
	// Testing with a sub-group of a group as that will test everything at once
	g := router.NewGroup("/base").NewGroup("/user")
	g.GET("/:param", makeHandler("GET"))
	g.POST("/:param", makeHandler("POST"))
	g.PATCH("/:param", makeHandler("PATCH"))
	g.PUT("/:param", makeHandler("PUT"))
	g.DELETE("/:param", makeHandler("DELETE"))

	testMethod := func(method, expect string) {
		result = ""
		w := httptest.NewRecorder()
		r, _ := reqGen(method, "/base/user/"+method, nil)
		router.ServeHTTP(w, r)
		if expect == "" && w.Code != http.StatusMethodNotAllowed {
			t.Errorf("Method %s not expected to match but saw code %d", method, w.Code)
		}

		if result != expect {
			t.Errorf("Method %s got result %s", method, result)
		}
	}

	testMethod("GET", "GET")
	testMethod("POST", "POST")
	testMethod("PATCH", "PATCH")
	testMethod("PUT", "PUT")
	testMethod("DELETE", "DELETE")
	if headCanUseGet {
		t.Log("Test implicit HEAD with HeadCanUseGet = true")
		testMethod("HEAD", "GET")
	} else {
		t.Log("Test implicit HEAD with HeadCanUseGet = false")
		testMethod("HEAD", "")
	}

	router.HEAD("/base/user/:param", makeHandler("HEAD"))
	testMethod("HEAD", "HEAD")
}

// Ensure that setting a GET handler doesn't overwrite an explciit HEAD handler.
func TestSetGetAfterHead(t *testing.T) {
	var result string
	makeHandler := func(method string) HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request, params map[string]string) {
			result = method
		}
	}

	router := New()
	router.HeadCanUseGet = true
	router.HEAD("/abc", makeHandler("HEAD"))
	router.GET("/abc", makeHandler("GET"))

	testMethod := func(method, expect string) {
		result = ""
		w := httptest.NewRecorder()
		r, _ := http.NewRequest(method, "/abc", nil)
		router.ServeHTTP(w, r)

		if result != expect {
			t.Errorf("Method %s got result %s", method, result)
		}
	}

	testMethod("HEAD", "HEAD")
	testMethod("GET", "GET")
}
<commit_msg>added group test<commit_after>package httptreemux

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestEmptyGroupAndMapping(t *testing.T) {
	defer func() {
		if err := recover(); err != nil {
			//everything is good, it paniced
		} else {
			t.Error(`Expected NewGroup("")`)
		}
	}()
	New().GET("", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {})
}
func TestSubGroupSlashMapping(t *testing.T) {
	r := New()
	r.NewGroup("/foo").GET("/", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {
		w.WriteHeader(200)
	})

	var req *http.Request
	var recorder *httptest.ResponseRecorder

	req, _ = http.NewRequest("GET", "/foo", nil)
	recorder = httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 301 { //should get redirected
		t.Error(`/foo on NewGroup("/foo").GET("/") should result in 301 response, got:`, recorder.Code)
	}

	req, _ = http.NewRequest("GET", "/foo/", nil)
	recorder = httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 200 {
		t.Error(`/foo/ on NewGroup("/foo").GET("/"") should result in 200 response, got:`, recorder.Code)
	}
}

func TestSubGroupEmptyMapping(t *testing.T) {
	r := New()
	r.NewGroup("/foo").GET("", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {
		w.WriteHeader(200)
	})
	req, _ := http.NewRequest("GET", "/foo", nil)
	recorder := httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != 200 {
		t.Error(`/foo on NewGroup("/foo").GET("") should result in 200 response, got:`, recorder.Code)
	}
}

func TestGroupCaseInsensitiveRouting(t *testing.T) {
	r := New()
	r.NewGroup("/MY-path").GET("", func(w http.ResponseWriter, _ *http.Request, _ map[string]string) {
		w.WriteHeader(200)
	})

	req, _ := http.NewRequest("GET", "/MY-PATH", nil)
	recorder := httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != http.StatusNotFound {
		t.Errorf("expected 404 response for case-insensitive request. Received: %d", recorder.Code)
	}

	// Now try with case-insensitive routing
	r.CaseInsensitive = true
	recorder = httptest.NewRecorder()
	r.ServeHTTP(recorder, req)
	if recorder.Code != http.StatusOK {
		t.Errorf("expected 200 response for case-insensitive request. Received: %d", recorder.Code)
	}
}

func TestGroupMethods(t *testing.T) {
	for _, scenario := range scenarios {
		t.Log(scenario.description)
		testGroupMethods(t, scenario.RequestCreator, false)
		testGroupMethods(t, scenario.RequestCreator, true)
	}
}

func TestInvalidHandle(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad handle path should have caused a panic")
		}
	}()
	New().NewGroup("/foo").GET("bar", nil)
}

func TestInvalidSubPath(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad sub-path should have caused a panic")
		}
	}()
	New().NewGroup("/foo").NewGroup("bar")
}

func TestInvalidPath(t *testing.T) {
	defer func() {
		if err := recover(); err == nil {
			t.Error("Bad path should have caused a panic")
		}
	}()
	New().NewGroup("foo")
}

//Liberally borrowed from router_test
func testGroupMethods(t *testing.T, reqGen RequestCreator, headCanUseGet bool) {
	var result string
	makeHandler := func(method string) HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request, params map[string]string) {
			result = method
		}
	}
	router := New()
	router.HeadCanUseGet = headCanUseGet
	// Testing with a sub-group of a group as that will test everything at once
	g := router.NewGroup("/base").NewGroup("/user")
	g.GET("/:param", makeHandler("GET"))
	g.POST("/:param", makeHandler("POST"))
	g.PATCH("/:param", makeHandler("PATCH"))
	g.PUT("/:param", makeHandler("PUT"))
	g.DELETE("/:param", makeHandler("DELETE"))

	testMethod := func(method, expect string) {
		result = ""
		w := httptest.NewRecorder()
		r, _ := reqGen(method, "/base/user/"+method, nil)
		router.ServeHTTP(w, r)
		if expect == "" && w.Code != http.StatusMethodNotAllowed {
			t.Errorf("Method %s not expected to match but saw code %d", method, w.Code)
		}

		if result != expect {
			t.Errorf("Method %s got result %s", method, result)
		}
	}

	testMethod("GET", "GET")
	testMethod("POST", "POST")
	testMethod("PATCH", "PATCH")
	testMethod("PUT", "PUT")
	testMethod("DELETE", "DELETE")
	if headCanUseGet {
		t.Log("Test implicit HEAD with HeadCanUseGet = true")
		testMethod("HEAD", "GET")
	} else {
		t.Log("Test implicit HEAD with HeadCanUseGet = false")
		testMethod("HEAD", "")
	}

	router.HEAD("/base/user/:param", makeHandler("HEAD"))
	testMethod("HEAD", "HEAD")
}

// Ensure that setting a GET handler doesn't overwrite an explciit HEAD handler.
func TestSetGetAfterHead(t *testing.T) {
	var result string
	makeHandler := func(method string) HandlerFunc {
		return func(w http.ResponseWriter, r *http.Request, params map[string]string) {
			result = method
		}
	}

	router := New()
	router.HeadCanUseGet = true
	router.HEAD("/abc", makeHandler("HEAD"))
	router.GET("/abc", makeHandler("GET"))

	testMethod := func(method, expect string) {
		result = ""
		w := httptest.NewRecorder()
		r, _ := http.NewRequest(method, "/abc", nil)
		router.ServeHTTP(w, r)

		if result != expect {
			t.Errorf("Method %s got result %s", method, result)
		}
	}

	testMethod("HEAD", "HEAD")
	testMethod("GET", "GET")
}
<|endoftext|>


<commit_before>package groupmebot

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type GroupMeBot struct {
	ID      string `json:"bot_id"`
	GroupID string `json:"group_id"`
	Host    string `json:"host"`
	Port    string `json:"port"`
	LogFile string `json:logfile`
	Server  string
	Hooks   map[string]func(InboundMessage) string
}

type InboundMessage struct {
	Avatar_url  string `json:"avatar_url"`
	Id          string `json:"id"`
	Name        string `json:"name"`
	Sender_id   string `json:"sender_id"`
	Sender_type string `json:"sender_type"`
	System      bool   `json:"system"`
	Text        string `json:"text"`
	User_id     string `json:"user_id"`
}

type OutboundMessage struct {
	ID   string `json:"bot_id"`
	Text string `json:"text"`
}

/// NewBotFromJson (json cfg file name)
/// This reads a json file containing the keys
/// See the example bot_cfg.json
/// Returns err from ioutil if file can not be read
func NewBotFromJson(filename string) (*GroupMeBot, error) {
	file, err := ioutil.ReadFile(filename)

	var bot GroupMeBot
	if err != nil {
		log.Fatal("Error reading bot configuration json file")
		return nil, err
	}

	// Parse out information from file
	json.Unmarshal(file, &bot)

	bot.Server = bot.Host + ":" + bot.Port
	log.Printf("Creating bot at %s\nLogging at %s\n", bot.Server, bot.LogFile)
	bot.Hooks = make(map[string]func(InboundMessage) string)

	return &bot, err
}

func (b *GroupMeBot) SendMessage(outMessage string) (*http.Response, error) {
	msg := OutboundMessage{b.ID, outMessage}
	payload, err := json.Marshal(msg)
	if err != nil {
		return nil, err
	}

	j_payload := string(payload)
	return http.Post("https://api.groupme.com/v3/bots/post", "application/json", strings.NewReader(j_payload))
}

func (b *GroupMeBot) AddHook(trigger string, response func(InboundMessage) string) {
	b.Hooks[trigger] = response
}

func (b *GroupMeBot) HandleMessage(msg InboundMessage) {
	resp := ""
	for trig, hook := range b.Hooks {
		matched, err := regexp.MatchString(trig, msg.Text)

		if matched {
			resp = hook(msg)
		} else if err != nil {
			log.Fatal("Error matching:", err)
		}

	}
	if len(resp) > 0 {
		log.Printf("Sending message: %v\n", resp)
		_, err := b.SendMessage(resp)
		if err != nil {
			log.Fatal("Error when sending.", err)
		}
	}

}

func (b *GroupMeBot) LogMessage(msg InboundMessage) {
	id := fmt.Sprintf("%s", msg.Sender_id)
	txt := fmt.Sprintf("%s", msg.Text)
	name := fmt.Sprintf("%s", msg.Name)
	values := []string{id, txt, name}

	log.Printf("%s: %s [Type: %s]\n", msg.Name, msg.Text, msg.Sender_type)

	f, err := os.OpenFile(b.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal("Couldn't open file to log messages")
	}

	defer f.Close()
	fwriter := bufio.NewWriter(f)
	csvWriter := csv.NewWriter(fwriter)

	csvWriter.Write(values)
	csvWriter.Flush()
	fwriter.Flush()
}

/*
 This is legitimate black magic, this is pretty cool, not usually able to do
 things like this in other languages. This is a function that takes
 a list of trigger functions and returns a function that can handle the Server
 Requests
*/
func (b *GroupMeBot) Handler() http.HandlerFunc {
	// Request Handler function

	return func(w http.ResponseWriter, req *http.Request) {
		if req.Method == "POST" {
			//log.Println("Bot recieving and handling message.")
			defer req.Body.Close()
			var msg InboundMessage
			err := json.NewDecoder(req.Body).Decode(&msg)
			if msg.Sender_type != "bot" {
				b.LogMessage(msg)
				// Find hook by running through hooklist
				b.HandleMessage(msg)
			}
			if err != nil {
				log.Fatal("Couldn't read all the body", err)
			}
		} else {
			//log.Println("Bot not responding to unknown message")
			//io.WriteString(w, "GOTEM")
		}
	}
}
<commit_msg>Changed Unmarshalling for messages that fail to process<commit_after>package groupmebot

import (
	"bufio"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"strings"
)

type GroupMeBot struct {
	ID      string `json:"bot_id"`
	GroupID string `json:"group_id"`
	Host    string `json:"host"`
	Port    string `json:"port"`
	LogFile string `json:logfile`
	Server  string
	Hooks   map[string]func(InboundMessage) string
}

type InboundMessage struct {
	Avatar_url  string `json:"avatar_url"`
	Id          string `json:"id"`
	Name        string `json:"name"`
	Sender_id   string `json:"sender_id"`
	Sender_type string `json:"sender_type"`
	System      bool   `json:"system"`
	Text        string `json:"text"`
	User_id     string `json:"user_id"`
}

type OutboundMessage struct {
	ID   string `json:"bot_id"`
	Text string `json:"text"`
}

/// NewBotFromJson (json cfg file name)
/// This reads a json file containing the keys
/// See the example bot_cfg.json
/// Returns err from ioutil if file can not be read
func NewBotFromJson(filename string) (*GroupMeBot, error) {
	file, err := ioutil.ReadFile(filename)

	var bot GroupMeBot
	if err != nil {
		log.Fatal("Error reading bot configuration json file")
		return nil, err
	}

	// Parse out information from file
	json.Unmarshal(file, &bot)

	bot.Server = bot.Host + ":" + bot.Port
	log.Printf("Creating bot at %s\nLogging at %s\n", bot.Server, bot.LogFile)
	bot.Hooks = make(map[string]func(InboundMessage) string)

	return &bot, err
}

func (b *GroupMeBot) SendMessage(outMessage string) (*http.Response, error) {
	msg := OutboundMessage{b.ID, outMessage}
	payload, err := json.Marshal(msg)
	if err != nil {
		return nil, err
	}

	j_payload := string(payload)
	return http.Post("https://api.groupme.com/v3/bots/post", "application/json", strings.NewReader(j_payload))
}

func (b *GroupMeBot) AddHook(trigger string, response func(InboundMessage) string) {
	b.Hooks[trigger] = response
}

func (b *GroupMeBot) HandleMessage(msg InboundMessage) {
	resp := ""
	for trig, hook := range b.Hooks {
		matched, err := regexp.MatchString(trig, msg.Text)

		if matched {
			resp = hook(msg)
		} else if err != nil {
			log.Fatal("Error matching:", err)
		}

	}
	if len(resp) > 0 {
		log.Printf("Sending message: %v\n", resp)
		_, err := b.SendMessage(resp)
		if err != nil {
			log.Fatal("Error when sending.", err)
		}
	}

}

func (b *GroupMeBot) LogMessage(msg InboundMessage) {
	id := fmt.Sprintf("%s", msg.Sender_id)
	txt := fmt.Sprintf("%s", msg.Text)
	name := fmt.Sprintf("%s", msg.Name)
	values := []string{id, txt, name}

	log.Printf("%s: %s [Type: %s]\n", msg.Name, msg.Text, msg.Sender_type)

	f, err := os.OpenFile(b.LogFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		log.Fatal("Couldn't open file to log messages")
	}

	defer f.Close()
	fwriter := bufio.NewWriter(f)
	csvWriter := csv.NewWriter(fwriter)

	csvWriter.Write(values)
	csvWriter.Flush()
	fwriter.Flush()
}

/*
 This is legitimate black magic, this is pretty cool, not usually able to do
 things like this in other languages. This is a function that takes
 a list of trigger functions and returns a function that can handle the Server
 Requests
*/
func (b *GroupMeBot) Handler() http.HandlerFunc {
	// Request Handler function

	return func(w http.ResponseWriter, req *http.Request) {
		if req.Method == "POST" {
			//log.Println("Bot recieving and handling message.")
			defer req.Body.Close()
			var msg InboundMessage
			err := json.NewDecoder(req.Body).Decode(&msg)
			if err != nil {
				log.Debug("Couldn't parse the request body")
				msg.Sender_type = "bot"
			}
			if msg.Sender_type != "bot" {
				b.LogMessage(msg)
				// Find hook by running through hooklist
				b.HandleMessage(msg)
			}
		} else {
			//log.Println("Bot not responding to unknown message")
			//io.WriteString(w, "GOTEM")
		}
	}
}
<|endoftext|>


<commit_before>package imap_server

import (
	"errors"
	"fmt"
	"time"
)

type DummyMailstore struct {
	user DummyUser
}

func newDummyMailbox(name string) DummyMailbox {
	return DummyMailbox{
		name:     name,
		messages: make([]Message, 0),
		nextuid:  1,
	}
}

func NewDummyMailstore() DummyMailstore {
	ms := DummyMailstore{
		user: DummyUser{
			authenticated: false,
			mailboxes:     make([]DummyMailbox, 2),
		},
	}
	ms.user.mailboxes[0] = newDummyMailbox("INBOX")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Test email", time.Now(),
		"Test email\r\n"+
			"Regards,\r\n"+
			"Me")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Another test email", time.Now(),
		"Another test email")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Last email", time.Now(),
		"Hello")

	ms.user.mailboxes[1] = newDummyMailbox("Trash")
	return ms
}

func (d DummyMailstore) Authenticate(username string, password string) (User, error) {
	if username != "username" {
		return DummyUser{}, errors.New("Invalid username. Use 'username'")
	}

	if password != "password" {
		return DummyUser{}, errors.New("Invalid password. Use 'password'")
	}

	d.user.authenticated = true
	return d.user, nil
}

type DummyUser struct {
	authenticated bool
	mailboxes     []DummyMailbox
}

func (u DummyUser) Mailboxes() []Mailbox {
	mailboxes := make([]Mailbox, len(u.mailboxes))
	index := 0
	for _, element := range u.mailboxes {
		mailboxes[index] = element
		index++
	}
	return mailboxes
}

func (u DummyUser) MailboxByName(name string) (Mailbox, error) {
	for _, mailbox := range u.mailboxes {
		if mailbox.Name() == name {
			return mailbox, nil
		}
	}
	return DummyMailbox{}, errors.New("Invalid mailbox")
}

type DummyMailbox struct {
	name     string
	nextuid  uint32
	messages []Message
}

func (m DummyMailbox) Name() string    { return m.name }
func (m DummyMailbox) NextUid() uint32 { return m.nextuid }
func (m DummyMailbox) Recent() uint32 {
	var count uint32 = 0
	for _, message := range m.messages {
		if message.IsRecent() {
			count++
		}
	}
	return count
}
func (m DummyMailbox) Messages() uint32 { return uint32(len(m.messages)) }
func (m DummyMailbox) Unseen() uint32 {
	var count uint32 = 0
	for _, message := range m.messages {
		if !message.IsSeen() {
			count++
		}
	}
	return count
}

func (m DummyMailbox) MessageBySequenceNumber(seqno uint32) Message {
	if seqno >= uint32(len(m.messages)) {
		return DummyMessage{}
	}
	return m.messages[seqno-1]
}

func (m DummyMailbox) MessageByUid(uidno uint32) Message {
	for _, message := range m.messages {
		if message.Uid() == uidno {
			return message
		}
	}

	// No message found
	return DummyMessage{}
}

func (m DummyMailbox) MessageSetByUid(set SequenceSet) []Message {
	msgs := make([]Message, 2)
	msgs[0] = m.MessageByUid(1)
	msgs[1] = m.MessageByUid(2)
	return msgs

}

func (m DummyMailbox) MessageSetBySequenceNumber(set SequenceSet) []Message {
	msgs := make([]Message, 2)
	msgs[0] = m.MessageBySequenceNumber(1)
	msgs[1] = m.MessageBySequenceNumber(2)
	return msgs

}

func (m *DummyMailbox) addEmail(from string, to string, subject string, date time.Time, body string) {
	uid := m.nextuid
	m.nextuid++

	hdr := make(map[string]string)
	hdr["Date"] = date.Format(rfc822Date)
	hdr["To"] = to
	hdr["From"] = from
	hdr["Subject"] = subject
	hdr["Message-ID"] = fmt.Sprintf("<%d@test.com>", uid)

	newMessage := DummyMessage{
		sequenceNumber: uint32(len(m.messages) + 1),
		uid:            uid,
		recent:         true,
		header:         hdr,
	}
	m.messages = append(m.messages, newMessage)
}

type DummyMessage struct {
	sequenceNumber uint32
	uid            uint32
	header         MIMEHeader
	seen           bool
	deleted        bool
	recent         bool
	answered       bool
	flagged        bool
	draft          bool
}

func (m DummyMessage) Header() (hdr MIMEHeader) {
	return m.header
}

func (m DummyMessage) Uid() uint32            { return m.uid }
func (m DummyMessage) SequenceNumber() uint32 { return m.sequenceNumber }

func (m DummyMessage) Size() uint32 {
	hdrStr := fmt.Sprintf("%s\r\n", m.Header())
	return uint32(len(hdrStr)) + uint32(len(m.Body()))
}

func (m DummyMessage) InternalDate() time.Time {
	tz := time.FixedZone("Australia/Brisbane", 10*60*60)
	return time.Date(2014, 10, 28, 0, 9, 0, 0, tz)
}

func (m DummyMessage) Body() string {
	return `This is the body of the email.
It is a short email`
}

func (m DummyMessage) Keywords() []string {
	f := make([]string, 0)
	//f[0] = "Test"
	return f
}

func (m DummyMessage) IsSeen() bool     { return m.seen }
func (m DummyMessage) IsAnswered() bool { return m.answered }
func (m DummyMessage) IsFlagged() bool  { return m.flagged }
func (m DummyMessage) IsDeleted() bool  { return m.deleted }
func (m DummyMessage) IsDraft() bool    { return m.draft }
func (m DummyMessage) IsRecent() bool   { return m.recent }
<commit_msg>Implement message sequence number fetching for sequence sets. No tests yet. #6<commit_after>package imap_server

import (
	"errors"
	"fmt"
	"time"
)

type DummyMailstore struct {
	user DummyUser
}

func newDummyMailbox(name string) DummyMailbox {
	return DummyMailbox{
		name:     name,
		messages: make([]Message, 0),
		nextuid:  1,
	}
}

func NewDummyMailstore() DummyMailstore {
	ms := DummyMailstore{
		user: DummyUser{
			authenticated: false,
			mailboxes:     make([]DummyMailbox, 2),
		},
	}
	ms.user.mailboxes[0] = newDummyMailbox("INBOX")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Test email", time.Now(),
		"Test email\r\n"+
			"Regards,\r\n"+
			"Me")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Another test email", time.Now(),
		"Another test email")
	ms.user.mailboxes[0].addEmail("me@test.com", "you@test.com", "Last email", time.Now(),
		"Hello")

	ms.user.mailboxes[1] = newDummyMailbox("Trash")
	return ms
}

func (d DummyMailstore) Authenticate(username string, password string) (User, error) {
	if username != "username" {
		return DummyUser{}, errors.New("Invalid username. Use 'username'")
	}

	if password != "password" {
		return DummyUser{}, errors.New("Invalid password. Use 'password'")
	}

	d.user.authenticated = true
	return d.user, nil
}

type DummyUser struct {
	authenticated bool
	mailboxes     []DummyMailbox
}

func (u DummyUser) Mailboxes() []Mailbox {
	mailboxes := make([]Mailbox, len(u.mailboxes))
	index := 0
	for _, element := range u.mailboxes {
		mailboxes[index] = element
		index++
	}
	return mailboxes
}

func (u DummyUser) MailboxByName(name string) (Mailbox, error) {
	for _, mailbox := range u.mailboxes {
		if mailbox.Name() == name {
			return mailbox, nil
		}
	}
	return DummyMailbox{}, errors.New("Invalid mailbox")
}

type DummyMailbox struct {
	name     string
	nextuid  uint32
	messages []Message
}

func (m DummyMailbox) Name() string    { return m.name }
func (m DummyMailbox) NextUid() uint32 { return m.nextuid }
func (m DummyMailbox) Recent() uint32 {
	var count uint32 = 0
	for _, message := range m.messages {
		if message.IsRecent() {
			count++
		}
	}
	return count
}
func (m DummyMailbox) Messages() uint32 { return uint32(len(m.messages)) }
func (m DummyMailbox) Unseen() uint32 {
	var count uint32 = 0
	for _, message := range m.messages {
		if !message.IsSeen() {
			count++
		}
	}
	return count
}

func (m DummyMailbox) MessageBySequenceNumber(seqno uint32) Message {
	if seqno >= uint32(len(m.messages)) {
		return DummyMessage{}
	}
	return m.messages[seqno-1]
}

func (m DummyMailbox) MessageByUid(uidno uint32) Message {
	for _, message := range m.messages {
		if message.Uid() == uidno {
			return message
		}
	}

	// No message found
	return DummyMessage{}
}

func (m DummyMailbox) MessageSetByUid(set SequenceSet) []Message {
	msgs := make([]Message, 2)
	msgs[0] = m.MessageByUid(1)
	msgs[1] = m.MessageByUid(2)
	return msgs

}

func (m DummyMailbox) MessageSetBySequenceNumber(set SequenceSet) []Message {
	msgs := make([]Message, 0)

	// For each sequence range in the sequence set
	for _, msgRange := range set {
		start, err := msgRange.min.Value()
		if err != nil {
			fmt.Printf("Error: %s\n", err.Error())
			return msgs
		}

		// If no max is specified, the sequence number must be either a fixed
		// sequence number or
		if msgRange.max.Nil() {
			var sequenceNo uint32
			if msgRange.min.Last() {
				// Fetch the last message in the mailbox
				// (sequence number = total number of messages in mailbox)
				sequenceNo = m.Messages()
			} else {
				// Fetch specific message by sequence number
				sequenceNo, err = msgRange.min.Value()
				if err != nil {
					fmt.Printf("Error: %s\n", err.Error())
					return msgs
				}
			}
			msgs = append(msgs, m.MessageBySequenceNumber(sequenceNo))
			continue
		}

		var end uint32
		if msgRange.max.Last() {
			end = uint32(len(m.messages))
		} else {
			end, err = msgRange.max.Value()
		}

		// Note this is very inefficient when
		// the message array is large. A proper
		// storage system using eg SQL might
		// instead perform a query here using
		// the range values instead.
		for index := uint32(start); index <= end; index += 1 {
			msgs = append(msgs, m.MessageBySequenceNumber(index-1))
		}
	}
	return msgs

}

func (m *DummyMailbox) addEmail(from string, to string, subject string, date time.Time, body string) {
	uid := m.nextuid
	m.nextuid++

	hdr := make(map[string]string)
	hdr["Date"] = date.Format(rfc822Date)
	hdr["To"] = to
	hdr["From"] = from
	hdr["Subject"] = subject
	hdr["Message-ID"] = fmt.Sprintf("<%d@test.com>", uid)

	newMessage := DummyMessage{
		sequenceNumber: uint32(len(m.messages) + 1),
		uid:            uid,
		recent:         true,
		header:         hdr,
	}
	m.messages = append(m.messages, newMessage)
}

type DummyMessage struct {
	sequenceNumber uint32
	uid            uint32
	header         MIMEHeader
	seen           bool
	deleted        bool
	recent         bool
	answered       bool
	flagged        bool
	draft          bool
}

func (m DummyMessage) Header() (hdr MIMEHeader) {
	return m.header
}

func (m DummyMessage) Uid() uint32            { return m.uid }
func (m DummyMessage) SequenceNumber() uint32 { return m.sequenceNumber }

func (m DummyMessage) Size() uint32 {
	hdrStr := fmt.Sprintf("%s\r\n", m.Header())
	return uint32(len(hdrStr)) + uint32(len(m.Body()))
}

func (m DummyMessage) InternalDate() time.Time {
	tz := time.FixedZone("Australia/Brisbane", 10*60*60)
	return time.Date(2014, 10, 28, 0, 9, 0, 0, tz)
}

func (m DummyMessage) Body() string {
	return `This is the body of the email.
It is a short email`
}

func (m DummyMessage) Keywords() []string {
	f := make([]string, 0)
	//f[0] = "Test"
	return f
}

func (m DummyMessage) IsSeen() bool     { return m.seen }
func (m DummyMessage) IsAnswered() bool { return m.answered }
func (m DummyMessage) IsFlagged() bool  { return m.flagged }
func (m DummyMessage) IsDeleted() bool  { return m.deleted }
func (m DummyMessage) IsDraft() bool    { return m.draft }
func (m DummyMessage) IsRecent() bool   { return m.recent }
<|endoftext|>


<commit_before>package engine

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/temoto/errors"
	"github.com/temoto/vender/helpers"
	"github.com/temoto/vender/log2"
)

type Engine struct {
	Log     *log2.Log
	lk      sync.Mutex
	actions map[string]Doer
}

func NewEngine(log *log2.Log) *Engine {
	self := &Engine{
		Log:     log,
		actions: make(map[string]Doer, 64),
	}
	return self
}

func (self *Engine) Register(action string, d Doer) {
	self.lk.Lock()
	self.actions[action] = d
	self.lk.Unlock()
}

func (self *Engine) RegisterNewSeq(name string, ds ...Doer) {
	tx := NewSeq(name)
	for _, d := range ds {
		tx.Append(d)
	}
	self.Register(name, tx)
}

func (self *Engine) RegisterParse(name, scenario string) error {
	d, err := self.ParseText(name, scenario)
	if err != nil {
		err = errors.Annotatef(err, "engine.RegisterParse() name=%s scenario=%s", name, scenario)
		return err
	}
	self.Register(name, d)
	return nil
}

var reActionArg = regexp.MustCompile(`^(.+)\((\d+)\)$`)

func (self *Engine) resolve(action string) Doer {
	// self.Log.Debugf(action)
	// TODO RLock?
	d, ok := self.actions[action]
	if ok {
		return d
	}

	match := reActionArg.FindStringSubmatch(action)
	if match == nil {
		return nil
	}

	normalized := match[1] + "(?)"
	d, ok = self.actions[normalized]
	if !ok {
		self.Log.Errorf("resolve action=%s normalized=%s not found", action, normalized)
		return nil
	}
	argn, _ := strconv.Atoi(match[2])
	return ArgApply(d, Arg(argn))
}

func (self *Engine) Resolve(action string) Doer {
	d := self.resolve(action)
	if d == nil {
		self.Log.Errorf("engine.Resolve action=%s not found", action)
	}
	return d
}

func (self *Engine) List() []string {
	self.lk.Lock()
	r := make([]string, 0, len(self.actions))
	for k := range self.actions {
		r = append(r, k)
	}
	self.lk.Unlock()
	return r
}

var reSleep = regexp.MustCompile(`sleep\((\d+m?s)\)`)

func (self *Engine) ResolveOrLazy(action string) (Doer, error) {
	d, ok := self.actions[action]
	if ok {
		return d, nil
	}

	if m := reSleep.FindStringSubmatch(action); len(m) == 2 {
		duration, err := time.ParseDuration(m[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		return Sleep{duration}, nil
	}

	return &Lazy{Name: action, r: self.resolve}, nil
}
func (self *Engine) MustResolveOrLazy(action string) Doer {
	d, err := self.ResolveOrLazy(action)
	if err != nil {
		return Fail{err}
	}
	return d
}

var reNotSpace = regexp.MustCompile(`\S+`)

func (self *Engine) ParseText(tag, text string) (Doer, error) {
	errs := make([]error, 0)
	words := reNotSpace.FindAllString(text, -1)

	tx := NewSeq(tag)
	for _, word := range words {
		d, err := self.ResolveOrLazy(word)
		if err != nil {
			return nil, errors.Annotatef(err, "scenario=%s unparsed=%s", text, word)
		}
		tx.Append(d)
	}
	return tx, helpers.FoldErrors(errs)
}

func (self *Engine) ExecList(ctx context.Context, tag string, list []string) error {
	self.Log.Debugf("engine.ExecList tag=%s list=%v", tag, list)

	errs := make([]error, 0)
	for i, text := range list {
		itemTag := fmt.Sprintf("%s:%d", tag, i)
		d, err := self.ParseText(itemTag, text)
		if err == nil {
			err = d.Do(ctx)
		}
		if err != nil {
			errs = append(errs, err)
		}
	}
	return helpers.FoldErrors(errs)
}
<commit_msg>engine: lock resolve<commit_after>package engine

import (
	"context"
	"fmt"
	"regexp"
	"strconv"
	"sync"
	"time"

	"github.com/temoto/errors"
	"github.com/temoto/vender/helpers"
	"github.com/temoto/vender/log2"
)

type Engine struct {
	Log     *log2.Log
	lk      sync.RWMutex
	actions map[string]Doer
}

func NewEngine(log *log2.Log) *Engine {
	self := &Engine{
		Log:     log,
		actions: make(map[string]Doer, 64),
	}
	return self
}

func (self *Engine) Register(action string, d Doer) {
	self.lk.Lock()
	self.actions[action] = d
	self.lk.Unlock()
}

func (self *Engine) RegisterNewSeq(name string, ds ...Doer) {
	tx := NewSeq(name)
	for _, d := range ds {
		tx.Append(d)
	}
	self.Register(name, tx)
}

func (self *Engine) RegisterParse(name, scenario string) error {
	d, err := self.ParseText(name, scenario)
	if err != nil {
		err = errors.Annotatef(err, "engine.RegisterParse() name=%s scenario=%s", name, scenario)
		return err
	}
	self.Register(name, d)
	return nil
}

var reActionArg = regexp.MustCompile(`^(.+)\((\d+)\)$`)

func (self *Engine) resolve(action string) Doer {
	// self.Log.Debugf(action)
	self.lk.RLock()
	defer self.lk.RUnlock()
	d, ok := self.actions[action]
	if ok {
		return d
	}

	match := reActionArg.FindStringSubmatch(action)
	if match == nil {
		return nil
	}

	normalized := match[1] + "(?)"
	d, ok = self.actions[normalized]
	if !ok {
		self.Log.Errorf("resolve action=%s normalized=%s not found", action, normalized)
		return nil
	}
	argn, _ := strconv.Atoi(match[2])
	return ArgApply(d, Arg(argn))
}

func (self *Engine) Resolve(action string) Doer {
	d := self.resolve(action)
	if d == nil {
		self.Log.Errorf("engine.Resolve action=%s not found", action)
	}
	return d
}

func (self *Engine) List() []string {
	self.lk.RLock()
	r := make([]string, 0, len(self.actions))
	for k := range self.actions {
		r = append(r, k)
	}
	self.lk.RUnlock()
	return r
}

var reSleep = regexp.MustCompile(`sleep\((\d+m?s)\)`)

func (self *Engine) ResolveOrLazy(action string) (Doer, error) {
	self.lk.RLock()
	defer self.lk.RUnlock()
	d, ok := self.actions[action]
	if ok {
		return d, nil
	}

	if m := reSleep.FindStringSubmatch(action); len(m) == 2 {
		duration, err := time.ParseDuration(m[1])
		if err != nil {
			return nil, errors.Trace(err)
		}
		return Sleep{duration}, nil
	}

	return &Lazy{Name: action, r: self.resolve}, nil
}
func (self *Engine) MustResolveOrLazy(action string) Doer {
	d, err := self.ResolveOrLazy(action)
	if err != nil {
		return Fail{err}
	}
	return d
}

var reNotSpace = regexp.MustCompile(`\S+`)

func (self *Engine) ParseText(tag, text string) (Doer, error) {
	errs := make([]error, 0)
	words := reNotSpace.FindAllString(text, -1)

	tx := NewSeq(tag)
	for _, word := range words {
		d, err := self.ResolveOrLazy(word)
		if err != nil {
			return nil, errors.Annotatef(err, "scenario=%s unparsed=%s", text, word)
		}
		tx.Append(d)
	}
	return tx, helpers.FoldErrors(errs)
}

func (self *Engine) ExecList(ctx context.Context, tag string, list []string) error {
	self.Log.Debugf("engine.ExecList tag=%s list=%v", tag, list)

	errs := make([]error, 0)
	for i, text := range list {
		itemTag := fmt.Sprintf("%s:%d", tag, i)
		d, err := self.ParseText(itemTag, text)
		if err == nil {
			err = d.Do(ctx)
		}
		if err != nil {
			errs = append(errs, err)
		}
	}
	return helpers.FoldErrors(errs)
}
<|endoftext|>


<commit_before>package htmlrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "/favicon.ico", Token: "link"},
		{URI: "/foo.css", Token: "link"},
		{URI: "/async.js", Token: "script"},
		{URI: "/foo.js", Token: "script"},
		{URI: "/relative", Token: "a"},
		{URI: "http://example.com/absolute", Token: "a"},
		{URI: "//google.com/protocol-relative", Token: "a"},
		{URI: "#yep", Token: "a"},
		{URI: "/sample.jpg", Token: "a"},
		{URI: "/logo.svg", Token: "source"},
		{URI: "/logo.webp", Token: "source"},
		{URI: "/logo.png", Token: "source"},
		{URI: "/logo.gif", Token: "img"},
		{URI: "/vid.mp4", Token: "video"},
	}

	// example HTML reader
	reader := strings.NewReader(`
		<!DOCTYPE html>

				<title>example</title>
				<link rel="icon" href="/favicon.ico">
				<link rel="stylesheet" type="text/css" href="/foo.css">

				yep

				<picture>
				   <source type="image/svg+xml" srcset="/logo.svg" />
				   <source type="image/webp" srcset="/logo.webp" />
				   <source type="image/png" srcset="/logo.png" />

				</picture>

nope

				<video src="/vid.mp4"></video>

	`)

	// get the refs from the implementation
	refs := All(reader)

	need, have := len(reqs), len(refs)
	if need != have {
		t.Errorf("Wrong number of refs returned. need: %d , have: %d", need, have)
	}

	for i := 0; i < len(reqs); i++ {
		uri := reqs[i].URI
		ref := refs[i]

		if uri != ref {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", uri, ref)
		}
	}
}
<commit_msg>make the test file a bit harder<commit_after>package htmlrefs

import (
	"strings"
	"testing"
)

func TestAll(t *testing.T) {

	// the required resources to be found in the example
	reqs := [...]Reference{
		{URI: "/favicon.ico", Token: "link"},
		{URI: "/foo.css", Token: "link"},
		{URI: "/async.js", Token: "script"},
		{URI: "/foo.js", Token: "script"},
		{URI: "/relative", Token: "a"},
		{URI: "http://example.com/absolute", Token: "a"},
		{URI: "//google.com/protocol-relative", Token: "a"},
		{URI: "#yep", Token: "a"},
		{URI: "/sample.jpg", Token: "a"},
		{URI: "/logo.svg", Token: "source"},
		{URI: "/logo.webp", Token: "source"},
		{URI: "/logo.png", Token: "source"},
		{URI: "/logo.gif", Token: "img"},
		{URI: "/vid.mp4", Token: "video"},
	}

	// example HTML reader
	reader := strings.NewReader(`
		<!DOCTYPE html>

				<title>example</title>
				<link rel="icon" href="/favicon.ico">
				<link rel="stylesheet" type="text/css" href="/foo.css" />

				yep

				<picture>
				   <source type="image/svg+xml" srcset="/logo.svg" />
				   <source type="image/webp" srcset="/logo.webp" />
				   <source type="image/png" srcset="/logo.png" />

				</picture>

nope

				<video src="/vid.mp4"></video>

	`)

	// get the refs from the implementation
	refs := All(reader)

	need, have := len(reqs), len(refs)
	if need != have {
		t.Errorf("Wrong number of refs returned. need: %d , have: %d", need, have)
	}

	for i := 0; i < len(reqs); i++ {
		uri := reqs[i].URI
		ref := refs[i]

		if uri != ref {
			t.Errorf("Mismatch URI detected. need: %s , have: %s", uri, ref)
		}
	}
}
<|endoftext|>


<commit_before>package sprig
/*
Sprig: Template functions for Go.

This package contains a number of utility functions for working with data
inside of Go `html/template` and `text/template` files.

To add these functions, use the `template.Funcs()` method:

	t := templates.New("foo").Funcs(sprig.FuncMap())

Note that you should add the function map before you parse any template files.

Principles:

The following principles were used in deciding on which functions to add, and
determining how to implement them.

- Template functions should be used to build layout. Therefore, the following
  types of operations are within the domain of template functions:
  - Formatting
  - Layout
  - Simple type conversions
  - Utilities that assist in handling common formatting and layout needs (e.g. arithmetic)
- Template functions should not return errors unless there is no way to print
  a sensible value. For example, converting a string to an integer should not
  produce an error if conversion fails. Instead, it should display a default
  value that can be displayed.
- Simple math is necessary for grid layouts, pagers, and so on. Complex math
  (anything other than arithmetic) should be done outside of templates.
- Template functins only deal with the data passed into them. They never retrieve
  data from a source.
- Finally, do not override core Go template functions.

Date Functions

- date: Format a date, where a date is an integer type or a time.Time type, and
  format is a time.Format formatting string.
- date_modify: Given a date, modify it with a duration: `date_modify "-1.5h" now`. If the duration doesn't
parse, it returns the time unaltered. See `time.ParseDuration` for info on duration strings.
- now: Current time.Time, for feeding into date-related functions.

String Functions

- trim: strings.TrimSpace
- upper: strings.ToUpper
- lower: strings.ToLower
- title: strings.Title
- repeat: strings.Repeat, but with the arguments switched: `repeat count str`. (This simplifies common pipelines)

Conversions:

- atoi: Convert a string to an integer. 0 if the integer could not be parsed.

Math Functions:

- add1: Increment an integer by 1
- add: Sum two integers
- sub: Subtract the second integer from the first
- div: Divide the first integer by the second
- mod: Module of first integer divided by second
- mul: Multiply two integers
- biggest: Return the biggest of two integers

Tests (Boolean Functions)

- gt: Greater than (integer)
- lt: Less than (integer)
- gte: Greater than or equal to (integer)
- lte: Less than or equal to (integer)

*/

import (
	"html/template"
	"time"
	"strings"
	"strconv"
)

func FuncMap() template.FuncMap {
	return template.FuncMap{
		"hello": func () string { return "Hello!" },

		// Date functions
		"date": date,
		"date_modify": dateModify,
		"now": func () time.Time { return time.Now() },

		// Strings
		"trim": strings.TrimSpace,
		"upper": strings.ToUpper,
		"lower": strings.ToLower,
		"title": strings.Title,
		// Switch or so that "foo" | repeat 5
		"repeat": func (count int, str string) string { return strings.Repeat(str, count) },

		// Wrap Atoi to stop errors.
		"atoi": func (a string) int { i, _ := strconv.Atoi(a); return i },

		"gt": func(a, b int) bool {return a > b},
		"gte": func(a, b int) bool {return a >= b},
		"lt": func(a, b int) bool {return a < b},
		"lte": func(a, b int) bool {return a <= b},

		// VERY basic arithmetic.
		"add1": func (i int) int {return i + 1},
		"add": func (a, b int) int { return a + b },
		"sub": func (a, b int) int { return a - b },
		"div": func (a, b int) int { return a / b },
		"mod": func (a, b int) int { return a % b },
		"mul": func (a, b int) int { return a * b },
		"biggest": biggest,
	}
}

// Given a format and a date, format the date string.
//
// Date can be a `time.Time` or an `int, int32, int64`.
// In the later case, it is treated as seconds since UNIX
// epoch.
func date(fmt string, date interface{}) string {
	switch date := date.(type) {
	default:
		return time.Now().Format(fmt)
	case time.Time:
		return date.Format(fmt)
	case int64:
		return time.Unix(date, 0).Format(fmt)
	case int:
		return time.Unix(int64(date), 0).Format(fmt)
	case int32:
		return time.Unix(int64(date), 0).Format(fmt)
	}
}

func dateModify(fmt string, date time.Time) time.Time {
	d, err := time.ParseDuration(fmt)
	if err != nil {
		return date
	}
	return date.Add(d)
}

func biggest(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Updated docs.<commit_after>/*
Sprig: Template functions for Go.

This package contains a number of utility functions for working with data
inside of Go `html/template` and `text/template` files.

To add these functions, use the `template.Funcs()` method:

	t := templates.New("foo").Funcs(sprig.FuncMap())

Note that you should add the function map before you parse any template files.

Principles:

The following principles were used in deciding on which functions to add, and
determining how to implement them.

- Template functions should be used to build layout. Therefore, the following
  types of operations are within the domain of template functions:
  - Formatting
  - Layout
  - Simple type conversions
  - Utilities that assist in handling common formatting and layout needs (e.g. arithmetic)
- Template functions should not return errors unless there is no way to print
  a sensible value. For example, converting a string to an integer should not
  produce an error if conversion fails. Instead, it should display a default
  value that can be displayed.
- Simple math is necessary for grid layouts, pagers, and so on. Complex math
  (anything other than arithmetic) should be done outside of templates.
- Template functins only deal with the data passed into them. They never retrieve
  data from a source.
- Finally, do not override core Go template functions.

Date Functions

- date: Format a date, where a date is an integer type or a time.Time type, and
  format is a time.Format formatting string.
- date_modify: Given a date, modify it with a duration: `date_modify "-1.5h" now`. If the duration doesn't
parse, it returns the time unaltered. See `time.ParseDuration` for info on duration strings.
- now: Current time.Time, for feeding into date-related functions.

String Functions

- trim: strings.TrimSpace
- upper: strings.ToUpper
- lower: strings.ToLower
- title: strings.Title
- repeat: strings.Repeat, but with the arguments switched: `repeat count str`. (This simplifies common pipelines)

Conversions:

- atoi: Convert a string to an integer. 0 if the integer could not be parsed.

Math Functions:

- add1: Increment an integer by 1
- add: Sum two integers
- sub: Subtract the second integer from the first
- div: Divide the first integer by the second
- mod: Module of first integer divided by second
- mul: Multiply two integers
- biggest: Return the biggest of two integers

Tests (Boolean Functions)

- gt: Greater than (integer)
- lt: Less than (integer)
- gte: Greater than or equal to (integer)
- lte: Less than or equal to (integer)

*/
package sprig

import (
	"html/template"
	"time"
	"strings"
	"strconv"
)

// Produce the function map.
//
// Use this to pass the functions into the template engine:
//
// 	tpl := template.New("foo").Funcs(sprig.FuncMap))
//
func FuncMap() template.FuncMap {
	return template.FuncMap{
		"hello": func () string { return "Hello!" },

		// Date functions
		"date": date,
		"date_modify": dateModify,
		"now": func () time.Time { return time.Now() },

		// Strings
		"trim": strings.TrimSpace,
		"upper": strings.ToUpper,
		"lower": strings.ToLower,
		"title": strings.Title,
		// Switch or so that "foo" | repeat 5
		"repeat": func (count int, str string) string { return strings.Repeat(str, count) },

		// Wrap Atoi to stop errors.
		"atoi": func (a string) int { i, _ := strconv.Atoi(a); return i },

		"gt": func(a, b int) bool {return a > b},
		"gte": func(a, b int) bool {return a >= b},
		"lt": func(a, b int) bool {return a < b},
		"lte": func(a, b int) bool {return a <= b},

		// VERY basic arithmetic.
		"add1": func (i int) int {return i + 1},
		"add": func (a, b int) int { return a + b },
		"sub": func (a, b int) int { return a - b },
		"div": func (a, b int) int { return a / b },
		"mod": func (a, b int) int { return a % b },
		"mul": func (a, b int) int { return a * b },
		"biggest": biggest,
	}
}

// Given a format and a date, format the date string.
//
// Date can be a `time.Time` or an `int, int32, int64`.
// In the later case, it is treated as seconds since UNIX
// epoch.
func date(fmt string, date interface{}) string {
	switch date := date.(type) {
	default:
		return time.Now().Format(fmt)
	case time.Time:
		return date.Format(fmt)
	case int64:
		return time.Unix(date, 0).Format(fmt)
	case int:
		return time.Unix(int64(date), 0).Format(fmt)
	case int32:
		return time.Unix(int64(date), 0).Format(fmt)
	}
}

func dateModify(fmt string, date time.Time) time.Time {
	d, err := time.ParseDuration(fmt)
	if err != nil {
		return date
	}
	return date.Add(d)
}

func biggest(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>