<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package helpers implements general utility functions that work with
// and on content.  The helper functions defined here lay down the
// foundation of how Hugo works with files and filepaths, and perform
// string operations on content.
package helpers

import (
	"bytes"
	"html/template"
	"os/exec"

	"github.com/russross/blackfriday"
	"github.com/spf13/viper"

	jww "github.com/spf13/jwalterweatherman"

	"strings"
	"sync"
)

// Length of the summary that Hugo extracts from a content.
var SummaryLength = 70

// Custom divider  let's user define where summarization ends.
var SummaryDivider = []byte("")

type Blackfriday struct {
	AngledQuotes   bool
	Fractions      bool
	PlainIdAnchors bool
	Extensions     []string
}

func NewBlackfriday() *Blackfriday {
	return &Blackfriday{
		AngledQuotes:   false,
		Fractions:      true,
		PlainIdAnchors: false,
	}
}

var blackfridayExtensionMap = map[string]int{
	"noIntraEmphasis":        blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
	"tables":                 blackfriday.EXTENSION_TABLES,
	"fencedCode":             blackfriday.EXTENSION_FENCED_CODE,
	"autolink":               blackfriday.EXTENSION_AUTOLINK,
	"strikethrough":          blackfriday.EXTENSION_STRIKETHROUGH,
	"laxHtmlBlocks":          blackfriday.EXTENSION_LAX_HTML_BLOCKS,
	"spaceHeaders":           blackfriday.EXTENSION_SPACE_HEADERS,
	"hardLineBreak":          blackfriday.EXTENSION_HARD_LINE_BREAK,
	"tabSizeEight":           blackfriday.EXTENSION_TAB_SIZE_EIGHT,
	"footnotes":              blackfriday.EXTENSION_FOOTNOTES,
	"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
	"headerIds":              blackfriday.EXTENSION_HEADER_IDS,
	"titleblock":             blackfriday.EXTENSION_TITLEBLOCK,
	"autoHeaderIds":          blackfriday.EXTENSION_AUTO_HEADER_IDS,
}

var stripHTMLReplacer = strings.NewReplacer("\n", " ", "
", "\n", "
", "\n", "
", "\n")

// StripHTML accepts a string, strips out all HTML tags and returns it.
func StripHTML(s string) string {

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		return s
	} else {
		s = stripHTMLReplacer.Replace(s)

		// Walk through the string removing all tags
		b := new(bytes.Buffer)
		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		return b.String()
	}
}

// StripEmptyNav strips out empty  tags from content.
func StripEmptyNav(in []byte) []byte {
	return bytes.Replace(in, []byte("\n\n\n"), []byte(``), -1)
}

// BytesToHTML converts bytes to type template.HTML.
func BytesToHTML(b []byte) template.HTML {
	return template.HTML(string(b))
}

func GetHtmlRenderer(defaultFlags int, ctx RenderingContext) blackfriday.Renderer {
	renderParameters := blackfriday.HtmlRendererParameters{
		FootnoteAnchorPrefix:       viper.GetString("FootnoteAnchorPrefix"),
		FootnoteReturnLinkContents: viper.GetString("FootnoteReturnLinkContents"),
	}

	b := len(ctx.DocumentId) != 0

	if b && !ctx.getConfig().PlainIdAnchors {
		renderParameters.FootnoteAnchorPrefix = ctx.DocumentId + ":" + renderParameters.FootnoteAnchorPrefix
		renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId
	}

	htmlFlags := defaultFlags
	htmlFlags |= blackfriday.HTML_USE_XHTML
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS

	if ctx.getConfig().AngledQuotes {
		htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
	}

	if ctx.getConfig().Fractions {
		htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	}

	return blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters)
}

func GetMarkdownExtensions(ctx RenderingContext) int {
	flags := 0 | blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS | blackfriday.EXTENSION_FOOTNOTES |
		blackfriday.EXTENSION_HEADER_IDS | blackfriday.EXTENSION_AUTO_HEADER_IDS
	for _, extension := range ctx.getConfig().Extensions {
		if flag, ok := blackfridayExtensionMap[extension]; ok {
			flags |= flag
		}
	}
	return flags
}

func MarkdownRender(ctx RenderingContext) []byte {
	return blackfriday.Markdown(ctx.Content, GetHtmlRenderer(0, ctx),
		GetMarkdownExtensions(ctx))
}

func MarkdownRenderWithTOC(ctx RenderingContext) []byte {
	return blackfriday.Markdown(ctx.Content,
		GetHtmlRenderer(blackfriday.HTML_TOC, ctx),
		GetMarkdownExtensions(ctx))
}

// ExtractTOC extracts Table of Contents from content.
func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
	origContent := make([]byte, len(content))
	copy(origContent, content)
	first := []byte(`

`)

	last := []byte(`

`)

	replacement := []byte(`

`)

	startOfTOC := bytes.Index(content, first)

	peekEnd := len(content)
	if peekEnd > 70+startOfTOC {
		peekEnd = 70 + startOfTOC
	}

	if startOfTOC < 0 {
		return StripEmptyNav(content), toc
	}
	// Need to peek ahead to see if this nav element is actually the right one.
	correctNav := bytes.Index(content[startOfTOC:peekEnd], []byte(`<li><a href="#`))
	if correctNav < 0 { // no match found
		return content, toc
	}
	lengthOfTOC := bytes.Index(content[startOfTOC:], last) + len(last)
	endOfTOC := startOfTOC + lengthOfTOC

	newcontent = append(content[:startOfTOC], content[endOfTOC:]...)
	toc = append(replacement, origContent[startOfTOC+len(first):endOfTOC]...)
	return
}

type RenderingContext struct {
	Content    []byte
	PageFmt    string
	DocumentId string
	Config     *Blackfriday
	configInit sync.Once
}

func (c *RenderingContext) getConfig() *Blackfriday {
	c.configInit.Do(func() {
		if c.Config == nil {
			c.Config = NewBlackfriday()
		}
	})
	return c.Config
}

func RenderBytesWithTOC(ctx RenderingContext) []byte {
	switch ctx.PageFmt {
	default:
		return MarkdownRenderWithTOC(ctx)
	case "markdown":
		return MarkdownRenderWithTOC(ctx)
	case "rst":
		return []byte(GetRstContent(ctx.Content))
	}
}

func RenderBytes(ctx RenderingContext) []byte {
	switch ctx.PageFmt {
	default:
		return MarkdownRender(ctx)
	case "markdown":
		return MarkdownRender(ctx)
	case "rst":
		return []byte(GetRstContent(ctx.Content))
	}
}

// TotalWords returns an int of the total number of words in a given content.
func TotalWords(s string) int {
	return len(strings.Fields(s))
}

// WordCount takes content and returns a map of words and count of each word.
func WordCount(s string) map[string]int {
	m := make(map[string]int)
	for _, f := range strings.Fields(s) {
		m[f] += 1
	}

	return m
}

// RemoveSummaryDivider removes summary-divider  from content.
func RemoveSummaryDivider(content []byte) []byte {
	return bytes.Replace(content, SummaryDivider, []byte(""), -1)
}

// TruncateWords takes content and an int and shortens down the number
// of words in the content down to the number of int.
func TruncateWords(s string, max int) string {
	words := strings.Fields(s)
	if max > len(words) {
		return strings.Join(words, " ")
	}

	return strings.Join(words[:max], " ")
}

// TruncateWordsToWholeSentence takes content and an int
// and returns entire sentences from content, delimited by the int
// and whether it's truncated or not.
func TruncateWordsToWholeSentence(words []string, max int) (string, bool) {
	if max >= len(words) {
		return strings.Join(words, " "), false
	}

	for counter, word := range words[max:] {
		if strings.HasSuffix(word, ".") ||
			strings.HasSuffix(word, "?") ||
			strings.HasSuffix(word, ".\"") ||
			strings.HasSuffix(word, "!") {
			upper := max + counter + 1
			return strings.Join(words[:upper], " "), (upper < len(words))
		}
	}

	return strings.Join(words[:max], " "), true
}

// GetRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func GetRstContent(content []byte) string {
	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)

	path, err := exec.LookPath("rst2html")
	if err != nil {
		path, err = exec.LookPath("rst2html.py")
		if err != nil {
			jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
				"                 Leaving reStructuredText content unrendered.")
			return (string(content))
		}
	}

	cmd := exec.Command(path, "--leave-comments")
	cmd.Stdin = bytes.NewReader(cleanContent)
	var out bytes.Buffer
	cmd.Stdout = &out
	if err := cmd.Run(); err != nil {
		jww.ERROR.Println(err)
	}

	rstLines := strings.Split(out.String(), "\n")
	for i, line := range rstLines {
		if strings.HasPrefix(line, "") {
			rstLines = (rstLines[i+1 : len(rstLines)-3])
		}
	}
	return strings.Join(rstLines, "\n")
}
<commit_msg>Use buffer pool in StripHTML<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package helpers implements general utility functions that work with
// and on content.  The helper functions defined here lay down the
// foundation of how Hugo works with files and filepaths, and perform
// string operations on content.
package helpers

import (
	"bytes"
	"html/template"
	"os/exec"

	"github.com/russross/blackfriday"
	bp "github.com/spf13/hugo/bufferpool"
	jww "github.com/spf13/jwalterweatherman"
	"github.com/spf13/viper"

	"strings"
	"sync"
)

// Length of the summary that Hugo extracts from a content.
var SummaryLength = 70

// Custom divider  let's user define where summarization ends.
var SummaryDivider = []byte("")

type Blackfriday struct {
	AngledQuotes   bool
	Fractions      bool
	PlainIdAnchors bool
	Extensions     []string
}

func NewBlackfriday() *Blackfriday {
	return &Blackfriday{
		AngledQuotes:   false,
		Fractions:      true,
		PlainIdAnchors: false,
	}
}

var blackfridayExtensionMap = map[string]int{
	"noIntraEmphasis":        blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
	"tables":                 blackfriday.EXTENSION_TABLES,
	"fencedCode":             blackfriday.EXTENSION_FENCED_CODE,
	"autolink":               blackfriday.EXTENSION_AUTOLINK,
	"strikethrough":          blackfriday.EXTENSION_STRIKETHROUGH,
	"laxHtmlBlocks":          blackfriday.EXTENSION_LAX_HTML_BLOCKS,
	"spaceHeaders":           blackfriday.EXTENSION_SPACE_HEADERS,
	"hardLineBreak":          blackfriday.EXTENSION_HARD_LINE_BREAK,
	"tabSizeEight":           blackfriday.EXTENSION_TAB_SIZE_EIGHT,
	"footnotes":              blackfriday.EXTENSION_FOOTNOTES,
	"noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
	"headerIds":              blackfriday.EXTENSION_HEADER_IDS,
	"titleblock":             blackfriday.EXTENSION_TITLEBLOCK,
	"autoHeaderIds":          blackfriday.EXTENSION_AUTO_HEADER_IDS,
}

var stripHTMLReplacer = strings.NewReplacer("\n", " ", "
", "\n", "
", "\n", "
", "\n")

// StripHTML accepts a string, strips out all HTML tags and returns it.
func StripHTML(s string) string {

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		return s
	} else {
		s = stripHTMLReplacer.Replace(s)

		// Walk through the string removing all tags
		b := bp.GetBuffer()
		defer bp.PutBuffer(b)

		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		return b.String()
	}
}

// StripEmptyNav strips out empty  tags from content.
func StripEmptyNav(in []byte) []byte {
	return bytes.Replace(in, []byte("\n\n\n"), []byte(``), -1)
}

// BytesToHTML converts bytes to type template.HTML.
func BytesToHTML(b []byte) template.HTML {
	return template.HTML(string(b))
}

func GetHtmlRenderer(defaultFlags int, ctx RenderingContext) blackfriday.Renderer {
	renderParameters := blackfriday.HtmlRendererParameters{
		FootnoteAnchorPrefix:       viper.GetString("FootnoteAnchorPrefix"),
		FootnoteReturnLinkContents: viper.GetString("FootnoteReturnLinkContents"),
	}

	b := len(ctx.DocumentId) != 0

	if b && !ctx.getConfig().PlainIdAnchors {
		renderParameters.FootnoteAnchorPrefix = ctx.DocumentId + ":" + renderParameters.FootnoteAnchorPrefix
		renderParameters.HeaderIDSuffix = ":" + ctx.DocumentId
	}

	htmlFlags := defaultFlags
	htmlFlags |= blackfriday.HTML_USE_XHTML
	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
	htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
	htmlFlags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS

	if ctx.getConfig().AngledQuotes {
		htmlFlags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
	}

	if ctx.getConfig().Fractions {
		htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
	}

	return blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters)
}

func GetMarkdownExtensions(ctx RenderingContext) int {
	flags := 0 | blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
		blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE |
		blackfriday.EXTENSION_AUTOLINK | blackfriday.EXTENSION_STRIKETHROUGH |
		blackfriday.EXTENSION_SPACE_HEADERS | blackfriday.EXTENSION_FOOTNOTES |
		blackfriday.EXTENSION_HEADER_IDS | blackfriday.EXTENSION_AUTO_HEADER_IDS
	for _, extension := range ctx.getConfig().Extensions {
		if flag, ok := blackfridayExtensionMap[extension]; ok {
			flags |= flag
		}
	}
	return flags
}

func MarkdownRender(ctx RenderingContext) []byte {
	return blackfriday.Markdown(ctx.Content, GetHtmlRenderer(0, ctx),
		GetMarkdownExtensions(ctx))
}

func MarkdownRenderWithTOC(ctx RenderingContext) []byte {
	return blackfriday.Markdown(ctx.Content,
		GetHtmlRenderer(blackfriday.HTML_TOC, ctx),
		GetMarkdownExtensions(ctx))
}

// ExtractTOC extracts Table of Contents from content.
func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
	origContent := make([]byte, len(content))
	copy(origContent, content)
	first := []byte(`

`)

	last := []byte(`

`)

	replacement := []byte(`

`)

	startOfTOC := bytes.Index(content, first)

	peekEnd := len(content)
	if peekEnd > 70+startOfTOC {
		peekEnd = 70 + startOfTOC
	}

	if startOfTOC < 0 {
		return StripEmptyNav(content), toc
	}
	// Need to peek ahead to see if this nav element is actually the right one.
	correctNav := bytes.Index(content[startOfTOC:peekEnd], []byte(`<li><a href="#`))
	if correctNav < 0 { // no match found
		return content, toc
	}
	lengthOfTOC := bytes.Index(content[startOfTOC:], last) + len(last)
	endOfTOC := startOfTOC + lengthOfTOC

	newcontent = append(content[:startOfTOC], content[endOfTOC:]...)
	toc = append(replacement, origContent[startOfTOC+len(first):endOfTOC]...)
	return
}

type RenderingContext struct {
	Content    []byte
	PageFmt    string
	DocumentId string
	Config     *Blackfriday
	configInit sync.Once
}

func (c *RenderingContext) getConfig() *Blackfriday {
	c.configInit.Do(func() {
		if c.Config == nil {
			c.Config = NewBlackfriday()
		}
	})
	return c.Config
}

func RenderBytesWithTOC(ctx RenderingContext) []byte {
	switch ctx.PageFmt {
	default:
		return MarkdownRenderWithTOC(ctx)
	case "markdown":
		return MarkdownRenderWithTOC(ctx)
	case "rst":
		return []byte(GetRstContent(ctx.Content))
	}
}

func RenderBytes(ctx RenderingContext) []byte {
	switch ctx.PageFmt {
	default:
		return MarkdownRender(ctx)
	case "markdown":
		return MarkdownRender(ctx)
	case "rst":
		return []byte(GetRstContent(ctx.Content))
	}
}

// TotalWords returns an int of the total number of words in a given content.
func TotalWords(s string) int {
	return len(strings.Fields(s))
}

// WordCount takes content and returns a map of words and count of each word.
func WordCount(s string) map[string]int {
	m := make(map[string]int)
	for _, f := range strings.Fields(s) {
		m[f] += 1
	}

	return m
}

// RemoveSummaryDivider removes summary-divider  from content.
func RemoveSummaryDivider(content []byte) []byte {
	return bytes.Replace(content, SummaryDivider, []byte(""), -1)
}

// TruncateWords takes content and an int and shortens down the number
// of words in the content down to the number of int.
func TruncateWords(s string, max int) string {
	words := strings.Fields(s)
	if max > len(words) {
		return strings.Join(words, " ")
	}

	return strings.Join(words[:max], " ")
}

// TruncateWordsToWholeSentence takes content and an int
// and returns entire sentences from content, delimited by the int
// and whether it's truncated or not.
func TruncateWordsToWholeSentence(words []string, max int) (string, bool) {
	if max >= len(words) {
		return strings.Join(words, " "), false
	}

	for counter, word := range words[max:] {
		if strings.HasSuffix(word, ".") ||
			strings.HasSuffix(word, "?") ||
			strings.HasSuffix(word, ".\"") ||
			strings.HasSuffix(word, "!") {
			upper := max + counter + 1
			return strings.Join(words[:upper], " "), (upper < len(words))
		}
	}

	return strings.Join(words[:max], " "), true
}

// GetRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func GetRstContent(content []byte) string {
	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)

	path, err := exec.LookPath("rst2html")
	if err != nil {
		path, err = exec.LookPath("rst2html.py")
		if err != nil {
			jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
				"                 Leaving reStructuredText content unrendered.")
			return (string(content))
		}
	}

	cmd := exec.Command(path, "--leave-comments")
	cmd.Stdin = bytes.NewReader(cleanContent)
	var out bytes.Buffer
	cmd.Stdout = &out
	if err := cmd.Run(); err != nil {
		jww.ERROR.Println(err)
	}

	rstLines := strings.Split(out.String(), "\n")
	for i, line := range rstLines {
		if strings.HasPrefix(line, "") {
			rstLines = (rstLines[i+1 : len(rstLines)-3])
		}
	}
	return strings.Join(rstLines, "\n")
}
<|endoftext|>


<commit_before>package helpers

import (
	"sync"

	"bytes"

	"io/ioutil"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/minio/minio-go"
)

var (
	minioBucket string
	minioClient *minio.Client
	minioLock   sync.Mutex
)

// uploads a file to the minio object storage
// objectName	: the name of the file to upload
// data			: the data for the new object
// metadata		: additional metadata attached to the object
func UploadFile(objectName string, data []byte, metadata map[string]string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("uploading " + objectName + " to minio storage")

	options := minio.PutObjectOptions{}

	// add content type
	filetype, err := SniffMime(data)
	if err == nil {
		options.ContentType = filetype
	}

	// add metadata
	if metadata != nil && len(metadata) > 0 {
		options.UserMetadata = metadata
	}

	// upload the data
	_, err = minioClient.PutObject(minioBucket, objectName, bytes.NewReader(data), -1, options)
	return err
}

// TODO: redis cache? file cache?
// retrieves a file from the minio object storage
// objectName	: the name of the file to retrieve
func RetrieveFile(objectName string) (data []byte, err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return data, err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio storage")

	// retrieve the object
	minioObject, err := minioClient.GetObject(minioBucket, objectName, minio.GetObjectOptions{})
	if err != nil {
		return data, err
	}

	// read the object into a byte slice
	data, err = ioutil.ReadAll(minioObject)
	if err != nil {
		return data, err
	}
	return data, nil
}

// deletes a file from the minio object storage
// objectName	: the file to delete
func DeleteFile(objectName string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio storage")

	// delete the object
	err = minioClient.RemoveObject(minioBucket, objectName)
	return err
}

// Initialize the minio client object, and creates the bucket if it doesn't exist yet
func setupMinioClient() (err error) {
	minioLock.Lock()
	minioBucket = GetConfig().Path("s3.bucket").Data().(string)
	minioClient, err = minio.New(
		GetConfig().Path("s3.endpoint").Data().(string),
		GetConfig().Path("s3.access_key").Data().(string),
		GetConfig().Path("s3.secret_secret_key").Data().(string),
		true,
	)
	minioLock.Unlock()

	bucketExists, err := minioClient.BucketExists(minioBucket)
	if err != nil {
		return err
	}

	if !bucketExists {
		err = minioClient.MakeBucket(minioBucket, "ams3")
		if err != nil {
			return err
		}
	}

	return err
}
<commit_msg>[storage] adds filecache<commit_after>package helpers

import (
	"sync"

	"bytes"

	"io/ioutil"

	"os"

	"path/filepath"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/kennygrant/sanitize"
	"github.com/minio/minio-go"
)

var (
	minioBucket string
	minioClient *minio.Client
	minioLock   sync.Mutex
)

// TODO: watch cache folder size

// uploads a file to the minio object storage
// objectName	: the name of the file to upload
// data			: the data for the new object
// metadata		: additional metadata attached to the object
func UploadFile(objectName string, data []byte, metadata map[string]string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("uploading " + objectName + " to minio storage")

	options := minio.PutObjectOptions{}

	// add content type
	filetype, err := SniffMime(data)
	if err == nil {
		options.ContentType = filetype
	}

	// add metadata
	if metadata != nil && len(metadata) > 0 {
		options.UserMetadata = metadata
	}

	// upload the data
	_, err = minioClient.PutObject(minioBucket, sanitize.BaseName(objectName), bytes.NewReader(data), -1, options)
	return err
}

// TODO: redis cache? file cache?
// retrieves a file from the minio object storage
// objectName	: the name of the file to retrieve
func RetrieveFile(objectName string) (data []byte, err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return data, err
		}
	}

	data = getBucketCache(objectName)
	if data != nil {
		cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio cache")
		return data, nil
	}

	cache.GetLogger().WithField("module", "storage").Infof("retrieving " + objectName + " from minio storage")

	// retrieve the object
	minioObject, err := minioClient.GetObject(minioBucket, sanitize.BaseName(objectName), minio.GetObjectOptions{})
	if err != nil {
		return data, err
	}

	// read the object into a byte slice
	data, err = ioutil.ReadAll(minioObject)
	if err != nil {
		return data, err
	}

	go func() {
		defer Recover()
		cache.GetLogger().WithField("module", "storage").Infof("caching " + objectName + " into minio cache")
		err = setBucketCache(objectName, data)
		RelaxLog(err)
	}()

	return data, nil
}

// deletes a file from the minio object storage
// objectName	: the file to delete
func DeleteFile(objectName string) (err error) {
	// setup minioClient if not yet done
	if minioClient == nil {
		err = setupMinioClient()
		if err != nil {
			return err
		}
	}

	cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio storage")

	go func() {
		defer Recover()
		cache.GetLogger().WithField("module", "storage").Infof("deleting " + objectName + " from minio cache")
		err = deleteBucketCache(objectName)
		RelaxLog(err)
	}()

	// delete the object
	err = minioClient.RemoveObject(minioBucket, sanitize.BaseName(objectName))
	return err
}

func getBucketCache(objectName string) (data []byte) {
	var err error

	if _, err = os.Stat(getObjectPath(objectName)); os.IsNotExist(err) {
		return nil
	}

	data, err = ioutil.ReadFile(getObjectPath(objectName))
	if err != nil {
		return nil
	}

	return data
}

func setBucketCache(objectName string, data []byte) (err error) {
	if _, err = os.Stat(filepath.Dir(getObjectPath(objectName))); os.IsNotExist(err) {
		err = os.MkdirAll(filepath.Dir(getObjectPath(objectName)), os.ModePerm)
		if err != nil {
			return err
		}
	}

	err = ioutil.WriteFile(getObjectPath(objectName), data, 0644)
	return err
}

func deleteBucketCache(objectName string) (err error) {
	if _, err = os.Stat(getObjectPath(objectName)); os.IsNotExist(err) {
		return nil
	}

	err = os.Remove(getObjectPath(objectName))
	return err
}

func getObjectPath(objectName string) (path string) {
	return GetConfig().Path("cache_folder").Data().(string) + "/minio-" + GetConfig().Path("s3.bucket").Data().(string) + "/" + sanitize.BaseName(objectName)
}

// Initialize the minio client object, and creates the bucket if it doesn't exist yet
func setupMinioClient() (err error) {
	minioLock.Lock()
	minioBucket = GetConfig().Path("s3.bucket").Data().(string)
	minioClient, err = minio.New(
		GetConfig().Path("s3.endpoint").Data().(string),
		GetConfig().Path("s3.access_key").Data().(string),
		GetConfig().Path("s3.secret_secret_key").Data().(string),
		true,
	)
	minioLock.Unlock()

	bucketExists, err := minioClient.BucketExists(minioBucket)
	if err != nil {
		return err
	}

	if !bucketExists {
		err = minioClient.MakeBucket(minioBucket, "ams3")
		if err != nil {
			return err
		}
	}

	return err
}
<|endoftext|>


<commit_before>package hlib

//
// ----------- cmt statements models ----------
//

// PathStmt declares a new path entry with name Name and value Value
type PathStmt struct {
	Value Value
}

func (stmt *PathStmt) is_stmt() {
}

type PathAppendStmt struct {
	Value Value
}

func (stmt *PathAppendStmt) is_stmt() {
}

type PathPrependStmt struct {
	Value Value
}

func (stmt *PathPrependStmt) is_stmt() {
}

type PathRemoveStmt struct {
	Value Value
}

func (stmt *PathRemoveStmt) is_stmt() {
}

type MacroStmt struct {
	Value Value
}

func (stmt *MacroStmt) is_stmt() {
}

type MacroAppendStmt struct {
	Value Value
}

func (stmt *MacroAppendStmt) is_stmt() {
}

type MacroRemoveStmt struct {
	Value Value
}

func (stmt *MacroRemoveStmt) is_stmt() {
}

// TagStmt defines a new CMT tag with name Name and content Content
type TagStmt struct {
	Name    string
	Content []string
}

func (stmt *TagStmt) is_stmt() {
}

type ApplyTagStmt struct {
	Value Value
}

func (stmt *ApplyTagStmt) is_stmt() {
}

type IncludeDirsStmt struct {
	Value []string
}

func (stmt *IncludeDirsStmt) is_stmt() {
}

type IncludePathStmt struct {
	Value []string
}

func (stmt *IncludePathStmt) is_stmt() {
}

type SetStmt struct {
	Value Value
}

func (stmt *SetStmt) is_stmt() {
}

type AliasStmt struct {
	Value Value
}

func (stmt *AliasStmt) is_stmt() {
}

type ActionStmt struct {
	Value Value
}

func (stmt *ActionStmt) is_stmt() {
}

type ApplyPatternStmt struct {
	Value Value
}

func (stmt *ApplyPatternStmt) is_stmt() {
}

type IgnorePatternStmt struct {
	Value Value
}

func (stmt *IgnorePatternStmt) is_stmt() {
}

// test interfaces
var _ Stmt = (*PathStmt)(nil)
var _ Stmt = (*PathAppendStmt)(nil)
var _ Stmt = (*PathPrependStmt)(nil)
var _ Stmt = (*PathRemoveStmt)(nil)
var _ Stmt = (*MacroStmt)(nil)
var _ Stmt = (*MacroAppendStmt)(nil)
var _ Stmt = (*MacroRemoveStmt)(nil)
var _ Stmt = (*TagStmt)(nil)
var _ Stmt = (*ApplyTagStmt)(nil)
var _ Stmt = (*IncludeDirsStmt)(nil)
var _ Stmt = (*IncludePathStmt)(nil)
var _ Stmt = (*SetStmt)(nil)
var _ Stmt = (*AliasStmt)(nil)
var _ Stmt = (*ApplyPatternStmt)(nil)
var _ Stmt = (*IgnorePatternStmt)(nil)
var _ Stmt = (*ActionStmt)(nil)

// EOF
<commit_msg>hlib: add make_fragment and macro_remove<commit_after>package hlib

//
// ----------- cmt statements models ----------
//

// PathStmt declares a new path entry with name Name and value Value
type PathStmt struct {
	Value Value
}

func (stmt *PathStmt) is_stmt() {
}

type PathAppendStmt struct {
	Value Value
}

func (stmt *PathAppendStmt) is_stmt() {
}

type PathPrependStmt struct {
	Value Value
}

func (stmt *PathPrependStmt) is_stmt() {
}

type PathRemoveStmt struct {
	Value Value
}

func (stmt *PathRemoveStmt) is_stmt() {
}

type MacroStmt struct {
	Value Value
}

func (stmt *MacroStmt) is_stmt() {
}

type MacroAppendStmt struct {
	Value Value
}

func (stmt *MacroAppendStmt) is_stmt() {
}

type MacroPrependStmt struct {
	Value Value
}

func (stmt *MacroPrependStmt) is_stmt() {
}

type MacroRemoveStmt struct {
	Value Value
}

func (stmt *MacroRemoveStmt) is_stmt() {
}

// TagStmt defines a new CMT tag with name Name and content Content
type TagStmt struct {
	Name    string
	Content []string
}

func (stmt *TagStmt) is_stmt() {
}

type ApplyTagStmt struct {
	Value Value
}

func (stmt *ApplyTagStmt) is_stmt() {
}

type IncludeDirsStmt struct {
	Value []string
}

func (stmt *IncludeDirsStmt) is_stmt() {
}

type IncludePathStmt struct {
	Value []string
}

func (stmt *IncludePathStmt) is_stmt() {
}

type SetStmt struct {
	Value Value
}

func (stmt *SetStmt) is_stmt() {
}

type AliasStmt struct {
	Value Value
}

func (stmt *AliasStmt) is_stmt() {
}

type ActionStmt struct {
	Value Value
}

func (stmt *ActionStmt) is_stmt() {
}

type ApplyPatternStmt struct {
	Name string
	Args []string
}

func (stmt *ApplyPatternStmt) is_stmt() {
}

type IgnorePatternStmt struct {
	Value Value
}

func (stmt *IgnorePatternStmt) is_stmt() {
}

type MakeFragmentStmt struct {
	Name string
}

func (stmt *MakeFragmentStmt) is_stmt() {
}

// test interfaces
var _ Stmt = (*PathStmt)(nil)
var _ Stmt = (*PathAppendStmt)(nil)
var _ Stmt = (*PathPrependStmt)(nil)
var _ Stmt = (*PathRemoveStmt)(nil)
var _ Stmt = (*MacroStmt)(nil)
var _ Stmt = (*MacroAppendStmt)(nil)
var _ Stmt = (*MacroPrependStmt)(nil)
var _ Stmt = (*MacroRemoveStmt)(nil)
var _ Stmt = (*TagStmt)(nil)
var _ Stmt = (*ApplyTagStmt)(nil)
var _ Stmt = (*IncludeDirsStmt)(nil)
var _ Stmt = (*IncludePathStmt)(nil)
var _ Stmt = (*SetStmt)(nil)
var _ Stmt = (*AliasStmt)(nil)
var _ Stmt = (*ApplyPatternStmt)(nil)
var _ Stmt = (*IgnorePatternStmt)(nil)
var _ Stmt = (*ActionStmt)(nil)
var _ Stmt = (*MakeFragmentStmt)(nil)

// EOF
<|endoftext|>


<commit_before>package wdclient

import (
	"context"
	"math/rand"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

type MasterClient struct {
	clientType     string
	clientHost     string
	grpcPort       uint32
	currentMaster  string
	masters        []string
	grpcDialOption grpc.DialOption

	vidMap
}

func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost string, clientGrpcPort uint32, clientDataCenter string, masters []string) *MasterClient {
	return &MasterClient{
		clientType:     clientType,
		clientHost:     clientHost,
		grpcPort:       clientGrpcPort,
		masters:        masters,
		grpcDialOption: grpcDialOption,
		vidMap:         newVidMap(clientDataCenter),
	}
}

func (mc *MasterClient) GetMaster() string {
	for mc.currentMaster == "" {
		time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
	}
	return mc.currentMaster
}

func (mc *MasterClient) WaitUntilConnected() {
	for mc.currentMaster == "" {
		time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
	}
}

func (mc *MasterClient) KeepConnectedToMaster() {
	glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters)
	for {
		mc.tryAllMasters()
		time.Sleep(time.Second)
	}
}

func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress string) (leader string) {
	for _, master := range mc.masters {
		if master == myMasterAddress {
			continue
		}
		if grpcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
			defer cancel()
			resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
			if err != nil {
				return err
			}
			leader = resp.Leader
			return nil
		}); grpcErr != nil {
			glog.V(0).Infof("connect to %s: %v", master, grpcErr)
		}
		if leader != "" {
			glog.V(0).Infof("existing leader is %s", leader)
			return
		}
	}
	glog.V(0).Infof("No existing leader found!")
	return
}

func (mc *MasterClient) tryAllMasters() {
	nextHintedLeader := ""
	for _, master := range mc.masters {

		nextHintedLeader = mc.tryConnectToMaster(master)
		for nextHintedLeader != "" {
			nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
		}

		mc.currentMaster = ""
		mc.vidMap = newVidMap("")
	}
}

func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) {
	glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master)
	gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {

		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.KeepConnected(ctx)
		if err != nil {
			glog.V(1).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err)
			return err
		}

		if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil {
			glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err)
			return err
		}

		glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master)
		mc.currentMaster = master

		for {
			volumeLocation, err := stream.Recv()
			if err != nil {
				glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err)
				return err
			}

			// maybe the leader is changed
			if volumeLocation.Leader != "" {
				glog.V(0).Infof("redirected to leader %v", volumeLocation.Leader)
				nextHintedLeader = volumeLocation.Leader
				return nil
			}

			// process new volume location
			loc := Location{
				Url:        volumeLocation.Url,
				PublicUrl:  volumeLocation.PublicUrl,
				DataCenter: volumeLocation.DataCenter,
			}
			for _, newVid := range volumeLocation.NewVids {
				glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid)
				mc.addLocation(newVid, loc)
			}
			for _, deletedVid := range volumeLocation.DeletedVids {
				glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid)
				mc.deleteLocation(deletedVid, loc)
			}
		}

	})
	if gprcErr != nil {
		glog.V(1).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr)
	}
	return
}

func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error {
	return util.Retry("master grpc", func() error {
		for mc.currentMaster == "" {
			time.Sleep(3 * time.Second)
		}
		return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			return fn(client)
		})
	})
}
<commit_msg>simplify a bit<commit_after>package wdclient

import (
	"context"
	"math/rand"
	"time"

	"github.com/chrislusf/seaweedfs/weed/util"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

type MasterClient struct {
	clientType     string
	clientHost     string
	grpcPort       uint32
	currentMaster  string
	masters        []string
	grpcDialOption grpc.DialOption

	vidMap
}

func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost string, clientGrpcPort uint32, clientDataCenter string, masters []string) *MasterClient {
	return &MasterClient{
		clientType:     clientType,
		clientHost:     clientHost,
		grpcPort:       clientGrpcPort,
		masters:        masters,
		grpcDialOption: grpcDialOption,
		vidMap:         newVidMap(clientDataCenter),
	}
}

func (mc *MasterClient) GetMaster() string {
	mc.WaitUntilConnected()
	return mc.currentMaster
}

func (mc *MasterClient) WaitUntilConnected() {
	for mc.currentMaster == "" {
		time.Sleep(time.Duration(rand.Int31n(200)) * time.Millisecond)
	}
}

func (mc *MasterClient) KeepConnectedToMaster() {
	glog.V(1).Infof("%s masterClient bootstraps with masters %v", mc.clientType, mc.masters)
	for {
		mc.tryAllMasters()
		time.Sleep(time.Second)
	}
}

func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress string) (leader string) {
	for _, master := range mc.masters {
		if master == myMasterAddress {
			continue
		}
		if grpcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
			defer cancel()
			resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
			if err != nil {
				return err
			}
			leader = resp.Leader
			return nil
		}); grpcErr != nil {
			glog.V(0).Infof("connect to %s: %v", master, grpcErr)
		}
		if leader != "" {
			glog.V(0).Infof("existing leader is %s", leader)
			return
		}
	}
	glog.V(0).Infof("No existing leader found!")
	return
}

func (mc *MasterClient) tryAllMasters() {
	nextHintedLeader := ""
	for _, master := range mc.masters {

		nextHintedLeader = mc.tryConnectToMaster(master)
		for nextHintedLeader != "" {
			nextHintedLeader = mc.tryConnectToMaster(nextHintedLeader)
		}

		mc.currentMaster = ""
		mc.vidMap = newVidMap("")
	}
}

func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) {
	glog.V(1).Infof("%s masterClient Connecting to master %v", mc.clientType, master)
	gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {

		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()

		stream, err := client.KeepConnected(ctx)
		if err != nil {
			glog.V(1).Infof("%s masterClient failed to keep connected to %s: %v", mc.clientType, master, err)
			return err
		}

		if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil {
			glog.V(0).Infof("%s masterClient failed to send to %s: %v", mc.clientType, master, err)
			return err
		}

		glog.V(1).Infof("%s masterClient Connected to %v", mc.clientType, master)
		mc.currentMaster = master

		for {
			volumeLocation, err := stream.Recv()
			if err != nil {
				glog.V(0).Infof("%s masterClient failed to receive from %s: %v", mc.clientType, master, err)
				return err
			}

			// maybe the leader is changed
			if volumeLocation.Leader != "" {
				glog.V(0).Infof("redirected to leader %v", volumeLocation.Leader)
				nextHintedLeader = volumeLocation.Leader
				return nil
			}

			// process new volume location
			loc := Location{
				Url:        volumeLocation.Url,
				PublicUrl:  volumeLocation.PublicUrl,
				DataCenter: volumeLocation.DataCenter,
			}
			for _, newVid := range volumeLocation.NewVids {
				glog.V(1).Infof("%s: %s masterClient adds volume %d", mc.clientType, loc.Url, newVid)
				mc.addLocation(newVid, loc)
			}
			for _, deletedVid := range volumeLocation.DeletedVids {
				glog.V(1).Infof("%s: %s masterClient removes volume %d", mc.clientType, loc.Url, deletedVid)
				mc.deleteLocation(deletedVid, loc)
			}
		}

	})
	if gprcErr != nil {
		glog.V(1).Infof("%s masterClient failed to connect with master %v: %v", mc.clientType, master, gprcErr)
	}
	return
}

func (mc *MasterClient) WithClient(fn func(client master_pb.SeaweedClient) error) error {
	return util.Retry("master grpc", func() error {
		for mc.currentMaster == "" {
			time.Sleep(3 * time.Second)
		}
		return pb.WithMasterClient(mc.currentMaster, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
			return fn(client)
		})
	})
}
<|endoftext|>


<commit_before>package httppool

import (
	"errors"
	"github.com/Comcast/webpa-common/logging"
	"os"
	"time"
)

const (
	DefaultWorkers   = 10
	DefaultQueueSize = 100
)

var (
	ErrorAlreadyRunning = errors.New("This client is already running")
)

// Client is an asynchronous, pooled HTTP transaction handler.  This type
// acts as a factory for Dispatcher implementations that manage a pool of
// workers, each handling HTTP transactions.  Support for rate limiting
// is also provided, via the Period member.
type Client struct {
	// Handler is any type that has a method with the signature Do(*http.Request) (*http.Response, error)
	Handler transactionHandler

	// Logger is the logging strategy used by this client.  If not supplied, all output will
	// go to the console.
	Logger logging.Logger

	// QueueSize specifies that maximum number of requests that can be queued.
	// If this value is zero or negative, DefaultQueueSize is used.
	QueueSize int

	// Workers is the number of pooled goroutines that handle tasks.
	// If this value is less than one (1), DefaultWorkers is used.
	Workers int

	// Period is the interval between requests on each worker.  If this
	// value is zero or negative, the workers will not be rate-limited.
	Period time.Duration
}

// Start starts the pool of goroutines and returns a DispatcherCloser which
// can be used to send tasks and shut down the pool.
func (client *Client) Start() (dispatcher DispatchCloser) {
	queueSize := client.QueueSize
	if queueSize < 1 {
		queueSize = DefaultQueueSize
	}

	logger := client.Logger
	if logger == nil {
		logger = &logging.LoggerWriter{os.Stdout}
	}

	var worker func(int)

	if client.Period > 0 {
		limited := &limitedClientDispatcher{
			pooledDispatcher: pooledDispatcher{
				handler: client.Handler,
				logger:  logger,
				tasks:   make(chan Task, queueSize),
			},
			ticker: time.NewTicker(client.Period),
		}

		worker = limited.worker
		dispatcher = limited
	} else {
		unlimited := &unlimitedClientDispatcher{
			pooledDispatcher: pooledDispatcher{
				handler: client.Handler,
				logger:  logger,
				tasks:   make(chan Task, queueSize),
			},
		}

		worker = unlimited.worker
		dispatcher = unlimited
	}

	workers := client.Workers
	if workers < 1 {
		workers = DefaultWorkers
	}

	for workerId := 0; workerId < workers; workerId++ {
		go worker(workerId)
	}

	return
}

// pooledDispatcher supplies the common state and logic for all
// Client-based dispatchers
type pooledDispatcher struct {
	handler transactionHandler
	logger  logging.Logger
	tasks   chan Task
}

func (pooled *pooledDispatcher) Close() error {
	pooled.logger.Debug("Close()")
	close(pooled.tasks)
	return nil
}

func (pooled *pooledDispatcher) Send(task Task) error {
	pooled.logger.Debug("Send(%v)", task)
	pooled.tasks <- task
	return nil
}

// handleTask takes care of using a task to create the request
// and then sending that request to the handler
func (pooled *pooledDispatcher) handleTask(workerId int, task Task) {
	pooled.logger.Debug("handleTask(%d, %v)", workerId, task)
	if request, err := task(); err == nil {
		if response, err := pooled.handler.Do(request); err == nil {
			pooled.logger.Debug("response: %v", response)
		} else {
			pooled.logger.Error("HTTP transaction failed: %s", err)
		}
	} else {
		pooled.logger.Error("Unable to create request: %s", err)
	}
}

// unlimitedClientDispatcher is a DispatcherCloser that provides
// access to a pool of goroutines that is not rate limited.
type unlimitedClientDispatcher struct {
	pooledDispatcher
}

func (unlimited *unlimitedClientDispatcher) worker(workerId int) {
	for task := range unlimited.tasks {
		unlimited.handleTask(workerId, task)
	}
}

// limitedClientDispatcher is a DispatcherCloser whose pooled goroutines
// are limited by a time channel.
type limitedClientDispatcher struct {
	pooledDispatcher
	ticker *time.Ticker
}

func (limited *limitedClientDispatcher) Close() error {
	defer limited.ticker.Stop()
	return limited.pooledDispatcher.Close()
}

func (limited *limitedClientDispatcher) worker(workerId int) {
	for task := range limited.tasks {
		<-limited.ticker.C
		limited.handleTask(workerId, task)
	}
}
<commit_msg>Refactored common defaulting logic into separate methods<commit_after>package httppool

import (
	"github.com/Comcast/webpa-common/logging"
	"net/http"
	"os"
	"time"
)

const (
	DefaultWorkers   = 10
	DefaultQueueSize = 100
)

// Client is an asynchronous, pooled HTTP transaction handler.  This type
// acts as a factory for Dispatcher implementations that manage a pool of
// workers, each handling HTTP transactions.  Support for rate limiting
// is also provided, via the Period member.
type Client struct {
	// Handler is any type that has a method with the signature Do(*http.Request) (*http.Response, error)
	// If not supplied, the http.DefaultClient is used.
	Handler transactionHandler

	// Logger is the logging strategy used by this client.  If not supplied, all output will
	// go to the console.
	Logger logging.Logger

	// QueueSize specifies that maximum number of requests that can be queued.
	// If this value is zero or negative, DefaultQueueSize is used.
	QueueSize int

	// Workers is the number of pooled goroutines that handle tasks.
	// If this value is less than one (1), DefaultWorkers is used.
	Workers int

	// Period is the interval between requests on each worker.  If this
	// value is zero or negative, the workers will not be rate-limited.
	Period time.Duration
}

func (client *Client) queueSize() int {
	if client.QueueSize > 0 {
		return client.QueueSize
	}

	return DefaultQueueSize
}

func (client *Client) workers() int {
	if client.Workers > 0 {
		return client.Workers
	}

	return DefaultWorkers
}

func (client *Client) logger() logging.Logger {
	if client.Logger != nil {
		return client.Logger
	}

	return &logging.LoggerWriter{os.Stdout}
}

func (client *Client) handler() transactionHandler {
	if client.Handler != nil {
		return client.Handler
	}

	return http.DefaultClient
}

// Start starts the pool of goroutines and returns a DispatcherCloser which
// can be used to send tasks and shut down the pool.
func (client *Client) Start() (dispatcher DispatchCloser) {
	logger := client.logger()
	logger.Debug("Start()")

	var worker func(int)
	if client.Period > 0 {
		limited := &limitedClientDispatcher{
			pooledDispatcher: pooledDispatcher{
				handler: client.handler(),
				logger:  logger,
				tasks:   make(chan Task, client.queueSize()),
			},
			ticker: time.NewTicker(client.Period),
		}

		worker = limited.worker
		dispatcher = limited
	} else {
		unlimited := &unlimitedClientDispatcher{
			pooledDispatcher: pooledDispatcher{
				handler: client.handler(),
				logger:  logger,
				tasks:   make(chan Task, client.queueSize()),
			},
		}

		worker = unlimited.worker
		dispatcher = unlimited
	}

	workers := client.workers()
	for workerId := 0; workerId < workers; workerId++ {
		go worker(workerId)
	}

	return
}

// pooledDispatcher supplies the common state and logic for all
// Client-based dispatchers
type pooledDispatcher struct {
	handler transactionHandler
	logger  logging.Logger
	tasks   chan Task
}

func (pooled *pooledDispatcher) Close() error {
	pooled.logger.Debug("Close()")
	close(pooled.tasks)
	return nil
}

func (pooled *pooledDispatcher) Send(task Task) error {
	pooled.logger.Debug("Send(%v)", task)
	pooled.tasks <- task
	return nil
}

// handleTask takes care of using a task to create the request
// and then sending that request to the handler
func (pooled *pooledDispatcher) handleTask(workerId int, task Task) {
	pooled.logger.Debug("handleTask(%d, %v)", workerId, task)
	if request, err := task(); err == nil {
		if response, err := pooled.handler.Do(request); err == nil {
			pooled.logger.Debug("response: %v", response)
		} else {
			pooled.logger.Error("HTTP transaction failed: %s", err)
		}
	} else {
		pooled.logger.Error("Unable to create request: %s", err)
	}
}

// unlimitedClientDispatcher is a DispatcherCloser that provides
// access to a pool of goroutines that is not rate limited.
type unlimitedClientDispatcher struct {
	pooledDispatcher
}

func (unlimited *unlimitedClientDispatcher) worker(workerId int) {
	for task := range unlimited.tasks {
		unlimited.handleTask(workerId, task)
	}
}

// limitedClientDispatcher is a DispatcherCloser whose pooled goroutines
// are limited by a time channel.
type limitedClientDispatcher struct {
	pooledDispatcher
	ticker *time.Ticker
}

func (limited *limitedClientDispatcher) Close() error {
	defer limited.ticker.Stop()
	return limited.pooledDispatcher.Close()
}

func (limited *limitedClientDispatcher) worker(workerId int) {
	for task := range limited.tasks {
		<-limited.ticker.C
		limited.handleTask(workerId, task)
	}
}
<|endoftext|>


<commit_before>package httputil

import (
	"net/http"

	log "gopkg.in/inconshreveable/log15.v2"
)

func LoggerMiddleware(logger log.Logger) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			next.ServeHTTP(w, r)
			log.Info(r.URL.String(), "method", r.Method, "len", r.ContentLength, "proto", r.Proto, "ip", GetIpAddress(r))
		})
	}
}
<commit_msg>httputil: updated request logger (added resp_time)<commit_after>package httputil

import (
	"net/http"
	"time"

	log "gopkg.in/inconshreveable/log15.v2"
)

func LoggerMiddleware(logger log.Logger) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			start := time.Now()
			next.ServeHTTP(w, r)
			log.Info(r.URL.String(), "method", r.Method, "len", r.ContentLength, "proto", r.Proto,
				"resp_time", time.Since(start), "ip", GetIpAddress(r))
		})
	}
}
<|endoftext|>


<commit_before>package swarm

import (
	"fmt"
	"strings"

	"github.com/docker/engine-api/types/swarm"
)

const (
	defaultListenAddr = "0.0.0.0:2377"
	// WORKER constant for worker name
	WORKER = "WORKER"
	// MANAGER constant for manager name
	MANAGER = "MANAGER"
)

var (
	defaultPolicies = []swarm.Policy{
		{Role: WORKER, Autoaccept: true},
		{Role: MANAGER, Autoaccept: false},
	}
)

// NodeAddrOption is a pflag.Value for listen and remote addresses
type NodeAddrOption struct {
	addr string
}

// String prints the representation of this flag
func (a *NodeAddrOption) String() string {
	return a.addr
}

// Set the value for this flag
func (a *NodeAddrOption) Set(value string) error {
	if !strings.Contains(value, ":") {
		return fmt.Errorf("Invalud url, a host and port are required")
	}

	parts := strings.Split(value, ":")
	if len(parts) != 2 {
		return fmt.Errorf("Invalud url, too many colons")
	}

	a.addr = value
	return nil
}

// Type returns the type of this flag
func (a *NodeAddrOption) Type() string {
	return "node-addr"
}

// NewNodeAddrOption returns a new node address option
func NewNodeAddrOption() NodeAddrOption {
	return NodeAddrOption{addr: defaultListenAddr}
}

// AutoAcceptOption is a value type for auto-accept policy
type AutoAcceptOption struct {
	values map[string]bool
}

// String prints a string representation of this option
func (o *AutoAcceptOption) String() string {
	keys := []string{}
	for key := range o.values {
		keys = append(keys, key)
	}
	return strings.Join(keys, " ")
}

// Set sets a new value on this option
func (o *AutoAcceptOption) Set(value string) error {
	value = strings.ToUpper(value)
	switch value {
	case "", "NONE":
		if accept, ok := o.values[WORKER]; ok && accept {
			return fmt.Errorf("value NONE is incompatible with %s", WORKER)
		}
		if accept, ok := o.values[MANAGER]; ok && accept {
			return fmt.Errorf("value NONE is incompatible with %s", MANAGER)
		}
		o.values[WORKER] = false
		o.values[MANAGER] = false
	case WORKER, MANAGER:
		if accept, ok := o.values[value]; ok && !accept {
			return fmt.Errorf("value NONE is incompatible with %s", value)
		}
		o.values[value] = true
	default:
		return fmt.Errorf("must be one of %s, %s, NONE", WORKER, MANAGER)
	}

	return nil
}

// Type returns the type of this option
func (o *AutoAcceptOption) Type() string {
	return "auto-accept"
}

// Policies returns a representation of this option for the api
func (o *AutoAcceptOption) Policies(secret *string) []swarm.Policy {
	policies := []swarm.Policy{}
	for _, p := range defaultPolicies {
		if len(o.values) != 0 {
			p.Autoaccept = o.values[string(p.Role)]
		}
		p.Secret = secret
		policies = append(policies, p)
	}
	return policies
}

// NewAutoAcceptOption returns a new auto-accept option
func NewAutoAcceptOption() AutoAcceptOption {
	return AutoAcceptOption{values: make(map[string]bool)}
}
<commit_msg>fix invalid typo in error message<commit_after>package swarm

import (
	"fmt"
	"strings"

	"github.com/docker/engine-api/types/swarm"
)

const (
	defaultListenAddr = "0.0.0.0:2377"
	// WORKER constant for worker name
	WORKER = "WORKER"
	// MANAGER constant for manager name
	MANAGER = "MANAGER"
)

var (
	defaultPolicies = []swarm.Policy{
		{Role: WORKER, Autoaccept: true},
		{Role: MANAGER, Autoaccept: false},
	}
)

// NodeAddrOption is a pflag.Value for listen and remote addresses
type NodeAddrOption struct {
	addr string
}

// String prints the representation of this flag
func (a *NodeAddrOption) String() string {
	return a.addr
}

// Set the value for this flag
func (a *NodeAddrOption) Set(value string) error {
	if !strings.Contains(value, ":") {
		return fmt.Errorf("Invalid url, a host and port are required")
	}

	parts := strings.Split(value, ":")
	if len(parts) != 2 {
		return fmt.Errorf("Invalid url, too many colons")
	}

	a.addr = value
	return nil
}

// Type returns the type of this flag
func (a *NodeAddrOption) Type() string {
	return "node-addr"
}

// NewNodeAddrOption returns a new node address option
func NewNodeAddrOption() NodeAddrOption {
	return NodeAddrOption{addr: defaultListenAddr}
}

// AutoAcceptOption is a value type for auto-accept policy
type AutoAcceptOption struct {
	values map[string]bool
}

// String prints a string representation of this option
func (o *AutoAcceptOption) String() string {
	keys := []string{}
	for key := range o.values {
		keys = append(keys, key)
	}
	return strings.Join(keys, " ")
}

// Set sets a new value on this option
func (o *AutoAcceptOption) Set(value string) error {
	value = strings.ToUpper(value)
	switch value {
	case "", "NONE":
		if accept, ok := o.values[WORKER]; ok && accept {
			return fmt.Errorf("value NONE is incompatible with %s", WORKER)
		}
		if accept, ok := o.values[MANAGER]; ok && accept {
			return fmt.Errorf("value NONE is incompatible with %s", MANAGER)
		}
		o.values[WORKER] = false
		o.values[MANAGER] = false
	case WORKER, MANAGER:
		if accept, ok := o.values[value]; ok && !accept {
			return fmt.Errorf("value NONE is incompatible with %s", value)
		}
		o.values[value] = true
	default:
		return fmt.Errorf("must be one of %s, %s, NONE", WORKER, MANAGER)
	}

	return nil
}

// Type returns the type of this option
func (o *AutoAcceptOption) Type() string {
	return "auto-accept"
}

// Policies returns a representation of this option for the api
func (o *AutoAcceptOption) Policies(secret *string) []swarm.Policy {
	policies := []swarm.Policy{}
	for _, p := range defaultPolicies {
		if len(o.values) != 0 {
			p.Autoaccept = o.values[string(p.Role)]
		}
		p.Secret = secret
		policies = append(policies, p)
	}
	return policies
}

// NewAutoAcceptOption returns a new auto-accept option
func NewAutoAcceptOption() AutoAcceptOption {
	return AutoAcceptOption{values: make(map[string]bool)}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

// These tests do not care about the tournaments,
// but the reference is required in NewMatch()
var tm *Tournament

func TestAddPlayer(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	assert.Equal(4, len(m.Players))
	assert.Equal(0, m.ActualPlayers())

	p := Player{Name: "I exist"}

	err := m.AddPlayer(p)
	assert.Nil(err)

	assert.Equal(1, m.ActualPlayers())
}

func TestAddFifthPlayer(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	m.Players = []Player{
		{Name: "a"},
		{Name: "b"},
		{Name: "c"},
		{Name: "d"},
	}
	p := Player{}

	err := m.AddPlayer(p)
	assert.NotNil(err)
	assert.Equal(4, len(m.Players))
}

func TestStartAlreadyStartedMatch(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Started = time.Now()

	err := m.Start()
	assert.NotNil(err)
}

func TestStart(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Players = []Player{
		{Name: "1"},
		{Name: "2"},
		{Name: "3"},
		{Name: "4"},
	}

	err := m.Start()
	assert.Nil(err)
	assert.Equal(false, m.Started.IsZero())
}

func TestEndGivesShotToWinner(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "tryout")
	m.Players = []Player{
		{Name: "1"},
		{Name: "2"},
		{Name: "3"},
		{Name: "4"},
	}

	err := m.Start()
	assert.Nil(err)
	m.Players[2].AddKill(10)

	err = m.End()
	assert.Nil(err)
	assert.Equal(1, m.Players[0].Shots)
}

func TestEndAlreadyEndedMatch(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Ended = time.Now()

	err := m.End()
	assert.NotNil(err)
}

func TestEnd(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	err := m.End()
	assert.Nil(err)
	assert.Equal(false, m.Ended.IsZero())
}

func TestString(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "test")
	m.AddPlayer(Player{Name: "1"})
	m.AddPlayer(Player{Name: "2"})
	m.AddPlayer(Player{Name: "3"})
	m.AddPlayer(Player{Name: "4"})
	ret := m.String()

	assert.Equal("<Test 1: 1 / 2 / 3 / 4 - not started>", ret)

	m2 := NewMatch(tm, 0, "final")
	m2.AddPlayer(Player{Name: "a"})
	m2.AddPlayer(Player{Name: "b"})
	m2.AddPlayer(Player{Name: "c"})
	m2.AddPlayer(Player{Name: "d"})
	m2.Start()
	ret2 := m2.String()

	assert.Equal("<Final: a / b / c / d - playing>", ret2)
}

func TestCorrectColorConflictsNoScores(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	m.AddPlayer(Player{Name: "d", PreferredColor: "pink"})

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.Equal("green", m.Players[0].PreferredColor)
	assert.NotEqual("green", m.Players[1].PreferredColor)
}

func TestCorrectColorConflictsNoScoresDoubleConflict(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	m.AddPlayer(Player{Name: "d", PreferredColor: "blue"})

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.Equal("green", m.Players[0].PreferredColor)
	assert.NotEqual("green", m.Players[1].PreferredColor)
	assert.Equal("blue", m.Players[2].PreferredColor)
	assert.NotEqual("blue", m.Players[3].PreferredColor)
}

func TestCorrectColorConflictPlayerTwoHasHigherScore(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	m.AddPlayer(Player{Name: "d", PreferredColor: "cyan"})

	// Add some score to player 2 so that it has preference over green.
	m.Players[1].AddKill(3)

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.NotEqual("green", m.Players[0].PreferredColor)
	assert.Equal("green", m.Players[1].PreferredColor)
	assert.Equal("blue", m.Players[2].PreferredColor)
	assert.Equal("cyan", m.Players[3].PreferredColor)
}
<commit_msg>Fix linting errors in match_test<commit_after>package main

import (
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

// These tests do not care about the tournaments,
// but the reference is required in NewMatch()
var tm *Tournament

func TestAddPlayer(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	assert.Equal(4, len(m.Players))
	assert.Equal(0, m.ActualPlayers())

	p := Player{Name: "I exist"}

	err := m.AddPlayer(p)
	assert.Nil(err)

	assert.Equal(1, m.ActualPlayers())
}

func TestAddFifthPlayer(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	m.Players = []Player{
		{Name: "a"},
		{Name: "b"},
		{Name: "c"},
		{Name: "d"},
	}
	p := Player{}

	err := m.AddPlayer(p)
	assert.NotNil(err)
	assert.Equal(4, len(m.Players))
}

func TestStartAlreadyStartedMatch(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Started = time.Now()

	err := m.Start()
	assert.NotNil(err)
}

func TestStart(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Players = []Player{
		{Name: "1"},
		{Name: "2"},
		{Name: "3"},
		{Name: "4"},
	}

	err := m.Start()
	assert.Nil(err)
	assert.Equal(false, m.Started.IsZero())
}

func TestEndGivesShotToWinner(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "tryout")
	m.Players = []Player{
		{Name: "1"},
		{Name: "2"},
		{Name: "3"},
		{Name: "4"},
	}

	err := m.Start()
	assert.Nil(err)
	m.Players[2].AddKill(10)

	err = m.End()
	assert.Nil(err)
	assert.Equal(1, m.Players[0].Shots)
}

func TestEndAlreadyEndedMatch(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")
	m.Ended = time.Now()

	err := m.End()
	assert.NotNil(err)
}

func TestEnd(t *testing.T) {
	assert := assert.New(t)
	m := NewMatch(tm, 1, "test")

	err := m.End()
	assert.Nil(err)
	assert.Equal(false, m.Ended.IsZero())
}

func TestString(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "test")
	_ = m.AddPlayer(Player{Name: "1"})
	_ = m.AddPlayer(Player{Name: "2"})
	_ = m.AddPlayer(Player{Name: "3"})
	_ = m.AddPlayer(Player{Name: "4"})
	ret := m.String()

	assert.Equal("<Test 1: 1 / 2 / 3 / 4 - not started>", ret)

	m2 := NewMatch(tm, 0, "final")
	_ = m2.AddPlayer(Player{Name: "a"})
	_ = m2.AddPlayer(Player{Name: "b"})
	_ = m2.AddPlayer(Player{Name: "c"})
	_ = m2.AddPlayer(Player{Name: "d"})
	_ = m2.Start()
	ret2 := m2.String()

	assert.Equal("<Final: a / b / c / d - playing>", ret2)
}

func TestCorrectColorConflictsNoScores(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	_ = m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	_ = m.AddPlayer(Player{Name: "d", PreferredColor: "pink"})

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.Equal("green", m.Players[0].PreferredColor)
	assert.NotEqual("green", m.Players[1].PreferredColor)
}

func TestCorrectColorConflictsNoScoresDoubleConflict(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	_ = m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	_ = m.AddPlayer(Player{Name: "d", PreferredColor: "blue"})

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.Equal("green", m.Players[0].PreferredColor)
	assert.NotEqual("green", m.Players[1].PreferredColor)
	assert.Equal("blue", m.Players[2].PreferredColor)
	assert.NotEqual("blue", m.Players[3].PreferredColor)
}

func TestCorrectColorConflictPlayerTwoHasHigherScore(t *testing.T) {
	assert := assert.New(t)

	m := NewMatch(tm, 0, "final")
	_ = m.AddPlayer(Player{Name: "a", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "b", PreferredColor: "green"})
	_ = m.AddPlayer(Player{Name: "c", PreferredColor: "blue"})
	_ = m.AddPlayer(Player{Name: "d", PreferredColor: "cyan"})

	// Add some score to player 2 so that it has preference over green.
	m.Players[1].AddKill(3)

	err := m.CorrectColorConflicts()
	assert.Nil(err)

	assert.NotEqual("green", m.Players[0].PreferredColor)
	assert.Equal("green", m.Players[1].PreferredColor)
	assert.Equal("blue", m.Players[2].PreferredColor)
	assert.Equal("cyan", m.Players[3].PreferredColor)
}
<|endoftext|>


<commit_before>package mem

import (
	"bytes"
	"encoding/json"
	"math"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/portworx/kvdb"
	"github.com/portworx/kvdb/common"
)

const (
	// Name is the name of this kvdb implementation.
	Name = "kv-mem"
)

func init() {
	if err := kvdb.Register(Name, New); err != nil {
		panic(err.Error())
	}
}

type memKV struct {
	m      map[string]*kvdb.KVPair
	w      map[string]*watchData
	wt     map[string]*watchData
	mutex  sync.Mutex
	index  uint64
	domain string
}

type watchData struct {
	cb     kvdb.WatchCB
	opaque interface{}
}

// New constructs a new kvdb.Kvdb.
func New(
	domain string,
	machines []string,
	options map[string]string,
) (kvdb.Kvdb, error) {
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	return &memKV{
		m:      make(map[string]*kvdb.KVPair),
		w:      make(map[string]*watchData),
		wt:     make(map[string]*watchData),
		domain: domain,
	}, nil
}

func (kv *memKV) String() string {
	return Name
}

func (kv *memKV) Get(key string) (*kvdb.KVPair, error) {

	key = kv.domain + key
	v, ok := kv.m[key]
	if !ok {
		return nil, kvdb.ErrNotFound
	}
	return v, nil
}

func (kv *memKV) Snapshot(prefix string) (kvdb.Kvdb, uint64, error) {
	kv.mutex.Lock()
	defer kv.mutex.Unlock()

	data := make(map[string]*kvdb.KVPair)
	for key, value := range kv.m {
		if strings.HasPrefix(key, prefix) && !strings.Contains(key, "/_") {
			continue
		}
		snap := &kvdb.KVPair{}
		*snap = *value
		snap.Value = make([]byte, len(value.Value))
		copy(snap.Value, value.Value)
		data[key] = snap
	}

	// only snapshot data, watches are not copied
	return &memKV{
		m:      data,
		w:      make(map[string]*watchData),
		wt:     make(map[string]*watchData),
		domain: kv.domain,
	}, kv.index, nil

}

func (kv *memKV) Put(
	key string,
	value interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	var kvp *kvdb.KVPair

	key = kv.domain + key
	kv.mutex.Lock()
	defer kv.mutex.Unlock()

	index := atomic.AddUint64(&kv.index, 1)
	if ttl != 0 {
		time.AfterFunc(time.Second*time.Duration(ttl), func() {
			// TODO: handle error
			_, _ = kv.Delete(key)
		})
	}
	b, err := common.ToBytes(value)
	if err != nil {
		return nil, err
	}
	if old, ok := kv.m[key]; ok {
		old.Value = b
		old.Action = kvdb.KVSet
		old.ModifiedIndex = index
		old.KVDBIndex = index
		kvp = old

	} else {
		kvp = &kvdb.KVPair{
			Key:           key,
			Value:         b,
			TTL:           int64(ttl),
			KVDBIndex:     index,
			ModifiedIndex: index,
			CreatedIndex:  index,
			Action:        kvdb.KVCreate,
		}
		kv.m[key] = kvp
	}

	kv.normalize(kvp)
	go kv.fireCB(key, *kvp, nil)
	return kvp, nil
}

func (kv *memKV) GetVal(key string, v interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(kvp.Value, v)
	return kvp, err
}

func (kv *memKV) Create(
	key string,
	value interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	result, err := kv.Get(key)
	if err != nil {
		return kv.Put(key, value, ttl)
	}
	return result, kvdb.ErrExist
}

func (kv *memKV) Update(
	key string,
	value interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	if _, err := kv.Get(key); err != nil {
		return nil, kvdb.ErrNotFound
	}
	return kv.Put(key, value, ttl)
}

func (kv *memKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	var kvp = make(kvdb.KVPairs, 0, 100)
	prefix = kv.domain + prefix

	for k, v := range kv.m {
		if strings.HasPrefix(k, prefix) && !strings.Contains(k, "/_") {
			kvpLocal := *v
			kv.normalize(&kvpLocal)
			kvp = append(kvp, &kvpLocal)
		}
	}

	return kvp, nil
}

func (kv *memKV) Delete(key string) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	kvp.KVDBIndex = atomic.AddUint64(&kv.index, 1)
	kvp.ModifiedIndex = kvp.KVDBIndex
	kvp.Action = kvdb.KVDelete
	go kv.fireCB(kv.domain+key, *kvp, nil)
	delete(kv.m, kv.domain+key)
	return kvp, nil
}

func (kv *memKV) DeleteTree(prefix string) error {

	kvp, err := kv.Enumerate(prefix)
	if err != nil {
		return err
	}
	for _, v := range kvp {
		// TODO: multiple errors
		if _, iErr := kv.Delete(v.Key); iErr != nil {
			err = iErr
		}
	}
	return err
}

func (kv *memKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *memKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte) (*kvdb.KVPair, error) {

	kv.mutex.Lock()

	result, err := kv.Get(kvp.Key)
	if err != nil {
		kv.mutex.Unlock()
		return nil, err
	}
	if prevValue != nil {
		if !bytes.Equal(result.Value, prevValue) {
			kv.mutex.Unlock()
			return nil, kvdb.ErrValueMismatch
		}
	}
	if flags == kvdb.KVModifiedIndex {
		if kvp.ModifiedIndex != result.ModifiedIndex {
			kv.mutex.Unlock()
			return nil, kvdb.ErrValueMismatch
		}
	}
	kv.mutex.Unlock()
	return kv.Put(kvp.Key, kvp.Value, 0)
}

func (kv *memKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	if flags != kvdb.KVFlags(0) {
		return nil, kvdb.ErrNotSupported
	}
	if result, err := kv.Get(kvp.Key); err != nil {
		return nil, err
	} else if !bytes.Equal(result.Value, kvp.Value) {
		return nil, kvdb.ErrNotFound
	}
	return kv.Delete(kvp.Key)
}

func (kv *memKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {
	kv.mutex.Lock()
	defer kv.mutex.Unlock()
	key = kv.domain + key
	if _, ok := kv.w[key]; ok {
		return kvdb.ErrExist
	}
	kv.w[key] = &watchData{cb: cb, opaque: opaque}
	return nil
}

func (kv *memKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	kv.mutex.Lock()
	defer kv.mutex.Unlock()
	prefix = kv.domain + prefix
	if _, ok := kv.wt[prefix]; ok {
		return kvdb.ErrExist
	}
	kv.wt[prefix] = &watchData{cb: cb, opaque: opaque}
	return nil
}

func (kv *memKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key
	duration := time.Duration(math.Min(float64(time.Second),
		float64((time.Duration(ttl)*time.Second)/10)))

	result, err := kv.Create(key, []byte("locked"), ttl)
	for err != nil {
		time.Sleep(duration)
		result, err = kv.Create(key, []byte("locked"), ttl)
	}

	if err != nil {
		return nil, err
	}
	return result, err
}

func (kv *memKV) Unlock(kvp *kvdb.KVPair) error {
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	return err
}

func (kv *memKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *memKV) normalize(kvp *kvdb.KVPair) {
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
}

func (kv *memKV) fireCB(key string, kvp kvdb.KVPair, err error) {
	for k, v := range kv.w {
		if k == key {
			err := v.cb(key, v.opaque, kvp, err)
			if err != nil {
				// TODO: handle error
				_ = v.cb("", v.opaque, nil, kvdb.ErrWatchStopped)
				delete(kv.w, key)

			}
			return
		}
	}
	for k, v := range kv.wt {
		if strings.HasPrefix(key, k) {
			err := v.cb(key, v.opaque, kvp, err)
			if err != nil {
				// TODO: handle error
				_ = v.cb("", v.opaque, nil, kvdb.ErrWatchStopped)
				delete(kv.wt, key)
			}
		}
	}
}
<commit_msg>pass in kvp pointer to cb<commit_after>package mem

import (
	"bytes"
	"encoding/json"
	"math"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/portworx/kvdb"
	"github.com/portworx/kvdb/common"
)

const (
	// Name is the name of this kvdb implementation.
	Name = "kv-mem"
)

func init() {
	if err := kvdb.Register(Name, New); err != nil {
		panic(err.Error())
	}
}

type memKV struct {
	m      map[string]*kvdb.KVPair
	w      map[string]*watchData
	wt     map[string]*watchData
	mutex  sync.Mutex
	index  uint64
	domain string
}

type watchData struct {
	cb     kvdb.WatchCB
	opaque interface{}
}

// New constructs a new kvdb.Kvdb.
func New(
	domain string,
	machines []string,
	options map[string]string,
) (kvdb.Kvdb, error) {
	if domain != "" && !strings.HasSuffix(domain, "/") {
		domain = domain + "/"
	}
	return &memKV{
		m:      make(map[string]*kvdb.KVPair),
		w:      make(map[string]*watchData),
		wt:     make(map[string]*watchData),
		domain: domain,
	}, nil
}

func (kv *memKV) String() string {
	return Name
}

func (kv *memKV) Get(key string) (*kvdb.KVPair, error) {

	key = kv.domain + key
	v, ok := kv.m[key]
	if !ok {
		return nil, kvdb.ErrNotFound
	}
	return v, nil
}

func (kv *memKV) Snapshot(prefix string) (kvdb.Kvdb, uint64, error) {
	kv.mutex.Lock()
	defer kv.mutex.Unlock()

	data := make(map[string]*kvdb.KVPair)
	for key, value := range kv.m {
		if strings.HasPrefix(key, prefix) && !strings.Contains(key, "/_") {
			continue
		}
		snap := &kvdb.KVPair{}
		*snap = *value
		snap.Value = make([]byte, len(value.Value))
		copy(snap.Value, value.Value)
		data[key] = snap
	}

	// only snapshot data, watches are not copied
	return &memKV{
		m:      data,
		w:      make(map[string]*watchData),
		wt:     make(map[string]*watchData),
		domain: kv.domain,
	}, kv.index, nil

}

func (kv *memKV) Put(
	key string,
	value interface{},
	ttl uint64,
) (*kvdb.KVPair, error) {

	var kvp *kvdb.KVPair

	key = kv.domain + key
	kv.mutex.Lock()
	defer kv.mutex.Unlock()

	index := atomic.AddUint64(&kv.index, 1)
	if ttl != 0 {
		time.AfterFunc(time.Second*time.Duration(ttl), func() {
			// TODO: handle error
			_, _ = kv.Delete(key)
		})
	}
	b, err := common.ToBytes(value)
	if err != nil {
		return nil, err
	}
	if old, ok := kv.m[key]; ok {
		old.Value = b
		old.Action = kvdb.KVSet
		old.ModifiedIndex = index
		old.KVDBIndex = index
		kvp = old

	} else {
		kvp = &kvdb.KVPair{
			Key:           key,
			Value:         b,
			TTL:           int64(ttl),
			KVDBIndex:     index,
			ModifiedIndex: index,
			CreatedIndex:  index,
			Action:        kvdb.KVCreate,
		}
		kv.m[key] = kvp
	}

	kv.normalize(kvp)
	go kv.fireCB(key, *kvp, nil)
	return kvp, nil
}

func (kv *memKV) GetVal(key string, v interface{}) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(kvp.Value, v)
	return kvp, err
}

func (kv *memKV) Create(
	key string,
	value interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	result, err := kv.Get(key)
	if err != nil {
		return kv.Put(key, value, ttl)
	}
	return result, kvdb.ErrExist
}

func (kv *memKV) Update(
	key string,
	value interface{},
	ttl uint64) (*kvdb.KVPair, error) {

	if _, err := kv.Get(key); err != nil {
		return nil, kvdb.ErrNotFound
	}
	return kv.Put(key, value, ttl)
}

func (kv *memKV) Enumerate(prefix string) (kvdb.KVPairs, error) {
	var kvp = make(kvdb.KVPairs, 0, 100)
	prefix = kv.domain + prefix

	for k, v := range kv.m {
		if strings.HasPrefix(k, prefix) && !strings.Contains(k, "/_") {
			kvpLocal := *v
			kv.normalize(&kvpLocal)
			kvp = append(kvp, &kvpLocal)
		}
	}

	return kvp, nil
}

func (kv *memKV) Delete(key string) (*kvdb.KVPair, error) {
	kvp, err := kv.Get(key)
	if err != nil {
		return nil, err
	}
	kvp.KVDBIndex = atomic.AddUint64(&kv.index, 1)
	kvp.ModifiedIndex = kvp.KVDBIndex
	kvp.Action = kvdb.KVDelete
	go kv.fireCB(kv.domain+key, *kvp, nil)
	delete(kv.m, kv.domain+key)
	return kvp, nil
}

func (kv *memKV) DeleteTree(prefix string) error {

	kvp, err := kv.Enumerate(prefix)
	if err != nil {
		return err
	}
	for _, v := range kvp {
		// TODO: multiple errors
		if _, iErr := kv.Delete(v.Key); iErr != nil {
			err = iErr
		}
	}
	return err
}

func (kv *memKV) Keys(prefix, key string) ([]string, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *memKV) CompareAndSet(
	kvp *kvdb.KVPair,
	flags kvdb.KVFlags,
	prevValue []byte) (*kvdb.KVPair, error) {

	kv.mutex.Lock()

	result, err := kv.Get(kvp.Key)
	if err != nil {
		kv.mutex.Unlock()
		return nil, err
	}
	if prevValue != nil {
		if !bytes.Equal(result.Value, prevValue) {
			kv.mutex.Unlock()
			return nil, kvdb.ErrValueMismatch
		}
	}
	if flags == kvdb.KVModifiedIndex {
		if kvp.ModifiedIndex != result.ModifiedIndex {
			kv.mutex.Unlock()
			return nil, kvdb.ErrValueMismatch
		}
	}
	kv.mutex.Unlock()
	return kv.Put(kvp.Key, kvp.Value, 0)
}

func (kv *memKV) CompareAndDelete(kvp *kvdb.KVPair, flags kvdb.KVFlags) (*kvdb.KVPair, error) {
	if flags != kvdb.KVFlags(0) {
		return nil, kvdb.ErrNotSupported
	}
	if result, err := kv.Get(kvp.Key); err != nil {
		return nil, err
	} else if !bytes.Equal(result.Value, kvp.Value) {
		return nil, kvdb.ErrNotFound
	}
	return kv.Delete(kvp.Key)
}

func (kv *memKV) WatchKey(
	key string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {
	kv.mutex.Lock()
	defer kv.mutex.Unlock()
	key = kv.domain + key
	if _, ok := kv.w[key]; ok {
		return kvdb.ErrExist
	}
	kv.w[key] = &watchData{cb: cb, opaque: opaque}
	return nil
}

func (kv *memKV) WatchTree(
	prefix string,
	waitIndex uint64,
	opaque interface{},
	cb kvdb.WatchCB) error {

	kv.mutex.Lock()
	defer kv.mutex.Unlock()
	prefix = kv.domain + prefix
	if _, ok := kv.wt[prefix]; ok {
		return kvdb.ErrExist
	}
	kv.wt[prefix] = &watchData{cb: cb, opaque: opaque}
	return nil
}

func (kv *memKV) Lock(key string, ttl uint64) (*kvdb.KVPair, error) {
	key = kv.domain + key
	duration := time.Duration(math.Min(float64(time.Second),
		float64((time.Duration(ttl)*time.Second)/10)))

	result, err := kv.Create(key, []byte("locked"), ttl)
	for err != nil {
		time.Sleep(duration)
		result, err = kv.Create(key, []byte("locked"), ttl)
	}

	if err != nil {
		return nil, err
	}
	return result, err
}

func (kv *memKV) Unlock(kvp *kvdb.KVPair) error {
	_, err := kv.CompareAndDelete(kvp, kvdb.KVFlags(0))
	return err
}

func (kv *memKV) TxNew() (kvdb.Tx, error) {
	return nil, kvdb.ErrNotSupported
}

func (kv *memKV) normalize(kvp *kvdb.KVPair) {
	kvp.Key = strings.TrimPrefix(kvp.Key, kv.domain)
}

func (kv *memKV) fireCB(key string, kvp kvdb.KVPair, err error) {
	for k, v := range kv.w {
		if k == key {
			err := v.cb(key, v.opaque, &kvp, err)
			if err != nil {
				// TODO: handle error
				_ = v.cb("", v.opaque, nil, kvdb.ErrWatchStopped)
				delete(kv.w, key)

			}
			return
		}
	}
	for k, v := range kv.wt {
		if strings.HasPrefix(key, k) {
			err := v.cb(key, v.opaque, &kvp, err)
			if err != nil {
				// TODO: handle error
				_ = v.cb("", v.opaque, nil, kvdb.ErrWatchStopped)
				delete(kv.wt, key)
			}
		}
	}
}
<|endoftext|>


<commit_before>package menu

/*
my $lastId = 0;

sub EVENT_SPAWN {
    #Get last ID
    $connect = plugin::LoadMysql();
    $query = "SELECT `id` FROM qs_player_speech ORDER BY `id` DESC LIMIT 1";
    $query_handle = $connect->prepare($query);
    $query_handle->execute();
    while (@row = $query_handle->fetchrow_array()){
        $lastId = $row[0];
    }
      quest::settimer("discord", 1);
}

sub EVENT_TIMER {
      $connect = plugin::LoadMysql();
    $query = "SELECT `from`, `message`, `id` FROM qs_player_speech WHERE `id` > ? AND `type` = 5 AND `to` = '!discord' LIMIT 1";
    $query_handle = $connect->prepare($query);
    $query_handle->execute($lastId);
    while (@row = $query_handle->fetchrow_array()){
        quest::we(260, $row[0]." says from discord, '".$row[1]."'");
        $lastId = $row[2];
    }
    return
}*/
<commit_msg>added disconnect to quest<commit_after>package menu

/*
my $lastId = 0;

sub EVENT_SPAWN {
    #Get last ID
    $connect = plugin::LoadMysql();
    $query = "SELECT `id` FROM qs_player_speech ORDER BY `id` DESC LIMIT 1";
    $query_handle = $connect->prepare($query);
    $query_handle->execute();
    while (@row = $query_handle->fetchrow_array()){
        $lastId = $row[0];
    }
      quest::settimer("discord", 1);
}

sub EVENT_TIMER {
    $connect = plugin::LoadMysql();
    $query = "SELECT `from`, `message`, `id` FROM qs_player_speech WHERE `id` > ? AND `type` = 5 AND `to` = '!discord' LIMIT 1";
    $query_handle = $connect->prepare($query);
    $query_handle->execute($lastId);
    while (@row = $query_handle->fetchrow_array()){
        quest::we(260, $row[0]." says from discord, '".$row[1]."'");
        $lastId = $row[2];
    }
    $connect->disconnect;
    return
}*/
<|endoftext|>


<commit_before>package slack_history

import (
	"../slack_list"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const api_url = "https://slack.com/api/"

type Message struct {
	Name string
	Time float64
	Text string
	File string
}
type Messages []Message

func (m Messages) Len() int {
	return len(m)
}
func (m Messages) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m Messages) Less(i, j int) bool {
	return m[i].Time < m[j].Time
}

func list2str(list []string) string {
	sort.Strings(list)
	str := strings.Join(list, "\n")
	return str
}
func get_history(api_token string, id string, api_kind string) []byte {
	// get api
	var raw []byte
	data := url.Values{}
	data.Set("token", api_token)
	data.Add("count", "300")
	data.Add("channel", id)
	client := &http.Client{}
	r, _ := http.NewRequest("POST", api_url+api_kind, bytes.NewBufferString(data.Encode()))
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	resp, _ := client.Do(r)
	raw, _ = ioutil.ReadAll(resp.Body)
	return raw
}
func parse_message(raw []byte, dict slack_list.Top) string {
	var ch_messages Messages
	var json_interface interface{}
	json.Unmarshal(raw, &json_interface)
	a := json_interface.(map[string]interface{})
	b := a["messages"]
	c := b.([]interface{})
	for _, v := range c {
		val := v.(map[string]interface{})
		time_f, _ := strconv.ParseFloat(val["ts"].(string), 64)
		file := ""
		if val["type"].(string) == "message" && val["user"] != nil {
			if f := val["file"]; f != nil {
				file = f.(map[string]interface{})["url_private_download"].(string)
			}
			ms := Message{
				Name: val["user"].(string),
				Time: time_f,
				Text: val["text"].(string),
				File: file,
			}
			ch_messages = append(ch_messages, ms)
		}
	}
	var messages string
	sort.Sort(ch_messages)
	for _, v := range ch_messages {
		t := time.Unix(int64(v.Time), 0)
		messages += fmt.Sprintf("%s (%s): %s\n", dict.Id2Name(v.Name), t.Format("2006/01/02 15:04"), v.Text)
		if v.File != "" {
			messages += fmt.Sprintf("  download link: %s\n", v.File)
		}
	}
	return messages
}
<commit_msg>Change download URL<commit_after>package slack_history

import (
	"../slack_list"
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const api_url = "https://slack.com/api/"

type Message struct {
	Name string
	Time float64
	Text string
	File string
}
type Messages []Message

func (m Messages) Len() int {
	return len(m)
}
func (m Messages) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}
func (m Messages) Less(i, j int) bool {
	return m[i].Time < m[j].Time
}

func list2str(list []string) string {
	sort.Strings(list)
	str := strings.Join(list, "\n")
	return str
}
func get_history(api_token string, id string, api_kind string) []byte {
	// get api
	var raw []byte
	data := url.Values{}
	data.Set("token", api_token)
	data.Add("count", "300")
	data.Add("channel", id)
	client := &http.Client{}
	r, _ := http.NewRequest("POST", api_url+api_kind, bytes.NewBufferString(data.Encode()))
	r.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	resp, _ := client.Do(r)
	raw, _ = ioutil.ReadAll(resp.Body)
	return raw
}
func parse_message(raw []byte, dict slack_list.Top) string {
	var ch_messages Messages
	var json_interface interface{}
	json.Unmarshal(raw, &json_interface)
	a := json_interface.(map[string]interface{})
	b := a["messages"]
	c := b.([]interface{})
	for _, v := range c {
		val := v.(map[string]interface{})
		time_f, _ := strconv.ParseFloat(val["ts"].(string), 64)
		file := ""
		if val["type"].(string) == "message" && val["user"] != nil {
			if f := val["file"]; f != nil {
				file = f.(map[string]interface{})["url_private"].(string)
			}
			ms := Message{
				Name: val["user"].(string),
				Time: time_f,
				Text: val["text"].(string),
				File: file,
			}
			ch_messages = append(ch_messages, ms)
		}
	}
	var messages string
	sort.Sort(ch_messages)
	for _, v := range ch_messages {
		t := time.Unix(int64(v.Time), 0)
		messages += fmt.Sprintf("%s (%s): %s\n", dict.Id2Name(v.Name), t.Format("2006/01/02 15:04"), v.Text)
		if v.File != "" {
			messages += fmt.Sprintf("  download link: %s\n", v.File)
		}
	}
	return messages
}
<|endoftext|>