<commit_before>package plugins

import (
    "fmt"
    "github.com/bwmarrin/discordgo"
    "github.com/sn0w/Karen/helpers"
    "regexp"
    "strings"
    "github.com/PuerkitoBio/goquery"
    "strconv"
    "math/rand"
)

type XKCD struct{}

func (x XKCD) Commands() []string {
    return []string{
        "xkcd",
    }
}

func (x XKCD) Init(session *discordgo.Session) {

}

func (x XKCD) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
    session.ChannelTyping(msg.ChannelID)

    var link string

    if content == "" {
        link = "https://xkcd.com/info.0.json"
    } else if regexp.MustCompile("^\\d+$").MatchString(content) {
        link = "https://xkcd.com/" + content + "/info.0.json"
    } else if strings.Contains(content, "rand") {
        // Get latest number
        doc, err := goquery.NewDocument("https://xkcd.com")
        helpers.Relax(err)

        var num string
        for _, attr := range doc.Find("ul.comicNav").Children().Get(1).FirstChild.Attr {
            if attr.Key == "href" {
                num = attr.Val
                break
            }
        }

        num = strings.Replace(num, "/", "", -1)

        max, err := strconv.ParseInt(num, 10, 32)
        if err != nil {
            session.ChannelMessageSend(msg.ChannelID, "Error getting latest comic. Try again later :frowning:")
            return
        }

        link = "https://xkcd.com/" + strconv.Itoa(rand.Intn(int(max - 1)) + 1) + "/info.0.json"
    }

    json := helpers.GetJSON(link)
    session.ChannelMessageSend(
        msg.ChannelID,
        fmt.Sprintf(
            "#%d from %s/%s/%s\n%s\n%s",
            int(json.Path("num").Data().(float64)),
            json.Path("day").Data().(string),
            json.Path("month").Data().(string),
            json.Path("year").Data().(string),
            json.Path("title").Data().(string),
            json.Path("img").Data().(string),
        ),
    )
    session.ChannelMessageSend(msg.ChannelID, json.Path("alt").Data().(string))
}
<commit_msg>Fix logical error in xkcd.go<commit_after>package plugins

import (
    "fmt"
    "github.com/bwmarrin/discordgo"
    "github.com/sn0w/Karen/helpers"
    "regexp"
    "strings"
    "github.com/PuerkitoBio/goquery"
    "strconv"
    "math/rand"
)

type XKCD struct{}

func (x XKCD) Commands() []string {
    return []string{
        "xkcd",
    }
}

func (x XKCD) Init(session *discordgo.Session) {

}

func (x XKCD) Action(command string, content string, msg *discordgo.Message, session *discordgo.Session) {
    session.ChannelTyping(msg.ChannelID)

    var link string

    if regexp.MustCompile("^\\d+$").MatchString(content) {
        link = "https://xkcd.com/" + content + "/info.0.json"
    } else if strings.Contains(content, "rand") {
        // Get latest number
        doc, err := goquery.NewDocument("https://xkcd.com")
        helpers.Relax(err)

        var num string
        for _, attr := range doc.Find("ul.comicNav").Children().Get(1).FirstChild.Attr {
            if attr.Key == "href" {
                num = attr.Val
                break
            }
        }

        num = strings.Replace(num, "/", "", -1)

        max, err := strconv.ParseInt(num, 10, 32)
        if err != nil {
            session.ChannelMessageSend(msg.ChannelID, "Error getting latest comic. Try again later :frowning:")
            return
        }

        link = "https://xkcd.com/" + strconv.Itoa(rand.Intn(int(max - 1)) + 1) + "/info.0.json"
    } else {
        link = "https://xkcd.com/info.0.json"
    }

    json := helpers.GetJSON(link)
    session.ChannelMessageSend(
        msg.ChannelID,
        fmt.Sprintf(
            "#%d from %s/%s/%s\n%s\n%s",
            int(json.Path("num").Data().(float64)),
            json.Path("day").Data().(string),
            json.Path("month").Data().(string),
            json.Path("year").Data().(string),
            json.Path("title").Data().(string),
            json.Path("img").Data().(string),
        ),
    )
    session.ChannelMessageSend(msg.ChannelID, json.Path("alt").Data().(string))
}
<|endoftext|>


<commit_before>package formatifier

import (
	"strings"
	"testing"
)

func TestToPhone(t *testing.T) {
	t.Parallel()
	result1, err1 := ToPhone("2155551212", "-")
	if err1 != nil {
		t.Error(err1)
	}
	if len(result1) != 14 {
		t.Error("Incorrect length of formatted string")
	} else if !strings.ContainsAny(result1, "( | )") {
		t.Error("Didn't find expected characters")
	}
	result2, err2 := ToPhone("12155551212", "-")
	if err2 != nil {
		t.Error(err2)
	}
	if len(result2) != 16 {
		t.Error("Incorrect length of formatted string")
	} else if !strings.ContainsAny(result2, "( | )") {
		t.Error("Didn't find expected characters")
	}
}

func TestToURL(t *testing.T) {
	t.Parallel()
	result1, err1 := ToURL("github.com", "www", false)
	if err1 != nil {
		t.Error(err1)
	}
	if !strings.ContainsAny(result1, "http | www") {
		t.Error("Didn't find expected protocol")
	}
	result2, err2 := ToURL("github.com", "www", true)
	if err2 != nil {
		t.Error(err2)
	}
	if !strings.ContainsAny(result2, "https | gist") {
		t.Error("Didn't find expected protocol")
	}
}

func TestToSSN(t *testing.T) {
	result, err := ToSSN("986732987", "-")
	if err != nil {
		t.Error(err)
	}
	resultLength := len(result)
	if resultLength != 11 {
		t.Error("Expected an 8 char string and got: ", resultLength)
	}
}

func TestToLockCombo(t *testing.T) {
	result, err := ToLockCombo("651096", "-")
	if err != nil {
		t.Error(err)
	}
	resultLength := len(result)
	if resultLength != 8 {
		t.Error("Expected an 8 char string and got: ", resultLength)
	}
}

func TestToISBN(t *testing.T) {
	t.Parallel()
	result, err := ToISBN("6517106483096", "-")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "651-7-10-648309-6") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToMorseCode(t *testing.T) {
	t.Parallel()
	result, err := ToMorseCode("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, ". . . ... _ . .. _ . ._ _ _ _ __ . _ _ . . _ .. _ .. .._ ._ . .") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToPirateSpeak(t *testing.T) {
	t.Parallel()
	result, err := ToPirateSpeak("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.ContainsAny(result, "ahoy | mate") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToIRSA(t *testing.T) {
	t.Parallel()
	result, err := ToIRSA("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "hotel echo lima lima oscar  |  mike yankee  |  foxtrot romeo india echo november delta ") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToLeet(t *testing.T) {
	t.Parallel()
	result, err := ToLeet("that is cool dude")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "7#47 15 kewl d00d") {
		t.Error("Didn't find the expected converted string")
	}
}
<commit_msg>comment udpate<commit_after>//
// formatifier package tests
//

package formatifier

import (
	"strings"
	"testing"
)

func TestToPhone(t *testing.T) {
	t.Parallel()
	result1, err1 := ToPhone("2155551212", "-")
	if err1 != nil {
		t.Error(err1)
	}
	if len(result1) != 14 {
		t.Error("Incorrect length of formatted string")
	} else if !strings.ContainsAny(result1, "( | )") {
		t.Error("Didn't find expected characters")
	}
	result2, err2 := ToPhone("12155551212", "-")
	if err2 != nil {
		t.Error(err2)
	}
	if len(result2) != 16 {
		t.Error("Incorrect length of formatted string")
	} else if !strings.ContainsAny(result2, "( | )") {
		t.Error("Didn't find expected characters")
	}
}

func TestToURL(t *testing.T) {
	t.Parallel()
	result1, err1 := ToURL("github.com", "www", false)
	if err1 != nil {
		t.Error(err1)
	}
	if !strings.ContainsAny(result1, "http | www") {
		t.Error("Didn't find expected protocol")
	}
	result2, err2 := ToURL("github.com", "www", true)
	if err2 != nil {
		t.Error(err2)
	}
	if !strings.ContainsAny(result2, "https | gist") {
		t.Error("Didn't find expected protocol")
	}
}

func TestToSSN(t *testing.T) {
	result, err := ToSSN("986732987", "-")
	if err != nil {
		t.Error(err)
	}
	resultLength := len(result)
	if resultLength != 11 {
		t.Error("Expected an 8 char string and got: ", resultLength)
	}
}

func TestToLockCombo(t *testing.T) {
	result, err := ToLockCombo("651096", "-")
	if err != nil {
		t.Error(err)
	}
	resultLength := len(result)
	if resultLength != 8 {
		t.Error("Expected an 8 char string and got: ", resultLength)
	}
}

func TestToISBN(t *testing.T) {
	t.Parallel()
	result, err := ToISBN("6517106483096", "-")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "651-7-10-648309-6") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToMorseCode(t *testing.T) {
	t.Parallel()
	result, err := ToMorseCode("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, ". . . ... _ . .. _ . ._ _ _ _ __ . _ _ . . _ .. _ .. .._ ._ . .") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToPirateSpeak(t *testing.T) {
	t.Parallel()
	result, err := ToPirateSpeak("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.ContainsAny(result, "ahoy | mate") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToIRSA(t *testing.T) {
	t.Parallel()
	result, err := ToIRSA("hello my friend")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "hotel echo lima lima oscar  |  mike yankee  |  foxtrot romeo india echo november delta ") {
		t.Error("Didn't find the expected converted string")
	}
}

func TestToLeet(t *testing.T) {
	t.Parallel()
	result, err := ToLeet("that is cool dude")
	if err != nil {
		t.Error(err)
	}
	if !strings.Contains(result, "7#47 15 kewl d00d") {
		t.Error("Didn't find the expected converted string")
	}
}
<|endoftext|>


<commit_before>package gw2api

import "net/url"

type Trait struct {
	ID             int           `json:"id"`
	Name           string        `json:"icon"`
	Icon           string        `json:"icon"`
	Description    string        `json:"description"`
	Specialization int           `json:"specialization"`
	Tier           int           `json:"tier"`
	Slot           string        `json:"slot"`
	Facts          []Fact        `json:"facts"`
	TraitedFacts   []TraitedFact `json:"traited_facts"`
	Skills         []Skill       `json:"skills"`
}

type Skill struct {
	ID           int           `json:"id"`
	Name         string        `json:"name"`
	Description  string        `json:"description"`
	Icon         string        `json:"icon"`
	Facts        []Fact        `json:"facts"`
	TraitedFacts []TraitedFact `json:"traited_facts"`
}

type FactType struct {
	//Common fields
	Text        string `json:"text"`
	Type        string `json:"type"`
	Icon        string `json:"icon"`
	Description string `json:"description"`
	Percent     int    `json:"percent"`
	Target      string `json:"target"`
	Duration    int    `json:"duration"`
	Status      string `json:"status"`
	ApplyCount  int    `json:"apply_count"`
	Value       int    `json:"value"`

	// AttributeAdjustFact

	// BuffFact

	// BuffConversionFact
	Source string `json:"source"`

	// ComboFieldFact
	FinisherType string `json:"finisher_type"`

	// DamageFact
	HitCount int `json:"hit_count"`

	// DistanceFact
	Distance int `json:"distance"`

	// NoDataFact

	// PercentFact

	// PrefixedBuffFact
	Prefix Fact `json:"prefix"`

	// RadiusFact

	// RangeFact

	// RechargeFact

	// TimeFact

	// UnblockableFact
}

type Fact struct {
	Text  string `json:"text"`
	Icon  string `json:"icon"`
	Type  string `json:"type"`
	Facts []Fact `json:"facts"`

	// Only as TraitedFact
	RequiresTrait int `json:"requires_trait"`
	Overrides     int `json:"overrides"`
}

type TraitedFact Fact

func (gw2 *GW2Api) Traits() (res []int, err error) {
	ver := "v2"
	tag := "traits"
	err = gw2.fetchEndpoint(ver, tag, nil, &res)
	return
}

func (gw2 *GW2Api) TraitIds(lang string, ids ...int) (traits []Trait, err error) {
	ver := "v2"
	tag := "traits"
	params := url.Values{}
	if lang != "" {
		params.Add("lang", lang)
	}
	params.Add("ids", commaList(stringSlice(ids)))
	err = gw2.fetchEndpoint(ver, tag, params, &traits)
	return
}

type Specialization struct {
	ID          int    `json:"id"`
	Name        string `json:"name"`
	Elite       bool   `json:"elite"`
	Icon        string `json:"icon"`
	Background  string `json:"background"`
	MinorTraits []int  `json:"minor_traits"`
	MajorTraits []int  `json:"major_traits"`
}

func (gw2 *GW2Api) Specializations() (res []string, err error) {
	ver := "v2"
	tag := "specializations"
	err = gw2.fetchEndpoint(ver, tag, nil, &res)
	return
}

func (gw2 *GW2Api) SpecializationIds(lang string, ids ...int) (specs []Specialization, err error) {
	ver := "v2"
	tag := "specializations"
	params := url.Values{}
	if lang != "" {
		params.Add("lang", lang)
	}
	params.Add("ids", commaList(stringSlice(ids)))
	err = gw2.fetchEndpoint(ver, tag, params, &specs)
	return
}
<commit_msg>sadly i cant improve this. using interfaces here would look weird in comparison<commit_after>package gw2api

import "net/url"

type Trait struct {
	ID             int           `json:"id"`
	Name           string        `json:"icon"`
	Icon           string        `json:"icon"`
	Description    string        `json:"description"`
	Specialization int           `json:"specialization"`
	Tier           int           `json:"tier"`
	Slot           string        `json:"slot"`
	Facts          []Fact        `json:"facts"`
	TraitedFacts   []TraitedFact `json:"traited_facts"`
	Skills         []Skill       `json:"skills"`
}

type Skill struct {
	ID           int           `json:"id"`
	Name         string        `json:"name"`
	Description  string        `json:"description"`
	Icon         string        `json:"icon"`
	Facts        []Fact        `json:"facts"`
	TraitedFacts []TraitedFact `json:"traited_facts"`
}

type FactType struct {
	//Common fields
	Text        string `json:"text"`
	Type        string `json:"type"`
	Icon        string `json:"icon"`
	Description string `json:"description"`
	Percent     int    `json:"percent"`
	Target      string `json:"target"`
	Duration    int    `json:"duration"`
	Status      string `json:"status"`
	ApplyCount  int    `json:"apply_count"`
	Value       int    `json:"value"`

	// AttributeAdjustFact

	// BuffFact

	// BuffConversionFact
	Source string `json:"source"`

	// ComboFieldFact
	FinisherType string `json:"finisher_type"`

	// DamageFact
	HitCount int `json:"hit_count"`

	// DistanceFact
	Distance int `json:"distance"`

	// NoDataFact

	// PercentFact

	// PrefixedBuffFact
	Prefix Fact `json:"prefix"`

	// RadiusFact

	// RangeFact

	// RechargeFact

	// TimeFact

	// UnblockableFact
}

type Fact struct {
	Text  string     `json:"text"`
	Icon  string     `json:"icon"`
	Type  string     `json:"type"`
	Facts []FactType `json:"facts"`

	// Only as TraitedFact
	RequiresTrait int `json:"requires_trait"`
	Overrides     int `json:"overrides"`
}

type TraitedFact Fact

func (gw2 *GW2Api) Traits() (res []int, err error) {
	ver := "v2"
	tag := "traits"
	err = gw2.fetchEndpoint(ver, tag, nil, &res)
	return
}

func (gw2 *GW2Api) TraitIds(lang string, ids ...int) (traits []Trait, err error) {
	ver := "v2"
	tag := "traits"
	params := url.Values{}
	if lang != "" {
		params.Add("lang", lang)
	}
	params.Add("ids", commaList(stringSlice(ids)))
	err = gw2.fetchEndpoint(ver, tag, params, &traits)
	return
}

type Specialization struct {
	ID          int    `json:"id"`
	Name        string `json:"name"`
	Elite       bool   `json:"elite"`
	Icon        string `json:"icon"`
	Background  string `json:"background"`
	MinorTraits []int  `json:"minor_traits"`
	MajorTraits []int  `json:"major_traits"`
}

func (gw2 *GW2Api) Specializations() (res []string, err error) {
	ver := "v2"
	tag := "specializations"
	err = gw2.fetchEndpoint(ver, tag, nil, &res)
	return
}

func (gw2 *GW2Api) SpecializationIds(lang string, ids ...int) (specs []Specialization, err error) {
	ver := "v2"
	tag := "specializations"
	params := url.Values{}
	if lang != "" {
		params.Add("lang", lang)
	}
	params.Add("ids", commaList(stringSlice(ids)))
	err = gw2.fetchEndpoint(ver, tag, params, &specs)
	return
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"fmt"
	"net/url"
	"strings"
)

func init() {
	Register("link", parseLink)
}

type Link struct {
	URL   *url.URL
	Label string
}

func (l Link) TemplateName() string { return "link" }

func parseLink(ctx *Context, fileName string, lineno int, text string) (Elem, error) {
	args := strings.Fields(text)
	url, err := url.Parse(args[1])
	if err != nil {
		return nil, err
	}
	label := ""
	if len(args) > 2 {
		label = strings.Join(args[2:], " ")
	} else {
		scheme := url.Scheme + "://"
		if url.Scheme == "mailto" {
			scheme = "mailto:"
		}
		label = strings.Replace(url.String(), scheme, "", 1)
	}
	return Link{url, label}, nil
}

func renderLink(url, text string) string {
	text = font(text)
	if text == "" {
		text = url
	}
	return fmt.Sprintf(`<a href="%s" target="_blank">%s</a>`, url, text)
}

// parseInlineLink parses an inline link at the start of s, and returns
// a rendered HTML link and the total length of the raw inline link.
// If no inline link is present, it returns all zeroes.
func parseInlineLink(s string) (link string, length int) {
	if !strings.HasPrefix(s, "[[") {
		return
	}
	end := strings.Index(s, "]]")
	if end == -1 {
		return
	}
	urlEnd := strings.Index(s, "]")
	rawURL := s[2:urlEnd]
	const badURLChars = `<>"{}|\^[] ` + "`" // per RFC2396 section 2.4.3
	if strings.ContainsAny(rawURL, badURLChars) {
		return
	}
	if urlEnd == end {
		simpleUrl := ""
		url, err := url.Parse(rawURL)
		if err == nil {
			// If the URL is http://foo.com, drop the http://
			// In other words, render [[http://golang.org]] as:
			//   golang.org
			if strings.HasPrefix(rawURL, url.Scheme+"://") {
				simpleUrl = strings.TrimPrefix(rawURL, url.Scheme+"://")
			} else if strings.HasPrefix(rawURL, url.Scheme+":") {
				simpleUrl = strings.TrimPrefix(rawURL, url.Scheme+":")
			}
		}
		return renderLink(rawURL, simpleUrl), end + 2
	}
	if s[urlEnd:urlEnd+2] != "][" {
		return
	}
	text := s[urlEnd+2 : end]
	return renderLink(rawURL, text), end + 2
}
<commit_msg>go.tools/present: Open links with local (non absolute) URLs in the same window.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"fmt"
	"net/url"
	"strings"
)

func init() {
	Register("link", parseLink)
}

type Link struct {
	URL   *url.URL
	Label string
}

func (l Link) TemplateName() string { return "link" }

func parseLink(ctx *Context, fileName string, lineno int, text string) (Elem, error) {
	args := strings.Fields(text)
	url, err := url.Parse(args[1])
	if err != nil {
		return nil, err
	}
	label := ""
	if len(args) > 2 {
		label = strings.Join(args[2:], " ")
	} else {
		scheme := url.Scheme + "://"
		if url.Scheme == "mailto" {
			scheme = "mailto:"
		}
		label = strings.Replace(url.String(), scheme, "", 1)
	}
	return Link{url, label}, nil
}

func renderLink(url, text string) string {
	text = font(text)
	if text == "" {
		text = url
	}
	// Open links in new window only when their url is absolute.
	target := "_blank"
	if url[0] == '/' {
		target = "_self"
	}

	return fmt.Sprintf(`<a href="%s" target="%s">%s</a>`, url, target, text)
}

// parseInlineLink parses an inline link at the start of s, and returns
// a rendered HTML link and the total length of the raw inline link.
// If no inline link is present, it returns all zeroes.
func parseInlineLink(s string) (link string, length int) {
	if !strings.HasPrefix(s, "[[") {
		return
	}
	end := strings.Index(s, "]]")
	if end == -1 {
		return
	}
	urlEnd := strings.Index(s, "]")
	rawURL := s[2:urlEnd]
	const badURLChars = `<>"{}|\^[] ` + "`" // per RFC2396 section 2.4.3
	if strings.ContainsAny(rawURL, badURLChars) {
		return
	}
	if urlEnd == end {
		simpleUrl := ""
		url, err := url.Parse(rawURL)
		if err == nil {
			// If the URL is http://foo.com, drop the http://
			// In other words, render [[http://golang.org]] as:
			//   golang.org
			if strings.HasPrefix(rawURL, url.Scheme+"://") {
				simpleUrl = strings.TrimPrefix(rawURL, url.Scheme+"://")
			} else if strings.HasPrefix(rawURL, url.Scheme+":") {
				simpleUrl = strings.TrimPrefix(rawURL, url.Scheme+":")
			}
		}
		return renderLink(rawURL, simpleUrl), end + 2
	}
	if s[urlEnd:urlEnd+2] != "][" {
		return
	}
	text := s[urlEnd+2 : end]
	return renderLink(rawURL, text), end + 2
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"testing"
)

var printTests = []struct {
	lines    []string
	cols     int
	rows     int
	expected string
}{
	{ // Normal
		[]string{"foobar\n"},
		80,
		20,
		"\nfoobar\n",
	},
	{ // Too many lines
		[]string{"foobar\n", "foobar\n", "foobar\n"},
		99,
		2,
		"\nfoobar\nfoobar",
	},
	{ // Lines too wide
		[]string{"foobar\n"},
		3,
		99,
		"\nfoo\n",
	},
	{
		[]string{"xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n"},
		6,
		4,
		"\nxxxYYY\nxxxYYY\nxxxYYY\nxxxYYY",
	},
}

func TestPrint(t *testing.T) {
	for _, tt := range printTests {
		target := new(bytes.Buffer)
		printer := NewPrinter(target, tt.cols, tt.rows)

		for _, line := range tt.lines {
			_, err := printer.Print(line)
			if err != nil {
				t.Errorf("error printing: %s", err)
			}
		}

		actual := target.String()
		if actual != tt.expected {
			t.Errorf("wrong output written. got=%q, expected=%q", actual, tt.expected)
		}
	}
}

func TestReset(t *testing.T) {
	testTarget := new(bytes.Buffer)
	testLine := "foobar\n"
	printer := NewPrinter(testTarget, 80, 1)

	_, err := printer.Print(testLine)
	if err != nil {
		t.Errorf("error printing: %s", err)
	}

	printer.Reset()

	_, err = printer.Print(testLine)
	if err != nil {
		t.Errorf("error printing: %s", err)
	}

	output := testTarget.String()
	// Cut of the newline char on the last line
	expected := "\n" + testLine + testLine[:len(testLine)-1]

	if output != expected {
		t.Errorf("wrong output written. expected: %q, got: %q", expected, output)
	}
}
<commit_msg>Add missing comment<commit_after>package main

import (
	"bytes"
	"testing"
)

var printTests = []struct {
	lines    []string
	cols     int
	rows     int
	expected string
}{
	{ // Normal
		[]string{"foobar\n"},
		80,
		20,
		"\nfoobar\n",
	},
	{ // Too many lines
		[]string{"foobar\n", "foobar\n", "foobar\n"},
		99,
		2,
		"\nfoobar\nfoobar",
	},
	{ // Lines too wide
		[]string{"foobar\n"},
		3,
		99,
		"\nfoo\n",
	},
	{ // Too many too wide lines
		[]string{"xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n", "xxxYYYzzz\n"},
		6,
		4,
		"\nxxxYYY\nxxxYYY\nxxxYYY\nxxxYYY",
	},
}

func TestPrint(t *testing.T) {
	for _, tt := range printTests {
		target := new(bytes.Buffer)
		printer := NewPrinter(target, tt.cols, tt.rows)

		for _, line := range tt.lines {
			_, err := printer.Print(line)
			if err != nil {
				t.Errorf("error printing: %s", err)
			}
		}

		actual := target.String()
		if actual != tt.expected {
			t.Errorf("wrong output written. got=%q, expected=%q", actual, tt.expected)
		}
	}
}

func TestReset(t *testing.T) {
	testTarget := new(bytes.Buffer)
	testLine := "foobar\n"
	printer := NewPrinter(testTarget, 80, 1)

	_, err := printer.Print(testLine)
	if err != nil {
		t.Errorf("error printing: %s", err)
	}

	printer.Reset()

	_, err = printer.Print(testLine)
	if err != nil {
		t.Errorf("error printing: %s", err)
	}

	output := testTarget.String()
	// Cut of the newline char on the last line
	expected := "\n" + testLine + testLine[:len(testLine)-1]

	if output != expected {
		t.Errorf("wrong output written. expected: %q, got: %q", expected, output)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"strings"

	"github.com/piotrkowalczuk/pqcomp"
)

type repositories struct {
	user            UserRepository
	userGroups      UserGroupsRepository
	userPermissions UserPermissionsRepository
	permission      PermissionRepository
	group           GroupRepository
	groupPermissions           GroupPermissionsRepository
}

func newRepositories(db *sql.DB) repositories {
	return repositories{
		user :newUserRepository(db),
		userGroups : newUserGroupsRepository(db),
		userPermissions: newUserPermissionsRepository(db),
		permission: newPermissionRepository(db),
		group : newGroupRepository(db),
		groupPermissions : newGroupPermissionsRepository(db),
	}
}

func execQueries(db *sql.DB, queries ...string) (err error) {
	exec := func(query string) {
		if err != nil {
			return
		}

		_, err = db.Exec(query)
	}

	for _, q := range queries {
		exec(q)
	}

	return
}

func setupDatabase(db *sql.DB) error {
	return execQueries(
		db,
		schemaSQL,
	)
}

func tearDownDatabase(db *sql.DB) error {
	return execQueries(
		db,
		`DROP SCHEMA IF EXISTS charon CASCADE`,
	)
}

func columns(names []string, prefix string) string {
	b := bytes.NewBuffer(nil)
	for i, n := range names {
		if i != 0 {
			b.WriteRune(',')
		}
		b.WriteString(prefix)
		b.WriteRune('.')
		b.WriteString(n)
	}

	return b.String()
}

func findQueryComp(db *sql.DB, table string, root, where *pqcomp.Composer, sort map[string]bool, columns []string) (*sql.Rows, error) {
	b := bytes.NewBufferString(`SELECT ` + strings.Join(columns, ", ") + ` FROM ` + table)

	if where.Len() != 0 {
		b.WriteString(` WHERE `)
		for where.Next() {
			if !where.First() {
				b.WriteString(" AND ")
			}

			fmt.Fprintf(b, "%s %s %s", where.Key(), where.Oper(), where.PlaceHolder())
		}
	}

	i := 0
SortLoop:
	for column, asc := range sort {
		if i != 0 {
			b.WriteString(", ")
		} else {
			b.WriteString(" ORDER BY ")
		}
		i++
		if asc {
			fmt.Fprintf(b, "%s ASC", column)
			continue SortLoop
		}

		fmt.Fprintf(b, "%s DESC ", column)
	}
	b.WriteString(" OFFSET $1 LIMIT $2")

	return db.Query(b.String(), root.Args()...)

}

func insertQueryComp(db *sql.DB, table string, insert *pqcomp.Composer, col []string) *sql.Row {
	b := bytes.NewBufferString(`INSERT INTO ` + table)

	if insert.Len() != 0 {
		b.WriteString(` (`)
		for insert.Next() {
			if !insert.First() {
				b.WriteString(", ")
			}

			fmt.Fprintf(b, "%s", insert.Key())
		}
		insert.Reset()
		b.WriteString(`) VALUES (`)
		for insert.Next() {
			if !insert.First() {
				b.WriteString(", ")
			}

			fmt.Fprintf(b, "%s", insert.PlaceHolder())
		}
		b.WriteString(`)`)
		if len(col) > 0 {
			b.WriteString(" RETURNING ")
			b.WriteString(strings.Join(col, ","))
		}
	}

	return db.QueryRow(b.String(), insert.Args()...)
}

func existsManyToManyQuery(table, column1, column2 string) string {
	return `
		SELECT EXISTS(
			SELECT 1 FROM  ` + table + ` AS ug
			WHERE ug.` + column1+ ` = $1
				AND ug.` + column2+ `= $2
		)
	`
}

func setManyToMany(db *sql.DB, table, column1, column2 string, id int64, ids []int64) (int64, int64, error) {
	var (
		err                    error
		aff, inserted, deleted int64
		tx                     *sql.Tx
		insert, exists         *sql.Stmt
		res                    sql.Result
		in                     []int64
		granted                bool
	)

	tx, err = db.Begin()
	if err != nil {
		return 0, 0, err
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			tx.Commit()
		}
	}()

	if len(ids) > 0 {
		insert, err = tx.Prepare(`INSERT INTO ` + table + ` (` + column1 + `, ` + column2 + `) VALUES ($1, $2)`)
		if err != nil {
			return 0, 0, err
		}
		exists, err = tx.Prepare(existsManyToManyQuery(table, column1,column2))
		if err != nil {
			return 0, 0, err
		}

		in = make([]int64, 0, len(ids))
		InsertLoop:
		for _, groupID := range ids {
			if err = exists.QueryRow(id, groupID).Scan(&granted); err != nil {
				return 0, 0, err
			}
			// Given combination already exists, ignore.
			if granted {
				in = append(in, groupID)
				granted = false
				continue InsertLoop
			}
			res, err = insert.Exec(id, groupID)
			if err != nil {
				return 0, 0, err
			}

			aff, err = res.RowsAffected()
			if err != nil {
				return 0, 0, err
			}
			inserted += aff

			in = append(in, groupID)
		}
	}

	delete := pqcomp.New(1, len(in))
	delete.AddArg(id)
	for _, id := range in {
		delete.AddExpr(column2, "IN", id)
	}

	query := bytes.NewBufferString(`DELETE FROM ` + table + ` WHERE ` + column1 + ` = $1`)
	for delete.Next() {
		if delete.First() {
			fmt.Fprint(query, " AND "+column2+" NOT IN (")
		} else {
			fmt.Fprint(query, ", ")

		}
		fmt.Fprintf(query, "%s", delete.PlaceHolder())
	}
	if len(in) > 0 {
		fmt.Fprint(query, ")")
	}

	res, err = tx.Exec(query.String(), delete.Args()...)
	if err != nil {
		return 0, 0, err
	}
	deleted, err = res.RowsAffected()
	if err != nil {
		return 0, 0, err
	}

	return inserted, deleted, nil
}<commit_msg>variable rename in generic query<commit_after>package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"strings"

	"github.com/piotrkowalczuk/pqcomp"
)

type repositories struct {
	user            UserRepository
	userGroups      UserGroupsRepository
	userPermissions UserPermissionsRepository
	permission      PermissionRepository
	group           GroupRepository
	groupPermissions           GroupPermissionsRepository
}

func newRepositories(db *sql.DB) repositories {
	return repositories{
		user :newUserRepository(db),
		userGroups : newUserGroupsRepository(db),
		userPermissions: newUserPermissionsRepository(db),
		permission: newPermissionRepository(db),
		group : newGroupRepository(db),
		groupPermissions : newGroupPermissionsRepository(db),
	}
}

func execQueries(db *sql.DB, queries ...string) (err error) {
	exec := func(query string) {
		if err != nil {
			return
		}

		_, err = db.Exec(query)
	}

	for _, q := range queries {
		exec(q)
	}

	return
}

func setupDatabase(db *sql.DB) error {
	return execQueries(
		db,
		schemaSQL,
	)
}

func tearDownDatabase(db *sql.DB) error {
	return execQueries(
		db,
		`DROP SCHEMA IF EXISTS charon CASCADE`,
	)
}

func columns(names []string, prefix string) string {
	b := bytes.NewBuffer(nil)
	for i, n := range names {
		if i != 0 {
			b.WriteRune(',')
		}
		b.WriteString(prefix)
		b.WriteRune('.')
		b.WriteString(n)
	}

	return b.String()
}

func findQueryComp(db *sql.DB, table string, root, where *pqcomp.Composer, sort map[string]bool, columns []string) (*sql.Rows, error) {
	b := bytes.NewBufferString(`SELECT ` + strings.Join(columns, ", ") + ` FROM ` + table)

	if where.Len() != 0 {
		b.WriteString(` WHERE `)
		for where.Next() {
			if !where.First() {
				b.WriteString(" AND ")
			}

			fmt.Fprintf(b, "%s %s %s", where.Key(), where.Oper(), where.PlaceHolder())
		}
	}

	i := 0
SortLoop:
	for column, asc := range sort {
		if i != 0 {
			b.WriteString(", ")
		} else {
			b.WriteString(" ORDER BY ")
		}
		i++
		if asc {
			fmt.Fprintf(b, "%s ASC", column)
			continue SortLoop
		}

		fmt.Fprintf(b, "%s DESC ", column)
	}
	b.WriteString(" OFFSET $1 LIMIT $2")

	return db.Query(b.String(), root.Args()...)

}

func insertQueryComp(db *sql.DB, table string, insert *pqcomp.Composer, col []string) *sql.Row {
	b := bytes.NewBufferString(`INSERT INTO ` + table)

	if insert.Len() != 0 {
		b.WriteString(` (`)
		for insert.Next() {
			if !insert.First() {
				b.WriteString(", ")
			}

			fmt.Fprintf(b, "%s", insert.Key())
		}
		insert.Reset()
		b.WriteString(`) VALUES (`)
		for insert.Next() {
			if !insert.First() {
				b.WriteString(", ")
			}

			fmt.Fprintf(b, "%s", insert.PlaceHolder())
		}
		b.WriteString(`)`)
		if len(col) > 0 {
			b.WriteString(" RETURNING ")
			b.WriteString(strings.Join(col, ","))
		}
	}

	return db.QueryRow(b.String(), insert.Args()...)
}

func existsManyToManyQuery(table, column1, column2 string) string {
	return `
		SELECT EXISTS(
			SELECT 1 FROM  ` + table + ` AS ug
			WHERE ug.` + column1+ ` = $1
				AND ug.` + column2+ `= $2
		)
	`
}

func setManyToMany(db *sql.DB, table, column1, column2 string, id int64, ids []int64) (int64, int64, error) {
	var (
		err                    error
		aff, inserted, deleted int64
		tx                     *sql.Tx
		insert, exists         *sql.Stmt
		res                    sql.Result
		in                     []int64
		granted                bool
	)

	tx, err = db.Begin()
	if err != nil {
		return 0, 0, err
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			tx.Commit()
		}
	}()

	if len(ids) > 0 {
		insert, err = tx.Prepare(`INSERT INTO ` + table + ` (` + column1 + `, ` + column2 + `) VALUES ($1, $2)`)
		if err != nil {
			return 0, 0, err
		}
		exists, err = tx.Prepare(existsManyToManyQuery(table, column1,column2))
		if err != nil {
			return 0, 0, err
		}

		in = make([]int64, 0, len(ids))
		InsertLoop:
		for _, idd := range ids {
			if err = exists.QueryRow(id, idd).Scan(&granted); err != nil {
				return 0, 0, err
			}
			// Given combination already exists, ignore.
			if granted {
				in = append(in, idd)
				granted = false
				continue InsertLoop
			}
			res, err = insert.Exec(id, idd)
			if err != nil {
				return 0, 0, err
			}

			aff, err = res.RowsAffected()
			if err != nil {
				return 0, 0, err
			}
			inserted += aff

			in = append(in, idd)
		}
	}

	delete := pqcomp.New(1, len(in))
	delete.AddArg(id)
	for _, id := range in {
		delete.AddExpr(column2, "IN", id)
	}

	query := bytes.NewBufferString(`DELETE FROM ` + table + ` WHERE ` + column1 + ` = $1`)
	for delete.Next() {
		if delete.First() {
			fmt.Fprint(query, " AND "+column2+" NOT IN (")
		} else {
			fmt.Fprint(query, ", ")

		}
		fmt.Fprintf(query, "%s", delete.PlaceHolder())
	}
	if len(in) > 0 {
		fmt.Fprint(query, ")")
	}

	res, err = tx.Exec(query.String(), delete.Args()...)
	if err != nil {
		return 0, 0, err
	}
	deleted, err = res.RowsAffected()
	if err != nil {
		return 0, 0, err
	}

	return inserted, deleted, nil
}<|endoftext|>


<commit_before>package mbotapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
)

type BotAPI struct {
	Token       string
	VerifyToken string
	Debug       bool
	Client      *http.Client
}

func NewBotAPI(token string, vtoken string) *BotAPI {
	return &BotAPI{
		Token:       token,
		VerifyToken: vtoken,
		Debug:       false,
		Client:      &http.Client{},
	}
}

func (bot *BotAPI) MakeRequest(b *bytes.Buffer) (APIResponse, error) {
	uri := fmt.Sprintf(APIEndpoint, bot.Token)

	req, _ := http.NewRequest("POST", uri, b)
	req.Header.Set("Content-Type", "application/json")
	resp, err := bot.Client.Do(req)
	if err != nil {
		return APIResponse{}, err
	}
	defer resp.Body.Close()

	var rsp APIResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&rsp)
	if err != nil {
		return APIResponse{}, nil
	}

	if resp.StatusCode != 200 {
		return rsp, errors.New(http.StatusText(resp.StatusCode))
	}
	return rsp, nil
}

func (bot *BotAPI) Send(u User, c interface{}, notif string) (APIResponse, error) {
	var r Request

	n := RegularNotif
	if notif != "" {
		n = notif
	}

	switch c.(type) {
	case Request:
		return APIResponse{}, errors.New("Use MakeRequest to send Request!!")
	case Message:
		r = Request{
			Recipient: u,
			Message:   c.(Message),
			NotifType: n,
		}

	case GenericTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(GenericTemplate),
				},
			},
		}

	case ButtonTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(ButtonTemplate),
				},
			},
		}

	case ReceiptTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(ReceiptTemplate),
				},
			},
		}

	default:
		return APIResponse{}, errors.New("Type is not supported")
	}

	if r == (Request{}) {
		return APIResponse{}, errors.New("Unknown Error")
	}
	payl, _ := json.Marshal(r)
	if bot.Debug {
		log.Printf("[INFO] Payload: %s", string(payl))
	}
	return bot.MakeRequest(bytes.NewBuffer(payl))
}

func (bot *BotAPI) SendFile(u User, path string) (APIResponse, error) {
	file, err := os.Open(path)
	if err != nil {
		return APIResponse{}, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("filedata", filepath.Base(path))
	if err != nil {
		return APIResponse{}, err
	}
	_, err = io.Copy(part, file)

	usr, _ := json.Marshal(u)
	_ = writer.WriteField("recipient", string(usr))
	img := NewImageMesssage("")
	im, _ := json.Marshal(img)
	_ = writer.WriteField("message", string(im))

	err = writer.Close()
	if err != nil {
		return APIResponse{}, err
	}

	return bot.MakeRequest(body)
}

func (bot *BotAPI) SetWebhook(pattern string) (<-chan Callback, *http.ServeMux) {
	callbackChan := make(chan Callback, 100)

	mux := http.NewServeMux()
	mux.HandleFunc(pattern, func(w http.ResponseWriter, req *http.Request) {
		switch req.Method {
		case "GET":
			if req.FormValue("hub.verify_token") == bot.VerifyToken {
				w.Write([]byte(req.FormValue("hub.challenge")))
				return
			}
			w.WriteHeader(http.StatusUnauthorized)
			return

		case "POST":
			defer req.Body.Close()

			var rsp Response
			decoder := json.NewDecoder(req.Body)
			decoder.Decode(&rsp)

			if rsp.Object == "page" {
				for _, e := range rsp.Entries {
					for _, c := range e.Messaging {
						callbackChan <- c
					}
				}
			}
			w.WriteHeader(http.StatusOK)
			return

		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		}
	})

	return callbackChan, mux

}
<commit_msg>spell check :(<commit_after>package mbotapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
)

type BotAPI struct {
	Token       string
	VerifyToken string
	Debug       bool
	Client      *http.Client
}

func NewBotAPI(token string, vtoken string) *BotAPI {
	return &BotAPI{
		Token:       token,
		VerifyToken: vtoken,
		Debug:       false,
		Client:      &http.Client{},
	}
}

func (bot *BotAPI) MakeRequest(b *bytes.Buffer) (APIResponse, error) {
	uri := fmt.Sprintf(APIEndpoint, bot.Token)

	req, _ := http.NewRequest("POST", uri, b)
	req.Header.Set("Content-Type", "application/json")
	resp, err := bot.Client.Do(req)
	if err != nil {
		return APIResponse{}, err
	}
	defer resp.Body.Close()

	var rsp APIResponse
	dec := json.NewDecoder(resp.Body)
	err = dec.Decode(&rsp)
	if err != nil {
		return APIResponse{}, nil
	}

	if resp.StatusCode != 200 {
		return rsp, errors.New(http.StatusText(resp.StatusCode))
	}
	return rsp, nil
}

func (bot *BotAPI) Send(u User, c interface{}, notif string) (APIResponse, error) {
	var r Request

	n := RegularNotif
	if notif != "" {
		n = notif
	}

	switch c.(type) {
	case Request:
		return APIResponse{}, errors.New("Use MakeRequest to send Request!!")
	case Message:
		r = Request{
			Recipient: u,
			Message:   c.(Message),
			NotifType: n,
		}

	case GenericTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(GenericTemplate),
				},
			},
		}

	case ButtonTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(ButtonTemplate),
				},
			},
		}

	case ReceiptTemplate:
		r = Request{
			Recipient: u,
			NotifType: n,
			Message: Message{
				Attachment: &Attachment{
					Type:    "template",
					Payload: c.(ReceiptTemplate),
				},
			},
		}

	default:
		return APIResponse{}, errors.New("Type is not supported")
	}

	if r == (Request{}) {
		return APIResponse{}, errors.New("Unknown Error")
	}
	payl, _ := json.Marshal(r)
	if bot.Debug {
		log.Printf("[INFO] Payload: %s", string(payl))
	}
	return bot.MakeRequest(bytes.NewBuffer(payl))
}

func (bot *BotAPI) SendFile(u User, path string) (APIResponse, error) {
	file, err := os.Open(path)
	if err != nil {
		return APIResponse{}, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile("filedata", filepath.Base(path))
	if err != nil {
		return APIResponse{}, err
	}
	_, err = io.Copy(part, file)

	usr, _ := json.Marshal(u)
	_ = writer.WriteField("recipient", string(usr))
	img := NewImageMessage("")
	im, _ := json.Marshal(img)
	_ = writer.WriteField("message", string(im))

	err = writer.Close()
	if err != nil {
		return APIResponse{}, err
	}

	return bot.MakeRequest(body)
}

func (bot *BotAPI) SetWebhook(pattern string) (<-chan Callback, *http.ServeMux) {
	callbackChan := make(chan Callback, 100)

	mux := http.NewServeMux()
	mux.HandleFunc(pattern, func(w http.ResponseWriter, req *http.Request) {
		switch req.Method {
		case "GET":
			if req.FormValue("hub.verify_token") == bot.VerifyToken {
				w.Write([]byte(req.FormValue("hub.challenge")))
				return
			}
			w.WriteHeader(http.StatusUnauthorized)
			return

		case "POST":
			defer req.Body.Close()

			var rsp Response
			decoder := json.NewDecoder(req.Body)
			decoder.Decode(&rsp)

			if rsp.Object == "page" {
				for _, e := range rsp.Entries {
					for _, c := range e.Messaging {
						callbackChan <- c
					}
				}
			}
			w.WriteHeader(http.StatusOK)
			return

		default:
			w.WriteHeader(http.StatusMethodNotAllowed)
			return
		}
	})

	return callbackChan, mux

}
<|endoftext|>


<commit_before>package accounting

import (
	"context"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/rc"
	"github.com/pkg/errors"
	"golang.org/x/time/rate"
)

// Globals
var (
	tokenBucketMu     sync.Mutex // protects the token bucket variables
	tokenBucket       *rate.Limiter
	prevTokenBucket   = tokenBucket
	bwLimitToggledOff = false
	currLimitMu       sync.Mutex // protects changes to the timeslot
	currLimit         fs.BwTimeSlot
)

const maxBurstSize = 1 * 1024 * 1024 // must be bigger than the biggest request

// make a new empty token bucket with the bandwidth given
func newTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter {
	newTokenBucket := rate.NewLimiter(rate.Limit(bandwidth), maxBurstSize)
	// empty the bucket
	err := newTokenBucket.WaitN(context.Background(), maxBurstSize)
	if err != nil {
		fs.Errorf(nil, "Failed to empty token bucket: %v", err)
	}
	return newTokenBucket
}

// StartTokenBucket starts the token bucket if necessary
func StartTokenBucket() {
	currLimitMu.Lock()
	currLimit := fs.Config.BwLimit.LimitAt(time.Now())
	currLimitMu.Unlock()

	if currLimit.Bandwidth > 0 {
		tokenBucket = newTokenBucket(currLimit.Bandwidth)
		fs.Infof(nil, "Starting bandwidth limiter at %vBytes/s", &currLimit.Bandwidth)

		// Start the SIGUSR2 signal handler to toggle bandwidth.
		// This function does nothing in windows systems.
		startSignalHandler()
	}
}

// StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
func StartTokenTicker() {
	// If the timetable has a single entry or was not specified, we don't need
	// a ticker to update the bandwidth.
	if len(fs.Config.BwLimit) <= 1 {
		return
	}

	ticker := time.NewTicker(time.Minute)
	go func() {
		for range ticker.C {
			limitNow := fs.Config.BwLimit.LimitAt(time.Now())
			currLimitMu.Lock()

			if currLimit.Bandwidth != limitNow.Bandwidth {
				tokenBucketMu.Lock()

				// If bwlimit is toggled off, the change should only
				// become active on the next toggle, which causes
				// an exchange of tokenBucket <-> prevTokenBucket
				var targetBucket **rate.Limiter
				if bwLimitToggledOff {
					targetBucket = &prevTokenBucket
				} else {
					targetBucket = &tokenBucket
				}

				// Set new bandwidth. If unlimited, set tokenbucket to nil.
				if limitNow.Bandwidth > 0 {
					*targetBucket = newTokenBucket(limitNow.Bandwidth)
					if bwLimitToggledOff {
						fs.Logf(nil, "Scheduled bandwidth change. "+
							"Limit will be set to %vBytes/s when toggled on again.", &limitNow.Bandwidth)
					} else {
						fs.Logf(nil, "Scheduled bandwidth change. Limit set to %vBytes/s", &limitNow.Bandwidth)
					}
				} else {
					*targetBucket = nil
					fs.Logf(nil, "Scheduled bandwidth change. Bandwidth limits disabled")
				}

				currLimit = limitNow
				tokenBucketMu.Unlock()
			}
			currLimitMu.Unlock()
		}
	}()
}

// limitBandwith sleeps for the correct amount of time for the passage
// of n bytes according to the current bandwidth limit
func limitBandwidth(n int) {
	tokenBucketMu.Lock()

	// Limit the transfer speed if required
	if tokenBucket != nil {
		err := tokenBucket.WaitN(context.Background(), n)
		if err != nil {
			fs.Errorf(nil, "Token bucket error: %v", err)
		}
	}

	tokenBucketMu.Unlock()
}

// SetBwLimit sets the current bandwidth limit
func SetBwLimit(bandwidth fs.SizeSuffix) {
	tokenBucketMu.Lock()
	defer tokenBucketMu.Unlock()
	if bandwidth > 0 {
		tokenBucket = newTokenBucket(bandwidth)
		fs.Logf(nil, "Bandwidth limit set to %v", bandwidth)
	} else {
		tokenBucket = nil
		fs.Logf(nil, "Bandwidth limit reset to unlimited")
	}
}

// Remote control for the token bucket
func init() {
	rc.Add(rc.Call{
		Path: "core/bwlimit",
		Fn: func(in rc.Params) (out rc.Params, err error) {
			ibwlimit, ok := in["rate"]
			if !ok {
				return out, errors.Errorf("parameter rate not found")
			}
			bwlimit, ok := ibwlimit.(string)
			if !ok {
				return out, errors.Errorf("value must be string rate=%v", ibwlimit)
			}
			var bws fs.BwTimetable
			err = bws.Set(bwlimit)
			if err != nil {
				return out, errors.Wrap(err, "bad bwlimit")
			}
			if len(bws) != 1 {
				return out, errors.New("need exactly 1 bandwidth setting")
			}
			bw := bws[0]
			SetBwLimit(bw.Bandwidth)
			return rc.Params{"rate": bw.Bandwidth.String()}, nil
		},
		Title: "Set the bandwidth limit.",
		Help: `
This sets the bandwidth limit to that passed in.

Eg

    rclone rc core/bwlimit rate=1M
    rclone rc core/bwlimit rate=off

The format of the parameter is exactly the same as passed to --bwlimit
except only one bandwidth may be specified.
`,
	})
}
<commit_msg>fs/accounting: increase maximum burst size of token bucket<commit_after>package accounting

import (
	"context"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/rc"
	"github.com/pkg/errors"
	"golang.org/x/time/rate"
)

// Globals
var (
	tokenBucketMu     sync.Mutex // protects the token bucket variables
	tokenBucket       *rate.Limiter
	prevTokenBucket   = tokenBucket
	bwLimitToggledOff = false
	currLimitMu       sync.Mutex // protects changes to the timeslot
	currLimit         fs.BwTimeSlot
)

const maxBurstSize = 4 * 1024 * 1024 // must be bigger than the biggest request

// make a new empty token bucket with the bandwidth given
func newTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter {
	newTokenBucket := rate.NewLimiter(rate.Limit(bandwidth), maxBurstSize)
	// empty the bucket
	err := newTokenBucket.WaitN(context.Background(), maxBurstSize)
	if err != nil {
		fs.Errorf(nil, "Failed to empty token bucket: %v", err)
	}
	return newTokenBucket
}

// StartTokenBucket starts the token bucket if necessary
func StartTokenBucket() {
	currLimitMu.Lock()
	currLimit := fs.Config.BwLimit.LimitAt(time.Now())
	currLimitMu.Unlock()

	if currLimit.Bandwidth > 0 {
		tokenBucket = newTokenBucket(currLimit.Bandwidth)
		fs.Infof(nil, "Starting bandwidth limiter at %vBytes/s", &currLimit.Bandwidth)

		// Start the SIGUSR2 signal handler to toggle bandwidth.
		// This function does nothing in windows systems.
		startSignalHandler()
	}
}

// StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
func StartTokenTicker() {
	// If the timetable has a single entry or was not specified, we don't need
	// a ticker to update the bandwidth.
	if len(fs.Config.BwLimit) <= 1 {
		return
	}

	ticker := time.NewTicker(time.Minute)
	go func() {
		for range ticker.C {
			limitNow := fs.Config.BwLimit.LimitAt(time.Now())
			currLimitMu.Lock()

			if currLimit.Bandwidth != limitNow.Bandwidth {
				tokenBucketMu.Lock()

				// If bwlimit is toggled off, the change should only
				// become active on the next toggle, which causes
				// an exchange of tokenBucket <-> prevTokenBucket
				var targetBucket **rate.Limiter
				if bwLimitToggledOff {
					targetBucket = &prevTokenBucket
				} else {
					targetBucket = &tokenBucket
				}

				// Set new bandwidth. If unlimited, set tokenbucket to nil.
				if limitNow.Bandwidth > 0 {
					*targetBucket = newTokenBucket(limitNow.Bandwidth)
					if bwLimitToggledOff {
						fs.Logf(nil, "Scheduled bandwidth change. "+
							"Limit will be set to %vBytes/s when toggled on again.", &limitNow.Bandwidth)
					} else {
						fs.Logf(nil, "Scheduled bandwidth change. Limit set to %vBytes/s", &limitNow.Bandwidth)
					}
				} else {
					*targetBucket = nil
					fs.Logf(nil, "Scheduled bandwidth change. Bandwidth limits disabled")
				}

				currLimit = limitNow
				tokenBucketMu.Unlock()
			}
			currLimitMu.Unlock()
		}
	}()
}

// limitBandwith sleeps for the correct amount of time for the passage
// of n bytes according to the current bandwidth limit
func limitBandwidth(n int) {
	tokenBucketMu.Lock()

	// Limit the transfer speed if required
	if tokenBucket != nil {
		err := tokenBucket.WaitN(context.Background(), n)
		if err != nil {
			fs.Errorf(nil, "Token bucket error: %v", err)
		}
	}

	tokenBucketMu.Unlock()
}

// SetBwLimit sets the current bandwidth limit
func SetBwLimit(bandwidth fs.SizeSuffix) {
	tokenBucketMu.Lock()
	defer tokenBucketMu.Unlock()
	if bandwidth > 0 {
		tokenBucket = newTokenBucket(bandwidth)
		fs.Logf(nil, "Bandwidth limit set to %v", bandwidth)
	} else {
		tokenBucket = nil
		fs.Logf(nil, "Bandwidth limit reset to unlimited")
	}
}

// Remote control for the token bucket
func init() {
	rc.Add(rc.Call{
		Path: "core/bwlimit",
		Fn: func(in rc.Params) (out rc.Params, err error) {
			ibwlimit, ok := in["rate"]
			if !ok {
				return out, errors.Errorf("parameter rate not found")
			}
			bwlimit, ok := ibwlimit.(string)
			if !ok {
				return out, errors.Errorf("value must be string rate=%v", ibwlimit)
			}
			var bws fs.BwTimetable
			err = bws.Set(bwlimit)
			if err != nil {
				return out, errors.Wrap(err, "bad bwlimit")
			}
			if len(bws) != 1 {
				return out, errors.New("need exactly 1 bandwidth setting")
			}
			bw := bws[0]
			SetBwLimit(bw.Bandwidth)
			return rc.Params{"rate": bw.Bandwidth.String()}, nil
		},
		Title: "Set the bandwidth limit.",
		Help: `
This sets the bandwidth limit to that passed in.

Eg

    rclone rc core/bwlimit rate=1M
    rclone rc core/bwlimit rate=off

The format of the parameter is exactly the same as passed to --bwlimit
except only one bandwidth may be specified.
`,
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy_test

import (
	"testing"

	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

func TestOgletest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Invariant-checking object proxy
////////////////////////////////////////////////////////////////////////

// A wrapper around ObjectProxy that calls CheckInvariants whenever invariants
// should hold. For catching logic errors early in the test.
type checkingObjectProxy struct {
	wrapped *gcsproxy.ObjectProxy
}

func (op *checkingObjectProxy) NoteLatest(o *storage.Object) error {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.NoteLatest(o)
}

func (op *checkingObjectProxy) Size() (uint64, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Size()
}

func (op *checkingObjectProxy) ReadAt(b []byte, o int64) (int, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.ReadAt(b, o)
}

func (op *checkingObjectProxy) WriteAt(b []byte, o int64) (int, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.WriteAt(b, o)
}

func (op *checkingObjectProxy) Truncate(n uint64) error {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Truncate(n)
}

func (op *checkingObjectProxy) Sync(ctx context.Context) (*storage.Object, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Sync(ctx)
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ObjectProxyTest struct {
	objectName string
	bucket     mock_gcs.MockBucket
	op         checkingObjectProxy
}

var _ SetUpInterface = &ObjectProxyTest{}

func (t *ObjectProxyTest) SetUp(ti *TestInfo) {
	t.objectName = "some/object"
	t.bucket = mock_gcs.NewMockBucket(ti.MockController, "bucket")

	var err error
	t.op.wrapped, err = gcsproxy.NewObjectProxy(
		t.bucket,
		t.objectName)

	if err != nil {
		panic(err)
	}
}

////////////////////////////////////////////////////////////////////////
// No source object
////////////////////////////////////////////////////////////////////////

// A test whose initial conditions are a fresh object proxy without a source
// object set.
type NoSourceObjectTest struct {
	ObjectProxyTest
}

var _ SetUpInterface = &NoSourceObjectTest{}

func init() { RegisterTestSuite(&NoSourceObjectTest{}) }

func (t *NoSourceObjectTest) NoteLatest_NegativeSize() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) NoteLatest_WrongName() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_InitialState() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_AfterTruncating() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_AfterReading() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Read_InitialState() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) WriteThenRead() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) GrowByTruncating() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_NoChanges() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_AfterWriting() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_AfterTruncating() {
	AssertTrue(false, "TODO")
}

////////////////////////////////////////////////////////////////////////
// Source object present
////////////////////////////////////////////////////////////////////////

// A test whose initial conditions are an object proxy branching from a source
// object in the bucket.
type SourceObjectPresentTest struct {
	ObjectProxyTest
	sourceObject storage.Object
}

var _ SetUpInterface = &SourceObjectPresentTest{}

func init() { RegisterTestSuite(&SourceObjectPresentTest{}) }

func (t *SourceObjectPresentTest) SetUp(ti *TestInfo) {
	t.ObjectProxyTest.SetUp(ti)

	// Set up the source object.
	t.sourceObject = storage.Object{
		Name:       t.objectName,
		Generation: 123,
		Size:       456,
	}

	if err := t.op.NoteLatest(&t.sourceObject); err != nil {
		panic(err)
	}
}

func (t *SourceObjectPresentTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<commit_msg>NoSourceObjectTest.NoteLatest_NegativeSize<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy_test

import (
	"testing"

	"github.com/jacobsa/gcloud/gcs/mock_gcs"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

func TestOgletest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Invariant-checking object proxy
////////////////////////////////////////////////////////////////////////

// A wrapper around ObjectProxy that calls CheckInvariants whenever invariants
// should hold. For catching logic errors early in the test.
type checkingObjectProxy struct {
	wrapped *gcsproxy.ObjectProxy
}

func (op *checkingObjectProxy) NoteLatest(o *storage.Object) error {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.NoteLatest(o)
}

func (op *checkingObjectProxy) Size() (uint64, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Size()
}

func (op *checkingObjectProxy) ReadAt(b []byte, o int64) (int, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.ReadAt(b, o)
}

func (op *checkingObjectProxy) WriteAt(b []byte, o int64) (int, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.WriteAt(b, o)
}

func (op *checkingObjectProxy) Truncate(n uint64) error {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Truncate(n)
}

func (op *checkingObjectProxy) Sync(ctx context.Context) (*storage.Object, error) {
	op.wrapped.CheckInvariants()
	defer op.wrapped.CheckInvariants()
	return op.wrapped.Sync(ctx)
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ObjectProxyTest struct {
	objectName string
	bucket     mock_gcs.MockBucket
	op         checkingObjectProxy
}

var _ SetUpInterface = &ObjectProxyTest{}

func (t *ObjectProxyTest) SetUp(ti *TestInfo) {
	t.objectName = "some/object"
	t.bucket = mock_gcs.NewMockBucket(ti.MockController, "bucket")

	var err error
	t.op.wrapped, err = gcsproxy.NewObjectProxy(
		t.bucket,
		t.objectName)

	if err != nil {
		panic(err)
	}
}

////////////////////////////////////////////////////////////////////////
// No source object
////////////////////////////////////////////////////////////////////////

// A test whose initial conditions are a fresh object proxy without a source
// object set.
type NoSourceObjectTest struct {
	ObjectProxyTest
}

var _ SetUpInterface = &NoSourceObjectTest{}

func init() { RegisterTestSuite(&NoSourceObjectTest{}) }

func (t *NoSourceObjectTest) NoteLatest_NegativeSize() {
	o := &storage.Object{
		Name:       t.objectName,
		Generation: 1234,
		Size:       -1,
	}

	err := t.op.NoteLatest(o)

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("size")))
	ExpectThat(err, Error(HasSubstr("-1")))
}

func (t *NoSourceObjectTest) NoteLatest_WrongName() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_InitialState() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_AfterTruncating() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Size_AfterReading() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Read_InitialState() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) WriteThenRead() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) GrowByTruncating() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_NoChanges() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_AfterWriting() {
	AssertTrue(false, "TODO")
}

func (t *NoSourceObjectTest) Sync_AfterTruncating() {
	AssertTrue(false, "TODO")
}

////////////////////////////////////////////////////////////////////////
// Source object present
////////////////////////////////////////////////////////////////////////

// A test whose initial conditions are an object proxy branching from a source
// object in the bucket.
type SourceObjectPresentTest struct {
	ObjectProxyTest
	sourceObject storage.Object
}

var _ SetUpInterface = &SourceObjectPresentTest{}

func init() { RegisterTestSuite(&SourceObjectPresentTest{}) }

func (t *SourceObjectPresentTest) SetUp(ti *TestInfo) {
	t.ObjectProxyTest.SetUp(ti)

	// Set up the source object.
	t.sourceObject = storage.Object{
		Name:       t.objectName,
		Generation: 123,
		Size:       456,
	}

	if err := t.op.NoteLatest(&t.sourceObject); err != nil {
		panic(err)
	}
}

func (t *SourceObjectPresentTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"fmt"
	"github.com/FactomProject/FactomCode/common"
	"github.com/FactomProject/FactomCode/factomlog"
	"github.com/FactomProject/FactomCode/util"
	"github.com/FactomProject/factoid/block"
	"github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"os"
	"sort"
)

func GetEntryCreditBalance(pubKey *[32]byte) (int32, error) {

	return eCreditMap[string(pubKey[:])], nil
}

func exportDChain(chain *common.DChain) {
	if len(chain.Blocks) == 0 || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	// get all ecBlocks from db
	dBlocks, _ := db.FetchAllDBlocks()
	sort.Sort(util.ByDBlockIDAccending(dBlocks))

	for _, block := range dBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.BlockHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportEChain(chain *common.EChain) {
	if procLog.Level() < factomlog.Info {
		return
	}

	eBlocks, _ := db.FetchAllEBlocksByChain(chain.ChainID)
	sort.Sort(util.ByEBlockIDAccending(*eBlocks))

	for _, block := range *eBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}

		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportECChain(chain *common.ECChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all ecBlocks from db
	ecBlocks, _ := db.FetchAllECBlocks()
	sort.Sort(util.ByECBlockIDAccending(ecBlocks))

	for _, block := range ecBlocks {
		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportAChain(chain *common.AdminChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	aBlocks, _ := db.FetchAllABlocks()
	sort.Sort(util.ByABlockIDAccending(aBlocks))

	for _, block := range aBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportFctChain(chain *common.FctChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	FBlocks, _ := db.FetchAllFBlocks()
	sort.Sort(util.ByFBlockIDAccending(FBlocks))

	for _, block := range FBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

// to export individual block once at a time - for debugging ------------------------
func exportDBlock(block *common.DirectoryBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := dchain.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.BlockHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportEBlock(block *common.EBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}

	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportECBlock(block *common.ECBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ECChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportABlock(block *common.AdminBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportFctBlock(block block.IFBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.GetChainID().String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
	if err != nil {
		panic(err)
	}

}

//--------------------------------------

func getPrePaidChainKey(entryHash *common.Hash, chainIDHash *common.Hash) string {
	return chainIDHash.String() + entryHash.String()
}

func copyCreditMap(
	originalMap map[string]int32,
	newMap map[string]int32) {
	newMap = make(map[string]int32)

	// copy every element from the original map
	for k, v := range originalMap {
		newMap[k] = v
	}

}

func printCreditMap() {
	fmt.Println("eCreditMap:")
	for key := range eCreditMap {
		procLog.Info("Key: %x Value %d\n", key, eCreditMap[key])
	}
}

func fileNotExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return true
	}
	return err != nil
}

// HaveBlockInDB returns whether or not the chain instance has the block represented
// by the passed hash.  This includes checking the various places a block can
// be like part of the main chain, on a side chain, or in the orphan pool.
//
// This function is NOT safe for concurrent access.
func HaveBlockInDB(hash *common.Hash) (bool, error) {
	util.Trace(spew.Sdump(hash))

	if hash == nil || dchain.Blocks == nil || len(dchain.Blocks) == 0 {
		return false, nil
	}

	// double check the block ids
	for i := 0; i < len(dchain.Blocks); i = i + 1 {
		if dchain.Blocks[i] == nil {
			continue
		}
		if dchain.Blocks[i].DBHash == nil {
			dchain.Blocks[i].DBHash, _ = common.CreateHash(dchain.Blocks[i])
		}
		if dchain.Blocks[i].DBHash.IsSameAs(hash) {
			return true, nil
		}
	}

	return false, nil
}
<commit_msg>Modified export block height for entry blocks<commit_after>// Copyright 2015 FactomProject Authors. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.

package process

import (
	"fmt"
	"github.com/FactomProject/FactomCode/common"
	"github.com/FactomProject/FactomCode/factomlog"
	"github.com/FactomProject/FactomCode/util"
	"github.com/FactomProject/factoid/block"
	"github.com/davecgh/go-spew/spew"
	"io/ioutil"
	"os"
	"sort"
)

func GetEntryCreditBalance(pubKey *[32]byte) (int32, error) {

	return eCreditMap[string(pubKey[:])], nil
}

func exportDChain(chain *common.DChain) {
	if len(chain.Blocks) == 0 || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	// get all ecBlocks from db
	dBlocks, _ := db.FetchAllDBlocks()
	sort.Sort(util.ByDBlockIDAccending(dBlocks))

	for _, block := range dBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.BlockHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportEChain(chain *common.EChain) {
	if procLog.Level() < factomlog.Info {
		return
	}

	eBlocks, _ := db.FetchAllEBlocksByChain(chain.ChainID)
	sort.Sort(util.ByEBlockIDAccending(*eBlocks))

	for _, block := range *eBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}

		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.EBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportECChain(chain *common.ECChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all ecBlocks from db
	ecBlocks, _ := db.FetchAllECBlocks()
	sort.Sort(util.ByECBlockIDAccending(ecBlocks))

	for _, block := range ecBlocks {
		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportAChain(chain *common.AdminChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	aBlocks, _ := db.FetchAllABlocks()
	sort.Sort(util.ByABlockIDAccending(aBlocks))

	for _, block := range aBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

func exportFctChain(chain *common.FctChain) {
	if procLog.Level() < factomlog.Info {
		return
	}
	// get all aBlocks from db
	FBlocks, _ := db.FetchAllFBlocks()
	sort.Sort(util.ByFBlockIDAccending(FBlocks))

	for _, block := range FBlocks {

		data, err := block.MarshalBinary()
		if err != nil {
			panic(err)
		}

		strChainID := chain.ChainID.String()
		if fileNotExists(dataStorePath + strChainID) {
			err := os.MkdirAll(dataStorePath+strChainID, 0777)
			if err == nil {
				procLog.Info("Created directory " + dataStorePath + strChainID)
			} else {
				procLog.Error(err)
			}
		}
		err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
		if err != nil {
			panic(err)
		}
	}
}

// to export individual block once at a time - for debugging ------------------------
func exportDBlock(block *common.DirectoryBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		//log.Println("no blocks to save for chain: " + string (*chain.ChainID))
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := dchain.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.BlockHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportEBlock(block *common.EBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}

	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportECBlock(block *common.ECBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ECChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportABlock(block *common.AdminBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.Header.ChainID.String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.Header.DBHeight), data, 0777)
	if err != nil {
		panic(err)
	}

}

func exportFctBlock(block block.IFBlock) {
	if block == nil || procLog.Level() < factomlog.Info {
		return
	}

	data, err := block.MarshalBinary()
	if err != nil {
		panic(err)
	}

	strChainID := block.GetChainID().String()
	if fileNotExists(dataStorePath + strChainID) {
		err := os.MkdirAll(dataStorePath+strChainID, 0777)
		if err == nil {
			procLog.Info("Created directory " + dataStorePath + strChainID)
		} else {
			procLog.Error(err)
		}
	}
	err = ioutil.WriteFile(fmt.Sprintf(dataStorePath+strChainID+"/store.%09d.block", block.GetDBHeight()), data, 0777)
	if err != nil {
		panic(err)
	}

}

//--------------------------------------

func getPrePaidChainKey(entryHash *common.Hash, chainIDHash *common.Hash) string {
	return chainIDHash.String() + entryHash.String()
}

func copyCreditMap(
	originalMap map[string]int32,
	newMap map[string]int32) {
	newMap = make(map[string]int32)

	// copy every element from the original map
	for k, v := range originalMap {
		newMap[k] = v
	}

}

func printCreditMap() {
	fmt.Println("eCreditMap:")
	for key := range eCreditMap {
		procLog.Info("Key: %x Value %d\n", key, eCreditMap[key])
	}
}

func fileNotExists(name string) bool {
	_, err := os.Stat(name)
	if os.IsNotExist(err) {
		return true
	}
	return err != nil
}

// HaveBlockInDB returns whether or not the chain instance has the block represented
// by the passed hash.  This includes checking the various places a block can
// be like part of the main chain, on a side chain, or in the orphan pool.
//
// This function is NOT safe for concurrent access.
func HaveBlockInDB(hash *common.Hash) (bool, error) {
	util.Trace(spew.Sdump(hash))

	if hash == nil || dchain.Blocks == nil || len(dchain.Blocks) == 0 {
		return false, nil
	}

	// double check the block ids
	for i := 0; i < len(dchain.Blocks); i = i + 1 {
		if dchain.Blocks[i] == nil {
			continue
		}
		if dchain.Blocks[i].DBHash == nil {
			dchain.Blocks[i].DBHash, _ = common.CreateHash(dchain.Blocks[i])
		}
		if dchain.Blocks[i].DBHash.IsSameAs(hash) {
			return true, nil
		}
	}

	return false, nil
}
<|endoftext|>


<commit_before>package meta

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
)

func parseIcy(rdr *bufio.Reader, c byte) (string, error) {
	numbytes := int(c) * 16
	bytes := make([]byte, numbytes)
	n, err := io.ReadFull(rdr, bytes)
	if err != nil {
		return "", err
	}
	if n != numbytes {
		return "", errors.New("didn't get enough data")
	}
	return strings.Split(strings.Split(string(bytes), "=")[1], ";")[0], nil
}

func extractMetadata(rdr io.Reader, skip int) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)
		bufrdr := bufio.NewReaderSize(rdr, skip)
		for {
			skipbytes := make([]byte, skip)

			_, err := io.ReadFull(bufrdr, skipbytes)
			if err != nil {
				return
			}
			c, err := bufrdr.ReadByte()
			if err != nil {
				return
			}
			if c > 0 {
				meta, err := parseIcy(bufrdr, c)
				if err != nil {
					return
				}
				ch <- meta
			}
		}
	}()
	return ch
}

// StreamMeta takes a url to stream frun and returns a channel of metadata
// strings or an error.
func StreamMeta(url string) (<-chan string, error) {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {

		return nil, err
	}

	req.Header.Add("Icy-MetaData", "1")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	amount := 0
	if _, err = fmt.Sscan(resp.Header.Get("Icy-Metaint"), &amount); err != nil {
		return nil, err
	}

	return extractMetadata(resp.Body, amount), nil
}
<commit_msg>should be slightly less heavy on the GC<commit_after>package meta

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
)

var tempBuffer []byte

func parseIcy(rdr *bufio.Reader, c byte) (string, error) {
	numbytes := int(c) * 16
	// Check that our reused buffer is sized ok.
	if len(tempBuffer) < numbytes {
		tempBuffer = make([]byte, numbytes)
	}

	// Create a slice of exactly the size we need, and work with it.
	bytes := tempBuffer[:numbytes]

	n, err := io.ReadFull(rdr, bytes)
	if err != nil {
		return "", err
	}
	if n != numbytes {
		return "", errors.New("didn't get enough data")
	}
	return strings.Split(strings.Split(string(bytes), "=")[1], ";")[0], nil
}

func extractMetadata(rdr io.Reader, skip int) <-chan string {
	ch := make(chan string)
	go func() {
		defer close(ch)
		bufrdr := bufio.NewReaderSize(rdr, skip)
		skipbytes := make([]byte, skip)
		for {
			_, err := io.ReadFull(bufrdr, skipbytes)
			if err != nil {
				return
			}
			c, err := bufrdr.ReadByte()
			if err != nil {
				return
			}
			if c > 0 {
				meta, err := parseIcy(bufrdr, c)
				if err != nil {
					return
				}
				ch <- meta
			}
		}
	}()
	return ch
}

// StreamMeta takes a url to stream frun and returns a channel of metadata
// strings or an error.
func StreamMeta(url string) (<-chan string, error) {
	client := &http.Client{}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {

		return nil, err
	}

	req.Header.Add("Icy-MetaData", "1")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	amount := 0
	if _, err = fmt.Sscan(resp.Header.Get("Icy-Metaint"), &amount); err != nil {
		return nil, err
	}

	return extractMetadata(resp.Body, amount), nil
}
<|endoftext|>


<commit_before>package microdata

import (
	"bytes"
	"code.google.com/p/go-html-transform/h5"
	"io"
	"strings"
)

type ValueList []interface{}
type PropertyMap map[string]ValueList

type Item struct {
	properties PropertyMap
	types []string
}

func NewItem() *Item {
	return &Item{
		properties: make(PropertyMap, 0),
		types: make([]string, 0),
	}
}

func (self *Item) SetString(property string, value string) {
	self.properties[property] = append(self.properties[property], value)
}

type Microdata struct {
	items []*Item
}

func NewMicrodata() *Microdata {
	return &Microdata{
		items: make([]*Item, 0),
	}
}

type Parser struct {
	p *h5.Parser
	data *Microdata
}

func NewParser(r io.Reader) *Parser {
	return &Parser {
		p : h5.NewParser(r),
		data: NewMicrodata(),
	}
}

func (self *Parser) Parse() (*Microdata, error) {
	err := self.p.Parse()
	if err != nil {
		return nil, err
	}
	tree := self.p.Tree()

	self.scanForItem(tree)

	return self.data, nil
}

func (self *Parser) scanForItem(node *h5.Node) {
	if node == nil {
		return
	}

	if _, exists := getAttr("itemscope", node); exists {
		item := NewItem()
		self.data.items = append(self.data.items, item)
		if itemtypes, exists := getAttr("itemtype", node); exists {
			for _, itemtype := range strings.Split(strings.TrimSpace(itemtypes), " ") {
				itemtype = strings.TrimSpace(itemtype)
				if itemtype != "" {
					item.types = append(item.types, itemtype)
				}
			}

		}

		if len(node.Children) > 0 {
	    	for _, child := range node.Children {
	        	self.readItem(item, child)
	        }
	    }

	} else {
		if len(node.Children) > 0 {
	    	for _, child := range node.Children {
	        	self.scanForItem(child)
	        }
		}
	}

}

func (self *Parser) readItem(item *Item, node *h5.Node) {
	if itemprop, exists := getAttr("itemprop", node); exists {
		var propertyValue string

		switch node.Data() {

		case "img","audio", "source", "video", "embed", "iframe", "track":
			if urlValue, exists := getAttr("src", node); exists {
				propertyValue = urlValue
			}
		case "a", "area", "link":
			if urlValue, exists := getAttr("href", node); exists {
				propertyValue = urlValue
			}
		case "data":
			if urlValue, exists := getAttr("value", node); exists {
				propertyValue = urlValue
			}
		case "time":
			if urlValue, exists := getAttr("datetime", node); exists {
				propertyValue = urlValue
			}

		default:
			var text bytes.Buffer
			node.Walk( func(n *h5.Node) {
					if n.Type == h5.TextNode {
						text.WriteString(n.Data())
					}

				})
			propertyValue = text.String()
		}

		for _, propertyName := range strings.Split(strings.TrimSpace(itemprop), " ") {
			propertyName = strings.TrimSpace(propertyName)
			if propertyName != "" {
				item.SetString(propertyName, propertyValue)
			}
		}
	}

	if len(node.Children) > 0 {
    	for _, child := range node.Children {
        	self.readItem(item, child)
        }
    }
}

func getAttr(name string, node *h5.Node) (string, bool) {
	for _, a := range node.Attr {
		if a.Name == name {
			return a.Value, true
		}
	}
	return "", false
}

<commit_msg>Added parsing of itemid<commit_after>package microdata

import (
	"bytes"
	"code.google.com/p/go-html-transform/h5"
	"io"
	"strings"
)

type ValueList []interface{}
type PropertyMap map[string]ValueList

type Item struct {
	properties PropertyMap
	types []string
	id string
}

func NewItem() *Item {
	return &Item{
		properties: make(PropertyMap, 0),
		types: make([]string, 0),
	}
}

func (self *Item) SetString(property string, value string) {
	self.properties[property] = append(self.properties[property], value)
}

type Microdata struct {
	items []*Item
}

func NewMicrodata() *Microdata {
	return &Microdata{
		items: make([]*Item, 0),
	}
}

type Parser struct {
	p *h5.Parser
	data *Microdata
}

func NewParser(r io.Reader) *Parser {
	return &Parser {
		p : h5.NewParser(r),
		data: NewMicrodata(),
	}
}

func (self *Parser) Parse() (*Microdata, error) {
	err := self.p.Parse()
	if err != nil {
		return nil, err
	}
	tree := self.p.Tree()

	self.scanForItem(tree)

	return self.data, nil
}

func (self *Parser) scanForItem(node *h5.Node) {
	if node == nil {
		return
	}

	if _, exists := getAttr("itemscope", node); exists {
		item := NewItem()
		self.data.items = append(self.data.items, item)
		if itemtypes, exists := getAttr("itemtype", node); exists {
			for _, itemtype := range strings.Split(strings.TrimSpace(itemtypes), " ") {
				itemtype = strings.TrimSpace(itemtype)
				if itemtype != "" {
					item.types = append(item.types, itemtype)
				}
			}
			// itemid only valid when itemscope and itemtype are both present
			if itemid, exists := getAttr("itemid", node); exists {
				item.id = strings.TrimSpace(itemid)
			}

		}

		if len(node.Children) > 0 {
	    	for _, child := range node.Children {
	        	self.readItem(item, child)
	        }
	    }

	} else {
		if len(node.Children) > 0 {
	    	for _, child := range node.Children {
	        	self.scanForItem(child)
	        }
		}
	}

}

func (self *Parser) readItem(item *Item, node *h5.Node) {
	if itemprop, exists := getAttr("itemprop", node); exists {
		var propertyValue string

		switch node.Data() {

		case "img","audio", "source", "video", "embed", "iframe", "track":
			if urlValue, exists := getAttr("src", node); exists {
				propertyValue = urlValue
			}
		case "a", "area", "link":
			if urlValue, exists := getAttr("href", node); exists {
				propertyValue = urlValue
			}
		case "data":
			if urlValue, exists := getAttr("value", node); exists {
				propertyValue = urlValue
			}
		case "time":
			if urlValue, exists := getAttr("datetime", node); exists {
				propertyValue = urlValue
			}

		default:
			var text bytes.Buffer
			node.Walk( func(n *h5.Node) {
					if n.Type == h5.TextNode {
						text.WriteString(n.Data())
					}

				})
			propertyValue = text.String()
		}

		for _, propertyName := range strings.Split(strings.TrimSpace(itemprop), " ") {
			propertyName = strings.TrimSpace(propertyName)
			if propertyName != "" {
				item.SetString(propertyName, propertyValue)
			}
		}
	}

	if len(node.Children) > 0 {
    	for _, child := range node.Children {
        	self.readItem(item, child)
        }
    }
}

func getAttr(name string, node *h5.Node) (string, bool) {
	for _, a := range node.Attr {
		if a.Name == name {
			return a.Value, true
		}
	}
	return "", false
}

<|endoftext|>


<commit_before>package movingmedian

import (
	"container/heap"
	"math"
)

type item struct {
	f   float64
	idx int
}

type itemHeap []*item

func (h itemHeap) Len() int { return len(h) }
func (h itemHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
	h[i].idx = i
	h[j].idx = j
}

func (h *itemHeap) Push(x interface{}) {
	e := x.(*item)
	e.idx = len(*h)
	*h = append(*h, e)
}

func (h *itemHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

type minFloat64Heap struct {
	itemHeap
}

func (h minFloat64Heap) Less(i, j int) bool { return h.itemHeap[i].f < h.itemHeap[j].f }

type maxFloat64Heap struct {
	itemHeap
}

func (h maxFloat64Heap) Less(i, j int) bool { return h.itemHeap[i].f > h.itemHeap[j].f }

type MovingMedian struct {
	queueIndex int
	nitems     int
	queue      []item
	maxHeap    maxFloat64Heap
	minHeap    minFloat64Heap
}

func NewMovingMedian(size int) MovingMedian {
	m := MovingMedian{
		queue:   make([]item, size),
		maxHeap: maxFloat64Heap{},
		minHeap: minFloat64Heap{},
	}

	heap.Init(&m.maxHeap)
	heap.Init(&m.minHeap)
	return m
}

func (m *MovingMedian) balanceHeaps() {
	if m.maxHeap.Len() > (m.minHeap.Len() + 1) {
		moveItem := heap.Pop(&m.maxHeap)
		heap.Push(&m.minHeap, moveItem)
	} else if m.minHeap.Len() > (m.maxHeap.Len() + 1) {
		moveItem := heap.Pop(&m.minHeap)
		heap.Push(&m.maxHeap, moveItem)
	}

}

func (m *MovingMedian) Push(v float64) {

	if m.nitems == len(m.queue) {
		old := &m.queue[m.queueIndex]
		heapIndex := old.idx

		if heapIndex < m.minHeap.Len() && old == m.minHeap.itemHeap[heapIndex] {
			heap.Remove(&m.minHeap, heapIndex)
		} else {
			heap.Remove(&m.maxHeap, heapIndex)
		}
	} else {
		m.nitems++
	}

	m.queue[m.queueIndex] = item{f: v}
	e := &m.queue[m.queueIndex]

	m.queueIndex++

	if m.queueIndex >= len(m.queue) {
		m.queueIndex = 0
	}

	if m.minHeap.Len() == 0 ||
		v > m.minHeap.itemHeap[0].f {
		heap.Push(&m.minHeap, e)
	} else {
		heap.Push(&m.maxHeap, e)
	}

	m.balanceHeaps()
}

func (m *MovingMedian) Median() float64 {
	if len(m.queue) == 0 {
		return math.NaN()
	}

	if (m.nitems % 2) == 0 {
		return (m.maxHeap.itemHeap[0].f + m.minHeap.itemHeap[0].f) / 2
	}

	if m.maxHeap.Len() > m.minHeap.Len() {
		return m.maxHeap.itemHeap[0].f
	}

	return m.minHeap.itemHeap[0].f
}
<commit_msg>Fix instead of removing and pushing when possible<commit_after>package movingmedian

import (
	"container/heap"
	"math"
)

type item struct {
	f   float64
	idx int
}

type itemHeap []*item

func (h itemHeap) Len() int { return len(h) }
func (h itemHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
	h[i].idx = i
	h[j].idx = j
}

func (h *itemHeap) Push(x interface{}) {
	e := x.(*item)
	e.idx = len(*h)
	*h = append(*h, e)
}

func (h *itemHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

type minItemHeap struct {
	itemHeap
}

func (h minItemHeap) Less(i, j int) bool { return h.itemHeap[i].f < h.itemHeap[j].f }

type maxItemHeap struct {
	itemHeap
}

func (h maxItemHeap) Less(i, j int) bool { return h.itemHeap[i].f > h.itemHeap[j].f }

type MovingMedian struct {
	queueIndex int
	nitems     int
	queue      []item
	maxHeap    maxItemHeap
	minHeap    minItemHeap
}

func NewMovingMedian(size int) MovingMedian {
	m := MovingMedian{
		queue:   make([]item, size),
		maxHeap: maxItemHeap{},
		minHeap: minItemHeap{},
	}

	heap.Init(&m.maxHeap)
	heap.Init(&m.minHeap)
	return m
}

func (m *MovingMedian) balanceHeaps() {
	if m.maxHeap.Len() > (m.minHeap.Len() + 1) {
		moveItem := heap.Pop(&m.maxHeap)
		heap.Push(&m.minHeap, moveItem)
	} else if m.minHeap.Len() > (m.maxHeap.Len() + 1) {
		moveItem := heap.Pop(&m.minHeap)
		heap.Push(&m.maxHeap, moveItem)
	}
}

func (m *MovingMedian) Push(v float64) {
	item := &m.queue[m.queueIndex]
	push := true
	if m.nitems == len(m.queue) {
		heapIndex := item.idx
		if heapIndex < m.minHeap.Len() && item == m.minHeap.itemHeap[heapIndex] {
			push = v < m.minHeap.itemHeap[0].f
			if push {
				heap.Remove(&m.minHeap, heapIndex)
			} else {
				item.f = v
				heap.Fix(&m.minHeap, heapIndex)
			}
		} else {
			push = v > m.maxHeap.itemHeap[0].f
			if push {
				heap.Remove(&m.maxHeap, heapIndex)
			} else {
				item.f = v
				heap.Fix(&m.maxHeap, heapIndex)
			}
		}
	} else {
		m.nitems++
	}

	m.queueIndex++
	if m.queueIndex >= len(m.queue) {
		m.queueIndex = 0
	}

	if push {
		item.f = v
		if m.minHeap.Len() == 0 ||
			v > m.minHeap.itemHeap[0].f {
			heap.Push(&m.minHeap, item)
		} else {
			heap.Push(&m.maxHeap, item)
		}

		m.balanceHeaps()
	}
}

func (m *MovingMedian) Median() float64 {
	if len(m.queue) == 0 {
		return math.NaN()
	}

	if (m.nitems % 2) == 0 {
		return (m.maxHeap.itemHeap[0].f + m.minHeap.itemHeap[0].f) / 2
	}

	if m.maxHeap.Len() > m.minHeap.Len() {
		return m.maxHeap.itemHeap[0].f
	}

	return m.minHeap.itemHeap[0].f
}
<|endoftext|>


<commit_before><commit_msg>update build/release/job status in CiBuilderEvent<commit_after><|endoftext|>