<commit_before>package parser

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"

	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/hashicorp/hcl/hcl/token"
)

func TestType(t *testing.T) {
	var literals = []struct {
		typ token.Type
		src string
	}{
		{token.STRING, `foo = "foo"`},
		{token.NUMBER, `foo = 123`},
		{token.NUMBER, `foo = -29`},
		{token.FLOAT, `foo = 123.12`},
		{token.FLOAT, `foo = -123.12`},
		{token.BOOL, `foo = true`},
		{token.HEREDOC, "foo = <<EOF\nHello\nWorld\nEOF"},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
		}

		lit, ok := item.Val.(*ast.LiteralType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
		}

		if lit.Token.Type != l.typ {
			t.Errorf("want: %s, got: %s", l.typ, lit.Token.Type)
		}
	}
}

func TestListType(t *testing.T) {
	var literals = []struct {
		src    string
		tokens []token.Type
	}{
		{
			`foo = ["123", 123]`,
			[]token.Type{token.STRING, token.NUMBER},
		},
		{
			`foo = [123, "123",]`,
			[]token.Type{token.NUMBER, token.STRING},
		},
		{
			`foo = []`,
			[]token.Type{},
		},
		{
			`foo = ["123", 123]`,
			[]token.Type{token.STRING, token.NUMBER},
		},
		{
			`foo = [1,
"string",
<<EOF
heredoc contents
EOF
]`,
			[]token.Type{token.NUMBER, token.STRING, token.HEREDOC},
		},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
		}

		list, ok := item.Val.(*ast.ListType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
		}

		tokens := []token.Type{}
		for _, li := range list.List {
			if tp, ok := li.(*ast.LiteralType); ok {
				tokens = append(tokens, tp.Token.Type)
			}
		}

		equals(t, l.tokens, tokens)
	}
}

func TestListOfMaps(t *testing.T) {
	src := `foo = [
    {key = "bar"},
    {key = "baz", key2 = "qux"},
  ]`
	p := newParser([]byte(src))

	file, err := p.Parse()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Here we make all sorts of assumptions about the input structure w/ type
	// assertions. The intent is only for this to be a "smoke test" ensuring
	// parsing actually performed its duty - giving this test something a bit
	// more robust than _just_ "no error occurred".
	expected := []string{`"bar"`, `"baz"`, `"qux"`}
	actual := make([]string, 0, 3)
	ol := file.Node.(*ast.ObjectList)
	objItem := ol.Items[0]
	list := objItem.Val.(*ast.ListType)
	for _, node := range list.List {
		obj := node.(*ast.ObjectType)
		for _, item := range obj.List.Items {
			val := item.Val.(*ast.LiteralType)
			actual = append(actual, val.Token.Text)
		}

	}
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("Expected: %#v, got %#v", expected, actual)
	}
}

func TestListOfMaps_requiresComma(t *testing.T) {
	src := `foo = [
    {key = "bar"}
    {key = "baz"}
  ]`
	p := newParser([]byte(src))

	_, err := p.Parse()
	if err == nil {
		t.Fatalf("Expected error, got none!")
	}

	expected := "error parsing list, expected comma or list end"
	if !strings.Contains(err.Error(), expected) {
		t.Fatalf("Expected err:\n  %s\nTo contain:\n  %s\n", err, expected)
	}
}

func TestObjectType(t *testing.T) {
	var literals = []struct {
		src      string
		nodeType []ast.Node
		itemLen  int
	}{
		{
			`foo = {}`,
			nil,
			0,
		},
		{
			`foo = {
				bar = "fatih"
			 }`,
			[]ast.Node{&ast.LiteralType{}},
			1,
		},
		{
			`foo = {
				bar = "fatih"
				baz = ["arslan"]
			 }`,
			[]ast.Node{
				&ast.LiteralType{},
				&ast.ListType{},
			},
			2,
		},
		{
			`foo = {
				bar {}
			 }`,
			[]ast.Node{
				&ast.ObjectType{},
			},
			1,
		},
		{
			`foo {
				bar {}
				foo = true
			 }`,
			[]ast.Node{
				&ast.ObjectType{},
				&ast.LiteralType{},
			},
			2,
		},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		// p.enableTrace = true
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
			continue
		}

		// we know that the ObjectKey name is foo for all cases, what matters
		// is the object
		obj, ok := item.Val.(*ast.ObjectType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
			continue
		}

		// check if the total length of items are correct
		equals(t, l.itemLen, len(obj.List.Items))

		// check if the types are correct
		for i, item := range obj.List.Items {
			equals(t, reflect.TypeOf(l.nodeType[i]), reflect.TypeOf(item.Val))
		}
	}
}

func TestObjectKey(t *testing.T) {
	keys := []struct {
		exp []token.Type
		src string
	}{
		{[]token.Type{token.IDENT}, `foo {}`},
		{[]token.Type{token.IDENT}, `foo = {}`},
		{[]token.Type{token.IDENT}, `foo = bar`},
		{[]token.Type{token.IDENT}, `foo = 123`},
		{[]token.Type{token.IDENT}, `foo = "${var.bar}`},
		{[]token.Type{token.STRING}, `"foo" {}`},
		{[]token.Type{token.STRING}, `"foo" = {}`},
		{[]token.Type{token.STRING}, `"foo" = "${var.bar}`},
		{[]token.Type{token.IDENT, token.IDENT}, `foo bar {}`},
		{[]token.Type{token.IDENT, token.STRING}, `foo "bar" {}`},
		{[]token.Type{token.STRING, token.IDENT}, `"foo" bar {}`},
		{[]token.Type{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`},
	}

	for _, k := range keys {
		p := newParser([]byte(k.src))
		keys, err := p.objectKey()
		if err != nil {
			t.Fatal(err)
		}

		tokens := []token.Type{}
		for _, o := range keys {
			tokens = append(tokens, o.Token.Type)
		}

		equals(t, k.exp, tokens)
	}

	errKeys := []struct {
		src string
	}{
		{`foo 12 {}`},
		{`foo bar = {}`},
		{`foo []`},
		{`12 {}`},
	}

	for _, k := range errKeys {
		p := newParser([]byte(k.src))
		_, err := p.objectKey()
		if err == nil {
			t.Errorf("case '%s' should give an error", k.src)
		}
	}
}

// Official HCL tests
func TestParse(t *testing.T) {
	cases := []struct {
		Name string
		Err  bool
	}{
		{
			"assign_colon.hcl",
			true,
		},
		{
			"comment.hcl",
			false,
		},
		{
			"comment_lastline.hcl",
			false,
		},
		{
			"comment_single.hcl",
			false,
		},
		{
			"empty.hcl",
			false,
		},
		{
			"list_comma.hcl",
			false,
		},
		{
			"multiple.hcl",
			false,
		},
		{
			"object_list_comma.hcl",
			false,
		},
		{
			"structure.hcl",
			false,
		},
		{
			"structure_basic.hcl",
			false,
		},
		{
			"structure_empty.hcl",
			false,
		},
		{
			"complex.hcl",
			false,
		},
		{
			"types.hcl",
			false,
		},
		{
			"array_comment.hcl",
			false,
		},
		{
			"array_comment_2.hcl",
			true,
		},
		{
			"missing_braces.hcl",
			true,
		},
		{
			"unterminated_object.hcl",
			true,
		},
		{
			"unterminated_object_2.hcl",
			true,
		},
		{
			"key_without_value.hcl",
			true,
		},
		{
			"object_key_without_value.hcl",
			true,
		},
	}

	const fixtureDir = "./test-fixtures"

	for _, tc := range cases {
		d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name))
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		_, err = Parse(d)
		if (err != nil) != tc.Err {
			t.Fatalf("Input: %s\n\nError: %s", tc.Name, err)
		}
	}
}

func TestParse_inline(t *testing.T) {
	cases := []struct {
		Value string
		Err   bool
	}{
		{"t t e{{}}", true},
		{"o{{}}", true},
		{"t t e d N{{}}", true},
		{"t t e d{{}}", true},
		{"N{}N{{}}", true},
		{"v\nN{{}}", true},
		{"v=/\n[,", true},
	}

	for _, tc := range cases {
		t.Logf("Testing: %q", tc.Value)
		_, err := Parse([]byte(tc.Value))
		if (err != nil) != tc.Err {
			t.Fatalf("Input: %q\n\nError: %s\n\nAST: %s", tc.Value, err)
		}
	}
}

// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
		tb.FailNow()
	}
}
<commit_msg>hcl/parser: test case for error in "10kb"<commit_after>package parser

import (
	"fmt"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"
	"testing"

	"github.com/hashicorp/hcl/hcl/ast"
	"github.com/hashicorp/hcl/hcl/token"
)

func TestType(t *testing.T) {
	var literals = []struct {
		typ token.Type
		src string
	}{
		{token.STRING, `foo = "foo"`},
		{token.NUMBER, `foo = 123`},
		{token.NUMBER, `foo = -29`},
		{token.FLOAT, `foo = 123.12`},
		{token.FLOAT, `foo = -123.12`},
		{token.BOOL, `foo = true`},
		{token.HEREDOC, "foo = <<EOF\nHello\nWorld\nEOF"},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
		}

		lit, ok := item.Val.(*ast.LiteralType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
		}

		if lit.Token.Type != l.typ {
			t.Errorf("want: %s, got: %s", l.typ, lit.Token.Type)
		}
	}
}

func TestListType(t *testing.T) {
	var literals = []struct {
		src    string
		tokens []token.Type
	}{
		{
			`foo = ["123", 123]`,
			[]token.Type{token.STRING, token.NUMBER},
		},
		{
			`foo = [123, "123",]`,
			[]token.Type{token.NUMBER, token.STRING},
		},
		{
			`foo = []`,
			[]token.Type{},
		},
		{
			`foo = ["123", 123]`,
			[]token.Type{token.STRING, token.NUMBER},
		},
		{
			`foo = [1,
"string",
<<EOF
heredoc contents
EOF
]`,
			[]token.Type{token.NUMBER, token.STRING, token.HEREDOC},
		},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
		}

		list, ok := item.Val.(*ast.ListType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
		}

		tokens := []token.Type{}
		for _, li := range list.List {
			if tp, ok := li.(*ast.LiteralType); ok {
				tokens = append(tokens, tp.Token.Type)
			}
		}

		equals(t, l.tokens, tokens)
	}
}

func TestListOfMaps(t *testing.T) {
	src := `foo = [
    {key = "bar"},
    {key = "baz", key2 = "qux"},
  ]`
	p := newParser([]byte(src))

	file, err := p.Parse()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Here we make all sorts of assumptions about the input structure w/ type
	// assertions. The intent is only for this to be a "smoke test" ensuring
	// parsing actually performed its duty - giving this test something a bit
	// more robust than _just_ "no error occurred".
	expected := []string{`"bar"`, `"baz"`, `"qux"`}
	actual := make([]string, 0, 3)
	ol := file.Node.(*ast.ObjectList)
	objItem := ol.Items[0]
	list := objItem.Val.(*ast.ListType)
	for _, node := range list.List {
		obj := node.(*ast.ObjectType)
		for _, item := range obj.List.Items {
			val := item.Val.(*ast.LiteralType)
			actual = append(actual, val.Token.Text)
		}

	}
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("Expected: %#v, got %#v", expected, actual)
	}
}

func TestListOfMaps_requiresComma(t *testing.T) {
	src := `foo = [
    {key = "bar"}
    {key = "baz"}
  ]`
	p := newParser([]byte(src))

	_, err := p.Parse()
	if err == nil {
		t.Fatalf("Expected error, got none!")
	}

	expected := "error parsing list, expected comma or list end"
	if !strings.Contains(err.Error(), expected) {
		t.Fatalf("Expected err:\n  %s\nTo contain:\n  %s\n", err, expected)
	}
}

func TestObjectType(t *testing.T) {
	var literals = []struct {
		src      string
		nodeType []ast.Node
		itemLen  int
	}{
		{
			`foo = {}`,
			nil,
			0,
		},
		{
			`foo = {
				bar = "fatih"
			 }`,
			[]ast.Node{&ast.LiteralType{}},
			1,
		},
		{
			`foo = {
				bar = "fatih"
				baz = ["arslan"]
			 }`,
			[]ast.Node{
				&ast.LiteralType{},
				&ast.ListType{},
			},
			2,
		},
		{
			`foo = {
				bar {}
			 }`,
			[]ast.Node{
				&ast.ObjectType{},
			},
			1,
		},
		{
			`foo {
				bar {}
				foo = true
			 }`,
			[]ast.Node{
				&ast.ObjectType{},
				&ast.LiteralType{},
			},
			2,
		},
	}

	for _, l := range literals {
		p := newParser([]byte(l.src))
		// p.enableTrace = true
		item, err := p.objectItem()
		if err != nil {
			t.Error(err)
			continue
		}

		// we know that the ObjectKey name is foo for all cases, what matters
		// is the object
		obj, ok := item.Val.(*ast.ObjectType)
		if !ok {
			t.Errorf("node should be of type LiteralType, got: %T", item.Val)
			continue
		}

		// check if the total length of items are correct
		equals(t, l.itemLen, len(obj.List.Items))

		// check if the types are correct
		for i, item := range obj.List.Items {
			equals(t, reflect.TypeOf(l.nodeType[i]), reflect.TypeOf(item.Val))
		}
	}
}

func TestObjectKey(t *testing.T) {
	keys := []struct {
		exp []token.Type
		src string
	}{
		{[]token.Type{token.IDENT}, `foo {}`},
		{[]token.Type{token.IDENT}, `foo = {}`},
		{[]token.Type{token.IDENT}, `foo = bar`},
		{[]token.Type{token.IDENT}, `foo = 123`},
		{[]token.Type{token.IDENT}, `foo = "${var.bar}`},
		{[]token.Type{token.STRING}, `"foo" {}`},
		{[]token.Type{token.STRING}, `"foo" = {}`},
		{[]token.Type{token.STRING}, `"foo" = "${var.bar}`},
		{[]token.Type{token.IDENT, token.IDENT}, `foo bar {}`},
		{[]token.Type{token.IDENT, token.STRING}, `foo "bar" {}`},
		{[]token.Type{token.STRING, token.IDENT}, `"foo" bar {}`},
		{[]token.Type{token.IDENT, token.IDENT, token.IDENT}, `foo bar baz {}`},
	}

	for _, k := range keys {
		p := newParser([]byte(k.src))
		keys, err := p.objectKey()
		if err != nil {
			t.Fatal(err)
		}

		tokens := []token.Type{}
		for _, o := range keys {
			tokens = append(tokens, o.Token.Type)
		}

		equals(t, k.exp, tokens)
	}

	errKeys := []struct {
		src string
	}{
		{`foo 12 {}`},
		{`foo bar = {}`},
		{`foo []`},
		{`12 {}`},
	}

	for _, k := range errKeys {
		p := newParser([]byte(k.src))
		_, err := p.objectKey()
		if err == nil {
			t.Errorf("case '%s' should give an error", k.src)
		}
	}
}

// Official HCL tests
func TestParse(t *testing.T) {
	cases := []struct {
		Name string
		Err  bool
	}{
		{
			"assign_colon.hcl",
			true,
		},
		{
			"comment.hcl",
			false,
		},
		{
			"comment_lastline.hcl",
			false,
		},
		{
			"comment_single.hcl",
			false,
		},
		{
			"empty.hcl",
			false,
		},
		{
			"list_comma.hcl",
			false,
		},
		{
			"multiple.hcl",
			false,
		},
		{
			"object_list_comma.hcl",
			false,
		},
		{
			"structure.hcl",
			false,
		},
		{
			"structure_basic.hcl",
			false,
		},
		{
			"structure_empty.hcl",
			false,
		},
		{
			"complex.hcl",
			false,
		},
		{
			"types.hcl",
			false,
		},
		{
			"array_comment.hcl",
			false,
		},
		{
			"array_comment_2.hcl",
			true,
		},
		{
			"missing_braces.hcl",
			true,
		},
		{
			"unterminated_object.hcl",
			true,
		},
		{
			"unterminated_object_2.hcl",
			true,
		},
		{
			"key_without_value.hcl",
			true,
		},
		{
			"object_key_without_value.hcl",
			true,
		},
	}

	const fixtureDir = "./test-fixtures"

	for _, tc := range cases {
		d, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name))
		if err != nil {
			t.Fatalf("err: %s", err)
		}

		_, err = Parse(d)
		if (err != nil) != tc.Err {
			t.Fatalf("Input: %s\n\nError: %s", tc.Name, err)
		}
	}
}

func TestParse_inline(t *testing.T) {
	cases := []struct {
		Value string
		Err   bool
	}{
		{"t t e{{}}", true},
		{"o{{}}", true},
		{"t t e d N{{}}", true},
		{"t t e d{{}}", true},
		{"N{}N{{}}", true},
		{"v\nN{{}}", true},
		{"v=/\n[,", true},
		{"v=10kb", true},
	}

	for _, tc := range cases {
		t.Logf("Testing: %q", tc.Value)
		_, err := Parse([]byte(tc.Value))
		if (err != nil) != tc.Err {
			t.Fatalf("Input: %q\n\nError: %s\n\nAST: %s", tc.Value, err)
		}
	}
}

// equals fails the test if exp is not equal to act.
func equals(tb testing.TB, exp, act interface{}) {
	if !reflect.DeepEqual(exp, act) {
		_, file, line, _ := runtime.Caller(1)
		fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
		tb.FailNow()
	}
}
<|endoftext|>


<commit_before>package luatest

import (
	"strings"
	"testing"

	"github.com/bmatsuo/lark/internal/module"
	"github.com/yuin/gopher-lua"
)

// Module is a lua module to be tested.
type Module struct {
	Module      module.Module
	TestScript  string
	PreloadDeps []*Module
}

// Preload runs the loader to register the module name.
func (m *Module) Preload(t testing.TB) *lua.LState {
	L := lua.NewState()
	module.Preload(L, m.Module)
	for _, m := range m.PreloadDeps {
		module.Preload(L, m.Module)
	}
	err := L.DoFile(m.TestScript)
	if err != nil {
		t.Error(err)
	}
	return L
}

// Test runs the specified test function
func (m *Module) Test(t testing.TB) {
	L := m.Preload(t)
	defer L.Close()

	testFuncs := getTestFuncs(L)
	errFn := L.NewFunction(errTraceback)
	for _, fname := range testFuncs {
		t.Logf("ENTER  %s", fname)
		L.Push(L.GetGlobal(fname))
		err := L.PCall(0, 0, errFn)
		if err != nil {
			t.Errorf("FAIL   %s\n%s", fname, err)
		} else {
			t.Logf("PASS   %s", fname)
		}
	}
}

func getTestFuncs(L *lua.LState) []string {
	return getGlobals(L, selTestFuncs)
}

func selTestFuncs(L *lua.LState, k, v lua.LValue) bool {
	s := string(k.(lua.LString))
	if !strings.HasPrefix(s, "test_") {
		return false
	}
	return true
}

func getGlobals(L *lua.LState, sel func(L *lua.LState, k, v lua.LValue) bool) []string {
	var sels []string
	globals := L.Get(lua.GlobalsIndex).(*lua.LTable)
	L.ForEach(globals, func(k, v lua.LValue) {
		s, ok := k.(lua.LString)
		if !ok {
			return
		}
		if sel(L, k, v) {
			sels = append(sels, string(s))
		}
	})
	return sels
}

func errTraceback(L *lua.LState) int {
	msg := L.Get(1)
	L.SetTop(0)
	L.Push(L.GetField(L.GetGlobal("debug"), "traceback"))
	L.Push(lua.LNumber(1))
	L.Push(msg)
	L.Push(lua.LNumber(2))
	L.Call(3, 1)
	return 1
}
<commit_msg>test: fix bug getting tracebacks<commit_after>package luatest

import (
	"strings"
	"testing"

	"github.com/bmatsuo/lark/internal/module"
	"github.com/yuin/gopher-lua"
)

// Module is a lua module to be tested.
type Module struct {
	Module      module.Module
	TestScript  string
	PreloadDeps []*Module
}

// Preload runs the loader to register the module name.
func (m *Module) Preload(t testing.TB) *lua.LState {
	L := lua.NewState()
	module.Preload(L, m.Module)
	for _, m := range m.PreloadDeps {
		module.Preload(L, m.Module)
	}
	err := L.DoFile(m.TestScript)
	if err != nil {
		t.Error(err)
	}
	return L
}

// Test runs the specified test function
func (m *Module) Test(t testing.TB) {
	L := m.Preload(t)
	defer L.Close()

	testFuncs := getTestFuncs(L)
	errFn := L.NewFunction(errTraceback)
	for _, fname := range testFuncs {
		t.Logf("ENTER  %s", fname)
		L.Push(L.GetGlobal(fname))
		err := L.PCall(0, 0, errFn)
		if err != nil {
			t.Errorf("FAIL   %s\n%s", fname, err)
		} else {
			t.Logf("PASS   %s", fname)
		}
	}
}

func getTestFuncs(L *lua.LState) []string {
	return getGlobals(L, selTestFuncs)
}

func selTestFuncs(L *lua.LState, k, v lua.LValue) bool {
	s := string(k.(lua.LString))
	if !strings.HasPrefix(s, "test_") {
		return false
	}
	return true
}

func getGlobals(L *lua.LState, sel func(L *lua.LState, k, v lua.LValue) bool) []string {
	var sels []string
	globals := L.Get(lua.GlobalsIndex).(*lua.LTable)
	L.ForEach(globals, func(k, v lua.LValue) {
		s, ok := k.(lua.LString)
		if !ok {
			return
		}
		if sel(L, k, v) {
			sels = append(sels, string(s))
		}
	})
	return sels
}

func errTraceback(L *lua.LState) int {
	msg := L.Get(1)
	L.SetTop(0)
	L.Push(L.GetField(L.GetGlobal("debug"), "traceback"))
	L.Push(msg)
	L.Push(lua.LNumber(2))
	L.Call(2, 1)
	return 1
}
<|endoftext|>


<commit_before>package render

import (
	"html/template"
	"net/http"
)

type (
	HTMLRender interface {
		Instance(string, interface{}) Render
	}

	HTMLProduction struct {
		Template *template.Template
	}

	HTMLDebug struct {
		Files []string
		Glob  string
	}

	HTML struct {
		Template *template.Template
		Name     string
		Data     interface{}
	}
)

func (r HTMLProduction) Instance(name string, data interface{}) Render {
	return HTML{
		Template: r.Template,
		Name:     name,
		Data:     data,
	}
}

func (r HTMLDebug) Instance(name string, data interface{}) Render {
	return HTML{
		Template: r.loadTemplate(),
		Name:     name,
		Data:     data,
	}
}
func (r HTMLDebug) loadTemplate() *template.Template {
	if len(r.Files) > 0 {
		return template.Must(template.ParseFiles(r.Files...))
	}
	if len(r.Glob) > 0 {
		return template.Must(template.ParseFiles(r.Files...))
	}
	panic("the HTML debug render was created without files or glob pattern")
}

func (r HTML) Write(w http.ResponseWriter) error {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	return r.Template.ExecuteTemplate(w, r.Name, r.Data)
}
<commit_msg>Fixes bug in HTMLDebug<commit_after>package render

import (
	"html/template"
	"net/http"
)

type (
	HTMLRender interface {
		Instance(string, interface{}) Render
	}

	HTMLProduction struct {
		Template *template.Template
	}

	HTMLDebug struct {
		Files []string
		Glob  string
	}

	HTML struct {
		Template *template.Template
		Name     string
		Data     interface{}
	}
)

func (r HTMLProduction) Instance(name string, data interface{}) Render {
	return HTML{
		Template: r.Template,
		Name:     name,
		Data:     data,
	}
}

func (r HTMLDebug) Instance(name string, data interface{}) Render {
	return HTML{
		Template: r.loadTemplate(),
		Name:     name,
		Data:     data,
	}
}
func (r HTMLDebug) loadTemplate() *template.Template {
	if len(r.Files) > 0 {
		return template.Must(template.ParseFiles(r.Files...))
	}
	if len(r.Glob) > 0 {
		return template.Must(template.ParseGlob(r.Glob))
	}
	panic("the HTML debug render was created without files or glob pattern")
}

func (r HTML) Write(w http.ResponseWriter) error {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	return r.Template.ExecuteTemplate(w, r.Name, r.Data)
}
<|endoftext|>


<commit_before>package raft

import (
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan struct{}
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	go r.heartbeat(s)

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLog())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLog())
		case <-s.stopCh:
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.localAddr,
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, 16)
	maxIndex := min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.localAddr,
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-s.stopCh:
			return
		}
	}
}
<commit_msg>Use config for buffer size<commit_after>package raft

import (
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan struct{}
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	go r.heartbeat(s)

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLog())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLog())
		case <-s.stopCh:
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.localAddr,
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex := min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.localAddr,
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-s.stopCh:
			return
		}
	}
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan uint64
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	stopHeartbeat := make(chan struct{})
	defer close(stopHeartbeat)
	r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.replicateTo(s, maxIndex)
			}
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
	var maxIndex uint64
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.trans.EncodePeer(r.localAddr),
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex = min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

CHECK_MORE:
	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return

	// SEND_SNAP is used when we fail to get a log, usually because the follower
	// is too far behind, and we must ship a snapshot down instead
SEND_SNAP:
	stop, err := r.sendLatestSnapshot(s)

	// Check if we should stop
	if stop {
		return true
	}

	// Check for an error
	if err != nil {
		log.Printf("[ERR] Failed to send snapshot to %v: %v", s.peer, err)
		return
	}

	// Check if there is more to replicate
	goto CHECK_MORE
}

// sendLatestSnapshot is used to send the latest snapshot we have
// down to our follower
func (r *Raft) sendLatestSnapshot(s *followerReplication) (bool, error) {
	// Get the snapshots
	snapshots, err := r.snapshots.List()
	if err != nil {
		log.Printf("[ERR] Failed to list snapshots: %v", err)
		return false, err
	}

	// Check we have at least a single snapshot
	if len(snapshots) == 0 {
		return false, fmt.Errorf("no snapshots found")
	}

	// Open the most recent snapshot
	snapId := snapshots[0].ID
	meta, snapshot, err := r.snapshots.Open(snapId)
	if err != nil {
		log.Printf("[ERR] Failed to open snapshot %v: %v", snapId, err)
		return false, err
	}
	defer snapshot.Close()

	// Setup the request
	req := InstallSnapshotRequest{
		Term:         s.currentTerm,
		Leader:       r.trans.EncodePeer(r.localAddr),
		LastLogIndex: meta.Index,
		LastLogTerm:  meta.Term,
		Peers:        meta.Peers,
		Size:         meta.Size,
	}

	// Make the call
	var resp InstallSnapshotResponse
	if err := r.trans.InstallSnapshot(s.peer, &req, &resp, snapshot); err != nil {
		log.Printf("[ERR] Failed to install snapshot %v: %v", snapId, err)
		return false, err
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true, nil
	}

	// Check for success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex; i <= meta.Index; i++ {
			s.inflight.Commit(i, s.peer)
		}

		// Update the indexes
		s.matchIndex = meta.Index
		s.nextIndex = s.matchIndex + 1
	} else {
		log.Printf("[WARN] InstallSnapshot to %v rejected", s.peer)
	}
	return false, nil
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.trans.EncodePeer(r.localAddr),
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-stopCh:
			return
		}
	}
}
<commit_msg>Do not double commit the matchIndex<commit_after>package raft

import (
	"fmt"
	"log"
	"net"
)

type followerReplication struct {
	peer     net.Addr
	inflight *inflight

	stopCh    chan uint64
	triggerCh chan struct{}

	currentTerm uint64
	matchIndex  uint64
	nextIndex   uint64
}

// replicate is a long running routine that is used to manage
// the process of replicating logs to our followers
func (r *Raft) replicate(s *followerReplication) {
	// Start an async heartbeating routing
	stopHeartbeat := make(chan struct{})
	defer close(stopHeartbeat)
	r.goFunc(func() { r.heartbeat(s, stopHeartbeat) })

	shouldStop := false
	for !shouldStop {
		select {
		case <-s.triggerCh:
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case <-randomTimeout(r.conf.CommitTimeout):
			shouldStop = r.replicateTo(s, r.getLastLogIndex())
		case maxIndex := <-s.stopCh:
			// Make a best effort to replicate up to this index
			if maxIndex > 0 {
				r.replicateTo(s, maxIndex)
			}
			return
		}
	}
}

// replicateTo is used to replicate the logs up to a given last index.
// If the follower log is behind, we take care to bring them up to date
func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {
	// Create the base request
	var l Log
	var req AppendEntriesRequest
	var resp AppendEntriesResponse
	var maxIndex uint64
START:
	req = AppendEntriesRequest{
		Term:              s.currentTerm,
		Leader:            r.trans.EncodePeer(r.localAddr),
		LeaderCommitIndex: r.getCommitIndex(),
	}

	// Get the previous log entry based on the nextIndex.
	// Guard for the first index, since there is no 0 log entry
	if s.nextIndex > 1 {
		if err := r.logs.GetLog(s.nextIndex-1, &l); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v",
				s.nextIndex-1, err)
			return
		}

		// Set the previous index and term (0 if nextIndex is 1)
		req.PrevLogEntry = l.Index
		req.PrevLogTerm = l.Term
	} else {
		req.PrevLogEntry = 0
		req.PrevLogTerm = 0
	}

	// Append up to MaxAppendEntries or up to the lastIndex
	req.Entries = make([]*Log, 0, r.conf.MaxAppendEntries)
	maxIndex = min(s.nextIndex+uint64(r.conf.MaxAppendEntries)-1, lastIndex)
	for i := s.nextIndex; i <= maxIndex; i++ {
		oldLog := new(Log)
		if err := r.logs.GetLog(i, oldLog); err != nil {
			if err == LogNotFound {
				goto SEND_SNAP
			}
			log.Printf("[ERR] Failed to get log at index %d: %v", i, err)
			return
		}
		req.Entries = append(req.Entries, oldLog)
	}

	// Make the RPC call
	if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
		log.Printf("[ERR] Failed to AppendEntries to %v: %v", s.peer, err)
		return
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true
	}

	// Update the s based on success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex + 1; i <= maxIndex; i++ {
			s.inflight.Commit(i, s.peer)
		}

		s.matchIndex = maxIndex
		s.nextIndex = maxIndex + 1
	} else {
		log.Printf("[WARN] AppendEntries to %v rejected, sending older logs", s.peer)
		s.nextIndex = max(min(s.nextIndex-1, resp.LastLog+1), 1)
		s.matchIndex = s.nextIndex - 1
	}

CHECK_MORE:
	// Check if there are more logs to replicate
	if s.nextIndex <= lastIndex {
		goto START
	}
	return

	// SEND_SNAP is used when we fail to get a log, usually because the follower
	// is too far behind, and we must ship a snapshot down instead
SEND_SNAP:
	stop, err := r.sendLatestSnapshot(s)

	// Check if we should stop
	if stop {
		return true
	}

	// Check for an error
	if err != nil {
		log.Printf("[ERR] Failed to send snapshot to %v: %v", s.peer, err)
		return
	}

	// Check if there is more to replicate
	goto CHECK_MORE
}

// sendLatestSnapshot is used to send the latest snapshot we have
// down to our follower
func (r *Raft) sendLatestSnapshot(s *followerReplication) (bool, error) {
	// Get the snapshots
	snapshots, err := r.snapshots.List()
	if err != nil {
		log.Printf("[ERR] Failed to list snapshots: %v", err)
		return false, err
	}

	// Check we have at least a single snapshot
	if len(snapshots) == 0 {
		return false, fmt.Errorf("no snapshots found")
	}

	// Open the most recent snapshot
	snapId := snapshots[0].ID
	meta, snapshot, err := r.snapshots.Open(snapId)
	if err != nil {
		log.Printf("[ERR] Failed to open snapshot %v: %v", snapId, err)
		return false, err
	}
	defer snapshot.Close()

	// Setup the request
	req := InstallSnapshotRequest{
		Term:         s.currentTerm,
		Leader:       r.trans.EncodePeer(r.localAddr),
		LastLogIndex: meta.Index,
		LastLogTerm:  meta.Term,
		Peers:        meta.Peers,
		Size:         meta.Size,
	}

	// Make the call
	var resp InstallSnapshotResponse
	if err := r.trans.InstallSnapshot(s.peer, &req, &resp, snapshot); err != nil {
		log.Printf("[ERR] Failed to install snapshot %v: %v", snapId, err)
		return false, err
	}

	// Check for a newer term, stop running
	if resp.Term > req.Term {
		return true, nil
	}

	// Check for success
	if resp.Success {
		// Mark any inflight logs as committed
		for i := s.matchIndex + 1; i <= meta.Index; i++ {
			s.inflight.Commit(i, s.peer)
		}

		// Update the indexes
		s.matchIndex = meta.Index
		s.nextIndex = s.matchIndex + 1
	} else {
		log.Printf("[WARN] InstallSnapshot to %v rejected", s.peer)
	}
	return false, nil
}

// hearbeat is used to periodically invoke AppendEntries on a peer
// to ensure they don't time out. This is done async of replicate(),
// since that routine could potentially be blocked on disk IO
func (r *Raft) heartbeat(s *followerReplication, stopCh chan struct{}) {
	req := AppendEntriesRequest{
		Term:   s.currentTerm,
		Leader: r.trans.EncodePeer(r.localAddr),
	}
	var resp AppendEntriesResponse
	for {
		select {
		case <-randomTimeout(r.conf.HeartbeatTimeout / 4):
			if err := r.trans.AppendEntries(s.peer, &req, &resp); err != nil {
				log.Printf("[ERR] Failed to heartbeat to %v: %v", s.peer, err)
			}
		case <-stopCh:
			return
		}
	}
}
<|endoftext|>


<commit_before>package handlers

import (
	"errors"
	"fmt"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/go-libsass/libs"
	sw "github.com/wellington/spritewell"
)

func init() {
	libsass.RegisterHandler("sprite($map, $name, $offsetX: 0px, $offsetY: 0px)", Sprite)
	libsass.RegisterHandler("sprite-map($glob, $spacing: 0px)", SpriteMap)
	libsass.RegisterHandler("sprite-file($map, $name)", SpriteFile)
	libsass.RegisterHandler("sprite-position($map, $file)", SpritePosition)
}

// SpritePosition returns the position of the image in the sprite-map.
// This is useful for passing directly to background-position
func SpritePosition(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob, name string
	err := libsass.Unmarshal(usv, &glob, &name)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		return setErrorAndReturn(errors.New("Context payload not found"), rsv)
	}
	sprites := payload.Sprite()
	sprites.RLock()
	defer sprites.RUnlock()
	imgs, ok := sprites.M[glob]
	if !ok {
		err := fmt.Errorf(
			"Variable not found matching glob: %s sprite:%s", glob, name)
		return setErrorAndReturn(err, rsv)
	}

	if imgs.Lookup(name) == -1 {
		return setErrorAndReturn(fmt.Errorf("image %s not found\n"+
			"   try one of these: %v", name, imgs.Paths), rsv)
	}

	// This is an odd name for what it does
	pos := imgs.GetPack(imgs.Lookup(name))

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	x := libs.SassNumber{Unit: "px", Value: float64(-pos.X)}
	y := libs.SassNumber{Unit: "px", Value: float64(-pos.Y)}

	str, err := libsass.Marshal(
		[]libs.SassNumber{x, y},
	)

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	if rsv != nil {
		*rsv = str
	}

	return nil
}

// SpriteFile proxies the sprite glob and image name through.
func SpriteFile(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	var glob, name string
	err := libsass.Unmarshal(usv, &glob, &name)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	infs := []interface{}{glob, name}
	res, err := libsass.Marshal(infs)
	if rsv != nil {
		*rsv = res
	}
	return nil
}

// Sprite returns the source and background position for an image in the
// spritesheet.
func Sprite(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob, name string
	var offsetX, offsetY libs.SassNumber
	err := libsass.Unmarshal(usv, &glob, &name, &offsetX, &offsetY)
	if err != nil {
		if err == libsass.ErrSassNumberNoUnit {
			err := fmt.Errorf(
				"Please specify unit for offset ie. (2px)")
			return setErrorAndReturn(err, rsv)
		}
		return setErrorAndReturn(err, rsv)
	}
	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		return setErrorAndReturn(errors.New("Context payload not found"), rsv)
	}
	sprites := payload.Sprite()
	sprites.RLock()
	defer sprites.RUnlock()
	imgs, ok := sprites.M[glob]
	if !ok {
		keys := make([]string, 0, len(sprites.M))
		for i := range sprites.M {
			keys = append(keys, i)
		}

		err := fmt.Errorf(
			"Variable not found matching glob: %s sprite:%s", glob, name)
		return setErrorAndReturn(err, rsv)
	}

	path, err := imgs.OutputPath()

	// FIXME: path directory can not be trusted, rebuild this from the context
	if ctx.HTTPPath == "" {
		ctxPath, _ := filepath.Rel(ctx.BuildDir, ctx.GenImgDir)
		path = strings.Join([]string{ctxPath, filepath.Base(path)}, "/")
	} else {
		u, err := url.Parse(ctx.HTTPPath)
		if err != nil {
			return setErrorAndReturn(err, rsv)
		}
		u.Path = strings.Join([]string{u.Path, "build", filepath.Base(path)}, "/")
		path = u.String()
	}
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	if imgs.Lookup(name) == -1 {
		return setErrorAndReturn(fmt.Errorf("image %s not found\n"+
			"   try one of these: %v", name, imgs.Paths), rsv)
	}
	// This is an odd name for what it does
	pos := imgs.GetPack(imgs.Lookup(name))

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	x := libs.SassNumber{Unit: "px", Value: float64(-pos.X)}
	x = x.Add(offsetX)

	y := libs.SassNumber{Unit: "px", Value: float64(-pos.Y)}
	y = y.Add(offsetY)

	str, err := libsass.Marshal(
		fmt.Sprintf(`url("%s") %s %s`,
			path, x, y,
		))
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	if rsv != nil {
		*rsv = str
	}
	return nil
}

// SpriteMap returns a sprite from the passed glob and sprite
// parameters.
func SpriteMap(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob string
	var spacing libs.SassNumber
	err := libsass.Unmarshal(usv, &glob, &spacing)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	imgs := sw.New(&sw.Options{
		ImageDir:  ctx.ImageDir,
		BuildDir:  ctx.BuildDir,
		GenImgDir: ctx.GenImgDir,
		Padding:   int(spacing.Value),
	})
	if cglob, err := strconv.Unquote(glob); err == nil {
		glob = cglob
	}

	key := glob + strconv.FormatInt(int64(spacing.Value), 10)

	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		err := errors.New("context payload not found")
		return setErrorAndReturn(err, rsv)
	}

	sprites := payload.Sprite()

	// TODO: benchmark a single write lock against this
	// read lock then write lock
	sprites.RLock()
	if _, ok := sprites.M[key]; ok {
		defer sprites.RUnlock()
		res, err := libsass.Marshal(key)
		if err != nil {
			return setErrorAndReturn(err, rsv)
		}
		if rsv != nil {
			*rsv = res
		}
		return nil
	}
	sprites.RUnlock()

	err = imgs.Decode(glob)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	imgs.Export()

	res, err := libsass.Marshal(key)
	sprites.Lock()
	sprites.M[key] = imgs
	sprites.Unlock()
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	if rsv != nil {
		*rsv = res
	}
	return nil
}
<commit_msg>report export errors<commit_after>package handlers

import (
	"errors"
	"fmt"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"

	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/go-libsass/libs"
	sw "github.com/wellington/spritewell"
)

func init() {
	libsass.RegisterHandler("sprite($map, $name, $offsetX: 0px, $offsetY: 0px)", Sprite)
	libsass.RegisterHandler("sprite-map($glob, $spacing: 0px)", SpriteMap)
	libsass.RegisterHandler("sprite-file($map, $name)", SpriteFile)
	libsass.RegisterHandler("sprite-position($map, $file)", SpritePosition)
}

// SpritePosition returns the position of the image in the sprite-map.
// This is useful for passing directly to background-position
func SpritePosition(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob, name string
	err := libsass.Unmarshal(usv, &glob, &name)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		return setErrorAndReturn(errors.New("Context payload not found"), rsv)
	}
	sprites := payload.Sprite()
	sprites.RLock()
	defer sprites.RUnlock()
	imgs, ok := sprites.M[glob]
	if !ok {
		err := fmt.Errorf(
			"Variable not found matching glob: %s sprite:%s", glob, name)
		return setErrorAndReturn(err, rsv)
	}

	if imgs.Lookup(name) == -1 {
		return setErrorAndReturn(fmt.Errorf("image %s not found\n"+
			"   try one of these: %v", name, imgs.Paths), rsv)
	}

	// This is an odd name for what it does
	pos := imgs.GetPack(imgs.Lookup(name))

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	x := libs.SassNumber{Unit: "px", Value: float64(-pos.X)}
	y := libs.SassNumber{Unit: "px", Value: float64(-pos.Y)}

	str, err := libsass.Marshal(
		[]libs.SassNumber{x, y},
	)

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	if rsv != nil {
		*rsv = str
	}

	return nil
}

// SpriteFile proxies the sprite glob and image name through.
func SpriteFile(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	var glob, name string
	err := libsass.Unmarshal(usv, &glob, &name)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	infs := []interface{}{glob, name}
	res, err := libsass.Marshal(infs)
	if rsv != nil {
		*rsv = res
	}
	return nil
}

// Sprite returns the source and background position for an image in the
// spritesheet.
func Sprite(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob, name string
	var offsetX, offsetY libs.SassNumber
	err := libsass.Unmarshal(usv, &glob, &name, &offsetX, &offsetY)
	if err != nil {
		if err == libsass.ErrSassNumberNoUnit {
			err := fmt.Errorf(
				"Please specify unit for offset ie. (2px)")
			return setErrorAndReturn(err, rsv)
		}
		return setErrorAndReturn(err, rsv)
	}
	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		return setErrorAndReturn(errors.New("Context payload not found"), rsv)
	}
	sprites := payload.Sprite()
	sprites.RLock()
	defer sprites.RUnlock()
	imgs, ok := sprites.M[glob]
	if !ok {
		keys := make([]string, 0, len(sprites.M))
		for i := range sprites.M {
			keys = append(keys, i)
		}

		err := fmt.Errorf(
			"Variable not found matching glob: %s sprite:%s", glob, name)
		return setErrorAndReturn(err, rsv)
	}

	path, err := imgs.OutputPath()

	// FIXME: path directory can not be trusted, rebuild this from the context
	if ctx.HTTPPath == "" {
		ctxPath, _ := filepath.Rel(ctx.BuildDir, ctx.GenImgDir)
		path = strings.Join([]string{ctxPath, filepath.Base(path)}, "/")
	} else {
		u, err := url.Parse(ctx.HTTPPath)
		if err != nil {
			return setErrorAndReturn(err, rsv)
		}
		u.Path = strings.Join([]string{u.Path, "build", filepath.Base(path)}, "/")
		path = u.String()
	}
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	if imgs.Lookup(name) == -1 {
		return setErrorAndReturn(fmt.Errorf("image %s not found\n"+
			"   try one of these: %v", name, imgs.Paths), rsv)
	}
	// This is an odd name for what it does
	pos := imgs.GetPack(imgs.Lookup(name))

	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	x := libs.SassNumber{Unit: "px", Value: float64(-pos.X)}
	x = x.Add(offsetX)

	y := libs.SassNumber{Unit: "px", Value: float64(-pos.Y)}
	y = y.Add(offsetY)

	str, err := libsass.Marshal(
		fmt.Sprintf(`url("%s") %s %s`,
			path, x, y,
		))
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	if rsv != nil {
		*rsv = str
	}
	return nil
}

// SpriteMap returns a sprite from the passed glob and sprite
// parameters.
func SpriteMap(v interface{}, usv libsass.SassValue, rsv *libsass.SassValue) error {
	ctx := v.(*libsass.Context)
	var glob string
	var spacing libs.SassNumber
	err := libsass.Unmarshal(usv, &glob, &spacing)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}
	imgs := sw.New(&sw.Options{
		ImageDir:  ctx.ImageDir,
		BuildDir:  ctx.BuildDir,
		GenImgDir: ctx.GenImgDir,
		Padding:   int(spacing.Value),
	})
	if cglob, err := strconv.Unquote(glob); err == nil {
		glob = cglob
	}

	key := glob + strconv.FormatInt(int64(spacing.Value), 10)

	payload, ok := ctx.Payload.(sw.Spriter)
	if !ok {
		err := errors.New("context payload not found")
		return setErrorAndReturn(err, rsv)
	}

	sprites := payload.Sprite()

	// TODO: benchmark a single write lock against this
	// read lock then write lock
	sprites.RLock()
	if _, ok := sprites.M[key]; ok {
		defer sprites.RUnlock()
		res, err := libsass.Marshal(key)
		if err != nil {
			return setErrorAndReturn(err, rsv)
		}
		if rsv != nil {
			*rsv = res
		}
		return nil
	}
	sprites.RUnlock()

	err = imgs.Decode(glob)
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	path, err := imgs.Export()
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	res, err := libsass.Marshal(key)
	sprites.Lock()
	sprites.M[key] = imgs
	sprites.Unlock()
	if err != nil {
		return setErrorAndReturn(err, rsv)
	}

	if rsv != nil {
		*rsv = res
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Jetstack Ltd.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"k8s.io/api/core/v1"
	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/kubernetes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	clientset "github.com/jetstack-experimental/cert-manager/pkg/client"
	"github.com/jetstack-experimental/cert-manager/test/util"
)

// Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
type Framework struct {
	BaseName string

	// A Kubernetes and Service Catalog client
	KubeClientSet          kubernetes.Interface
	CertManagerClientSet   clientset.Interface
	APIExtensionsClientSet apiextcs.Interface

	// Namespace in which all test resources should reside
	Namespace *v1.Namespace

	// To make sure that this framework cleans up after itself, no matter what,
	// we install a Cleanup action before each test and clear it after.  If we
	// should abort, the AfterSuite hook should run all Cleanup actions.
	cleanupHandle CleanupActionHandle
}

// NewFramework makes a new framework and sets up a BeforeEach/AfterEach for
// you (you can write additional before/after each functions).
func NewDefaultFramework(baseName string) *Framework {
	f := &Framework{
		BaseName: baseName,
	}

	BeforeEach(f.BeforeEach)
	AfterEach(f.AfterEach)

	return f
}

// BeforeEach gets a client and makes a namespace.
func (f *Framework) BeforeEach() {
	f.cleanupHandle = AddCleanupAction(f.AfterEach)

	By("Creating a kubernetes client")
	kubeConfig, err := LoadConfig(TestContext.KubeConfig, TestContext.KubeContext)
	Expect(err).NotTo(HaveOccurred())

	f.KubeClientSet, err = kubernetes.NewForConfig(kubeConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Creating an API extensions client")
	f.APIExtensionsClientSet, err = apiextcs.NewForConfig(kubeConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Creating a cert manager client")
	certManagerConfig, err := LoadConfig(TestContext.CertManagerConfig, TestContext.CertManagerContext)
	Expect(err).NotTo(HaveOccurred())
	f.CertManagerClientSet, err = clientset.NewForConfig(certManagerConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Building a namespace api object")
	namespace, err := CreateKubeNamespace(f.BaseName, f.KubeClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Building an Issuer CustomResourceDefinition api object")
	err = CreateIssuerCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Building an Certificate CustomResourceDefinition api object")
	err = CreateCertificateCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	f.Namespace = namespace
}

// AfterEach deletes the namespace, after reading its events.
func (f *Framework) AfterEach() {
	RemoveCleanupAction(f.cleanupHandle)

	By("Deleting test namespace")
	err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace.Name)
	Expect(err).NotTo(HaveOccurred())

	By("Deleting Issuer CustomResourceDefinition")
	err = DeleteIssuerCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Deleting Certificate CustomResourceDefinition")
	err = DeleteCertificateCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	err = util.WaitForCRDToNotExist(f.APIExtensionsClientSet.ApiextensionsV1beta1().CustomResourceDefinitions(), issuerCrd().Name)
	Expect(err).NotTo(HaveOccurred())
	err = util.WaitForCRDToNotExist(f.APIExtensionsClientSet.ApiextensionsV1beta1().CustomResourceDefinitions(), certificateCrd().Name)
	Expect(err).NotTo(HaveOccurred())
}

// Wrapper function for ginkgo describe.  Adds namespacing.
func CertManagerDescribe(text string, body func()) bool {
	return Describe("[cert-manager] "+text, body)
}
<commit_msg>Fix e2e tests<commit_after>/*
Copyright 2017 Jetstack Ltd.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"k8s.io/api/core/v1"
	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/kubernetes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	clientset "github.com/jetstack-experimental/cert-manager/pkg/client/clientset"
	"github.com/jetstack-experimental/cert-manager/test/util"
)

// Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
type Framework struct {
	BaseName string

	// A Kubernetes and Service Catalog client
	KubeClientSet          kubernetes.Interface
	CertManagerClientSet   clientset.Interface
	APIExtensionsClientSet apiextcs.Interface

	// Namespace in which all test resources should reside
	Namespace *v1.Namespace

	// To make sure that this framework cleans up after itself, no matter what,
	// we install a Cleanup action before each test and clear it after.  If we
	// should abort, the AfterSuite hook should run all Cleanup actions.
	cleanupHandle CleanupActionHandle
}

// NewFramework makes a new framework and sets up a BeforeEach/AfterEach for
// you (you can write additional before/after each functions).
func NewDefaultFramework(baseName string) *Framework {
	f := &Framework{
		BaseName: baseName,
	}

	BeforeEach(f.BeforeEach)
	AfterEach(f.AfterEach)

	return f
}

// BeforeEach gets a client and makes a namespace.
func (f *Framework) BeforeEach() {
	f.cleanupHandle = AddCleanupAction(f.AfterEach)

	By("Creating a kubernetes client")
	kubeConfig, err := LoadConfig(TestContext.KubeConfig, TestContext.KubeContext)
	Expect(err).NotTo(HaveOccurred())

	f.KubeClientSet, err = kubernetes.NewForConfig(kubeConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Creating an API extensions client")
	f.APIExtensionsClientSet, err = apiextcs.NewForConfig(kubeConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Creating a cert manager client")
	certManagerConfig, err := LoadConfig(TestContext.CertManagerConfig, TestContext.CertManagerContext)
	Expect(err).NotTo(HaveOccurred())
	f.CertManagerClientSet, err = clientset.NewForConfig(certManagerConfig)
	Expect(err).NotTo(HaveOccurred())

	By("Building a namespace api object")
	namespace, err := CreateKubeNamespace(f.BaseName, f.KubeClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Building an Issuer CustomResourceDefinition api object")
	err = CreateIssuerCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Building an Certificate CustomResourceDefinition api object")
	err = CreateCertificateCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	f.Namespace = namespace
}

// AfterEach deletes the namespace, after reading its events.
func (f *Framework) AfterEach() {
	RemoveCleanupAction(f.cleanupHandle)

	By("Deleting test namespace")
	err := DeleteKubeNamespace(f.KubeClientSet, f.Namespace.Name)
	Expect(err).NotTo(HaveOccurred())

	By("Deleting Issuer CustomResourceDefinition")
	err = DeleteIssuerCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	By("Deleting Certificate CustomResourceDefinition")
	err = DeleteCertificateCRD(f.APIExtensionsClientSet)
	Expect(err).NotTo(HaveOccurred())

	err = util.WaitForCRDToNotExist(f.APIExtensionsClientSet.ApiextensionsV1beta1().CustomResourceDefinitions(), issuerCrd().Name)
	Expect(err).NotTo(HaveOccurred())
	err = util.WaitForCRDToNotExist(f.APIExtensionsClientSet.ApiextensionsV1beta1().CustomResourceDefinitions(), certificateCrd().Name)
	Expect(err).NotTo(HaveOccurred())
}

// Wrapper function for ginkgo describe.  Adds namespacing.
func CertManagerDescribe(text string, body func()) bool {
	return Describe("[cert-manager] "+text, body)
}
<|endoftext|>


<commit_before>/* https://leetcode.com/problems/valid-anagram/#/description
Given two strings s and t, write a function to determine if t is an anagram of s.

For example,
s = "anagram", t = "nagaram", return true.
s = "rat", t = "car", return false.

Note:
You may assume the string contains only lowercase alphabets.

Follow up:
What if the inputs contain unicode characters? How would you adapt your solution to such case?
*/

package leetcode

func isAnagram(s string, t string) bool {
	m, n := len(s), len(t)
	if m != n {
		return false
	}

	maps := make(map[byte]int)
	for i := range s {
		maps[s[i]]++
		maps[t[i]]--
	}

	for _, v := range maps {
		if v != 0 {
			return false
		}
	}
	return true
}
<commit_msg>add another answer: 5 fast<commit_after>/* https://leetcode.com/problems/valid-anagram/#/description
Given two strings s and t, write a function to determine if t is an anagram of s.

For example,
s = "anagram", t = "nagaram", return true.
s = "rat", t = "car", return false.

Note:
You may assume the string contains only lowercase alphabets.

Follow up:
What if the inputs contain unicode characters? How would you adapt your solution to such case?
*/

package leetcode

func isAnagram(s string, t string) bool {
	if len(s) != len(t) {
		return false
	}
	record := make([]int, 26)
	count := len(s)
	for _, v := range s {
		record[v-'a']++
	}
	for _, v := range t {
		if record[v-'a'] >= 1 {
			count--
		}
		record[v-'a']--
	}
	return count == 0
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"golang.org/x/net/html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

func redirectFunc(req *http.Request, via []*http.Request) error {
	if len(via) > 10 {
		return errors.New("too many redirects")
	}
	req.Header = via[0].Header
	return nil
}

func RequestURL(url, method, username, password string) *http.Response {
	log.Printf("requesting %s.\n", url)

	client := &http.Client{
		CheckRedirect: redirectFunc,
	}

	var res *http.Response
	var err error
	if username == "" {
		res, err = client.Get(url)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			log.Fatal(err)
		}

		log.Printf("setting basic auth to %s:%s\n", username, password)
		req.SetBasicAuth(username, password)

		res, err = client.Do(req)
		if err != nil {
			log.Fatal(err)
		}
	}

	if res.StatusCode == 401 {
		log.Fatalf("status code %d for %s: auth challenge is [%s]\n", res.StatusCode, url, res.Header.Get("WWW-Authenticate"))
	}
	if res.StatusCode != 200 {
		log.Fatalf("status code %d for %s\n", res.StatusCode, url)
	}

	return res
}

func OpenURL(url, username, password string) io.ReadCloser {
	return RequestURL(url, "GET", username, password).Body
}

func SizeURL(url, username, password string) uint64 {
	size, _ := strconv.ParseUint(
		RequestURL(url, "HEAD", username, password).Header.Get("Content-Length"),
		10, 64,
	)
	return size
}

func ReadURL(url, username, password string) []byte {
	body := OpenURL(url, username, password)
	defer body.Close()

	data, err := ioutil.ReadAll(body)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("got %d bytes for %s\n", len(data), url)
	return data
}

func SaveBytes(data []byte, dst string) {
	file, err := os.Create(dst)
	if err != nil {
		log.Fatal(err)
	}

	defer file.Close()

	n, err := file.Write(data)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("saved %d bytes into %s\n", n, dst)
}

func SaveURL(url, username, password, dst string, file_size uint64, total_done uint64, total_size uint64) {
	log.Printf("%s -> %s\n", url, dst)

	reader := OpenURL(url, username, password)
	defer reader.Close()

	writer, err := os.Create(dst)
	if err != nil {
		log.Fatal(err)
	}
	defer writer.Close()

	var file_done uint64 = 0
	for {
		written, err := io.CopyN(writer, reader, 102400)
		file_done += uint64(written)
		log.Printf(
			"fetching %s (%d/%d bytes of file = %0.2f%%, %d/%d bytes total = %0.2f%%)\n",
			dst,
			file_done, file_size, float32(file_done) / float32(file_size),
			total_done + file_done, total_size, float32(total_done + file_done) / float32(total_size),
		)
		if err != nil {
			break
		}
	}
}

func ExtractMboxURLs(n *html.Node) []string {
	urls := make([]string, 0)
	if n.Type == html.ElementNode && n.Data == "a" {
		for _, attr := range n.Attr {
			if attr.Key == "href" && strings.Contains(attr.Val, "mbox") {
				urls = append(urls, attr.Val)
			}
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		urls = append(urls, ExtractMboxURLs(c)...)
	}
	return urls
}

type NameSize struct {
	Name string
	Size uint64
}

func main() {
	url := "http://postgresql.org/list/pgsql-hackers/"
	reader := OpenURL(url, "", "")
	defer reader.Close()
	doc, err := html.Parse(reader)
	if err != nil {
		log.Fatalf("could not parse %s", url)
	}
	urls := ExtractMboxURLs(doc)

	sizes := make(map[string]uint64)
	sizechan := make(chan NameSize)
	for _, url := range urls {
		go func(url string) {
			size := SizeURL(
				"http://postgresql.org" + url,
				"archives",
				"antispam",
			)
			log.Printf("%d : %s\n", size, url)
			sizechan <- NameSize{url, size}
		}(url)
	}
	for i := 0; i < len(urls); i++ {
		namesize := <-sizechan
		sizes[namesize.Name] = namesize.Size
	}

	var total_size uint64 = 0
	for url, size := range sizes {
		total_size += size
		log.Printf("%d = %s\n", size, url)
	}

	log.Printf("%d bytes total\n", total_size)

	var total_done uint64 = 0
	for _, url := range urls {
		file_name := filepath.Base(url)
		file_size := sizes[url]
		SaveURL(
			"http://postgresql.org" + url,
			"archives",
			"antispam",
			file_name,
			file_size,
			total_done,
			total_size,
		)
		total_done += file_size
	}
	log.Printf("done.\n")
}
<commit_msg>Fix percentage calculations.<commit_after>package main

import (
	"errors"
	"golang.org/x/net/html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

func redirectFunc(req *http.Request, via []*http.Request) error {
	if len(via) > 10 {
		return errors.New("too many redirects")
	}
	req.Header = via[0].Header
	return nil
}

func RequestURL(url, method, username, password string) *http.Response {
	log.Printf("requesting %s.\n", url)

	client := &http.Client{
		CheckRedirect: redirectFunc,
	}

	var res *http.Response
	var err error
	if username == "" {
		res, err = client.Get(url)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			log.Fatal(err)
		}

		log.Printf("setting basic auth to %s:%s\n", username, password)
		req.SetBasicAuth(username, password)

		res, err = client.Do(req)
		if err != nil {
			log.Fatal(err)
		}
	}

	if res.StatusCode == 401 {
		log.Fatalf("status code %d for %s: auth challenge is [%s]\n", res.StatusCode, url, res.Header.Get("WWW-Authenticate"))
	}
	if res.StatusCode != 200 {
		log.Fatalf("status code %d for %s\n", res.StatusCode, url)
	}

	return res
}

func OpenURL(url, username, password string) io.ReadCloser {
	return RequestURL(url, "GET", username, password).Body
}

func SizeURL(url, username, password string) uint64 {
	size, _ := strconv.ParseUint(
		RequestURL(url, "HEAD", username, password).Header.Get("Content-Length"),
		10, 64,
	)
	return size
}

func ReadURL(url, username, password string) []byte {
	body := OpenURL(url, username, password)
	defer body.Close()

	data, err := ioutil.ReadAll(body)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("got %d bytes for %s\n", len(data), url)
	return data
}

func SaveBytes(data []byte, dst string) {
	file, err := os.Create(dst)
	if err != nil {
		log.Fatal(err)
	}

	defer file.Close()

	n, err := file.Write(data)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("saved %d bytes into %s\n", n, dst)
}

func SaveURL(url, username, password, dst string, file_size uint64, total_done uint64, total_size uint64) {
	log.Printf("%s -> %s\n", url, dst)

	reader := OpenURL(url, username, password)
	defer reader.Close()

	writer, err := os.Create(dst)
	if err != nil {
		log.Fatal(err)
	}
	defer writer.Close()

	var file_done uint64 = 0
	for {
		written, err := io.CopyN(writer, reader, 102400)
		file_done += uint64(written)
		log.Printf(
			"fetching %s (%d/%d bytes of file = %0.2f%%, %d/%d bytes total = %0.2f%%)\n",
			dst,
			file_done, file_size, 100 * float32(file_done) / float32(file_size),
			total_done + file_done, total_size, 100 * float32(total_done + file_done) / float32(total_size),
		)
		if err != nil {
			break
		}
	}
}

func ExtractMboxURLs(n *html.Node) []string {
	urls := make([]string, 0)
	if n.Type == html.ElementNode && n.Data == "a" {
		for _, attr := range n.Attr {
			if attr.Key == "href" && strings.Contains(attr.Val, "mbox") {
				urls = append(urls, attr.Val)
			}
		}
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		urls = append(urls, ExtractMboxURLs(c)...)
	}
	return urls
}

type NameSize struct {
	Name string
	Size uint64
}

func main() {
	url := "http://postgresql.org/list/pgsql-hackers/"
	reader := OpenURL(url, "", "")
	defer reader.Close()
	doc, err := html.Parse(reader)
	if err != nil {
		log.Fatalf("could not parse %s", url)
	}
	urls := ExtractMboxURLs(doc)

	sizes := make(map[string]uint64)
	sizechan := make(chan NameSize)
	for _, url := range urls {
		go func(url string) {
			size := SizeURL(
				"http://postgresql.org" + url,
				"archives",
				"antispam",
			)
			log.Printf("%d : %s\n", size, url)
			sizechan <- NameSize{url, size}
		}(url)
	}
	for i := 0; i < len(urls); i++ {
		namesize := <-sizechan
		sizes[namesize.Name] = namesize.Size
	}

	var total_size uint64 = 0
	for url, size := range sizes {
		total_size += size
		log.Printf("%d = %s\n", size, url)
	}

	log.Printf("%d bytes total\n", total_size)

	var total_done uint64 = 0
	for _, url := range urls {
		file_name := filepath.Base(url)
		file_size := sizes[url]
		SaveURL(
			"http://postgresql.org" + url,
			"archives",
			"antispam",
			file_name,
			file_size,
			total_done,
			total_size,
		)
		total_done += file_size
	}
	log.Printf("done.\n")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"image"
	"image/jpeg"
	_ "image/png"
	"io/ioutil"
	"vip/fetch"
	"vip/test"

	. "gopkg.in/check.v1"
)

var (
	sizes = map[int]int{
		250:  333,
		500:  666,
		160:  213,
		720:  960,
		1024: 1365,
		683:  910,
		431:  574,
	}
	noExifSizes = map[int]int{
		250:  156,
		500:  312,
		160:  100,
		720:  450,
		1024: 640,
		683:  426,
		431:  269,
	}
)

var (
	_ = Suite(&ResizeSuite{})
)

type ResizeSuite struct{}

func (s *ResizeSuite) SetUpSuite(c *C) {
	setUpSuite(c)
}

func (s *ResizeSuite) SetUpTest(c *C) {
	setUpTest(c)

	storage = test.NewStore()
}

func (s *ResizeSuite) BenchmarkThumbnailResize(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	ctx := &fetch.CacheContext{
		Width: 160,
	}

	for i := 0; i < c.N; i++ {
		// Need a new io.Reader on every iteration
		buf := bytes.NewReader(file)
		_, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)
	}
}

func (s *ResizeSuite) BenchmarkLargeResize(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	ctx := &fetch.CacheContext{
		Width: 720,
	}

	for i := 0; i < c.N; i++ {
		// Need a new io.Reader on every iteration
		buf := bytes.NewReader(file)
		_, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)
	}
}

func (s *ResizeSuite) TestResizeImage(c *C) {
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			Width: width,
		}

		data := bytes.NewBuffer(file)

		orig, _, err := fetch.GetRotatedImage(data)
		c.Assert(err, IsNil)

		buf := new(bytes.Buffer)
		jpeg.Encode(buf, orig, nil)

		resized, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)

		image, _, err := image.Decode(resized)
		c.Assert(err, IsNil)
		c.Assert(image.Bounds().Size().X, Equals, width)
		c.Assert(image.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) TestResizeNoExifImage(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	for width, height := range noExifSizes {
		ctx := &fetch.CacheContext{
			Width: width,
		}

		buf := bytes.NewReader(file)
		resized, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)

		image, _, err := image.Decode(resized)
		c.Assert(err, IsNil)
		c.Assert(image.Bounds().Size().X, Equals, width)
		c.Assert(image.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) insertMockImage() (*fetch.CacheContext, error) {
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	if err != nil {
		return nil, err
	}

	data := bytes.NewBuffer(file)

	image, _, err := fetch.GetRotatedImage(data)
	if err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	jpeg.Encode(buf, image, nil)

	// Push the file data into the mock datastore
	storage.PutReader("test_bucket", "test_id", buf, int64(len(file)), "image/jpeg")

	return &fetch.CacheContext{
		ImageId: "test_id",
		Bucket:  "test_bucket",
	}, err
}

func (s *ResizeSuite) TestOriginalColdCache(c *C) {
	// Open the file once to get it's size
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	c.Assert(err, IsNil)

	img, _, err := image.Decode(bytes.NewReader(file))
	c.Assert(err, IsNil)

	// Since this image should be rotated, height should equal
	// width after it's uploaded.
	originalSize := img.Bounds().Size().Y

	// A single, unresized image is in the database/store
	ctx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	// Run the image resize request
	data, err := fetch.ImageData(storage, ctx)
	c.Assert(err, IsNil)

	// Verify the size of the resulting byte slice
	img, _, err = image.Decode(bytes.NewReader(data))
	c.Assert(err, IsNil)
	c.Assert(img.Bounds().Size().X, Equals, originalSize)
}

func (s *ResizeSuite) TestResizeColdCache(c *C) {
	// A single, unresized image is in the database/store
	mockCtx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			ImageId: mockCtx.ImageId,
			Bucket:  mockCtx.Bucket,
			Width:   width,
		}

		// Run the image resize request
		data, err := fetch.ImageData(storage, ctx)
		c.Assert(err, IsNil)

		// Verify the size of the resulting byte slice
		img, _, err := image.Decode(bytes.NewReader(data))
		c.Assert(err, IsNil)
		c.Assert(img.Bounds().Size().X, Equals, width)
		c.Assert(img.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) TestResizeCropColdCache(c *C) {
	// A single, unresized image is in the database/store
	mockCtx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			ImageId: mockCtx.ImageId,
			Bucket:  mockCtx.Bucket,
			Width:   width,
			Crop:    true,
		}

		// Run the image resize request
		data, err := fetch.ImageData(storage, ctx)
		c.Assert(err, IsNil)

		// Verify the size of the resulting byte slice
		img, _, err := image.Decode(bytes.NewReader(data))
		c.Assert(err, IsNil)
		c.Assert(img.Bounds().Size().X, Equals, img.Bounds().Size().Y)
		c.Assert(img.Bounds().Size().X > 0, Equals, true)
		c.Assert(img.Bounds().Size().X <= height, Equals, true)
	}
}
<commit_msg>Use Check vs Assert for tests in a loop.<commit_after>package main

import (
	"bytes"
	"image"
	"image/jpeg"
	_ "image/png"
	"io/ioutil"
	"vip/fetch"
	"vip/test"

	. "gopkg.in/check.v1"
)

var (
	sizes = map[int]int{
		250:  333,
		500:  666,
		160:  213,
		720:  960,
		1024: 1365,
		683:  910,
		431:  574,
	}
	noExifSizes = map[int]int{
		250:  156,
		500:  312,
		160:  100,
		720:  450,
		1024: 640,
		683:  426,
		431:  269,
	}
)

var (
	_ = Suite(&ResizeSuite{})
)

type ResizeSuite struct{}

func (s *ResizeSuite) SetUpSuite(c *C) {
	setUpSuite(c)
}

func (s *ResizeSuite) SetUpTest(c *C) {
	setUpTest(c)

	storage = test.NewStore()
}

func (s *ResizeSuite) BenchmarkThumbnailResize(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	ctx := &fetch.CacheContext{
		Width: 160,
	}

	for i := 0; i < c.N; i++ {
		// Need a new io.Reader on every iteration
		buf := bytes.NewReader(file)
		_, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)
	}
}

func (s *ResizeSuite) BenchmarkLargeResize(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	ctx := &fetch.CacheContext{
		Width: 720,
	}

	for i := 0; i < c.N; i++ {
		// Need a new io.Reader on every iteration
		buf := bytes.NewReader(file)
		_, err := fetch.Resize(buf, ctx)
		c.Assert(err, IsNil)
	}
}

func (s *ResizeSuite) TestResizeImage(c *C) {
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			Width: width,
		}

		data := bytes.NewBuffer(file)

		orig, _, err := fetch.GetRotatedImage(data)
		c.Check(err, IsNil)

		buf := new(bytes.Buffer)
		jpeg.Encode(buf, orig, nil)

		resized, err := fetch.Resize(buf, ctx)
		c.Check(err, IsNil)

		image, _, err := image.Decode(resized)
		c.Check(err, IsNil)
		c.Check(image.Bounds().Size().X, Equals, width)
		c.Check(image.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) TestResizeNoExifImage(c *C) {
	file, err := ioutil.ReadFile("test/AWESOME.jpg")
	c.Assert(err, IsNil)

	for width, height := range noExifSizes {
		ctx := &fetch.CacheContext{
			Width: width,
		}

		buf := bytes.NewReader(file)
		resized, err := fetch.Resize(buf, ctx)
		c.Check(err, IsNil)

		image, _, err := image.Decode(resized)
		c.Check(err, IsNil)
		c.Check(image.Bounds().Size().X, Equals, width)
		c.Check(image.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) insertMockImage() (*fetch.CacheContext, error) {
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	if err != nil {
		return nil, err
	}

	data := bytes.NewBuffer(file)

	image, _, err := fetch.GetRotatedImage(data)
	if err != nil {
		return nil, err
	}

	buf := new(bytes.Buffer)
	jpeg.Encode(buf, image, nil)

	// Push the file data into the mock datastore
	storage.PutReader("test_bucket", "test_id", buf, int64(len(file)), "image/jpeg")

	return &fetch.CacheContext{
		ImageId: "test_id",
		Bucket:  "test_bucket",
	}, err
}

func (s *ResizeSuite) TestOriginalColdCache(c *C) {
	// Open the file once to get it's size
	file, err := ioutil.ReadFile("test/exif_test_img.jpg")
	c.Assert(err, IsNil)

	img, _, err := image.Decode(bytes.NewReader(file))
	c.Assert(err, IsNil)

	// Since this image should be rotated, height should equal
	// width after it's uploaded.
	originalSize := img.Bounds().Size().Y

	// A single, unresized image is in the database/store
	ctx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	// Run the image resize request
	data, err := fetch.ImageData(storage, ctx)
	c.Assert(err, IsNil)

	// Verify the size of the resulting byte slice
	img, _, err = image.Decode(bytes.NewReader(data))
	c.Assert(err, IsNil)
	c.Assert(img.Bounds().Size().X, Equals, originalSize)
}

func (s *ResizeSuite) TestResizeColdCache(c *C) {
	// A single, unresized image is in the database/store
	mockCtx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			ImageId: mockCtx.ImageId,
			Bucket:  mockCtx.Bucket,
			Width:   width,
		}

		// Run the image resize request
		data, err := fetch.ImageData(storage, ctx)
		c.Assert(err, IsNil)

		// Verify the size of the resulting byte slice
		img, _, err := image.Decode(bytes.NewReader(data))
		c.Assert(err, IsNil)
		c.Assert(img.Bounds().Size().X, Equals, width)
		c.Assert(img.Bounds().Size().Y, Equals, height)
	}
}

func (s *ResizeSuite) TestResizeCropColdCache(c *C) {
	// A single, unresized image is in the database/store
	mockCtx, err := s.insertMockImage()
	c.Assert(err, IsNil)

	for width, height := range sizes {
		ctx := &fetch.CacheContext{
			ImageId: mockCtx.ImageId,
			Bucket:  mockCtx.Bucket,
			Width:   width,
			Crop:    true,
		}

		// Run the image resize request
		data, err := fetch.ImageData(storage, ctx)
		c.Assert(err, IsNil)

		// Verify the size of the resulting byte slice
		img, _, err := image.Decode(bytes.NewReader(data))
		c.Assert(err, IsNil)
		c.Assert(img.Bounds().Size().X, Equals, img.Bounds().Size().Y)
		c.Assert(img.Bounds().Size().X > 0, Equals, true)
		c.Assert(img.Bounds().Size().X <= height, Equals, true)
	}
}
<|endoftext|>


<commit_before>package hosttree

import (
	"crypto/rand"
	"errors"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// errWeightTooHeavy is returned from a SelectRandom() call if a weight that exceeds
	// the total weight of the tree is requested.
	errWeightTooHeavy = errors.New("requested a too-heavy weight")

	// errNegativeWeight is returned from an Insert() call if an entry with a
	// negative weight is added to the tree. Entries must always have a positive
	// weight.
	errNegativeWeight = errors.New("cannot insert using a negative weight")

	// errNilEntry is returned if a fetch call results in a nil tree entry. nodes
	// should always have a non-nil entry, unless they have been Delete()ed.
	errNilEntry = errors.New("node has a nil entry")

	// errHostExists is returned if an Insert is called with a public key that
	// already exists in the tree.
	errHostExists = errors.New("host already exists in the tree")

	// errNoSuchHost is returned if Remove is called with a public key that does
	// not exist in the tree.
	errNoSuchHost = errors.New("no host with specified public key")
)

type (
	// WeightFunc is a function used to weight a given HostDBEntry in the tree.
	WeightFunc func(modules.HostDBEntry) types.Currency

	// HostTree is used to store and select host database entries. Each HostTree
	// is initialized with a weighting func that is able to assign a weight to
	// each entry. The entries can then be selected at random, weighted by the
	// weight func.
	HostTree struct {
		root *node

		// hosts is a map of public keys to nodes.
		hosts map[string]*node

		// weight applies a weight to a hostEntry
		weight WeightFunc

		mu sync.Mutex
	}

	// hostEntry is an entry in the host tree.
	hostEntry struct {
		modules.HostDBEntry
		weight types.Currency
	}

	// node is a node in the tree.
	node struct {
		parent *node
		left   *node
		right  *node

		count int  // cumulative count of this node and all children
		taken bool // `taken` indicates whether there is an active host at this node or not.

		weight types.Currency
		entry  *hostEntry
	}
)

// createNode creates a new node using the provided `parent` and `entry`.
func createNode(parent *node, entry *hostEntry) *node {
	return &node{
		parent: parent,
		weight: entry.weight,
		count:  1,

		taken: true,
		entry: entry,
	}
}

// New creates a new, empty, HostTree. It takes one argument, a `WeightFunc`,
// which is used to determine the weight of a node on Insert.
func New(wf WeightFunc) *HostTree {
	return &HostTree{
		root: &node{
			count: 1,
		},
		weight: wf,
		hosts:  make(map[string]*node),
	}
}

// recursiveInsert inserts an entry into the appropriate place in the tree. The
// running time of recursiveInsert is log(n) in the maximum number of elements
// that have ever been in the tree.
func (n *node) recursiveInsert(entry *hostEntry) (nodesAdded int, newnode *node) {
	// If there is no parent and no children, and the node is not taken, assign
	// this entry to this node.
	if n.parent == nil && n.left == nil && n.right == nil && !n.taken {
		n.entry = entry
		n.taken = true
		n.weight = entry.weight
		newnode = n
		return
	}

	n.weight = n.weight.Add(entry.weight)

	// If the current node is empty, add the entry but don't increase the
	// count.
	if !n.taken {
		n.taken = true
		n.entry = entry
		newnode = n
		return
	}

	// Insert the element into the lest populated side.
	if n.left == nil {
		n.left = createNode(n, entry)
		nodesAdded = 1
		newnode = n.left
	} else if n.right == nil {
		n.right = createNode(n, entry)
		nodesAdded = 1
		newnode = n.right
	} else if n.left.count <= n.right.count {
		nodesAdded, newnode = n.left.recursiveInsert(entry)
	} else {
		nodesAdded, newnode = n.right.recursiveInsert(entry)
	}

	n.count += nodesAdded
	return
}

// nodeAtWeight grabs an element in the tree that appears at the given weight.
// Though the tree has an arbitrary sorting, a sufficiently random weight will
// pull a random element. The tree is searched through in a post-ordered way.
func (n *node) nodeAtWeight(weight types.Currency) (*node, error) {
	// Sanity check - weight must be less than the total weight of the tree.
	if weight.Cmp(n.weight) > 0 {
		return nil, errWeightTooHeavy
	}

	// Check if the left or right child should be returned.
	if n.left != nil {
		if weight.Cmp(n.left.weight) < 0 {
			return n.left.nodeAtWeight(weight)
		}
		weight = weight.Sub(n.left.weight) // Search from the 0th index of the right side.
	}
	if n.right != nil && weight.Cmp(n.right.weight) < 0 {
		return n.right.nodeAtWeight(weight)
	}

	// Should we panic here instead?
	if !n.taken {
		return nil, errNilEntry
	}

	// Return the root entry.
	return n, nil
}

// remove takes a node and removes it from the tree by climbing through the
// list of parents. remove does not delete nodes.
func (n *node) remove() {
	n.weight = n.weight.Sub(n.entry.weight)
	n.taken = false
	current := n.parent
	for current != nil {
		current.weight = current.weight.Sub(n.entry.weight)
		current = current.parent
	}
}

// Insert inserts the entry provided to `entry` into the host tree. Insert will
// return an error if the input host already exists.
func (ht *HostTree) Insert(hdbe modules.HostDBEntry) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	entry := &hostEntry{
		HostDBEntry: hdbe,
		weight:      ht.weight(hdbe),
	}

	if _, exists := ht.hosts[string(entry.PublicKey.Key)]; exists {
		return errHostExists
	}

	_, node := ht.root.recursiveInsert(entry)

	ht.hosts[string(entry.PublicKey.Key)] = node
	return nil
}

// Remove removes the host with the public key provided by `pk`.
func (ht *HostTree) Remove(pk types.SiaPublicKey) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	node, exists := ht.hosts[string(pk.Key)]
	if !exists {
		return errNoSuchHost
	}
	node.remove()
	delete(ht.hosts, string(pk.Key))

	return nil
}

// Modify updates a host entry at the given public key, replacing the old entry
// with the entry provided by `newEntry`.
func (ht *HostTree) Modify(hdbe modules.HostDBEntry) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	node, exists := ht.hosts[string(hdbe.PublicKey.Key)]
	if !exists {
		return errNoSuchHost
	}

	node.remove()

	entry := &hostEntry{
		HostDBEntry: hdbe,
		weight:      ht.weight(hdbe),
	}

	_, node = ht.root.recursiveInsert(entry)

	ht.hosts[string(entry.PublicKey.Key)] = node
	return nil
}

// SelectRandom grabs a random n hosts from the tree. There will be no repeats, but
// the length of the slice returned may be less than n, and may even be zero.
// The hosts that are returned first have the higher priority. Hosts passed to
// 'ignore' will not be considered; pass `nil` if no blacklist is desired.
func (ht *HostTree) SelectRandom(n int, ignore []types.SiaPublicKey) ([]modules.HostDBEntry, error) {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	var hosts []modules.HostDBEntry
	var removedEntries []*hostEntry

	for _, pubkey := range ignore {
		node, exists := ht.hosts[string(pubkey.Key)]
		if !exists {
			continue
		}
		node.remove()
		delete(ht.hosts, string(pubkey.Key))
		removedEntries = append(removedEntries, node.entry)
	}

	for len(hosts) < n && len(ht.hosts) > 0 {
		randWeight, err := rand.Int(rand.Reader, ht.root.weight.Big())
		if err != nil {
			return hosts, err
		}
		node, err := ht.root.nodeAtWeight(types.NewCurrency(randWeight))
		if err != nil {
			return hosts, err
		}

		if node.entry.HostDBEntry.AcceptingContracts {
			hosts = append(hosts, node.entry.HostDBEntry)
		}

		removedEntries = append(removedEntries, node.entry)
		node.remove()
		delete(ht.hosts, string(node.entry.PublicKey.Key))
	}

	for _, entry := range removedEntries {
		_, node := ht.root.recursiveInsert(entry)

		ht.hosts[string(entry.PublicKey.Key)] = node
	}

	return hosts, nil
}
<commit_msg>rename weight -> weightFn<commit_after>package hosttree

import (
	"crypto/rand"
	"errors"
	"sync"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// errWeightTooHeavy is returned from a SelectRandom() call if a weight that exceeds
	// the total weight of the tree is requested.
	errWeightTooHeavy = errors.New("requested a too-heavy weight")

	// errNegativeWeight is returned from an Insert() call if an entry with a
	// negative weight is added to the tree. Entries must always have a positive
	// weight.
	errNegativeWeight = errors.New("cannot insert using a negative weight")

	// errNilEntry is returned if a fetch call results in a nil tree entry. nodes
	// should always have a non-nil entry, unless they have been Delete()ed.
	errNilEntry = errors.New("node has a nil entry")

	// errHostExists is returned if an Insert is called with a public key that
	// already exists in the tree.
	errHostExists = errors.New("host already exists in the tree")

	// errNoSuchHost is returned if Remove is called with a public key that does
	// not exist in the tree.
	errNoSuchHost = errors.New("no host with specified public key")
)

type (
	// WeightFunc is a function used to weight a given HostDBEntry in the tree.
	WeightFunc func(modules.HostDBEntry) types.Currency

	// HostTree is used to store and select host database entries. Each HostTree
	// is initialized with a weighting func that is able to assign a weight to
	// each entry. The entries can then be selected at random, weighted by the
	// weight func.
	HostTree struct {
		root *node

		// hosts is a map of public keys to nodes.
		hosts map[string]*node

		// weightFn calculates the weight of a hostEntry
		weightFn WeightFunc

		mu sync.Mutex
	}

	// hostEntry is an entry in the host tree.
	hostEntry struct {
		modules.HostDBEntry
		weight types.Currency
	}

	// node is a node in the tree.
	node struct {
		parent *node
		left   *node
		right  *node

		count int  // cumulative count of this node and all children
		taken bool // `taken` indicates whether there is an active host at this node or not.

		weight types.Currency
		entry  *hostEntry
	}
)

// createNode creates a new node using the provided `parent` and `entry`.
func createNode(parent *node, entry *hostEntry) *node {
	return &node{
		parent: parent,
		weight: entry.weight,
		count:  1,

		taken: true,
		entry: entry,
	}
}

// New creates a new, empty, HostTree. It takes one argument, a `WeightFunc`,
// which is used to determine the weight of a node on Insert.
func New(wf WeightFunc) *HostTree {
	return &HostTree{
		root: &node{
			count: 1,
		},
		weightFn: wf,
		hosts:    make(map[string]*node),
	}
}

// recursiveInsert inserts an entry into the appropriate place in the tree. The
// running time of recursiveInsert is log(n) in the maximum number of elements
// that have ever been in the tree.
func (n *node) recursiveInsert(entry *hostEntry) (nodesAdded int, newnode *node) {
	// If there is no parent and no children, and the node is not taken, assign
	// this entry to this node.
	if n.parent == nil && n.left == nil && n.right == nil && !n.taken {
		n.entry = entry
		n.taken = true
		n.weight = entry.weight
		newnode = n
		return
	}

	n.weight = n.weight.Add(entry.weight)

	// If the current node is empty, add the entry but don't increase the
	// count.
	if !n.taken {
		n.taken = true
		n.entry = entry
		newnode = n
		return
	}

	// Insert the element into the lest populated side.
	if n.left == nil {
		n.left = createNode(n, entry)
		nodesAdded = 1
		newnode = n.left
	} else if n.right == nil {
		n.right = createNode(n, entry)
		nodesAdded = 1
		newnode = n.right
	} else if n.left.count <= n.right.count {
		nodesAdded, newnode = n.left.recursiveInsert(entry)
	} else {
		nodesAdded, newnode = n.right.recursiveInsert(entry)
	}

	n.count += nodesAdded
	return
}

// nodeAtWeight grabs an element in the tree that appears at the given weight.
// Though the tree has an arbitrary sorting, a sufficiently random weight will
// pull a random element. The tree is searched through in a post-ordered way.
func (n *node) nodeAtWeight(weight types.Currency) (*node, error) {
	// Sanity check - weight must be less than the total weight of the tree.
	if weight.Cmp(n.weight) > 0 {
		return nil, errWeightTooHeavy
	}

	// Check if the left or right child should be returned.
	if n.left != nil {
		if weight.Cmp(n.left.weight) < 0 {
			return n.left.nodeAtWeight(weight)
		}
		weight = weight.Sub(n.left.weight) // Search from the 0th index of the right side.
	}
	if n.right != nil && weight.Cmp(n.right.weight) < 0 {
		return n.right.nodeAtWeight(weight)
	}

	// Should we panic here instead?
	if !n.taken {
		return nil, errNilEntry
	}

	// Return the root entry.
	return n, nil
}

// remove takes a node and removes it from the tree by climbing through the
// list of parents. remove does not delete nodes.
func (n *node) remove() {
	n.weight = n.weight.Sub(n.entry.weight)
	n.taken = false
	current := n.parent
	for current != nil {
		current.weight = current.weight.Sub(n.entry.weight)
		current = current.parent
	}
}

// Insert inserts the entry provided to `entry` into the host tree. Insert will
// return an error if the input host already exists.
func (ht *HostTree) Insert(hdbe modules.HostDBEntry) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	entry := &hostEntry{
		HostDBEntry: hdbe,
		weight:      ht.weightFn(hdbe),
	}

	if _, exists := ht.hosts[string(entry.PublicKey.Key)]; exists {
		return errHostExists
	}

	_, node := ht.root.recursiveInsert(entry)

	ht.hosts[string(entry.PublicKey.Key)] = node
	return nil
}

// Remove removes the host with the public key provided by `pk`.
func (ht *HostTree) Remove(pk types.SiaPublicKey) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	node, exists := ht.hosts[string(pk.Key)]
	if !exists {
		return errNoSuchHost
	}
	node.remove()
	delete(ht.hosts, string(pk.Key))

	return nil
}

// Modify updates a host entry at the given public key, replacing the old entry
// with the entry provided by `newEntry`.
func (ht *HostTree) Modify(hdbe modules.HostDBEntry) error {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	node, exists := ht.hosts[string(hdbe.PublicKey.Key)]
	if !exists {
		return errNoSuchHost
	}

	node.remove()

	entry := &hostEntry{
		HostDBEntry: hdbe,
		weight:      ht.weightFn(hdbe),
	}

	_, node = ht.root.recursiveInsert(entry)

	ht.hosts[string(entry.PublicKey.Key)] = node
	return nil
}

// SelectRandom grabs a random n hosts from the tree. There will be no repeats, but
// the length of the slice returned may be less than n, and may even be zero.
// The hosts that are returned first have the higher priority. Hosts passed to
// 'ignore' will not be considered; pass `nil` if no blacklist is desired.
func (ht *HostTree) SelectRandom(n int, ignore []types.SiaPublicKey) ([]modules.HostDBEntry, error) {
	ht.mu.Lock()
	defer ht.mu.Unlock()

	var hosts []modules.HostDBEntry
	var removedEntries []*hostEntry

	for _, pubkey := range ignore {
		node, exists := ht.hosts[string(pubkey.Key)]
		if !exists {
			continue
		}
		node.remove()
		delete(ht.hosts, string(pubkey.Key))
		removedEntries = append(removedEntries, node.entry)
	}

	for len(hosts) < n && len(ht.hosts) > 0 {
		randWeight, err := rand.Int(rand.Reader, ht.root.weight.Big())
		if err != nil {
			return hosts, err
		}
		node, err := ht.root.nodeAtWeight(types.NewCurrency(randWeight))
		if err != nil {
			return hosts, err
		}

		if node.entry.HostDBEntry.AcceptingContracts {
			hosts = append(hosts, node.entry.HostDBEntry)
		}

		removedEntries = append(removedEntries, node.entry)
		node.remove()
		delete(ht.hosts, string(node.entry.PublicKey.Key))
	}

	for _, entry := range removedEntries {
		_, node := ht.root.recursiveInsert(entry)

		ht.hosts[string(entry.PublicKey.Key)] = node
	}

	return hosts, nil
}
<|endoftext|>