<commit_before>package hashstructure

import (
	"testing"
)

func TestHash_identity(t *testing.T) {
	cases := []interface{}{
		nil,
		"foo",
		42,
		true,
		false,
		[]string{"foo", "bar"},
		[]interface{}{1, nil, "foo"},
		map[string]string{"foo": "bar"},
		map[interface{}]string{"foo": "bar"},
		map[interface{}]interface{}{"foo": "bar", "bar": 0},
		struct {
			Foo string
			Bar []interface{}
		}{
			Foo: "foo",
			Bar: []interface{}{nil, nil, nil},
		},
		&struct {
			Foo string
			Bar []interface{}
		}{
			Foo: "foo",
			Bar: []interface{}{nil, nil, nil},
		},
	}

	for _, tc := range cases {
		// We run the test 100 times to try to tease out variability
		// in the runtime in terms of ordering.
		valuelist := make([]uint64, 100)
		for i, _ := range valuelist {
			v, err := Hash(tc, nil)
			if err != nil {
				t.Fatalf("Error: %s\n\n%#v", err, tc)
			}

			valuelist[i] = v
		}

		// Zero is always wrong
		if valuelist[0] == 0 {
			t.Fatalf("zero hash: %#v", tc)
		}

		// Make sure all the values match
		t.Logf("%#v: %d", tc, valuelist[0])
		for i := 1; i < len(valuelist); i++ {
			if valuelist[i] != valuelist[0] {
				t.Fatalf("non-matching: %d, %d\n\n%#v", i, 0, tc)
			}
		}
	}
}

func TestHash_equal(t *testing.T) {
	type testFoo struct{ Name string }
	type testBar struct{ Name string }

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			map[string]string{"foo": "bar"},
			map[interface{}]string{"foo": "bar"},
			true,
		},

		{
			map[string]interface{}{"1": "1"},
			map[string]interface{}{"1": "1", "2": "2"},
			false,
		},

		{
			struct{ Fname, Lname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"bar", "foo"},
			false,
		},

		{
			struct{ Lname, Fname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"foo", "bar"},
			false,
		},

		{
			struct{ Lname, Fname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"bar", "foo"},
			true,
		},

		{
			testFoo{"foo"},
			testBar{"foo"},
			false,
		},

		{
			struct {
				Foo        string
				unexported string
			}{
				Foo:        "bar",
				unexported: "baz",
			},
			struct {
				Foo        string
				unexported string
			}{
				Foo:        "bar",
				unexported: "bang",
			},
			true,
		},
	}

	for _, tc := range cases {
		t.Logf("Hashing: %#v", tc.One)
		one, err := Hash(tc.One, nil)
		t.Logf("Result: %d", one)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		t.Logf("Hashing: %#v", tc.Two)
		two, err := Hash(tc.Two, nil)
		t.Logf("Result: %d", two)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_equalIgnore(t *testing.T) {
	type Test1 struct {
		Name string
		UUID string `hash:"ignore"`
	}

	type Test2 struct {
		Name string
		UUID string `hash:"-"`
	}

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			Test1{Name: "foo", UUID: "foo"},
			Test1{Name: "foo", UUID: "bar"},
			true,
		},

		{
			Test1{Name: "foo", UUID: "foo"},
			Test1{Name: "foo", UUID: "foo"},
			true,
		},

		{
			Test2{Name: "foo", UUID: "foo"},
			Test2{Name: "foo", UUID: "bar"},
			true,
		},

		{
			Test2{Name: "foo", UUID: "foo"},
			Test2{Name: "foo", UUID: "foo"},
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_equalNil(t *testing.T) {
	type Test struct {
		Str   *string
		Int   *int
		Map   map[string]string
		Slice []string
	}

	cases := []struct {
		One, Two interface{}
		ZeroNil  bool
		Match    bool
	}{
		{
			Test{
				Str:   nil,
				Int:   nil,
				Map:   nil,
				Slice: nil,
			},
			Test{
				Str:   new(string),
				Int:   new(int),
				Map:   make(map[string]string),
				Slice: make([]string, 0),
			},
			true,
			true,
		},
		{
			Test{
				Str:   nil,
				Int:   nil,
				Map:   nil,
				Slice: nil,
			},
			Test{
				Str:   new(string),
				Int:   new(int),
				Map:   make(map[string]string),
				Slice: make([]string, 0),
			},
			false,
			false,
		},
		{
			nil,
			0,
			true,
			true,
		},
		{
			nil,
			0,
			false,
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, &HashOptions{ZeroNil: tc.ZeroNil})
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, &HashOptions{ZeroNil: tc.ZeroNil})
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_equalSet(t *testing.T) {
	type Test struct {
		Name    string
		Friends []string `hash:"set"`
	}

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			Test{Name: "foo", Friends: []string{"bar", "foo"}},
			true,
		},

		{
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_includable(t *testing.T) {
	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			testIncludable{Value: "foo"},
			testIncludable{Value: "foo"},
			true,
		},

		{
			testIncludable{Value: "foo", Ignore: "bar"},
			testIncludable{Value: "foo"},
			true,
		},

		{
			testIncludable{Value: "foo", Ignore: "bar"},
			testIncludable{Value: "bar"},
			false,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_includableMap(t *testing.T) {
	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			true,
		},

		{
			testIncludableMap{Map: map[string]string{"foo": "bar", "ignore": "true"}},
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			true,
		},

		{
			testIncludableMap{Map: map[string]string{"foo": "bar", "ignore": "true"}},
			testIncludableMap{Map: map[string]string{"bar": "baz"}},
			false,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

type testIncludable struct {
	Value  string
	Ignore string
}

func (t testIncludable) HashInclude(field string, v interface{}) (bool, error) {
	return field != "Ignore", nil
}

type testIncludableMap struct {
	Map map[string]string
}

func (t testIncludableMap) HashIncludeMap(field string, k, v interface{}) (bool, error) {
	if field != "Map" {
		return true, nil
	}

	if s, ok := k.(string); ok && s == "ignore" {
		return false, nil
	}

	return true, nil
}
<commit_msg>tests for unexported embedded structs<commit_after>package hashstructure

import (
	"fmt"
	"testing"
)

func TestHash_identity(t *testing.T) {
	cases := []interface{}{
		nil,
		"foo",
		42,
		true,
		false,
		[]string{"foo", "bar"},
		[]interface{}{1, nil, "foo"},
		map[string]string{"foo": "bar"},
		map[interface{}]string{"foo": "bar"},
		map[interface{}]interface{}{"foo": "bar", "bar": 0},
		struct {
			Foo string
			Bar []interface{}
		}{
			Foo: "foo",
			Bar: []interface{}{nil, nil, nil},
		},
		&struct {
			Foo string
			Bar []interface{}
		}{
			Foo: "foo",
			Bar: []interface{}{nil, nil, nil},
		},
	}

	for _, tc := range cases {
		// We run the test 100 times to try to tease out variability
		// in the runtime in terms of ordering.
		valuelist := make([]uint64, 100)
		for i, _ := range valuelist {
			v, err := Hash(tc, nil)
			if err != nil {
				t.Fatalf("Error: %s\n\n%#v", err, tc)
			}

			valuelist[i] = v
		}

		// Zero is always wrong
		if valuelist[0] == 0 {
			t.Fatalf("zero hash: %#v", tc)
		}

		// Make sure all the values match
		t.Logf("%#v: %d", tc, valuelist[0])
		for i := 1; i < len(valuelist); i++ {
			if valuelist[i] != valuelist[0] {
				t.Fatalf("non-matching: %d, %d\n\n%#v", i, 0, tc)
			}
		}
	}
}

func TestHash_equal(t *testing.T) {
	type testFoo struct{ Name string }
	type testBar struct{ Name string }

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			map[string]string{"foo": "bar"},
			map[interface{}]string{"foo": "bar"},
			true,
		},

		{
			map[string]interface{}{"1": "1"},
			map[string]interface{}{"1": "1", "2": "2"},
			false,
		},

		{
			struct{ Fname, Lname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"bar", "foo"},
			false,
		},

		{
			struct{ Lname, Fname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"foo", "bar"},
			false,
		},

		{
			struct{ Lname, Fname string }{"foo", "bar"},
			struct{ Fname, Lname string }{"bar", "foo"},
			true,
		},

		{
			testFoo{"foo"},
			testBar{"foo"},
			false,
		},

		{
			struct {
				Foo        string
				unexported string
			}{
				Foo:        "bar",
				unexported: "baz",
			},
			struct {
				Foo        string
				unexported string
			}{
				Foo:        "bar",
				unexported: "bang",
			},
			true,
		},

		{
			struct {
				testFoo
				Foo string
			}{
				Foo:     "bar",
				testFoo: testFoo{Name: "baz"},
			},
			struct {
				testFoo
				Foo string
			}{
				Foo: "bar",
			},
			true,
		},

		{
			struct {
				Foo string
			}{
				Foo: "bar",
			},
			struct {
				testFoo
				Foo string
			}{
				Foo: "bar",
			},
			true,
		},
	}

	for i, tc := range cases {
		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
			t.Logf("Hashing: %#v", tc.One)
			one, err := Hash(tc.One, nil)
			t.Logf("Result: %d", one)
			if err != nil {
				t.Fatalf("Failed to hash %#v: %s", tc.One, err)
			}
			t.Logf("Hashing: %#v", tc.Two)
			two, err := Hash(tc.Two, nil)
			t.Logf("Result: %d", two)
			if err != nil {
				t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
			}

			// Zero is always wrong
			if one == 0 {
				t.Fatalf("zero hash: %#v", tc.One)
			}

			// Compare
			if (one == two) != tc.Match {
				t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
			}
		})
	}
}

func TestHash_equalIgnore(t *testing.T) {
	type Test1 struct {
		Name string
		UUID string `hash:"ignore"`
	}

	type Test2 struct {
		Name string
		UUID string `hash:"-"`
	}

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			Test1{Name: "foo", UUID: "foo"},
			Test1{Name: "foo", UUID: "bar"},
			true,
		},

		{
			Test1{Name: "foo", UUID: "foo"},
			Test1{Name: "foo", UUID: "foo"},
			true,
		},

		{
			Test2{Name: "foo", UUID: "foo"},
			Test2{Name: "foo", UUID: "bar"},
			true,
		},

		{
			Test2{Name: "foo", UUID: "foo"},
			Test2{Name: "foo", UUID: "foo"},
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_equalNil(t *testing.T) {
	type Test struct {
		Str   *string
		Int   *int
		Map   map[string]string
		Slice []string
	}

	cases := []struct {
		One, Two interface{}
		ZeroNil  bool
		Match    bool
	}{
		{
			Test{
				Str:   nil,
				Int:   nil,
				Map:   nil,
				Slice: nil,
			},
			Test{
				Str:   new(string),
				Int:   new(int),
				Map:   make(map[string]string),
				Slice: make([]string, 0),
			},
			true,
			true,
		},
		{
			Test{
				Str:   nil,
				Int:   nil,
				Map:   nil,
				Slice: nil,
			},
			Test{
				Str:   new(string),
				Int:   new(int),
				Map:   make(map[string]string),
				Slice: make([]string, 0),
			},
			false,
			false,
		},
		{
			nil,
			0,
			true,
			true,
		},
		{
			nil,
			0,
			false,
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, &HashOptions{ZeroNil: tc.ZeroNil})
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, &HashOptions{ZeroNil: tc.ZeroNil})
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_equalSet(t *testing.T) {
	type Test struct {
		Name    string
		Friends []string `hash:"set"`
	}

	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			Test{Name: "foo", Friends: []string{"bar", "foo"}},
			true,
		},

		{
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			Test{Name: "foo", Friends: []string{"foo", "bar"}},
			true,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_includable(t *testing.T) {
	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			testIncludable{Value: "foo"},
			testIncludable{Value: "foo"},
			true,
		},

		{
			testIncludable{Value: "foo", Ignore: "bar"},
			testIncludable{Value: "foo"},
			true,
		},

		{
			testIncludable{Value: "foo", Ignore: "bar"},
			testIncludable{Value: "bar"},
			false,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

func TestHash_includableMap(t *testing.T) {
	cases := []struct {
		One, Two interface{}
		Match    bool
	}{
		{
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			true,
		},

		{
			testIncludableMap{Map: map[string]string{"foo": "bar", "ignore": "true"}},
			testIncludableMap{Map: map[string]string{"foo": "bar"}},
			true,
		},

		{
			testIncludableMap{Map: map[string]string{"foo": "bar", "ignore": "true"}},
			testIncludableMap{Map: map[string]string{"bar": "baz"}},
			false,
		},
	}

	for _, tc := range cases {
		one, err := Hash(tc.One, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.One, err)
		}
		two, err := Hash(tc.Two, nil)
		if err != nil {
			t.Fatalf("Failed to hash %#v: %s", tc.Two, err)
		}

		// Zero is always wrong
		if one == 0 {
			t.Fatalf("zero hash: %#v", tc.One)
		}

		// Compare
		if (one == two) != tc.Match {
			t.Fatalf("bad, expected: %#v\n\n%#v\n\n%#v", tc.Match, tc.One, tc.Two)
		}
	}
}

type testIncludable struct {
	Value  string
	Ignore string
}

func (t testIncludable) HashInclude(field string, v interface{}) (bool, error) {
	return field != "Ignore", nil
}

type testIncludableMap struct {
	Map map[string]string
}

func (t testIncludableMap) HashIncludeMap(field string, k, v interface{}) (bool, error) {
	if field != "Map" {
		return true, nil
	}

	if s, ok := k.(string); ok && s == "ignore" {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>package rtfdoc

import (
	"fmt"
	"unicode"
	"unicode/utf16"
)

// convertTextToUTF16 функция берет на вход строку и возвращает обратно строку,
// в которой русские буквы заменяются на эквиваленты в кодировке UTF-16
func convertNonASCIIToUTF16(text string) string {
	res := ""
	for _, r := range text {
		// if isCyrillicLetter(r) {
		if unicode.Is(unicode.Cyrillic, r) {
			res += fmt.Sprintf("\\u%d\\'3f", utf16.Encode([]rune{r})[0])
		} else {
			res += string(r)
		}
	}
	return res
}
<commit_msg>added № to cyrillic symbols<commit_after>package rtfdoc

import (
	"fmt"
	"unicode"
	"unicode/utf16"
)

// convertTextToUTF16 функция берет на вход строку и возвращает обратно строку,
// в которой русские буквы заменяются на эквиваленты в кодировке UTF-16
func convertNonASCIIToUTF16(text string) string {
	res := ""
	for _, r := range text {
		// if isCyrillicLetter(r) {
		if unicode.Is(unicode.Cyrillic, r) || r == '№' {
			res += fmt.Sprintf("\\u%d\\'3f", utf16.Encode([]rune{r})[0])
		} else {
			res += string(r)
		}
	}
	return res
}
<|endoftext|>


<commit_before>package mongo

import (
	"fmt"

	"github.com/globalsign/mgo/bson"
)

// GetClanNameTextIndexCommand returns a mongo command to create the clan names text index.
func GetClanNameTextIndexCommand(gameID string, background bool) bson.D {
	return bson.D{
		{Name: "createIndexes", Value: fmt.Sprintf("clans_%s", gameID)},
		{Name: "indexes", Value: []interface{}{
			bson.M{
				"key": bson.M{
					"name":         "text",
					"namePrefixes": "text",
				},
				"name":             fmt.Sprintf("clans_%s_name_text_namePrefixes_text_index", gameID),
				"background":       background,
				"default_language": "none",
			},
		}},
	}
}
<commit_msg>Improve text search route (#92)<commit_after>package mongo

import (
	"fmt"

	"github.com/globalsign/mgo/bson"
)

// GetClanNameTextIndexCommand returns a mongo command to create the clan names text index.
func GetClanNameTextIndexCommand(gameID string, background bool) bson.D {
	return bson.D{
		{Name: "createIndexes", Value: fmt.Sprintf("clans_%s", gameID)},
		{Name: "indexes", Value: []interface{}{
			bson.M{
				"key": bson.M{
					"name":         "text",
					"namePrefixes": "text",
				},
				"weights": bson.M{
					"name":         256,
					"namePrefixes": 1,
				},
				"name":             fmt.Sprintf("clans_%s_name_text_namePrefixes_text_index", gameID),
				"background":       background,
				"default_language": "none",
			},
		}},
	}
}
<|endoftext|>


<commit_before>package util

import (
	"io/ioutil"
	"net/http"
	"strings"
)

// FindUserIP gets the user's public IP by querying whatismyip.akamai.com
func FindUserIP() (string, error) {
	resp, err := http.Get("http://whatismyip.akamai.com")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	return strings.TrimSpace(string(bytes)), nil
}
<commit_msg>make user IP check more robust<commit_after>package util

import (
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

// FindUserIP gets the user's public IP by querying whatismyip.akamai.com
func FindUserIP() (string, error) {
	const retries = 10
	for i := 0; i < retries; i++ {
		resp, err := http.Get("http://whatismyip.akamai.com")
		if err != nil {
			return "", err
		}

		bytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		ip := strings.TrimSpace(string(bytes))
		if ip != "" {
			return ip, nil
		}
		time.Sleep(time.Duration(i) * time.Second)
	}
	return "", errors.New("timed out getting user IP")
}
<|endoftext|>


<commit_before>/*
 Brassica is a simple and resource efficient CMS for low dynamic
 private and small business sites with mostly static pages and simple
 structure.

 This package implements the main application and http server.
*/
package main

import (
	"datenkarussell.de/monsti/l10n"
	"datenkarussell.de/monsti/template"
	"datenkarussell.de/monsti/util"
	"datenkarussell.de/monsti/worker"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

// Site configuration.
type site struct {
	// Name of the site for internal use.
	Name string
	// Title as used in HTML head.
	Title string
	// The hosts which should deliver this site.
	Hosts []string
	// Name and email address of site owner.
	//
	// The owner's address is used as recipient of contact form submissions.
	Owner struct {
		Name, Email string
	}
	// Key to authenticate session cookies.
	SessionAuthKey string
	// Locale used to translate monsti's web interface.
	Locale string
	// Absolute paths to site specific directories.
	Directories struct {
		// Site content
		Data string
		// Site specific static files
		Statics string
	}
}

// Settings for the application and the sites.
type settings struct {
	// Settings for sending mail (outgoing SMTP).
	Mail struct {
		// Host may be specified as address:port
		Host, Username, Password string
	}
	// Absolute paths to used directories.
	Directories struct {
		// Config files
		Config string
		// Monsti's static files
		Statics string
		// HTML Templates
		Templates string
		// Locales, i.e. the gettext machine objects (.mo)
		Locales string
	}
	// List of node types to be activated.
	NodeTypes []string
	// Sites hosted by this monsti instance.
	Sites map[string]site
}

func main() {
	logger := log.New(os.Stderr, "monsti", log.LstdFlags)
	flag.Parse()
	if flag.NArg() != 1 {
		fmt.Printf("Usage: %v <config_directory>\n", filepath.Base(os.Args[0]))
		os.Exit(1)
	}
	cfgPath := util.GetConfigPath("monsti", flag.Arg(0))
	var settings settings
	err := util.ParseYAML(cfgPath, &settings)
	if err != nil {
		fmt.Println("Could not load configuration file: " + err.Error())
		os.Exit(1)
	}
	l10n.DefaultSettings.Domain = "monsti"
	l10n.DefaultSettings.Directory = settings.Directories.Locales
	handler := nodeHandler{
		Renderer:   template.Renderer{Root: settings.Directories.Templates},
		Settings:   settings,
		NodeQueues: make(map[string]chan worker.Ticket),
		Log:        logger}
	for _, ntype := range settings.NodeTypes {
		handler.AddNodeProcess(ntype, logger)
	}
	http.Handle("/static/", http.FileServer(http.Dir(
		filepath.Dir(settings.Directories.Statics))))
	handler.Hosts = make(map[string]string)
	for site_title, site := range settings.Sites {
		for _, host := range site.Hosts {
			handler.Hosts[host] = site_title
			http.Handle(host+"/site-static/", http.FileServer(http.Dir(
				filepath.Dir(site.Directories.Statics))))
		}
	}
	http.Handle("/", &handler)
	host := ":8080"
	c := make(chan int)
	go func() {
		http.ListenAndServe(host, nil)
		c <- 1
	}()
	log.Printf("Monsti is up and running. Listening on %q.", host)
	<-c
}
<commit_msg>Fix project name.<commit_after>/*
 Monsti is a simple and resource efficient CMS for low dynamic
 private and small business sites with mostly static pages and simple
 structure.

 This package implements the main application and http server.
*/
package main

import (
	"datenkarussell.de/monsti/l10n"
	"datenkarussell.de/monsti/template"
	"datenkarussell.de/monsti/util"
	"datenkarussell.de/monsti/worker"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
)

// Site configuration.
type site struct {
	// Name of the site for internal use.
	Name string
	// Title as used in HTML head.
	Title string
	// The hosts which should deliver this site.
	Hosts []string
	// Name and email address of site owner.
	//
	// The owner's address is used as recipient of contact form submissions.
	Owner struct {
		Name, Email string
	}
	// Key to authenticate session cookies.
	SessionAuthKey string
	// Locale used to translate monsti's web interface.
	Locale string
	// Absolute paths to site specific directories.
	Directories struct {
		// Site content
		Data string
		// Site specific static files
		Statics string
	}
}

// Settings for the application and the sites.
type settings struct {
	// Settings for sending mail (outgoing SMTP).
	Mail struct {
		// Host may be specified as address:port
		Host, Username, Password string
	}
	// Absolute paths to used directories.
	Directories struct {
		// Config files
		Config string
		// Monsti's static files
		Statics string
		// HTML Templates
		Templates string
		// Locales, i.e. the gettext machine objects (.mo)
		Locales string
	}
	// List of node types to be activated.
	NodeTypes []string
	// Sites hosted by this monsti instance.
	Sites map[string]site
}

func main() {
	logger := log.New(os.Stderr, "monsti", log.LstdFlags)
	flag.Parse()
	if flag.NArg() != 1 {
		fmt.Printf("Usage: %v <config_directory>\n", filepath.Base(os.Args[0]))
		os.Exit(1)
	}
	cfgPath := util.GetConfigPath("monsti", flag.Arg(0))
	var settings settings
	err := util.ParseYAML(cfgPath, &settings)
	if err != nil {
		fmt.Println("Could not load configuration file: " + err.Error())
		os.Exit(1)
	}
	l10n.DefaultSettings.Domain = "monsti"
	l10n.DefaultSettings.Directory = settings.Directories.Locales
	handler := nodeHandler{
		Renderer:   template.Renderer{Root: settings.Directories.Templates},
		Settings:   settings,
		NodeQueues: make(map[string]chan worker.Ticket),
		Log:        logger}
	for _, ntype := range settings.NodeTypes {
		handler.AddNodeProcess(ntype, logger)
	}
	http.Handle("/static/", http.FileServer(http.Dir(
		filepath.Dir(settings.Directories.Statics))))
	handler.Hosts = make(map[string]string)
	for site_title, site := range settings.Sites {
		for _, host := range site.Hosts {
			handler.Hosts[host] = site_title
			http.Handle(host+"/site-static/", http.FileServer(http.Dir(
				filepath.Dir(site.Directories.Statics))))
		}
	}
	http.Handle("/", &handler)
	host := ":8080"
	c := make(chan int)
	go func() {
		http.ListenAndServe(host, nil)
		c <- 1
	}()
	log.Printf("Monsti is up and running. Listening on %q.", host)
	<-c
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"path"
	"sort"
	"strconv"
	"sync"
	"time"

	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
)

// A server is an http.Handler that serves content within staticDir at root and
// the dynamically-generated dashboards at their respective endpoints.
type server struct {
	mux       *http.ServeMux
	staticDir string

	cMu              sync.RWMutex // Used to protect the fields below.
	corpus           *maintner.Corpus
	helpWantedIssues []int32
	userMapping      map[int]*maintner.GitHubUser // Gerrit Owner ID => GitHub user
	activities       []activity                   // All contribution activities
	totalPoints      int
}

func newServer(mux *http.ServeMux, staticDir string) *server {
	s := &server{
		mux:         mux,
		staticDir:   staticDir,
		userMapping: map[int]*maintner.GitHubUser{},
	}
	s.mux.Handle("/", http.FileServer(http.Dir(s.staticDir)))
	s.mux.HandleFunc("/favicon.ico", s.handleFavicon)
	s.mux.HandleFunc("/release", handleRelease)
	for _, p := range []string{"/imfeelinghelpful", "/imfeelinglucky"} {
		s.mux.HandleFunc(p, s.handleRandomHelpWantedIssue)
	}
	s.mux.HandleFunc("/_/activities", s.handleActivities)
	return s
}

// initCorpus fetches a full maintner corpus, overwriting any existing data.
func (s *server) initCorpus(ctx context.Context) error {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	corpus, err := godata.Get(ctx)
	if err != nil {
		return fmt.Errorf("godata.Get: %v", err)
	}
	s.corpus = corpus
	return nil
}

// corpusUpdateLoop continuously updates the server’s corpus until ctx’s Done
// channel is closed.
func (s *server) corpusUpdateLoop(ctx context.Context) {
	log.Println("Starting corpus update loop ...")
	for {
		log.Println("Updating help wanted issues ...")
		s.updateHelpWantedIssues()
		log.Println("Updating activities ...")
		s.updateActivities()
		err := s.corpus.UpdateWithLocker(ctx, &s.cMu)
		if err != nil {
			if err == maintner.ErrSplit {
				log.Println("Corpus out of sync. Re-fetching corpus.")
				s.initCorpus(ctx)
			} else {
				log.Printf("corpus.Update: %v; sleeping 15s", err)
				time.Sleep(15 * time.Second)
				continue
			}
		}

		select {
		case <-ctx.Done():
			return
		default:
			continue
		}
	}
}

const (
	labelIDHelpWanted         = 150880243
	issuesURLBase             = "https://github.com/golang/go/issues/"
	issueNumGerritUserMapping = 20945 // Special sign-up issue.
)

func (s *server) updateHelpWantedIssues() {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	repo := s.corpus.GitHub().Repo("golang", "go")
	if repo == nil {
		log.Println(`s.corpus.GitHub().Repo("golang", "go") = nil`)
		return
	}

	ids := []int32{}
	repo.ForeachIssue(func(i *maintner.GitHubIssue) error {
		if i.Closed {
			return nil
		}
		if _, ok := i.Labels[labelIDHelpWanted]; ok {
			ids = append(ids, i.Number)
		}
		return nil
	})
	s.helpWantedIssues = ids
}

// intFromStr returns the first integer within s, allowing for non-numeric
// characters to be present.
func intFromStr(s string) (int, bool) {
	var (
		foundNum bool
		r        int
	)
	for i := 0; i < len(s); i++ {
		if s[i] >= '0' && s[i] <= '9' {
			foundNum = true
			r = r*10 + int(s[i]-'0')
		} else if foundNum {
			return r, true
		}
	}
	if foundNum {
		return r, true
	}
	return 0, false
}

// Keep these in sync with the frontend JS.
const (
	activityTypeRegister     = "REGISTER"
	activityTypeCreateChange = "CREATE_CHANGE"
	activityTypeAmendChange  = "AMEND_CHANGE"
	activityTypeMergeChange  = "MERGE_CHANGE"
)

var pointsPerActivity = map[string]int{
	activityTypeRegister:     1,
	activityTypeCreateChange: 2,
	activityTypeAmendChange:  2,
	activityTypeMergeChange:  3,
}

// An activity represents something a contributor has done. e.g. register on
// the GitHub issue, create a change, amend a change, etc.
type activity struct {
	Type    string    `json:"type"`
	Created time.Time `json:"created"`
	User    string    `json:"gitHubUser"`
	Points  int       `json:"points"`
}

func (s *server) updateActivities() {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	repo := s.corpus.GitHub().Repo("golang", "go")
	if repo == nil {
		log.Println(`s.corpus.GitHub().Repo("golang", "go") = nil`)
		return
	}
	issue := repo.Issue(issueNumGerritUserMapping)
	if issue == nil {
		log.Printf("repo.Issue(%d) = nil", issueNumGerritUserMapping)
		return
	}
	latest := issue.Created
	if len(s.activities) > 0 {
		latest = s.activities[len(s.activities)-1].Created
	}

	var newActivities []activity
	issue.ForeachComment(func(c *maintner.GitHubComment) error {
		if !c.Created.After(latest) {
			return nil
		}
		id, ok := intFromStr(c.Body)
		if !ok {
			return fmt.Errorf("intFromStr(%q) = %v", c.Body, ok)
		}
		s.userMapping[id] = c.User

		newActivities = append(newActivities, activity{
			Type:    activityTypeRegister,
			Created: c.Created,
			User:    c.User.Login,
			Points:  pointsPerActivity[activityTypeRegister],
		})
		s.totalPoints += pointsPerActivity[activityTypeRegister]
		return nil
	})

	s.corpus.Gerrit().ForeachProjectUnsorted(func(p *maintner.GerritProject) error {
		p.ForeachCLUnsorted(func(cl *maintner.GerritCL) error {
			if !cl.Commit.CommitTime.After(latest) {
				return nil
			}
			user := s.userMapping[cl.OwnerID()]
			if user == nil {
				return nil
			}

			newActivities = append(newActivities, activity{
				Type:    activityTypeCreateChange,
				Created: cl.Created,
				User:    user.Login,
				Points:  pointsPerActivity[activityTypeCreateChange],
			})
			s.totalPoints += pointsPerActivity[activityTypeCreateChange]
			if cl.Version > 1 {
				newActivities = append(newActivities, activity{
					Type:    activityTypeAmendChange,
					Created: cl.Commit.CommitTime,
					User:    user.Login,
					Points:  pointsPerActivity[activityTypeAmendChange],
				})
				s.totalPoints += pointsPerActivity[activityTypeAmendChange]
			}
			if cl.Status == "merged" {
				newActivities = append(newActivities, activity{
					Type:    activityTypeMergeChange,
					Created: cl.Commit.CommitTime,
					User:    user.Login,
					Points:  pointsPerActivity[activityTypeMergeChange],
				})
				s.totalPoints += pointsPerActivity[activityTypeMergeChange]
			}
			return nil
		})
		return nil
	})

	sort.Sort(byCreated(newActivities))
	s.activities = append(s.activities, newActivities...)
}

type byCreated []activity

func (a byCreated) Len() int           { return len(a) }
func (a byCreated) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byCreated) Less(i, j int) bool { return a[i].Created.Before(a[j].Created) }

func (s *server) handleActivities(w http.ResponseWriter, r *http.Request) {
	i, _ := strconv.Atoi(r.FormValue("since"))
	since := time.Unix(int64(i)/1000, 0)

	recentActivity := []activity{}
	for _, a := range s.activities {
		if a.Created.After(since) {
			recentActivity = append(recentActivity, a)
		}
	}

	s.cMu.RLock()
	defer s.cMu.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	result := struct {
		Activities  []activity `json:"activities"`
		TotalPoints int        `json:"totalPoints"`
	}{
		Activities:  recentActivity,
		TotalPoints: s.totalPoints,
	}
	if err := json.NewEncoder(w).Encode(result); err != nil {
		log.Printf("Encode(%+v) = %v", result, err)
		return
	}
}

func (s *server) handleRandomHelpWantedIssue(w http.ResponseWriter, r *http.Request) {
	s.cMu.RLock()
	defer s.cMu.RUnlock()
	if len(s.helpWantedIssues) == 0 {
		http.Redirect(w, r, issuesURLBase, http.StatusSeeOther)
		return
	}
	rid := s.helpWantedIssues[rand.Intn(len(s.helpWantedIssues))]
	http.Redirect(w, r, issuesURLBase+strconv.Itoa(int(rid)), http.StatusSeeOther)
}

func (s *server) handleFavicon(w http.ResponseWriter, r *http.Request) {
	// Need to specify content type for consistent tests, without this it's
	// determined from mime.types on the box the test is running on
	w.Header().Set("Content-Type", "image/x-icon")
	http.ServeFile(w, r, path.Join(s.staticDir, "/favicon.ico"))
}

// ServeHTTP satisfies the http.Handler interface.
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.TLS != nil {
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; preload")
	}
	s.mux.ServeHTTP(w, r)
}

var (
	pageStoreMu sync.Mutex
	pageStore   = map[string][]byte{}
)

func getPage(name string) ([]byte, error) {
	pageStoreMu.Lock()
	defer pageStoreMu.Unlock()
	p, ok := pageStore[name]
	if ok {
		return p, nil
	}
	return nil, fmt.Errorf("page key %s not found", name)
}

func writePage(key string, content []byte) error {
	pageStoreMu.Lock()
	defer pageStoreMu.Unlock()
	pageStore[key] = content
	return nil
}

func servePage(w http.ResponseWriter, r *http.Request, key string) {
	b, err := getPage(key)
	if err != nil {
		log.Printf("getPage(%q) = %v", key, err)
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Write(b)
}

func handleRelease(w http.ResponseWriter, r *http.Request) {
	servePage(w, r, "release")
}
<commit_msg>devapp: update issue number for contributor workshop<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"path"
	"sort"
	"strconv"
	"sync"
	"time"

	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
)

// A server is an http.Handler that serves content within staticDir at root and
// the dynamically-generated dashboards at their respective endpoints.
type server struct {
	mux       *http.ServeMux
	staticDir string

	cMu              sync.RWMutex // Used to protect the fields below.
	corpus           *maintner.Corpus
	helpWantedIssues []int32
	userMapping      map[int]*maintner.GitHubUser // Gerrit Owner ID => GitHub user
	activities       []activity                   // All contribution activities
	totalPoints      int
}

func newServer(mux *http.ServeMux, staticDir string) *server {
	s := &server{
		mux:         mux,
		staticDir:   staticDir,
		userMapping: map[int]*maintner.GitHubUser{},
	}
	s.mux.Handle("/", http.FileServer(http.Dir(s.staticDir)))
	s.mux.HandleFunc("/favicon.ico", s.handleFavicon)
	s.mux.HandleFunc("/release", handleRelease)
	for _, p := range []string{"/imfeelinghelpful", "/imfeelinglucky"} {
		s.mux.HandleFunc(p, s.handleRandomHelpWantedIssue)
	}
	s.mux.HandleFunc("/_/activities", s.handleActivities)
	return s
}

// initCorpus fetches a full maintner corpus, overwriting any existing data.
func (s *server) initCorpus(ctx context.Context) error {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	corpus, err := godata.Get(ctx)
	if err != nil {
		return fmt.Errorf("godata.Get: %v", err)
	}
	s.corpus = corpus
	return nil
}

// corpusUpdateLoop continuously updates the server’s corpus until ctx’s Done
// channel is closed.
func (s *server) corpusUpdateLoop(ctx context.Context) {
	log.Println("Starting corpus update loop ...")
	for {
		log.Println("Updating help wanted issues ...")
		s.updateHelpWantedIssues()
		log.Println("Updating activities ...")
		s.updateActivities()
		err := s.corpus.UpdateWithLocker(ctx, &s.cMu)
		if err != nil {
			if err == maintner.ErrSplit {
				log.Println("Corpus out of sync. Re-fetching corpus.")
				s.initCorpus(ctx)
			} else {
				log.Printf("corpus.Update: %v; sleeping 15s", err)
				time.Sleep(15 * time.Second)
				continue
			}
		}

		select {
		case <-ctx.Done():
			return
		default:
			continue
		}
	}
}

const (
	labelIDHelpWanted         = 150880243
	issuesURLBase             = "https://github.com/golang/go/issues/"
	issueNumGerritUserMapping = 21017 // Special sign-up issue.
)

func (s *server) updateHelpWantedIssues() {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	repo := s.corpus.GitHub().Repo("golang", "go")
	if repo == nil {
		log.Println(`s.corpus.GitHub().Repo("golang", "go") = nil`)
		return
	}

	ids := []int32{}
	repo.ForeachIssue(func(i *maintner.GitHubIssue) error {
		if i.Closed {
			return nil
		}
		if _, ok := i.Labels[labelIDHelpWanted]; ok {
			ids = append(ids, i.Number)
		}
		return nil
	})
	s.helpWantedIssues = ids
}

// intFromStr returns the first integer within s, allowing for non-numeric
// characters to be present.
func intFromStr(s string) (int, bool) {
	var (
		foundNum bool
		r        int
	)
	for i := 0; i < len(s); i++ {
		if s[i] >= '0' && s[i] <= '9' {
			foundNum = true
			r = r*10 + int(s[i]-'0')
		} else if foundNum {
			return r, true
		}
	}
	if foundNum {
		return r, true
	}
	return 0, false
}

// Keep these in sync with the frontend JS.
const (
	activityTypeRegister     = "REGISTER"
	activityTypeCreateChange = "CREATE_CHANGE"
	activityTypeAmendChange  = "AMEND_CHANGE"
	activityTypeMergeChange  = "MERGE_CHANGE"
)

var pointsPerActivity = map[string]int{
	activityTypeRegister:     1,
	activityTypeCreateChange: 2,
	activityTypeAmendChange:  2,
	activityTypeMergeChange:  3,
}

// An activity represents something a contributor has done. e.g. register on
// the GitHub issue, create a change, amend a change, etc.
type activity struct {
	Type    string    `json:"type"`
	Created time.Time `json:"created"`
	User    string    `json:"gitHubUser"`
	Points  int       `json:"points"`
}

func (s *server) updateActivities() {
	s.cMu.Lock()
	defer s.cMu.Unlock()
	repo := s.corpus.GitHub().Repo("golang", "go")
	if repo == nil {
		log.Println(`s.corpus.GitHub().Repo("golang", "go") = nil`)
		return
	}
	issue := repo.Issue(issueNumGerritUserMapping)
	if issue == nil {
		log.Printf("repo.Issue(%d) = nil", issueNumGerritUserMapping)
		return
	}
	latest := issue.Created
	if len(s.activities) > 0 {
		latest = s.activities[len(s.activities)-1].Created
	}

	var newActivities []activity
	issue.ForeachComment(func(c *maintner.GitHubComment) error {
		if !c.Created.After(latest) {
			return nil
		}
		id, ok := intFromStr(c.Body)
		if !ok {
			return fmt.Errorf("intFromStr(%q) = %v", c.Body, ok)
		}
		s.userMapping[id] = c.User

		newActivities = append(newActivities, activity{
			Type:    activityTypeRegister,
			Created: c.Created,
			User:    c.User.Login,
			Points:  pointsPerActivity[activityTypeRegister],
		})
		s.totalPoints += pointsPerActivity[activityTypeRegister]
		return nil
	})

	s.corpus.Gerrit().ForeachProjectUnsorted(func(p *maintner.GerritProject) error {
		p.ForeachCLUnsorted(func(cl *maintner.GerritCL) error {
			if !cl.Commit.CommitTime.After(latest) {
				return nil
			}
			user := s.userMapping[cl.OwnerID()]
			if user == nil {
				return nil
			}

			newActivities = append(newActivities, activity{
				Type:    activityTypeCreateChange,
				Created: cl.Created,
				User:    user.Login,
				Points:  pointsPerActivity[activityTypeCreateChange],
			})
			s.totalPoints += pointsPerActivity[activityTypeCreateChange]
			if cl.Version > 1 {
				newActivities = append(newActivities, activity{
					Type:    activityTypeAmendChange,
					Created: cl.Commit.CommitTime,
					User:    user.Login,
					Points:  pointsPerActivity[activityTypeAmendChange],
				})
				s.totalPoints += pointsPerActivity[activityTypeAmendChange]
			}
			if cl.Status == "merged" {
				newActivities = append(newActivities, activity{
					Type:    activityTypeMergeChange,
					Created: cl.Commit.CommitTime,
					User:    user.Login,
					Points:  pointsPerActivity[activityTypeMergeChange],
				})
				s.totalPoints += pointsPerActivity[activityTypeMergeChange]
			}
			return nil
		})
		return nil
	})

	sort.Sort(byCreated(newActivities))
	s.activities = append(s.activities, newActivities...)
}

type byCreated []activity

func (a byCreated) Len() int           { return len(a) }
func (a byCreated) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byCreated) Less(i, j int) bool { return a[i].Created.Before(a[j].Created) }

func (s *server) handleActivities(w http.ResponseWriter, r *http.Request) {
	i, _ := strconv.Atoi(r.FormValue("since"))
	since := time.Unix(int64(i)/1000, 0)

	recentActivity := []activity{}
	for _, a := range s.activities {
		if a.Created.After(since) {
			recentActivity = append(recentActivity, a)
		}
	}

	s.cMu.RLock()
	defer s.cMu.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	result := struct {
		Activities  []activity `json:"activities"`
		TotalPoints int        `json:"totalPoints"`
	}{
		Activities:  recentActivity,
		TotalPoints: s.totalPoints,
	}
	if err := json.NewEncoder(w).Encode(result); err != nil {
		log.Printf("Encode(%+v) = %v", result, err)
		return
	}
}

func (s *server) handleRandomHelpWantedIssue(w http.ResponseWriter, r *http.Request) {
	s.cMu.RLock()
	defer s.cMu.RUnlock()
	if len(s.helpWantedIssues) == 0 {
		http.Redirect(w, r, issuesURLBase, http.StatusSeeOther)
		return
	}
	rid := s.helpWantedIssues[rand.Intn(len(s.helpWantedIssues))]
	http.Redirect(w, r, issuesURLBase+strconv.Itoa(int(rid)), http.StatusSeeOther)
}

func (s *server) handleFavicon(w http.ResponseWriter, r *http.Request) {
	// Need to specify content type for consistent tests, without this it's
	// determined from mime.types on the box the test is running on
	w.Header().Set("Content-Type", "image/x-icon")
	http.ServeFile(w, r, path.Join(s.staticDir, "/favicon.ico"))
}

// ServeHTTP satisfies the http.Handler interface.
func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.TLS != nil {
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; preload")
	}
	s.mux.ServeHTTP(w, r)
}

var (
	pageStoreMu sync.Mutex
	pageStore   = map[string][]byte{}
)

func getPage(name string) ([]byte, error) {
	pageStoreMu.Lock()
	defer pageStoreMu.Unlock()
	p, ok := pageStore[name]
	if ok {
		return p, nil
	}
	return nil, fmt.Errorf("page key %s not found", name)
}

func writePage(key string, content []byte) error {
	pageStoreMu.Lock()
	defer pageStoreMu.Unlock()
	pageStore[key] = content
	return nil
}

func servePage(w http.ResponseWriter, r *http.Request, key string) {
	b, err := getPage(key)
	if err != nil {
		log.Printf("getPage(%q) = %v", key, err)
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Write(b)
}

func handleRelease(w http.ResponseWriter, r *http.Request) {
	servePage(w, r, "release")
}
<|endoftext|>


<commit_before>package gorm

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

type mysql struct {
	commonDialect
}

func init() {
	RegisterDialect("mysql", &mysql{})
}

func (mysql) GetName() string {
	return "mysql"
}

func (mysql) Quote(key string) string {
	return fmt.Sprintf("`%s`", key)
}

// Get Data Type for MySQL Dialect
func (mysql) DataTypeOf(field *StructField) string {
	var dataValue, sqlType, size, additionalType = ParseFieldStructForDialect(field)

	if sqlType == "" {
		switch dataValue.Kind() {
		case reflect.Bool:
			sqlType = "boolean"
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "int AUTO_INCREMENT"
			} else {
				sqlType = "int"
			}
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "int unsigned AUTO_INCREMENT"
			} else {
				sqlType = "int unsigned"
			}
		case reflect.Int64:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "bigint AUTO_INCREMENT"
			} else {
				sqlType = "bigint"
			}
		case reflect.Uint64:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "bigint unsigned AUTO_INCREMENT"
			} else {
				sqlType = "bigint unsigned"
			}
		case reflect.Float32, reflect.Float64:
			sqlType = "double"
		case reflect.String:
			if size > 0 && size < 65532 {
				sqlType = fmt.Sprintf("varchar(%d)", size)
			} else {
				sqlType = "longtext"
			}
		case reflect.Struct:
			if _, ok := dataValue.Interface().(time.Time); ok {
				if _, ok := field.TagSettings["NOT NULL"]; ok {
					sqlType = "timestamp"
				} else {
					sqlType = "timestamp NULL"
				}
			}
		default:
			if _, ok := dataValue.Interface().([]byte); ok {
				if size > 0 && size < 65532 {
					sqlType = fmt.Sprintf("varbinary(%d)", size)
				} else {
					sqlType = "longblob"
				}
			}
		}
	}

	if sqlType == "" {
		panic(fmt.Sprintf("invalid sql type %s (%s) for mysql", dataValue.Type().Name(), dataValue.Kind().String()))
	}

	if strings.TrimSpace(additionalType) == "" {
		return sqlType
	}
	return fmt.Sprintf("%v %v", sqlType, additionalType)
}

func (s mysql) RemoveIndex(tableName string, indexName string) error {
	_, err := s.db.Exec(fmt.Sprintf("DROP INDEX %v ON %v", indexName, s.Quote(tableName)))
	return err
}

func (s mysql) HasForeignKey(tableName string, foreignKeyName string) bool {
	var count int
	s.db.QueryRow("SELECT count(*) FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=? AND TABLE_NAME=? AND CONSTRAINT_NAME=? AND CONSTRAINT_TYPE='FOREIGN KEY'", s.currentDatabase(), foreignKeyName).Scan(&count)
	return count > 0
}

func (s mysql) currentDatabase() (name string) {
	s.db.QueryRow("SELECT DATABASE()").Scan(&name)
	return
}

func (mysql) SelectFromDummyTable() string {
	return "FROM DUAL"
}
<commit_msg>fix mysql HasForeignKey<commit_after>package gorm

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

type mysql struct {
	commonDialect
}

func init() {
	RegisterDialect("mysql", &mysql{})
}

func (mysql) GetName() string {
	return "mysql"
}

func (mysql) Quote(key string) string {
	return fmt.Sprintf("`%s`", key)
}

// Get Data Type for MySQL Dialect
func (mysql) DataTypeOf(field *StructField) string {
	var dataValue, sqlType, size, additionalType = ParseFieldStructForDialect(field)

	if sqlType == "" {
		switch dataValue.Kind() {
		case reflect.Bool:
			sqlType = "boolean"
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "int AUTO_INCREMENT"
			} else {
				sqlType = "int"
			}
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uintptr:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "int unsigned AUTO_INCREMENT"
			} else {
				sqlType = "int unsigned"
			}
		case reflect.Int64:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "bigint AUTO_INCREMENT"
			} else {
				sqlType = "bigint"
			}
		case reflect.Uint64:
			if _, ok := field.TagSettings["AUTO_INCREMENT"]; ok || field.IsPrimaryKey {
				sqlType = "bigint unsigned AUTO_INCREMENT"
			} else {
				sqlType = "bigint unsigned"
			}
		case reflect.Float32, reflect.Float64:
			sqlType = "double"
		case reflect.String:
			if size > 0 && size < 65532 {
				sqlType = fmt.Sprintf("varchar(%d)", size)
			} else {
				sqlType = "longtext"
			}
		case reflect.Struct:
			if _, ok := dataValue.Interface().(time.Time); ok {
				if _, ok := field.TagSettings["NOT NULL"]; ok {
					sqlType = "timestamp"
				} else {
					sqlType = "timestamp NULL"
				}
			}
		default:
			if _, ok := dataValue.Interface().([]byte); ok {
				if size > 0 && size < 65532 {
					sqlType = fmt.Sprintf("varbinary(%d)", size)
				} else {
					sqlType = "longblob"
				}
			}
		}
	}

	if sqlType == "" {
		panic(fmt.Sprintf("invalid sql type %s (%s) for mysql", dataValue.Type().Name(), dataValue.Kind().String()))
	}

	if strings.TrimSpace(additionalType) == "" {
		return sqlType
	}
	return fmt.Sprintf("%v %v", sqlType, additionalType)
}

func (s mysql) RemoveIndex(tableName string, indexName string) error {
	_, err := s.db.Exec(fmt.Sprintf("DROP INDEX %v ON %v", indexName, s.Quote(tableName)))
	return err
}

func (s mysql) HasForeignKey(tableName string, foreignKeyName string) bool {
	var count int
	s.db.QueryRow("SELECT count(*) FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=? AND TABLE_NAME=? AND CONSTRAINT_NAME=? AND CONSTRAINT_TYPE='FOREIGN KEY'", s.currentDatabase(), tableName, foreignKeyName).Scan(&count)
	return count > 0
}

func (s mysql) currentDatabase() (name string) {
	s.db.QueryRow("SELECT DATABASE()").Scan(&name)
	return
}

func (mysql) SelectFromDummyTable() string {
	return "FROM DUAL"
}
<|endoftext|>


<commit_before>// Copyright ©2013 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	check "launchpad.net/gocheck"
	"os"
	"reflect"
	"testing"
)

var (
	bam    = flag.String("bam", "", "output first failing bam data to this file for inspection")
	allbam = flag.String("allbam", "", "output all bam data to this file base for inspection")
)

func Test(t *testing.T) { check.TestingT(t) }

type S struct{}

var _ = check.Suite(&S{})

func (s *S) TestRead(c *check.C) {
	for i, t := range []struct {
		in     []byte
		header *Header
		lines  int
	}{
		{
			in:     bamHG00096_1000,
			header: headerHG00096_1000,
			lines:  1000,
		},
	} {
		br, err := NewReader(bytes.NewBuffer(t.in), false)
		c.Assert(err, check.Equals, nil)
		c.Check(br.Header(), check.DeepEquals, t.header)
		if !reflect.DeepEqual(br.Header(), t.header) {
			c.Check(br.Header().Refs(), check.DeepEquals, t.header.Refs())
			c.Check(br.Header().RGs(), check.DeepEquals, t.header.RGs())
			c.Check(br.Header().Progs(), check.DeepEquals, t.header.Progs())
			c.Check(br.Header().Comments, check.DeepEquals, t.header.Comments)
		}
		var lines int
		for {
			_, err := br.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			lines++
		}
		c.Check(lines, check.Equals, t.lines)
		if *allbam != "" {
			bf, err := os.Create(fmt.Sprintf("%s-%d.bam", *allbam, i))
			c.Assert(err, check.Equals, nil)
			bf.Write(t.in)
			bf.Close()
		}
		if c.Failed() && *bam != "" {
			bf, err := os.Create(*bam)
			c.Assert(err, check.Equals, nil)
			bf.Write(t.in)
			bf.Close()
			c.FailNow()
		}
	}
}

func (s *S) TestRoundTrip(c *check.C) {
	for _, t := range []struct {
		in     []byte
		header *Header
		lines  int
	}{
		{
			in:     bamHG00096_1000,
			header: headerHG00096_1000,
			lines:  1000,
		},
	} {
		br, err := NewReader(bytes.NewBuffer(t.in), false)
		c.Assert(err, check.Equals, nil)

		var buf bytes.Buffer
		bw, err := NewWriter(&buf, br.Header().Clone())
		for {
			r, err := br.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			bw.Write(r)
		}
		c.Assert(bw.Close(), check.Equals, nil)

		br, err = NewReader(bytes.NewBuffer(t.in), false)
		c.Assert(err, check.Equals, nil)
		brr, err := NewReader(&buf, false)
		c.Assert(err, check.Equals, nil)
		c.Check(brr.Header().String(), check.Equals, br.Header().String())
		c.Check(brr.Header(), check.DeepEquals, br.Header())
		if !reflect.DeepEqual(brr.Header(), br.Header()) {
			c.Check(brr.Header().Refs(), check.DeepEquals, br.Header().Refs())
			c.Check(brr.Header().RGs(), check.DeepEquals, br.Header().RGs())
			c.Check(brr.Header().Progs(), check.DeepEquals, br.Header().Progs())
			c.Check(brr.Header().Comments, check.DeepEquals, br.Header().Comments)
		}
		for {
			r, err := br.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
			}
			rr, err := brr.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			c.Check(rr, check.DeepEquals, r)
		}
	}
}
<commit_msg>Add benchmark and test for limited reader<commit_after>// Copyright ©2013 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"bytes"
	"code.google.com/p/biogo.bam/bgzf/egzip"
	"flag"
	"fmt"
	"io"
	check "launchpad.net/gocheck"
	"os"
	"reflect"
	"testing"
)

var (
	bam    = flag.String("bam", "", "output first failing bam data to this file for inspection")
	allbam = flag.String("allbam", "", "output all bam data to this file base for inspection")
)

func Test(t *testing.T) { check.TestingT(t) }

type S struct{}

var _ = check.Suite(&S{})

func (s *S) TestRead(c *check.C) {
	for i, t := range []struct {
		in     []byte
		limit  bool
		header *Header
		lines  int
	}{
		{
			in:     bamHG00096_1000,
			limit:  false,
			header: headerHG00096_1000,
			lines:  1000,
		},
		{
			in:     bamHG00096_1000,
			limit:  true,
			header: headerHG00096_1000,
			lines:  1000,
		},
	} {
		br, err := NewReader(bytes.NewBuffer(t.in), t.limit)
		c.Assert(err, check.Equals, nil)
		c.Check(br.Header(), check.DeepEquals, t.header)
		if !reflect.DeepEqual(br.Header(), t.header) {
			c.Check(br.Header().Refs(), check.DeepEquals, t.header.Refs())
			c.Check(br.Header().RGs(), check.DeepEquals, t.header.RGs())
			c.Check(br.Header().Progs(), check.DeepEquals, t.header.Progs())
			c.Check(br.Header().Comments, check.DeepEquals, t.header.Comments)
		}
		var lines int
		for {
			_, err := br.Read()
			if err != nil {
				if err == egzip.NewBlock {
					continue
				}
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			lines++
		}
		c.Check(lines, check.Equals, t.lines)
		if *allbam != "" {
			bf, err := os.Create(fmt.Sprintf("%s-%d.bam", *allbam, i))
			c.Assert(err, check.Equals, nil)
			bf.Write(t.in)
			bf.Close()
		}
		if c.Failed() && *bam != "" {
			bf, err := os.Create(*bam)
			c.Assert(err, check.Equals, nil)
			bf.Write(t.in)
			bf.Close()
			c.FailNow()
		}
	}
}

func (s *S) TestRoundTrip(c *check.C) {
	for _, t := range []struct {
		in     []byte
		header *Header
		lines  int
	}{
		{
			in:     bamHG00096_1000,
			header: headerHG00096_1000,
			lines:  1000,
		},
	} {
		br, err := NewReader(bytes.NewBuffer(t.in), false)
		c.Assert(err, check.Equals, nil)

		var buf bytes.Buffer
		bw, err := NewWriter(&buf, br.Header().Clone())
		for {
			r, err := br.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			bw.Write(r)
		}
		c.Assert(bw.Close(), check.Equals, nil)

		br, err = NewReader(bytes.NewBuffer(t.in), false)
		c.Assert(err, check.Equals, nil)
		brr, err := NewReader(&buf, false)
		c.Assert(err, check.Equals, nil)
		c.Check(brr.Header().String(), check.Equals, br.Header().String())
		c.Check(brr.Header(), check.DeepEquals, br.Header())
		if !reflect.DeepEqual(brr.Header(), br.Header()) {
			c.Check(brr.Header().Refs(), check.DeepEquals, br.Header().Refs())
			c.Check(brr.Header().RGs(), check.DeepEquals, br.Header().RGs())
			c.Check(brr.Header().Progs(), check.DeepEquals, br.Header().Progs())
			c.Check(brr.Header().Comments, check.DeepEquals, br.Header().Comments)
		}
		for {
			r, err := br.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
			}
			rr, err := brr.Read()
			if err != nil {
				c.Assert(err, check.Equals, io.EOF)
				break
			}
			c.Check(rr, check.DeepEquals, r)
		}
	}
}

func BenchmarkRoundTrip(b *testing.B) {
	for i := 0; i < b.N; i++ {
		br, _ := NewReader(bytes.NewBuffer(bamHG00096_1000), false)

		var buf bytes.Buffer
		bw, _ := NewWriter(&buf, br.Header().Clone())
		for {
			r, err := br.Read()
			if err != nil {
				break
			}
			bw.Write(r)
		}
	}
}
<|endoftext|>


<commit_before>package mpb_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strings"
	"sync/atomic"
	"testing"
	"time"
	"unicode/utf8"

	"github.com/vbauerster/mpb/v6"
	"github.com/vbauerster/mpb/v6/decor"
)

func TestBarCompleted(t *testing.T) {
	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(ioutil.Discard))
	total := 80
	bar := p.AddBar(int64(total))

	var count int
	for !bar.Completed() {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
		count++
	}

	p.Wait()
	if count != total {
		t.Errorf("got count: %d, expected %d\n", count, total)
	}
}

func TestBarID(t *testing.T) {
	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(ioutil.Discard))
	total := 100
	wantID := 11
	bar := p.AddBar(int64(total), mpb.BarID(wantID))

	go func() {
		for i := 0; i < total; i++ {
			time.Sleep(50 * time.Millisecond)
			bar.Increment()
		}
	}()

	gotID := bar.ID()
	if gotID != wantID {
		t.Errorf("Expected bar id: %d, got %d\n", wantID, gotID)
	}

	bar.Abort(true)
	p.Wait()
}

func TestBarSetRefill(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithOutput(&buf), mpb.WithWidth(100))

	total := 100
	till := 30
	refillRune, _ := utf8.DecodeLastRuneInString(mpb.BarDefaultStyle)

	bar := p.AddBar(int64(total), mpb.BarFillerTrim())

	bar.SetRefill(int64(till))
	bar.IncrBy(till)

	for i := 0; i < total-till; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	wantBar := fmt.Sprintf("[%s%s]",
		strings.Repeat(string(refillRune), till-1),
		strings.Repeat("=", total-till-1),
	)

	got := string(getLastLine(buf.Bytes()))

	if !strings.Contains(got, wantBar) {
		t.Errorf("Want bar: %q, got bar: %q\n", wantBar, got)
	}
}

func TestBarHas100PercentWithOnCompleteDecorator(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(&buf))

	total := 50

	bar := p.AddBar(int64(total),
		mpb.AppendDecorators(
			decor.OnComplete(
				decor.Percentage(), "done",
			),
		),
	)

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	hundred := "100 %"
	if !bytes.Contains(buf.Bytes(), []byte(hundred)) {
		t.Errorf("Bar's buffer does not contain: %q\n", hundred)
	}
}

func TestBarHas100PercentWithBarRemoveOnComplete(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(&buf))

	total := 50

	bar := p.AddBar(int64(total),
		mpb.BarRemoveOnComplete(),
		mpb.AppendDecorators(decor.Percentage()),
	)

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	hundred := "100 %"
	if !bytes.Contains(buf.Bytes(), []byte(hundred)) {
		t.Errorf("Bar's buffer does not contain: %q\n", hundred)
	}
}

func TestBarStyle(t *testing.T) {
	var buf bytes.Buffer
	customFormat := "╢▌▌░╟"
	total := 80
	p := mpb.New(mpb.WithWidth(total), mpb.WithOutput(&buf))
	bar := p.Add(int64(total), mpb.NewBarFiller(customFormat), mpb.BarFillerTrim())

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	runes := []rune(customFormat)
	wantBar := fmt.Sprintf("%s%s%s",
		string(runes[0]),
		strings.Repeat(string(runes[1]), total-2),
		string(runes[len(runes)-1]),
	)
	got := string(getLastLine(buf.Bytes()))

	if !strings.Contains(got, wantBar) {
		t.Errorf("Want bar: %q:%d, got bar: %q:%d\n", wantBar, utf8.RuneCountInString(wantBar), got, utf8.RuneCountInString(got))
	}
}

func TestBarPanicBeforeComplete(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New(
		mpb.WithWidth(80),
		mpb.WithDebugOutput(&buf),
		mpb.WithOutput(ioutil.Discard),
	)

	total := 100
	panicMsg := "Upps!!!"
	var pCount uint32
	bar := p.AddBar(int64(total),
		mpb.PrependDecorators(panicDecorator(panicMsg,
			func(st decor.Statistics) bool {
				if st.Current >= 42 {
					atomic.AddUint32(&pCount, 1)
					return true
				}
				return false
			},
		)),
	)

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
	}

	p.Wait()

	if pCount != 1 {
		t.Errorf("Decor called after panic %d times\n", pCount-1)
	}

	barStr := buf.String()
	if !strings.Contains(barStr, panicMsg) {
		t.Errorf("%q doesn't contain %q\n", barStr, panicMsg)
	}
}

func TestBarPanicAfterComplete(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New(
		mpb.WithWidth(80),
		mpb.WithDebugOutput(&buf),
		mpb.WithOutput(ioutil.Discard),
	)

	total := 100
	panicMsg := "Upps!!!"
	var pCount uint32
	bar := p.AddBar(int64(total),
		mpb.PrependDecorators(panicDecorator(panicMsg,
			func(st decor.Statistics) bool {
				if st.Completed {
					atomic.AddUint32(&pCount, 1)
					return true
				}
				return false
			},
		)),
	)

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
	}

	p.Wait()

	if pCount > 2 {
		t.Error("Decor called after panic more than 2 times\n")
	}

	barStr := buf.String()
	if !strings.Contains(barStr, panicMsg) {
		t.Errorf("%q doesn't contain %q\n", barStr, panicMsg)
	}
}

func panicDecorator(panicMsg string, cond func(decor.Statistics) bool) decor.Decorator {
	return decor.Any(func(st decor.Statistics) string {
		if cond(st) {
			panic(panicMsg)
		}
		return ""
	})
}
<commit_msg>refactoring: bar_test<commit_after>package mpb_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strings"
	"sync/atomic"
	"testing"
	"time"
	"unicode/utf8"

	"github.com/vbauerster/mpb/v6"
	"github.com/vbauerster/mpb/v6/decor"
)

func TestBarCompleted(t *testing.T) {
	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(ioutil.Discard))
	total := 80
	bar := p.AddBar(int64(total))

	var count int
	for !bar.Completed() {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
		count++
	}

	p.Wait()
	if count != total {
		t.Errorf("got count: %d, expected %d\n", count, total)
	}
}

func TestBarID(t *testing.T) {
	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(ioutil.Discard))
	total := 100
	wantID := 11
	bar := p.AddBar(int64(total), mpb.BarID(wantID))

	go func() {
		for i := 0; i < total; i++ {
			time.Sleep(50 * time.Millisecond)
			bar.Increment()
		}
	}()

	gotID := bar.ID()
	if gotID != wantID {
		t.Errorf("Expected bar id: %d, got %d\n", wantID, gotID)
	}

	bar.Abort(true)
	p.Wait()
}

func TestBarSetRefill(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithOutput(&buf), mpb.WithWidth(100))

	total := 100
	till := 30
	refiller := "+"

	bar := p.Add(int64(total), mpb.NewBarFiller(mpb.BarStyle().Refiller(refiller)), mpb.BarFillerTrim())

	bar.SetRefill(int64(till))
	bar.IncrBy(till)

	for i := 0; i < total-till; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	wantBar := fmt.Sprintf("[%s%s]",
		strings.Repeat(refiller, till-1),
		strings.Repeat("=", total-till-1),
	)

	got := string(getLastLine(buf.Bytes()))

	if !strings.Contains(got, wantBar) {
		t.Errorf("Want bar: %q, got bar: %q\n", wantBar, got)
	}
}

func TestBarHas100PercentWithOnCompleteDecorator(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(&buf))

	total := 50

	bar := p.AddBar(int64(total),
		mpb.AppendDecorators(
			decor.OnComplete(
				decor.Percentage(), "done",
			),
		),
	)

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	hundred := "100 %"
	if !bytes.Contains(buf.Bytes(), []byte(hundred)) {
		t.Errorf("Bar's buffer does not contain: %q\n", hundred)
	}
}

func TestBarHas100PercentWithBarRemoveOnComplete(t *testing.T) {
	var buf bytes.Buffer

	p := mpb.New(mpb.WithWidth(80), mpb.WithOutput(&buf))

	total := 50

	bar := p.AddBar(int64(total),
		mpb.BarRemoveOnComplete(),
		mpb.AppendDecorators(decor.Percentage()),
	)

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	hundred := "100 %"
	if !bytes.Contains(buf.Bytes(), []byte(hundred)) {
		t.Errorf("Bar's buffer does not contain: %q\n", hundred)
	}
}

func TestBarStyle(t *testing.T) {
	var buf bytes.Buffer
	customFormat := "╢▌▌░╟"
	runes := []rune(customFormat)
	total := 80
	p := mpb.New(mpb.WithWidth(total), mpb.WithOutput(&buf))
	bs := mpb.BarStyle()
	bs.Lbound(string(runes[0]))
	bs.Filler(string(runes[1]))
	bs.Tip(string(runes[2]))
	bs.Padding(string(runes[3]))
	bs.Rbound(string(runes[4]))
	bar := p.Add(int64(total), mpb.NewBarFiller(bs), mpb.BarFillerTrim())

	for i := 0; i < total; i++ {
		bar.Increment()
		time.Sleep(10 * time.Millisecond)
	}

	p.Wait()

	wantBar := fmt.Sprintf("%s%s%s%s",
		string(runes[0]),
		strings.Repeat(string(runes[1]), total-3),
		string(runes[2]),
		string(runes[4]),
	)
	got := string(getLastLine(buf.Bytes()))

	if !strings.Contains(got, wantBar) {
		t.Errorf("Want bar: %q:%d, got bar: %q:%d\n", wantBar, utf8.RuneCountInString(wantBar), got, utf8.RuneCountInString(got))
	}
}

func TestBarPanicBeforeComplete(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New(
		mpb.WithWidth(80),
		mpb.WithDebugOutput(&buf),
		mpb.WithOutput(ioutil.Discard),
	)

	total := 100
	panicMsg := "Upps!!!"
	var pCount uint32
	bar := p.AddBar(int64(total),
		mpb.PrependDecorators(panicDecorator(panicMsg,
			func(st decor.Statistics) bool {
				if st.Current >= 42 {
					atomic.AddUint32(&pCount, 1)
					return true
				}
				return false
			},
		)),
	)

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
	}

	p.Wait()

	if pCount != 1 {
		t.Errorf("Decor called after panic %d times\n", pCount-1)
	}

	barStr := buf.String()
	if !strings.Contains(barStr, panicMsg) {
		t.Errorf("%q doesn't contain %q\n", barStr, panicMsg)
	}
}

func TestBarPanicAfterComplete(t *testing.T) {
	var buf bytes.Buffer
	p := mpb.New(
		mpb.WithWidth(80),
		mpb.WithDebugOutput(&buf),
		mpb.WithOutput(ioutil.Discard),
	)

	total := 100
	panicMsg := "Upps!!!"
	var pCount uint32
	bar := p.AddBar(int64(total),
		mpb.PrependDecorators(panicDecorator(panicMsg,
			func(st decor.Statistics) bool {
				if st.Completed {
					atomic.AddUint32(&pCount, 1)
					return true
				}
				return false
			},
		)),
	)

	for i := 0; i < total; i++ {
		time.Sleep(10 * time.Millisecond)
		bar.Increment()
	}

	p.Wait()

	if pCount > 2 {
		t.Error("Decor called after panic more than 2 times\n")
	}

	barStr := buf.String()
	if !strings.Contains(barStr, panicMsg) {
		t.Errorf("%q doesn't contain %q\n", barStr, panicMsg)
	}
}

func panicDecorator(panicMsg string, cond func(decor.Statistics) bool) decor.Decorator {
	return decor.Any(func(st decor.Statistics) string {
		if cond(st) {
			panic(panicMsg)
		}
		return ""
	})
}
<|endoftext|>


<commit_before>// Package baseconv converts a string in an arbitrary base to any other
// arbitrary base.
package baseconv

import (
	"errors"
	"fmt"
	"unicode/utf8"
)

// Convert num from specified base to a different base.
func Convert(num, fromBase, toBase string) (string, error) {
	if num == "" {
		return "", errors.New("invalid number")
	}

	if len(fromBase) < 2 {
		return "", errors.New("invalid fromBase")
	}

	if len(toBase) < 2 {
		return "", errors.New("invalid toBase")
	}

	// rune counts
	fromLenRunes := utf8.RuneCountInString(fromBase)
	toLenRunes := utf8.RuneCountInString(toBase)
	numLen := utf8.RuneCountInString(num)

	// loop over unicode runes in original string and store representative
	// values in number -- number[i] = index(num[i], fromBase)
	number, ipos := make([]int, numLen), 0
	for i, r := range num {
		jpos, found := 0, false
		for _, s := range fromBase {
			if r == s {
				number[ipos] = jpos
				found = true
				break
			}

			jpos++
		}

		// if character wasn't found in fromBase, then error
		if !found {
			return "", fmt.Errorf("invalid character '%c' at position %d (%d)", r, ipos, i)
		}

		ipos++
	}

	// split the runes in toBase
	todigits, idx := make([]rune, toLenRunes), 0
	for _, r := range toBase {
		todigits[idx] = r
		idx++
	}

	// loop until whole number is converted
	var result []rune
	for {
		divide, newlen := 0, 0

		// perform division manually (which is why this works with big numbers)
		for i := 0; i < numLen; i++ {
			divide = divide*fromLenRunes + number[i]
			if divide >= toLenRunes {
				number[newlen] = divide / toLenRunes
				divide = divide % toLenRunes
				newlen++
			} else if newlen > 0 {
				number[newlen] = 0
				newlen++
			}
		}

		numLen = newlen
		result = append(result, todigits[divide])

		if newlen == 0 {
			break
		}
	}

	// reverse result
	for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
		result[i], result[j] = result[j], result[i]
	}

	return string(result), nil
}

const (
	// DigitsBin represents binary digits
	DigitsBin = "01"

	// DigitsOct represents octal Digits
	DigitsOct = "01234567"

	// DigitsDec represents decimal digits
	DigitsDec = "0123456789"

	// DigitsHex represents hex digits
	DigitsHex = "0123456789abcdef"

	// Digits36 represents base36 digits
	Digits36 = "0123456789abcdefghijklmnopqrstuvwxyz"

	// Digits62 represents base62 digits
	Digits62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

	// Digits64 represents base64 digits
	Digits64 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
)

// EncodeBinFromDec encodes a string to DigitsBin from DigitsDec.
func EncodeBinFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsBin)
}

// DecodeBinToDec decodes a string from DigitsBin to DigitsDec.
func DecodeBinToDec(num string) (string, error) {
	return Convert(num, DigitsBin, DigitsDec)
}

// EncodeOctFromDec encodes a string to DigitsOct from DigitsDec.
func EncodeOctFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsOct)
}

// DecodeOctToDec decodes a string from DigitsOct to DigitsDec.
func DecodeOctToDec(num string) (string, error) {
	return Convert(num, DigitsOct, DigitsDec)
}

// EncodeHexFromDec encodes a string to DigitsHex from DigitsDec.
func EncodeHexFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsHex)
}

// DecodeHexToDec decodes a string from DigitsHex to DigitsDec.
func DecodeHexToDec(num string) (string, error) {
	return Convert(num, DigitsHex, DigitsDec)
}

// Encode36FromDec encodes a string to Digits36 from DigitsDec.
func Encode36FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits36)
}

// Decode36ToDec decodes a string from Digits36 to DigitsDec.
func Decode36ToDec(num string) (string, error) {
	return Convert(num, Digits36, DigitsDec)
}

// Encode62FromDec encodes a string to Digits62 to DigitsDec.
func Encode62FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits62)
}

// Decode62ToDec decodes a string from Digits62 to DigitsDec.
func Decode62ToDec(num string) (string, error) {
	return Convert(num, Digits62, DigitsDec)
}

// Encode64FromDec encodes a string to Digits64 to DigitsDec.
func Encode64FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits64)
}

// Decode64ToDec decodes a string from Digits64 to DigitsDec.
func Decode64ToDec(num string) (string, error) {
	return Convert(num, Digits64, DigitsDec)
}
<commit_msg>Converting errors to sentinel form<commit_after>// Package baseconv converts a string in an arbitrary base to any other
// arbitrary base.
package baseconv

import (
	"fmt"
	"unicode/utf8"
)

// Error is a base conversion error.
type Error string

// Error satisfies the error interface.
func (err Error) Error() string {
	return string(err)
}

// InvalidCharacterError is an invalid character error.
type InvalidCharacterError struct {
	r      rune
	pos, n int
}

// Error satisfies the error interface.
func (err *InvalidCharacterError) Error() string {
	return fmt.Sprintf("invalid character '%c' at position %d (%d)", err.r, err.pos, err.n)
}

// Error values.
const (
	// ErrInvalidNumber is the invalid number error.
	ErrInvalidNumber Error = "invalid number"

	// ErrInvalidFromBase is the invalid from base error.
	ErrInvalidFromBase Error = "invalid fromBase"

	// ErrInvalidToBase is the invalid to base error.
	ErrInvalidToBase Error = "invalid toBase"
)

// Convert num from specified base to a different base.
func Convert(num, fromBase, toBase string) (string, error) {
	if num == "" {
		return "", ErrInvalidNumber
	}

	if len(fromBase) < 2 {
		return "", ErrInvalidFromBase
	}

	if len(toBase) < 2 {
		return "", ErrInvalidToBase
	}

	// rune counts
	fromLenRunes := utf8.RuneCountInString(fromBase)
	toLenRunes := utf8.RuneCountInString(toBase)
	numLen := utf8.RuneCountInString(num)

	// loop over unicode runes in original string and store representative
	// values in number -- number[i] = index(num[i], fromBase)
	number, ipos := make([]int, numLen), 0
	for i, r := range num {
		jpos, found := 0, false
		for _, s := range fromBase {
			if r == s {
				number[ipos] = jpos
				found = true
				break
			}

			jpos++
		}

		// if character wasn't found in fromBase, then error
		if !found {
			return "", &InvalidCharacterError{r, ipos, i}
		}

		ipos++
	}

	// split the runes in toBase
	todigits, idx := make([]rune, toLenRunes), 0
	for _, r := range toBase {
		todigits[idx] = r
		idx++
	}

	// loop until whole number is converted
	var result []rune
	for {
		divide, newlen := 0, 0

		// perform division manually (which is why this works with big numbers)
		for i := 0; i < numLen; i++ {
			divide = divide*fromLenRunes + number[i]
			if divide >= toLenRunes {
				number[newlen] = divide / toLenRunes
				divide = divide % toLenRunes
				newlen++
			} else if newlen > 0 {
				number[newlen] = 0
				newlen++
			}
		}

		numLen = newlen
		result = append(result, todigits[divide])

		if newlen == 0 {
			break
		}
	}

	// reverse result
	for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
		result[i], result[j] = result[j], result[i]
	}

	return string(result), nil
}

const (
	// DigitsBin represents binary digits
	DigitsBin = "01"

	// DigitsOct represents octal Digits
	DigitsOct = "01234567"

	// DigitsDec represents decimal digits
	DigitsDec = "0123456789"

	// DigitsHex represents hex digits
	DigitsHex = "0123456789abcdef"

	// Digits36 represents base36 digits
	Digits36 = "0123456789abcdefghijklmnopqrstuvwxyz"

	// Digits62 represents base62 digits
	Digits62 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

	// Digits64 represents base64 digits
	Digits64 = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_"
)

// EncodeBinFromDec encodes a string to DigitsBin from DigitsDec.
func EncodeBinFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsBin)
}

// DecodeBinToDec decodes a string from DigitsBin to DigitsDec.
func DecodeBinToDec(num string) (string, error) {
	return Convert(num, DigitsBin, DigitsDec)
}

// EncodeOctFromDec encodes a string to DigitsOct from DigitsDec.
func EncodeOctFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsOct)
}

// DecodeOctToDec decodes a string from DigitsOct to DigitsDec.
func DecodeOctToDec(num string) (string, error) {
	return Convert(num, DigitsOct, DigitsDec)
}

// EncodeHexFromDec encodes a string to DigitsHex from DigitsDec.
func EncodeHexFromDec(num string) (string, error) {
	return Convert(num, DigitsDec, DigitsHex)
}

// DecodeHexToDec decodes a string from DigitsHex to DigitsDec.
func DecodeHexToDec(num string) (string, error) {
	return Convert(num, DigitsHex, DigitsDec)
}

// Encode36FromDec encodes a string to Digits36 from DigitsDec.
func Encode36FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits36)
}

// Decode36ToDec decodes a string from Digits36 to DigitsDec.
func Decode36ToDec(num string) (string, error) {
	return Convert(num, Digits36, DigitsDec)
}

// Encode62FromDec encodes a string to Digits62 to DigitsDec.
func Encode62FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits62)
}

// Decode62ToDec decodes a string from Digits62 to DigitsDec.
func Decode62ToDec(num string) (string, error) {
	return Convert(num, Digits62, DigitsDec)
}

// Encode64FromDec encodes a string to Digits64 to DigitsDec.
func Encode64FromDec(num string) (string, error) {
	return Convert(num, DigitsDec, Digits64)
}

// Decode64ToDec decodes a string from Digits64 to DigitsDec.
func Decode64ToDec(num string) (string, error) {
	return Convert(num, Digits64, DigitsDec)
}
<|endoftext|>


<commit_before>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"text/template"

	"sqlflow.org/gomaxcompute"
)

type alpsFiller struct {
	// Training or Predicting
	IsTraining bool

	// Input & Output
	TrainInputTable    string
	EvalInputTable     string
	PredictInputTable  string
	ModelDir           string
	ScratchDir         string
	PredictOutputTable string

	// Schema & Decode info
	Fields string
	X      string
	Y      string

	// Train
	ModelCreatorCode string
	TrainClause      *resolvedTrainClause

	// Feature map
	FeatureMapTable     string
	FeatureMapPartition string

	// ODPS
	OdpsConf *gomaxcompute.Config
}

func modelCreatorCode(resolved *resolvedTrainClause, args []string) (string, error) {
	cl := make([]string, 0)
	for _, a := range resolved.ModelConstructorParams {
		code, err := a.GenerateCode()
		if err != nil {
			return "", err
		}
		cl = append(cl, code)
	}
	if args != nil {
		for _, arg := range args {
			cl = append(cl, arg)
		}
	}
	modelName := resolved.ModelName
	if resolved.IsPreMadeModel {
		modelName = fmt.Sprintf("tf.estimator.%s", resolved.ModelName)
	}
	return fmt.Sprintf("%s(%s)", modelName, strings.Join(cl, ",")), nil
}

func newALPSTrainFiller(pr *extendedSelect, db *DB) (*alpsFiller, error) {
	resolved, err := resolveTrainClause(&pr.trainClause)
	if err != nil {
		return nil, err
	}

	featureMapTable := ""
	featureMapPartition := ""

	csCode := make([]string, 0)
	for _, css := range resolved.ColumnSpecs {
		for _, cs := range css {
			csCode = append(csCode, cs.ToString())
			if cs.FeatureMap.Table != "" {
				featureMapTable = cs.FeatureMap.Table
			}
			if cs.FeatureMap.Partition != "" {
				featureMapPartition = cs.FeatureMap.Partition
			}
		}
	}

	var odpsConfig = &gomaxcompute.Config{}
	if db != nil {
		odpsConfig, err = gomaxcompute.ParseDSN(db.dataSourceName)
		if err != nil {
			return nil, err
		}
	}

	args := make([]string, 0)
	args = append(args, "config=run_config")
	for target, fcs := range resolved.FeatureColumns {
		code, err := generateFeatureColumnCode(fcs)
		if err != nil {
			return nil, err
		}
		args = append(args, fmt.Sprintf("%s=%s", target, code))
	}
	modelCode, err := modelCreatorCode(resolved, args)
	if err != nil {
		return nil, err
	}

	tableName := pr.tables[0]

	fields := make([]string, len(pr.fields))
	for idx, f := range pr.fields {
		fields[idx] = fmt.Sprintf("\"%s\"", f)
	}

	y := &columnSpec{
		ColumnName: pr.label,
		IsSparse:   false,
		Shape:      []int{1},
		DType:      "int",
		Delimiter:  ","}

	//TODO(uuleon): the scratchDir will be deleted after model uploading
	scratchDir, err := ioutil.TempDir("/tmp", "alps_scratch_dir_")
	if err != nil {
		return nil, err
	}
	modelDir := fmt.Sprintf("%s/model/", scratchDir)

	return &alpsFiller{
		IsTraining:          true,
		TrainInputTable:     tableName,
		EvalInputTable:      tableName, //FIXME(uuleon): Train and Eval should use different dataset.
		ScratchDir:          scratchDir,
		ModelDir:            modelDir,
		Fields:              fmt.Sprintf("[%s]", strings.Join(fields, ",")),
		X:                   fmt.Sprintf("[%s]", strings.Join(csCode, ",")),
		Y:                   y.ToString(),
		OdpsConf:            odpsConfig,
		ModelCreatorCode:    modelCode,
		TrainClause:         resolved,
		FeatureMapTable:     featureMapTable,
		FeatureMapPartition: featureMapPartition}, nil
}

func newALPSPredictFiller(pr *extendedSelect) (*alpsFiller, error) {
	return nil, fmt.Errorf("alps predict not supported")
}

func genALPSFiller(w io.Writer, pr *extendedSelect, db *DB) (*alpsFiller, error) {
	if pr.train {
		return newALPSTrainFiller(pr, db)
	}
	return newALPSPredictFiller(pr)
}

func submitALPS(w *PipeWriter, pr *extendedSelect, db *DB, cwd string) error {
	var program bytes.Buffer

	filler, err := genALPSFiller(&program, pr, db)
	if err != nil {
		return err
	}

	if err = alpsTemplate.Execute(&program, filler); err != nil {
		return fmt.Errorf("submitALPS: failed executing template: %v", err)
	}

	code := program.String()

	cw := &logChanWriter{wr: w}
	cmd := tensorflowCmd(cwd, "maxcompute")
	cmd.Stdin = &program
	cmd.Stdout = cw
	cmd.Stderr = cw
	if e := cmd.Run(); e != nil {
		return fmt.Errorf("code %v failed %v", code, e)
	}

	if pr.train {
		// TODO(uuleon): save model to DB
	}

	return nil
}

const alpsTemplateText = `
# coding: utf-8
# Copyright (c) Antfin, Inc. All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow as tf

from alps.conf.closure import Closure
from alps.framework.train.training import build_run_config
from alps.framework.exporter import ExportStrategy
from alps.framework.exporter.arks_exporter import ArksExporter
from alps.client.base import run_experiment
from alps.framework.engine import LocalEngine
from alps.framework.column.column import DenseColumn, SparseColumn
from alps.framework.exporter.compare_fn import best_auc_fn
from alps.io import DatasetX
from alps.io.base import OdpsConf, FeatureMap
from alps.framework.experiment import EstimatorBuilder, Experiment, TrainConf, EvalConf
from alps.io.reader.odps_reader import OdpsReader

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'    # for debug usage.
tf.logging.set_verbosity(tf.logging.INFO)

class SQLFlowEstimatorBuilder(EstimatorBuilder):
    def _build(self, experiment, run_config):
        return {{.ModelCreatorCode}}

if __name__ == "__main__":

	odpsConf=OdpsConf(
		accessid="{{.OdpsConf.AccessID}}",
		accesskey="{{.OdpsConf.AccessKey}}",
		endpoint="{{.OdpsConf.Endpoint}}"
	)

	trainDs = DatasetX(
		num_epochs={{.TrainClause.Epoch}},
		batch_size={{.TrainClause.BatchSize}},
		shuffle="{{.TrainClause.EnableShuffle}}" == "true",
		shuffle_buffer_size={{.TrainClause.ShuffleBufferSize}},
{{if eq .TrainClause.EnableCache true}}
		cache_file={{.TrainClause.CachePath}},
{{end}}
		reader=OdpsReader(
			odps=odpsConf,
			project="{{.OdpsConf.Project}}",
			table="{{.TrainInputTable}}",
			field_names={{.Fields}},
			features={{.X}},
			labels={{.Y}},
{{if ne .FeatureMapTable ""}}
			feature_map=FeatureMap(table="{{.FeatureMapTable}}",
{{if ne .FeatureMapPartition ""}}
				partition="{{.FeatureMapPartition}}"
{{end}}
			)
{{end}}
		),
		drop_remainder="{{.TrainClause.DropRemainder}}" == "true"
	)

	evalDs = DatasetX(
		num_epochs=1,
		batch_size={{.TrainClause.BatchSize}},
		reader=OdpsReader(
			odps=odpsConf,
			project="{{.OdpsConf.Project}}",
			table="{{.EvalInputTable}}",
			field_names={{.Fields}},
			features={{.X}},
			labels={{.Y}}
		)
	)

	export_path = "{{.ModelDir}}"

	experiment = Experiment(
		user="sqlflow",
		engine=LocalEngine(),
		train=TrainConf(input=trainDs,
{{if ne .TrainClause.MaxSteps -1}}
						max_steps={{.TrainClause.MaxSteps}},
{{end}}
		),
		eval=EvalConf(input=evalDs,
{{if ne .TrainClause.EvalSteps -1}}
					  steps={{.TrainClause.EvalSteps}},
{{end}}
					  start_delay_secs={{.TrainClause.EvalStartDelay}},
					  throttle_secs={{.TrainClause.EvalThrottle}},
		),
		exporter=ArksExporter(deploy_path=export_path, strategy=ExportStrategy.BEST, compare_fn=Closure(best_auc_fn)),
		model_dir="{{.ScratchDir}}",
		model_builder=SQLFlowEstimatorBuilder())

	run_experiment(experiment)

`

var alpsTemplate = template.Must(template.New("alps").Parse(alpsTemplateText))
<commit_msg>refine and fix codegen_alps (#533)<commit_after>// Copyright 2019 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"text/template"

	"sqlflow.org/gomaxcompute"
)

type alpsFiller struct {
	// Training or Predicting
	IsTraining bool

	// Input & Output
	TrainInputTable    string
	EvalInputTable     string
	PredictInputTable  string
	ModelDir           string
	ScratchDir         string
	PredictOutputTable string

	// Schema & Decode info
	Fields string
	X      string
	Y      string

	// Train
	ModelCreatorCode string
	TrainClause      *resolvedTrainClause

	// Feature map
	FeatureMapTable     string
	FeatureMapPartition string

	// ODPS
	OdpsConf *gomaxcompute.Config
}

func modelCreatorCode(resolved *resolvedTrainClause, args []string) (string, error) {
	cl := make([]string, 0)
	for _, a := range resolved.ModelConstructorParams {
		code, err := a.GenerateCode()
		if err != nil {
			return "", err
		}
		cl = append(cl, code)
	}
	if args != nil {
		for _, arg := range args {
			cl = append(cl, arg)
		}
	}
	modelName := resolved.ModelName
	if resolved.IsPreMadeModel {
		modelName = fmt.Sprintf("tf.estimator.%s", resolved.ModelName)
	}
	return fmt.Sprintf("%s(%s)", modelName, strings.Join(cl, ",")), nil
}

func newALPSTrainFiller(pr *extendedSelect, db *DB) (*alpsFiller, error) {
	resolved, err := resolveTrainClause(&pr.trainClause)
	if err != nil {
		return nil, err
	}

	featureMapTable := ""
	featureMapPartition := ""

	csCode := make([]string, 0)
	for _, css := range resolved.ColumnSpecs {
		for _, cs := range css {
			csCode = append(csCode, cs.ToString())
			if cs.FeatureMap.Table != "" {
				featureMapTable = cs.FeatureMap.Table
			}
			if cs.FeatureMap.Partition != "" {
				featureMapPartition = cs.FeatureMap.Partition
			}
		}
	}

	var odpsConfig = &gomaxcompute.Config{}
	if db != nil {
		odpsConfig, err = gomaxcompute.ParseDSN(db.dataSourceName)
		if err != nil {
			return nil, err
		}
	}

	args := make([]string, 0)
	args = append(args, "config=run_config")
	for target, fcs := range resolved.FeatureColumns {
		code, err := generateFeatureColumnCode(fcs)
		if err != nil {
			return nil, err
		}
		args = append(args, fmt.Sprintf("%s=%s", target, code))
	}
	modelCode, err := modelCreatorCode(resolved, args)
	if err != nil {
		return nil, err
	}

	tableName := pr.tables[0]

	fields := make([]string, len(pr.fields))
	for idx, f := range pr.fields {
		fields[idx] = fmt.Sprintf("\"%s\"", f)
	}

	y := &columnSpec{
		ColumnName: pr.label,
		IsSparse:   false,
		Shape:      []int{1},
		DType:      "int",
		Delimiter:  ","}

	//TODO(uuleon): the scratchDir will be deleted after model uploading
	scratchDir, err := ioutil.TempDir("/tmp", "alps_scratch_dir_")
	if err != nil {
		return nil, err
	}
	modelDir := fmt.Sprintf("%s/model/", scratchDir)

	return &alpsFiller{
		IsTraining:          true,
		TrainInputTable:     tableName,
		EvalInputTable:      tableName, //FIXME(uuleon): Train and Eval should use different dataset.
		ScratchDir:          scratchDir,
		ModelDir:            modelDir,
		Fields:              fmt.Sprintf("[%s]", strings.Join(fields, ",")),
		X:                   fmt.Sprintf("[%s]", strings.Join(csCode, ",")),
		Y:                   y.ToString(),
		OdpsConf:            odpsConfig,
		ModelCreatorCode:    modelCode,
		TrainClause:         resolved,
		FeatureMapTable:     featureMapTable,
		FeatureMapPartition: featureMapPartition}, nil
}

func newALPSPredictFiller(pr *extendedSelect) (*alpsFiller, error) {
	return nil, fmt.Errorf("alps predict not supported")
}

func genALPSFiller(w io.Writer, pr *extendedSelect, db *DB) (*alpsFiller, error) {
	if pr.train {
		return newALPSTrainFiller(pr, db)
	}
	return newALPSPredictFiller(pr)
}

func submitALPS(w *PipeWriter, pr *extendedSelect, db *DB, cwd string) error {
	var program bytes.Buffer

	filler, err := genALPSFiller(&program, pr, db)
	if err != nil {
		return err
	}

	if err = alpsTemplate.Execute(&program, filler); err != nil {
		return fmt.Errorf("submitALPS: failed executing template: %v", err)
	}
	code := program.String()

	cw := &logChanWriter{wr: w}
	cmd := tensorflowCmd(cwd, "maxcompute")
	cmd.Stdin = &program
	cmd.Stdout = cw
	cmd.Stderr = cw
	if e := cmd.Run(); e != nil {
		return fmt.Errorf("code %v failed %v", code, e)
	}

	if pr.train {
		// TODO(uuleon): save model to DB
	}

	return nil
}

const alpsTemplateText = `
# coding: utf-8
# Copyright (c) Antfin, Inc. All rights reserved.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow as tf

from alps.conf.closure import Closure
from alps.framework.train.training import build_run_config
from alps.framework.exporter import ExportStrategy
from alps.framework.exporter.arks_exporter import ArksExporter
from alps.client.base import run_experiment
from alps.framework.engine import LocalEngine
from alps.framework.column.column import DenseColumn, SparseColumn
from alps.framework.exporter.compare_fn import best_auc_fn
from alps.io import DatasetX
from alps.io.base import OdpsConf, FeatureMap
from alps.framework.experiment import EstimatorBuilder, Experiment, TrainConf, EvalConf
from alps.io.reader.odps_reader import OdpsReader

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'    # for debug usage.
tf.logging.set_verbosity(tf.logging.INFO)

class SQLFlowEstimatorBuilder(EstimatorBuilder):
    def _build(self, experiment, run_config):
        return {{.ModelCreatorCode}}

if __name__ == "__main__":
    odpsConf=OdpsConf(
        accessid="{{.OdpsConf.AccessID}}",
        accesskey="{{.OdpsConf.AccessKey}}",
        endpoint="{{.OdpsConf.Endpoint}}"
    )

    trainDs = DatasetX(
        num_epochs={{.TrainClause.Epoch}},
        batch_size={{.TrainClause.BatchSize}},
        shuffle="{{.TrainClause.EnableShuffle}}" == "true",
        shuffle_buffer_size={{.TrainClause.ShuffleBufferSize}},
{{if .TrainClause.EnableCache}}
        cache_file={{.TrainClause.CachePath}},
{{end}}
        reader=OdpsReader(
            odps=odpsConf,
            project="{{.OdpsConf.Project}}",
            table="{{.TrainInputTable}}",
            field_names={{.Fields}},
            features={{.X}},
			labels={{.Y}},
{{if ne .FeatureMapTable ""}}
            feature_map=FeatureMap(table="{{.FeatureMapTable}}",
{{if ne .FeatureMapPartition ""}}
                partition="{{.FeatureMapPartition}}"
{{end}}
            )
{{end}}
        ),
        drop_remainder="{{.TrainClause.DropRemainder}}" == "true"
    )

    evalDs = DatasetX(
        num_epochs=1,
        batch_size={{.TrainClause.BatchSize}},
        reader=OdpsReader(
        odps=odpsConf,
            project="{{.OdpsConf.Project}}",
            table="{{.EvalInputTable}}",
            field_names={{.Fields}},
            features={{.X}},
            labels={{.Y}}
        )
    )

    export_path = "{{.ModelDir}}"

    experiment = Experiment(
        user="sqlflow",
        engine=LocalEngine(),
        train=TrainConf(input=trainDs,
{{if (ne .TrainClause.MaxSteps -1)}}
                        max_steps={{.TrainClause.MaxSteps}},
{{end}}
        ),
        eval=EvalConf(input=evalDs,
{{if (ne .TrainClause.EvalSteps -1)}}
                      steps={{.TrainClause.EvalSteps}},
{{end}}
                      start_delay_secs={{.TrainClause.EvalStartDelay}},
                      throttle_secs={{.TrainClause.EvalThrottle}},
        ),
        exporter=ArksExporter(deploy_path=export_path, strategy=ExportStrategy.BEST, compare_fn=Closure(best_auc_fn)),
        model_dir="{{.ScratchDir}}",
        model_builder=SQLFlowEstimatorBuilder())

    run_experiment(experiment)

`

var alpsTemplate = template.Must(template.New("alps").Parse(alpsTemplateText))
<|endoftext|>


<commit_before>// Package bitutils provides a collection of utilities to deal with bits.
package bitutils

import (
	"fmt"
	"strconv"
)

// W is the length of a machine word.
const W = 64

// Magic constants.
const (
	lowers2 = 0x5555555555555555
	lowers4 = 0x3333333333333333
	lowers8 = 0x0f0f0f0f0f0f0f0f
	lowest8 = 0x0101010101010101
)

// Word represents a 64-bit binary string.
type Word uint64

var (
	Shifted  [W + 1]Word // Shifted[i] has a 1 only at i.
	ShiftedC [W + 1]Word // ShiftedC[i] has a 0 only at i.
	Lowers   [W + 1]Word // Lowers[i] has 1s in its i LSBs.
	Uppers   [W + 1]Word // Uppers[i] has 1s in its i MSBs.
)

func init() {
	for i := 0; i < len(Shifted); i++ {
		Shifted[i] = Word(1) << uint(i)
		ShiftedC[i] = ^Shifted[i]
		Lowers[i] = Shifted[i] - 1
		Uppers[i] = Lowers[i] << uint(W-i)
	}
}

// ParseWord returns a Word from a string.
func ParseWord(s string) (Word, error) {
	w, err := strconv.ParseUint(s, 2, 64)
	return Word(w), err
}

// String returns binary string w[0]w[1]...w[63].
func (w Word) String() string {
	return fmt.Sprintf("%064b", w)
}

// Count1 returns the number of ones contained in w.
func (w Word) Count1() int {
	w -= (w >> 1) & lowers2
	w = (w & lowers4) + ((w >> 2) & lowers4)
	w = (w + (w >> 4)) & lowers8
	return int((w * lowest8) >> 56)
}

// Count0 returns the number of zeros contained in w.
func (w Word) Count0() int {
	w = ^w
	return w.Count1()
}

// Count returns the number of b[0]'s contained in w.
func (w Word) Count(b int) int {
	w = w ^ (^Word(0) + Word(b))
	return w.Count1()
}

// Get returns w[i].
func (w Word) Get(i int) Word {
	w = w >> uint(i)
	return w & Shifted[0]
}

// Set1 sets w[i] to 1.
func (w Word) Set1(i int) Word {
	return w | Shifted[i]
}

// Set0 sets w[i] to 0.
func (w Word) Set0(i int) Word {
	return w & ShiftedC[i]
}

// Flip flips w[i].
func (w Word) Flip(i int) Word {
	return w ^ Shifted[i]
}

// Least1 returns a word that indicates the least 1 in w.
func (w Word) Least1() Word {
	if w == 0 {
		return 0
	}
	w = ((w - 1) ^ w) & w
	return w
}

// LeastIndex1 returns the index of the least 1 in w if exists and -1
// otherwise.
func (w Word) LeastIndex1() int {
	if w == 0 {
		return -1
	}
	w = (w - 1) ^ w
	return w.Count1() - 1
}

// Rank1 returns the number of ones in w[0]...w[i].
func (w Word) Rank1(i int) int {
	w = w << uint(W-i-1)
	return w.Count1()
}

// Rank0 returns the number of zeros in w[0]...w[i].
func (w Word) Rank0(i int) int {
	w = ^w << uint(W-i-1)
	return w.Count1()
}
<commit_msg>Export some constants<commit_after>// Package bitutils provides a collection of utilities to deal with bits.
package bitutils

import (
	"fmt"
	"strconv"
)

// W is the length of a machine word.
const W = 64

// Magic constants.
const (
	Lowers2 = 0x5555555555555555
	Lowers4 = 0x3333333333333333
	Lowers8 = 0x0f0f0f0f0f0f0f0f
	Lowest8 = 0x0101010101010101
)

// Word represents a 64-bit binary string.
type Word uint64

var (
	Shifted  [W + 1]Word // Shifted[i] has a 1 only at i.
	ShiftedC [W + 1]Word // ShiftedC[i] has a 0 only at i.
	Lowers   [W + 1]Word // Lowers[i] has 1s in its i LSBs.
	Uppers   [W + 1]Word // Uppers[i] has 1s in its i MSBs.
)

func init() {
	for i := 0; i < len(Shifted); i++ {
		Shifted[i] = Word(1) << uint(i)
		ShiftedC[i] = ^Shifted[i]
		Lowers[i] = Shifted[i] - 1
		Uppers[i] = Lowers[i] << uint(W-i)
	}
}

// ParseWord returns a Word from a string.
func ParseWord(s string) (Word, error) {
	w, err := strconv.ParseUint(s, 2, 64)
	return Word(w), err
}

// String returns binary string w[0]w[1]...w[63].
func (w Word) String() string {
	return fmt.Sprintf("%064b", w)
}

// Count1 returns the number of ones contained in w.
func (w Word) Count1() int {
	w -= (w >> 1) & Lowers2
	w = (w & Lowers4) + ((w >> 2) & Lowers4)
	w = (w + (w >> 4)) & Lowers8
	return int((w * Lowest8) >> 56)
}

// Count0 returns the number of zeros contained in w.
func (w Word) Count0() int {
	w = ^w
	return w.Count1()
}

// Count returns the number of b[0]'s contained in w.
func (w Word) Count(b int) int {
	w = w ^ (^Word(0) + Word(b))
	return w.Count1()
}

// Get returns w[i].
func (w Word) Get(i int) Word {
	w = w >> uint(i)
	return w & Shifted[0]
}

// Set1 sets w[i] to 1.
func (w Word) Set1(i int) Word {
	return w | Shifted[i]
}

// Set0 sets w[i] to 0.
func (w Word) Set0(i int) Word {
	return w & ShiftedC[i]
}

// Flip flips w[i].
func (w Word) Flip(i int) Word {
	return w ^ Shifted[i]
}

// Least1 returns a word that indicates the least 1 in w.
func (w Word) Least1() Word {
	if w == 0 {
		return 0
	}
	w = ((w - 1) ^ w) & w
	return w
}

// LeastIndex1 returns the index of the least 1 in w if exists and -1
// otherwise.
func (w Word) LeastIndex1() int {
	if w == 0 {
		return -1
	}
	w = (w - 1) ^ w
	return w.Count1() - 1
}

// Rank1 returns the number of ones in w[0]...w[i].
func (w Word) Rank1(i int) int {
	w = w << uint(W-i-1)
	return w.Count1()
}

// Rank0 returns the number of zeros in w[0]...w[i].
func (w Word) Rank0(i int) int {
	w = ^w << uint(W-i-1)
	return w.Count1()
}
<|endoftext|>